@vellumai/assistant 0.8.1 → 0.8.3

package/src/permissions/question-prompter.test.ts

@@ -0,0 +1,416 @@
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type {
+  QuestionRequest,
+  ServerMessage,
+} from "../daemon/message-protocol.js";
+import type {
+  QuestionBatchSubmission,
+  QuestionPromptResult,
+} from "./question-prompter.js";
+
+// Use a tiny timeout so the setTimeout branch fires quickly in tests
+const mockConfig = {
+  timeouts: { permissionTimeoutSec: 0.05 },
+};
+mock.module("../config/loader.js", () => ({
+  getConfig: () => mockConfig,
+  loadConfig: () => mockConfig,
+  invalidateConfigCache: () => {},
+}));
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () => ({
+    info: () => {},
+    warn: () => {},
+    error: () => {},
+    debug: () => {},
+    trace: () => {},
+    fatal: () => {},
+    child: () => ({
+      info: () => {},
+      warn: () => {},
+      error: () => {},
+      debug: () => {},
+    }),
+  }),
+}));
+
+// Use a real Map so QuestionPrompter can store and retrieve callbacks.
+interface MockInteraction {
+  rpcResolve?: (v: unknown) => void;
+  rpcReject?: (e: unknown) => void;
+  timer?: ReturnType<typeof setTimeout>;
+  metadata?: {
+    orderedIds: string[];
+    optionsById: Record<string, string[]>;
+  };
+}
+const _piStore = new Map<string, MockInteraction>();
+mock.module("../runtime/pending-interactions.js", () => ({
+  register: (id: string, entry: MockInteraction) => _piStore.set(id, entry),
+  resolve: (id: string) => {
+    const e = _piStore.get(id);
+    if (e?.timer != null) clearTimeout(e.timer);
+    _piStore.delete(id);
+    return e;
+  },
+  get: (id: string) => _piStore.get(id),
+  getAll: () => [..._piStore.values()],
+  getByConversation: () => [],
+  getByKind: () => [],
+  removeByConversation: () => {},
+  clear: () => _piStore.clear(),
+}));
+
+const {
+  QuestionPrompter,
+  QuestionBatchValidationError,
+  buildBatchEntries,
+} = await import("./question-prompter.js");
+
+function makePrompter() {
+  const sent: ServerMessage[] = [];
+  const prompter = new QuestionPrompter({
+    broadcastMessage: (msg) => sent.push(msg),
+  });
+  return { prompter, sent };
+}
+
+/**
+ * Drive a pending question interaction the same way the
+ * `/v1/question-response` route does: look up the metadata, run
+ * `buildBatchEntries`, deregister, then fire `rpcResolve`. Centralizing the
+ * sequence in one helper keeps the tests focused on observable behavior and
+ * mirrors the production resolution path.
+ */
+function resolveBatch(
+  requestId: string,
+  submissions: QuestionBatchSubmission[],
+): QuestionPromptResult {
+  const interaction = _piStore.get(requestId);
+  if (!interaction?.metadata) {
+    throw new Error(`No pending question interaction for ${requestId}`);
+  }
+  const { orderedIds, optionsById } = interaction.metadata;
+  const entries = buildBatchEntries(
+    orderedIds,
+    (qid, oid) => (optionsById[qid] ?? []).includes(oid),
+    new Set(Object.keys(optionsById)),
+    submissions,
+  );
+  const result: QuestionPromptResult = { entries, overall: "completed" };
+  if (interaction.timer != null) clearTimeout(interaction.timer);
+  _piStore.delete(requestId);
+  interaction.rpcResolve?.(result);
+  return result;
+}
+
+/**
+ * Close a pending question card: every entry reported as `skipped`, overall
+ * status `closed`. Mirrors the route's `kind: "close"` branch.
+ */
+function closeBatch(requestId: string): QuestionPromptResult {
+  const interaction = _piStore.get(requestId);
+  if (!interaction?.metadata) {
+    throw new Error(`No pending question interaction for ${requestId}`);
+  }
+  const result: QuestionPromptResult = {
+    entries: interaction.metadata.orderedIds.map((id) => ({
+      questionId: id,
+      decision: "skipped" as const,
+    })),
+    overall: "closed",
+  };
+  if (interaction.timer != null) clearTimeout(interaction.timer);
+  _piStore.delete(requestId);
+  interaction.rpcResolve?.(result);
+  return result;
+}
+
+const fruitOptions = [
+  { id: "a", label: "Apple" },
+  { id: "b", label: "Banana" },
+];
+
+const singleQuestionParams = {
+  conversationId: "conv-1",
+  questions: [
+    {
+      question: "Pick one",
+      options: fruitOptions,
+    },
+  ],
+};
+
+const threeQuestionParams = {
+  conversationId: "conv-1",
+  questions: [
+    { question: "Q1?", options: fruitOptions },
+    {
+      question: "Q2?",
+      options: [
+        { id: "x", label: "X" },
+        { id: "y", label: "Y" },
+      ],
+    },
+    {
+      question: "Q3?",
+      options: [
+        { id: "p", label: "P" },
+        { id: "q", label: "Q" },
+      ],
+      freeTextPlaceholder: "or type",
+    },
+  ],
+};
+
+describe("QuestionPrompter", () => {
+  beforeEach(() => {
+    _piStore.clear();
+  });
+
+  test("happy path: option resolution via the shared batch helpers", async () => {
+    const { prompter, sent } = makePrompter();
+
+    const promise = prompter.prompt(singleQuestionParams);
+
+    expect(sent).toHaveLength(1);
+    const req = sent[0] as QuestionRequest;
+    expect(req.type).toBe("question_request");
+    expect(req.questions).toHaveLength(1);
+    expect(req.questions[0]?.id).toBe("q1");
+
+    resolveBatch(req.requestId, [
+      { questionId: "q1", kind: "option", optionId: "a" },
+    ]);
+
+    const result = await promise;
+    expect(result).toEqual({
+      entries: [{ questionId: "q1", decision: "option", optionId: "a" }],
+      overall: "completed",
+    });
+    expect(_piStore.has(req.requestId)).toBe(false);
+  });
+
+  test("free-text resolution", async () => {
+    const { prompter, sent } = makePrompter();
+
+    const promise = prompter.prompt({
+      conversationId: "conv-1",
+      questions: [
+        {
+          question: "Pick one",
+          options: fruitOptions,
+          freeTextPlaceholder: "Type a fruit",
+        },
+      ],
+    });
+
+    const req = sent[0] as QuestionRequest;
+    expect(req.freeTextPlaceholder).toBe("Type a fruit");
+    expect(req.questions[0]?.freeTextPlaceholder).toBe("Type a fruit");
+
+    resolveBatch(req.requestId, [
+      { questionId: "q1", kind: "free_text", text: "Cherry" },
+    ]);
+
+    const result = await promise;
+    expect(result).toEqual({
+      entries: [{ questionId: "q1", decision: "free_text", text: "Cherry" }],
+      overall: "completed",
+    });
+  });
+
+  test("batched broadcast: assigns sequential q1..qN ids and mirrors questions[0] in flat fields", async () => {
+    const { prompter, sent } = makePrompter();
+
+    void prompter.prompt(threeQuestionParams);
+
+    expect(sent).toHaveLength(1);
+    const req = sent[0] as QuestionRequest;
+    expect(req.questions.map((q) => q.id)).toEqual(["q1", "q2", "q3"]);
+    // Flat fields mirror the first entry for backwards compat.
+    expect(req.question).toBe("Q1?");
+    expect(req.options).toEqual(fruitOptions);
+  });
+
+  test("three-question batch: two options + one free-text → ordered entries", async () => {
+    const { prompter, sent } = makePrompter();
+
+    const promise = prompter.prompt(threeQuestionParams);
+    const req = sent[0] as QuestionRequest;
+
+    resolveBatch(req.requestId, [
+      { questionId: "q2", kind: "option", optionId: "y" },
+      { questionId: "q1", kind: "option", optionId: "a" },
+      { questionId: "q3", kind: "free_text", text: "noon-ish" },
+    ]);
+
+    const result = await promise;
+    expect(result.overall).toBe("completed");
+    // Result entries are in the original questions[] order, regardless of
+    // the order submissions arrive in.
+    expect(result.entries).toEqual([
+      { questionId: "q1", decision: "option", optionId: "a" },
+      { questionId: "q2", decision: "option", optionId: "y" },
+      { questionId: "q3", decision: "free_text", text: "noon-ish" },
+    ]);
+  });
+
+  test("three-question batch: all skipped via submitted entries", async () => {
+    const { prompter, sent } = makePrompter();
+
+    const promise = prompter.prompt(threeQuestionParams);
+    const req = sent[0] as QuestionRequest;
+
+    resolveBatch(req.requestId, [
+      { questionId: "q1", kind: "skip" },
+      { questionId: "q2", kind: "skip" },
+      { questionId: "q3", kind: "skip" },
+    ]);
+
+    const result = await promise;
+    expect(result.overall).toBe("completed");
+    expect(result.entries.every((e) => e.decision === "skipped")).toBe(true);
+  });
+
+  test("close path: all entries skipped with overall=closed", async () => {
+    const { prompter, sent } = makePrompter();
+
+    const promise = prompter.prompt(threeQuestionParams);
+    const req = sent[0] as QuestionRequest;
+
+    closeBatch(req.requestId);
+
+    const result = await promise;
+    expect(result.overall).toBe("closed");
+    expect(result.entries).toEqual([
+      { questionId: "q1", decision: "skipped" },
+      { questionId: "q2", decision: "skipped" },
+      { questionId: "q3", decision: "skipped" },
+    ]);
+  });
+
+  test("buildBatchEntries rejects unknown questionId", () => {
+    expect(() =>
+      buildBatchEntries(
+        ["q1"],
+        () => true,
+        new Set(["q1"]),
+        [{ questionId: "qX", kind: "option", optionId: "a" }],
+      ),
+    ).toThrow(QuestionBatchValidationError);
+  });
+
+  test("buildBatchEntries rejects missing entry", () => {
+    expect(() =>
+      buildBatchEntries(
+        ["q1", "q2", "q3"],
+        () => true,
+        new Set(["q1", "q2", "q3"]),
+        [
+          { questionId: "q1", kind: "option", optionId: "a" },
+          { questionId: "q2", kind: "option", optionId: "x" },
+        ],
+      ),
+    ).toThrow(QuestionBatchValidationError);
+  });
+
+  test("buildBatchEntries rejects unknown optionId", () => {
+    expect(() =>
+      buildBatchEntries(
+        ["q1"],
+        (qid, oid) => qid === "q1" && (oid === "a" || oid === "b"),
+        new Set(["q1"]),
+        [{ questionId: "q1", kind: "option", optionId: "nope" }],
+      ),
+    ).toThrow(QuestionBatchValidationError);
+  });
+
+  test("timeout fires with overall: timed_out and timed_out entries", async () => {
+    const { prompter } = makePrompter();
+    const result = await prompter.prompt(threeQuestionParams);
+    expect(result.overall).toBe("timed_out");
+    expect(result.entries).toEqual([
+      { questionId: "q1", decision: "timed_out" },
+      { questionId: "q2", decision: "timed_out" },
+      { questionId: "q3", decision: "timed_out" },
+    ]);
+  });
+
+  test("abort signal triggers overall: aborted with per-entry aborted decisions", async () => {
+    const { prompter, sent } = makePrompter();
+    const ac = new AbortController();
+
+    const promise = prompter.prompt({
+      ...threeQuestionParams,
+      signal: ac.signal,
+    });
+    const req = sent[0] as QuestionRequest;
+    expect(_piStore.has(req.requestId)).toBe(true);
+
+    ac.abort();
+    const result = await promise;
+    expect(result.overall).toBe("aborted");
+    expect(result.entries).toEqual([
+      { questionId: "q1", decision: "aborted" },
+      { questionId: "q2", decision: "aborted" },
+      { questionId: "q3", decision: "aborted" },
+    ]);
+    expect(_piStore.has(req.requestId)).toBe(false);
+  });
+
+  test("abort after removeByConversation still resolves the Promise (no hang)", async () => {
+    // Regression test for the race exposed by post-merge review of #30581:
+    // when `removeByConversation()` (auto-deny on enqueue) deregisters the
+    // question interaction before the abort signal fires, the abort handler
+    // must still resolve the prompt Promise. Previously, the handler used
+    // `pendingInteractions.resolve(id) === undefined` as the idempotency
+    // guard — which returned `undefined` after the registry was cleared,
+    // causing the handler to early-return and the Promise to hang forever.
+    // Now an internal `settled` flag guards every resolution path.
+    const { prompter, sent } = makePrompter();
+    const ac = new AbortController();
+
+    const promise = prompter.prompt({
+      ...threeQuestionParams,
+      signal: ac.signal,
+    });
+    const req = sent[0] as QuestionRequest;
+    expect(_piStore.has(req.requestId)).toBe(true);
+
+    // Simulate `removeByConversation` clearing the registry entry before
+    // the abort signal fires.
+    const interaction = _piStore.get(req.requestId);
+    if (interaction?.timer != null) clearTimeout(interaction.timer);
+    _piStore.delete(req.requestId);
+
+    ac.abort();
+    const result = await promise;
+    expect(result.overall).toBe("aborted");
+    expect(result.entries).toEqual([
+      { questionId: "q1", decision: "aborted" },
+      { questionId: "q2", decision: "aborted" },
+      { questionId: "q3", decision: "aborted" },
+    ]);
+  });
+
+  test("pre-aborted signal short-circuits before broadcasting with aborted entries", async () => {
+    const { prompter, sent } = makePrompter();
+    const ac = new AbortController();
+    ac.abort();
+
+    const result = await prompter.prompt({
+      ...threeQuestionParams,
+      signal: ac.signal,
+    });
+    expect(result.overall).toBe("aborted");
+    expect(result.entries).toEqual([
+      { questionId: "q1", decision: "aborted" },
+      { questionId: "q2", decision: "aborted" },
+      { questionId: "q3", decision: "aborted" },
+    ]);
+    expect(sent).toHaveLength(0);
+  });
+});

package/src/permissions/question-prompter.ts

@@ -0,0 +1,294 @@
+import { v4 as uuid } from "uuid";
+
+import { getConfig } from "../config/loader.js";
+import type {
+  QuestionOption,
+  QuestionRequest,
+  ServerMessage,
+} from "../daemon/message-protocol.js";
+import * as pendingInteractions from "../runtime/pending-interactions.js";
+import { AssistantError, ErrorCode } from "../util/errors.js";
+import { getLogger } from "../util/logger.js";
+
+const log = getLogger("question-prompter");
+
+/**
+ * Thrown when a batched submission fails validation (unknown questionId,
+ * missing entries, unknown optionId, duplicate questionId). The route layer
+ * maps this to a 400.
+ */
+export class QuestionBatchValidationError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "QuestionBatchValidationError";
+  }
+}
+
+/**
+ * One per-question entry in a batched question result.
+ *
+ * `decision` records how the user responded to that specific question:
+ *  - `"option"` / `"free_text"` — direct answer.
+ *  - `"skipped"` — the user explicitly skipped this question, or the card
+ *    was closed before any answer was submitted.
+ *  - `"timed_out"` — the prompt timer fired before the client submitted.
+ *  - `"aborted"` — the prompter's abort signal fired before any answer
+ *    was submitted.
+ *
+ * `questionId` matches the daemon-assigned id (`q1`, `q2`...) that the
+ * prompter attached to the broadcast.
+ */
+export interface QuestionPromptEntryResult {
+  questionId: string;
+  decision: "option" | "free_text" | "skipped" | "timed_out" | "aborted";
+  optionId?: string;
+  text?: string;
+}
+
+/**
+ * Aggregate result for a single `prompt()` call. `entries` is ordered to
+ * match the original `questions` array; `overall` summarizes how the
+ * card lifecycle ended.
+ */
+export interface QuestionPromptResult {
+  entries: QuestionPromptEntryResult[];
+  overall: "completed" | "closed" | "timed_out" | "aborted";
+}
+
+export interface QuestionPromptParamsEntry {
+  question: string;
+  description?: string;
+  options: QuestionOption[];
+  freeTextPlaceholder?: string;
+}
+
+export interface QuestionPromptParams {
+  conversationId: string;
+  /** One or more clarifying questions to broadcast as a single card. */
+  questions: QuestionPromptParamsEntry[];
+  toolUseId?: string;
+  signal?: AbortSignal;
+}
+
+/** One per-question submission inside a batch from the client. */
+export type QuestionBatchSubmission =
+  | { questionId: string; kind: "option"; optionId: string }
+  | { questionId: string; kind: "free_text"; text: string }
+  | { questionId: string; kind: "skip" };
+
+/**
+ * Validate a batched submission against the original ordered ids and per-id
+ * option-id sets, and return the ordered per-entry result. The lookup helpers
+ * are passed in so callers can back the metadata with whatever container
+ * they prefer (Set/Map, plain Record, etc.).
+ *
+ * Throws {@link QuestionBatchValidationError} if validation fails.
+ */
+export function buildBatchEntries(
+  orderedIds: readonly string[],
+  isKnownOption: (questionId: string, optionId: string) => boolean,
+  knownQuestionIds: ReadonlySet<string>,
+  submissions: readonly QuestionBatchSubmission[],
+): QuestionPromptEntryResult[] {
+  const submittedIds = new Set<string>();
+  for (const s of submissions) {
+    if (!knownQuestionIds.has(s.questionId)) {
+      throw new QuestionBatchValidationError(
+        `Unknown questionId in batch: ${s.questionId}`,
+      );
+    }
+    if (submittedIds.has(s.questionId)) {
+      throw new QuestionBatchValidationError(
+        `Duplicate questionId in batch: ${s.questionId}`,
+      );
+    }
+    submittedIds.add(s.questionId);
+    if (s.kind === "option" && !isKnownOption(s.questionId, s.optionId)) {
+      throw new QuestionBatchValidationError(
+        `Unknown optionId "${s.optionId}" for question ${s.questionId}`,
+      );
+    }
+  }
+  for (const id of orderedIds) {
+    if (!submittedIds.has(id)) {
+      throw new QuestionBatchValidationError(
+        `Missing response for questionId ${id}`,
+      );
+    }
+  }
+
+  const byId = new Map<string, QuestionBatchSubmission>();
+  for (const s of submissions) byId.set(s.questionId, s);
+
+  return orderedIds.map((id) => {
+    const s = byId.get(id)!;
+    if (s.kind === "option") {
+      return { questionId: id, decision: "option", optionId: s.optionId };
+    }
+    if (s.kind === "free_text") {
+      return { questionId: id, decision: "free_text", text: s.text };
+    }
+    return { questionId: id, decision: "skipped" };
+  });
+}
+
+/**
+ * Shape of the per-batch bookkeeping stashed on `PendingInteraction.metadata`.
+ * The route reads this to validate batched submissions without needing a
+ * reference to the prompter that registered them.
+ */
+export interface QuestionBatchMetadata {
+  orderedIds: string[];
+  optionsById: Record<string, string[]>;
+}
+
+/**
+ * Broadcast an ask-question request to all connected clients and wait for the
+ * user's reply. All lifecycle state (rpcResolve, rpcReject, timer, batch
+ * metadata) lives on the `pendingInteractions` entry — `/v1/question-response`
+ * resolves the entry directly without holding a reference back to the prompter
+ * that registered it.
+ *
+ * Batching: a single `prompt()` call broadcasts one or more questions, and
+ * the prompter waits for exactly one resolution call carrying the full
+ * ordered response array. The web UI collects per-question answers
+ * locally, lets the user revise freely while the card is open, and POSTs
+ * the whole batch to `/v1/question-response` when the user is done — no
+ * per-question accumulator, no partial state machine.
+ *
+ * Timeout reuses `getConfig().timeouts.permissionTimeoutSec` (default 5 min) —
+ * questions are user-prompts in the same UX family as permission prompts and
+ * secret prompts, so they share the same idle-timeout knob.
+ */
+export class QuestionPrompter {
+  constructor(
+    private deps: { broadcastMessage(msg: ServerMessage): void },
+  ) {}
+
+  async prompt(params: QuestionPromptParams): Promise<QuestionPromptResult> {
+    const { conversationId, questions, toolUseId, signal } = params;
+
+    if (questions.length === 0) {
+      throw new AssistantError(
+        "QuestionPrompter.prompt requires at least one question",
+        ErrorCode.INTERNAL_ERROR,
+      );
+    }
+
+    // Assign per-question ids (`q1`, `q2`, ...) — daemon-side only; the LLM
+    // never sees these. Build the on-wire entries in the same pass.
+    const entries = questions.map((q, i) => ({
+      id: `q${i + 1}`,
+      question: q.question,
+      description: q.description,
+      options: q.options,
+      freeTextPlaceholder: q.freeTextPlaceholder,
+    }));
+    const orderedIds = entries.map((e) => e.id);
+    const optionsById: Record<string, string[]> = {};
+    for (const e of entries) {
+      optionsById[e.id] = e.options.map((o) => o.id);
+    }
+
+    if (signal?.aborted) {
+      return {
+        entries: orderedIds.map((id) => ({
+          questionId: id,
+          decision: "aborted",
+        })),
+        overall: "aborted",
+      };
+    }
+
+    const requestId = uuid();
+
+    return new Promise<QuestionPromptResult>((resolve, reject) => {
+      const timeoutMs = getConfig().timeouts.permissionTimeoutSec * 1000;
+
+      // Closure-scoped idempotency guard. Every resolution path (timeout,
+      // abort, route resolution via `rpcResolve`/`rpcReject`) routes through
+      // `finish()`, which tears down the timer + abort listener exactly
+      // once. We cannot use `pendingInteractions.resolve(requestId) ===
+      // undefined` as the guard because `removeByConversation()` (called
+      // during auto-deny on enqueue) can deregister the entry before any of
+      // our local handlers fire — using the registry as the guard in that
+      // case would leave the Promise unresolved and the tool hung.
+      let settled = false;
+      let onAbort: (() => void) | undefined;
+      const finish = (fn: () => void): void => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        if (signal && onAbort) {
+          signal.removeEventListener("abort", onAbort);
+        }
+        // Idempotent: a no-op if the entry was already removed (e.g. by
+        // `removeByConversation`) or by an earlier path.
+        pendingInteractions.resolve(requestId);
+        fn();
+      };
+
+      const timer = setTimeout(() => {
+        log.warn({ requestId, conversationId }, "Question prompt timed out");
+        finish(() =>
+          resolve({
+            entries: orderedIds.map((id) => ({
+              questionId: id,
+              decision: "timed_out",
+            })),
+            overall: "timed_out",
+          }),
+        );
+      }, timeoutMs);
+
+      if (signal) {
+        onAbort = () => {
+          finish(() =>
+            resolve({
+              entries: orderedIds.map((id) => ({
+                questionId: id,
+                decision: "aborted",
+              })),
+              overall: "aborted",
+            }),
+          );
+        };
+        signal.addEventListener("abort", onAbort, { once: true });
+      }
+
+      // Stash the per-question metadata on the interaction so the route can
+      // validate batched submissions without holding a prompter reference.
+      // Route resolution funnels through `finish()` so the same teardown +
+      // idempotency guard applies whether the response comes from the route,
+      // a timeout, or an abort.
+      pendingInteractions.register(requestId, {
+        conversationId,
+        kind: "question",
+        rpcResolve: (value: unknown) =>
+          finish(() => resolve(value as QuestionPromptResult)),
+        rpcReject: (err: unknown) => finish(() => reject(err)),
+        timer,
+        toolUseId,
+        metadata: { orderedIds, optionsById } satisfies QuestionBatchMetadata,
+      });
+
+      // Populate both shapes on the wire: `questions[]` is the canonical
+      // batched payload, and the flat fields mirror `questions[0]` for
+      // backwards compat with clients that haven't adopted `questions[]`.
+      const head = entries[0]!;
+      const msg: QuestionRequest = {
+        type: "question_request",
+        requestId,
+        questions: entries,
+        question: head.question,
+        description: head.description,
+        options: head.options,
+        freeTextPlaceholder: head.freeTextPlaceholder,
+        conversationId,
+        toolUseId,
+      };
+
+      this.deps.broadcastMessage(msg);
+    });
+  }
+}