@vellumai/assistant 0.11.4-dev.202608192308.82015f7 → 0.11.4-dev.202608200019.f77fe6a

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.11.4-dev.202608192308.82015f7",
+  "version": "0.11.4-dev.202608200019.f77fe6a",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/config/feature-flag-registry.json

@@ -145,7 +145,7 @@
       "scope": "client",
       "key": "fork-from-message",
       "label": "Fork from Message",
-      "description": "Show the 'Fork from here' option in message overflow menus",
+      "description": "Show the fork affordance: 'Fork from here' on messages and 'Fork Conversation' in the thread menu. Staff-only: the client also requires the viewer to be Vellum staff, so enabling this for a non-staff account still shows nothing.",
       "defaultEnabled": false
     },
     {

package/src/daemon/__tests__/provider-rejection-log-fields.test.ts

@@ -0,0 +1,272 @@
+/**
+ * Tests that an unclassified 4xx provider rejection logs the upstream
+ * response body (bounded) plus the offending request content-part coordinates,
+ * so the failing message can be identified from the daemon log alone.
+ */
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type { AssistantEvent } from "../../api/index.js";
+import { ProviderError } from "../../util/errors.js";
+import * as loggerModule from "../../util/logger.js";
+import type {
+  EventHandlerDeps,
+  EventHandlerState,
+} from "../conversation-agent-loop-handlers.js";
+import {
+  buildProviderRejectionLogFields,
+  MAX_LOGGED_UPSTREAM_BODY_CHARS,
+} from "../provider-rejection-log-fields.js";
+
+const logRecords: Record<string, unknown>[] = [];
+
+mock.module("../../util/logger.js", () => ({
+  ...loggerModule,
+  getLogger: () => ({
+    error: (record: Record<string, unknown>) => {
+      logRecords.push(record);
+    },
+    warn: () => {},
+    info: () => {},
+    debug: () => {},
+    trace: () => {},
+  }),
+}));
+
+// The handlers module binds its logger at import time, so it is loaded
+// dynamically once the logger stub is installed above.
+const { createEventHandlerState, dispatchAgentEvent } =
+  await import("../conversation-agent-loop-handlers.js");
+
+function createDeps(): EventHandlerDeps {
+  return {
+    ctx: {
+      conversationId: "conv-provider-rejection",
+      provider: { name: "openai" },
+      streamThinking: false,
+      emitActivityState: () => {},
+      markWorkspaceTopLevelDirty: () => {},
+      currentTurnSurfaces: [],
+    } as unknown as EventHandlerDeps["ctx"],
+    onEvent: (_msg: AssistantEvent) => {},
+    reqId: "req-provider-rejection",
+    isFirstMessage: false,
+    shouldGenerateTitle: false,
+    rlog: new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }) as unknown as EventHandlerDeps["rlog"],
+    turnChannelContext: {
+      userMessageChannel: "vellum",
+      assistantMessageChannel: "vellum",
+    } as EventHandlerDeps["turnChannelContext"],
+    turnInterfaceContext: {
+      userMessageInterface: "macos",
+      assistantMessageInterface: "macos",
+    } as EventHandlerDeps["turnInterfaceContext"],
+  } as EventHandlerDeps;
+}
+
+describe("buildProviderRejectionLogFields", () => {
+  test("carries the verbatim upstream body", () => {
+    // GIVEN a provider rejection that captured the upstream 400 body
+    const rawBody = JSON.stringify({
+      error: {
+        message: "Invalid body: failed to parse JSON value from the request.",
+        type: "invalid_request_error",
+        code: "invalid_body",
+      },
+    });
+    const error = new ProviderError(
+      "API error (400): Invalid body",
+      "openai",
+      400,
+      {
+        rawBody,
+        apiErrorCode: "invalid_body",
+        apiErrorType: "invalid_request_error",
+        requestId: "req_abc123",
+      },
+    );
+
+    // WHEN diagnostic log fields are built for it
+    const fields = buildProviderRejectionLogFields(error);
+
+    // THEN the body and the upstream metadata ride the record
+    expect(fields.upstreamErrorBody).toBe(rawBody);
+    expect(fields.upstreamErrorBodyTruncated).toBeUndefined();
+    expect(fields.apiErrorCode).toBe("invalid_body");
+    expect(fields.apiErrorType).toBe("invalid_request_error");
+    expect(fields.requestId).toBe("req_abc123");
+  });
+
+  test("bounds the body to 4 KB and flags the truncation", () => {
+    // GIVEN an upstream body larger than the logged ceiling
+    const rawBody = "x".repeat(MAX_LOGGED_UPSTREAM_BODY_CHARS + 500);
+    const error = new ProviderError("API error (400): huge", "openai", 400, {
+      rawBody,
+    });
+
+    // WHEN diagnostic log fields are built for it
+    const fields = buildProviderRejectionLogFields(error);
+
+    // THEN the logged body stops at the ceiling
+    expect(fields.upstreamErrorBody).toHaveLength(
+      MAX_LOGGED_UPSTREAM_BODY_CHARS,
+    );
+    // AND the record says the body was cut
+    expect(fields.upstreamErrorBodyTruncated).toBe(true);
+  });
+
+  test("parses the offending content-part coordinates for oversized strings", () => {
+    // GIVEN an oversized-content-part rejection naming the request position
+    const error = new ProviderError(
+      "API error (400): Invalid 'input[191].content[1].text': string too long. Expected a string with maximum length 10485760, but got a string with length 11436754 instead.",
+      "openai",
+      400,
+      { apiErrorCode: "string_above_max_length" },
+    );
+
+    // WHEN diagnostic log fields are built for it
+    const fields = buildProviderRejectionLogFields(error);
+
+    // THEN the message and content indices are logged for a follow-up query
+    expect(fields.offendingMessageIndex).toBe(191);
+    expect(fields.offendingContentIndex).toBe(1);
+  });
+
+  test("parses coordinates out of the Chat Completions `messages[N]` shape", () => {
+    // GIVEN a rejection using the Chat Completions request shape
+    const error = new ProviderError(
+      "API error (400): Invalid 'messages[7].content[0].text': string too long.",
+      "openai",
+      400,
+      { apiErrorCode: "string_above_max_length" },
+    );
+
+    // WHEN diagnostic log fields are built for it
+    const fields = buildProviderRejectionLogFields(error);
+
+    // THEN the same coordinates are extracted
+    expect(fields.offendingMessageIndex).toBe(7);
+    expect(fields.offendingContentIndex).toBe(0);
+  });
+
+  test("leaves coordinates absent for codes that carry no content-part pointer", () => {
+    // GIVEN a rejection whose code is outside the narrow tap
+    const error = new ProviderError(
+      "API error (400): Invalid 'input[3].content[0].text': something else.",
+      "openai",
+      400,
+      { apiErrorCode: "model_not_found" },
+    );
+
+    // WHEN diagnostic log fields are built for it
+    const fields = buildProviderRejectionLogFields(error);
+
+    // THEN no coordinates are inferred from a coincidental match
+    expect(fields.offendingMessageIndex).toBeUndefined();
+    expect(fields.offendingContentIndex).toBeUndefined();
+  });
+
+  test("scrubs secrets out of the upstream body", () => {
+    // GIVEN an upstream body that echoed back a credential
+    const projectKey = `sk-proj-${"A".repeat(48)}`;
+    const error = new ProviderError("API error (400): bad key", "openai", 400, {
+      rawBody: JSON.stringify({
+        error: { message: `Incorrect API key provided: ${projectKey}` },
+        headers: { authorization: "Bearer some-token-value" },
+      }),
+    });
+
+    // WHEN diagnostic log fields are built for it
+    const fields = buildProviderRejectionLogFields(error);
+
+    // THEN the credential never reaches the log record
+    expect(fields.upstreamErrorBody).not.toContain(projectKey);
+    expect(fields.upstreamErrorBody).not.toContain("some-token-value");
+    expect(fields.upstreamErrorBody).toContain("[REDACTED]");
+  });
+
+  test("reports a null body for errors that are not provider errors", () => {
+    // GIVEN a non-provider error
+    const error = new Error("socket hang up");
+
+    // WHEN diagnostic log fields are built for it
+    const fields = buildProviderRejectionLogFields(error);
+
+    // THEN the body is explicitly null rather than missing
+    expect(fields.upstreamErrorBody).toBeNull();
+  });
+});
+
+describe("unclassified 4xx log line", () => {
+  let state: EventHandlerState;
+
+  beforeEach(() => {
+    logRecords.length = 0;
+    state = createEventHandlerState();
+  });
+
+  test("includes the upstream response body", async () => {
+    // GIVEN an unclassified 4xx provider rejection carrying the upstream body
+    const rawBody = JSON.stringify({
+      error: {
+        message:
+          "Invalid body: failed to parse JSON value from the request. Common errors include trailing commas.",
+        type: "invalid_request_error",
+        code: "invalid_body",
+      },
+    });
+    const error = new ProviderError(
+      "API error (400): Invalid body: failed to parse JSON value from the request.",
+      "openai",
+      400,
+      { rawBody, apiErrorCode: "invalid_body" },
+    );
+
+    // WHEN the agent loop dispatches the error event
+    await dispatchAgentEvent(state, createDeps(), { type: "error", error });
+
+    // THEN the log record carries the body alongside the existing fields
+    const record = logRecords.find((r) => r.upstreamErrorBody !== undefined);
+    expect(record).toBeDefined();
+    expect(record?.upstreamErrorBody).toBe(rawBody);
+    expect(record?.conversationId).toBe("conv-provider-rejection");
+    expect(record?.statusCode).toBe(400);
+    expect(record?.provider).toBe("openai");
+  });
+
+  test("scrubs secrets out of the error message it logs", async () => {
+    // GIVEN a rejection whose normalized message echoes a credential
+    const error = new ProviderError(
+      "API error (400): rejected request with Authorization: Bearer some-token-value",
+      "openai",
+      400,
+      { rawBody: "{}" },
+    );
+
+    // WHEN the agent loop dispatches the error event
+    await dispatchAgentEvent(state, createDeps(), { type: "error", error });
+
+    // THEN the credential is redacted out of the logged message
+    const record = logRecords.find((r) => r.upstreamErrorBody !== undefined);
+    expect(record?.errorMessage).not.toContain("some-token-value");
+    expect(record?.errorMessage).toContain("Bearer [REDACTED]");
+  });
+
+  test("bounds a huge upstream body to 4 KB", async () => {
+    // GIVEN an unclassified 4xx rejection whose body exceeds the ceiling
+    const error = new ProviderError("API error (400): huge", "openai", 400, {
+      rawBody: "y".repeat(MAX_LOGGED_UPSTREAM_BODY_CHARS * 3),
+    });
+
+    // WHEN the agent loop dispatches the error event
+    await dispatchAgentEvent(state, createDeps(), { type: "error", error });
+
+    // THEN the logged body is capped
+    const record = logRecords.find((r) => r.upstreamErrorBody !== undefined);
+    expect(record?.upstreamErrorBody).toHaveLength(
+      MAX_LOGGED_UPSTREAM_BODY_CHARS,
+    );
+    expect(record?.upstreamErrorBodyTruncated).toBe(true);
+  });
+});

package/src/daemon/conversation-agent-loop-handlers.ts

@@ -89,6 +89,7 @@ import {
 } from "../usage/pricing.js";
 import { ProviderError } from "../util/errors.js";
 import { faviconUrlForDomain } from "../util/favicon.js";
+import { redactLogString } from "../util/log-redact.js";
 import { getLogger } from "../util/logger.js";
 import { withSqliteRetry } from "../util/sqlite-retry.js";
 import type { DirectiveRequest } from "./assistant-attachments.js";
@@ -143,6 +144,7 @@ import type {
   WebSearchResultItem,
 } from "./message-types/web-activity.js";
 import { referenceMediaBlocksForPersist } from "./persist-media-references.js";
+import { buildProviderRejectionLogFields } from "./provider-rejection-log-fields.js";
 import { turnOrRestingTrust } from "./trust-context-types.js";
 import type { TurnLatencyTracker } from "./turn-latency-tracker.js";
 
@@ -2708,7 +2710,8 @@ function handleError(
           event.error instanceof ProviderError
             ? event.error.provider
             : undefined,
-        errorMessage: event.error.message,
+        errorMessage: redactLogString(event.error.message),
+        ...buildProviderRejectionLogFields(event.error),
       },
       "Provider rejected request with unclassified 4xx error",
     );

package/src/daemon/provider-rejection-log-fields.ts

@@ -0,0 +1,124 @@
+/**
+ * Log-field builder for provider rejections that carry no actionable
+ * classification of their own (unclassified 4xx).
+ *
+ * The extracted fields on `ProviderError` (`apiErrorCode`, `message`) drop the
+ * detail an operator needs to find the offending request: which request field,
+ * which message, which byte pattern. The verbatim upstream body carries it, so
+ * it rides the log record, bounded so a large provider error page cannot flood
+ * the log stream.
+ */
+
+import { ProviderError } from "../util/errors.js";
+import { redactLogString } from "../util/log-redact.js";
+
+/**
+ * Ceiling for the logged upstream body. Bodies are already capped at 16 KiB
+ * when captured (`providers/openai/api-error-normalization.ts`); this second,
+ * tighter bound is what a log record carries.
+ */
+export const MAX_LOGGED_UPSTREAM_BODY_CHARS = 4096;
+
+/**
+ * Upstream error codes whose message names the offending request content part,
+ * e.g. `Invalid 'input[191].content[1].text': string too long`. Kept narrow:
+ * codes outside this set have no such pointer, so parsing their message would
+ * only produce coincidental matches.
+ */
+const CONTENT_PART_POINTER_CODES = new Set([
+  "string_above_max_length",
+  "invalid_body",
+]);
+
+/**
+ * Positional pointer into the request payload. `input[N]` is the Responses API
+ * shape, `messages[N]` the Chat Completions shape; both index the request's
+ * message array in send order, so `N` is an offset into the conversation
+ * history that produced the request.
+ */
+const CONTENT_PART_POINTER = /\b(?:input|messages)\[(\d+)\]\.content\[(\d+)\]/i;
+
+export interface ProviderRejectionLogFields {
+  /** Verbatim upstream non-2xx body, truncated to the logged ceiling. */
+  upstreamErrorBody: string | null;
+  /** True when the body was cut to fit the ceiling. */
+  upstreamErrorBodyTruncated?: boolean;
+  apiErrorCode?: string;
+  apiErrorType?: string;
+  apiErrorParam?: string;
+  requestId?: string;
+  /** Index of the offending message within the request's message array. */
+  offendingMessageIndex?: number;
+  /** Index of the offending content part within that message. */
+  offendingContentIndex?: number;
+}
+
+/**
+ * Extract the offending `input[N].content[M]` coordinates from an upstream
+ * error message, for the codes that carry such a pointer.
+ */
+function parseContentPartPointer(
+  error: ProviderError,
+): Pick<
+  ProviderRejectionLogFields,
+  "offendingMessageIndex" | "offendingContentIndex"
+> {
+  const code = error.apiErrorCode?.toLowerCase();
+  if (!code || !CONTENT_PART_POINTER_CODES.has(code)) {
+    return {};
+  }
+  const match = CONTENT_PART_POINTER.exec(
+    `${error.message} ${error.rawBody ?? ""}`,
+  );
+  if (!match) {
+    return {};
+  }
+  const messageIndex = Number(match[1]);
+  const contentIndex = Number(match[2]);
+  if (
+    !Number.isSafeInteger(messageIndex) ||
+    !Number.isSafeInteger(contentIndex)
+  ) {
+    return {};
+  }
+  return {
+    offendingMessageIndex: messageIndex,
+    offendingContentIndex: contentIndex,
+  };
+}
+
+/** Diagnostic fields describing what the provider actually rejected. */
+export function buildProviderRejectionLogFields(
+  error: Error,
+): ProviderRejectionLogFields {
+  if (!(error instanceof ProviderError)) {
+    return { upstreamErrorBody: null };
+  }
+
+  // The body is arbitrary upstream text under a custom field name, so it is
+  // outside the reach of the pino `err`/`req`/`res` serializers and is scrubbed
+  // here before it reaches the record.
+  const rawBody =
+    error.rawBody === undefined ? undefined : redactLogString(error.rawBody);
+  const truncated =
+    rawBody !== undefined && rawBody.length > MAX_LOGGED_UPSTREAM_BODY_CHARS;
+
+  return {
+    upstreamErrorBody:
+      rawBody === undefined
+        ? null
+        : rawBody.slice(0, MAX_LOGGED_UPSTREAM_BODY_CHARS),
+    ...(truncated ? { upstreamErrorBodyTruncated: true } : {}),
+    ...(error.apiErrorCode !== undefined
+      ? { apiErrorCode: error.apiErrorCode }
+      : {}),
+    ...(error.apiErrorType !== undefined
+      ? { apiErrorType: error.apiErrorType }
+      : {}),
+    ...(error.apiErrorParam !== undefined
+      ? { apiErrorParam: error.apiErrorParam }
+      : {}),
+    ...(error.requestId !== undefined ? { requestId: error.requestId } : {}),
+    ...parseContentPartPointer(error),
+  };
+}