@vellumai/assistant 0.11.2 → 0.11.3-dev.202608080427.92f668a

package/src/tts/speakable-segments.ts

@@ -7,20 +7,90 @@
  * as soon as it is complete, so speech starts before the full response lands.
  */
 
+import { isHighSurrogate, isLowSurrogate } from "../util/unicode.js";
+
 const DEFAULT_CHAR_THRESHOLD = 180;
 const EAGER_CHAR_THRESHOLD = 60;
 
-const SENTENCE_ENDING_PUNCTUATION = new Set([".", "!", "?"]);
+// Sentence enders in scripts without inter-word whitespace (CJK, Thai) or
+// with their own terminators (Devanagari danda, Arabic-script marks). Unlike
+// ASCII `. ! ?`, these are unambiguous: a boundary after one is valid even
+// with no following whitespace.
+export const NON_LATIN_SENTENCE_ENDING_PUNCTUATION = new Set([
+  "。", // ideographic full stop
+  "｡", // halfwidth ideographic full stop
+  "！", // fullwidth exclamation mark
+  "？", // fullwidth question mark
+  "．", // fullwidth full stop (digit-guarded below: also a decimal point)
+  "।", // Devanagari danda
+  "॥", // Devanagari double danda
+  "؟", // Arabic question mark
+  "۔", // Arabic full stop
+]);
+const SENTENCE_ENDING_PUNCTUATION = new Set([
+  ".",
+  "!",
+  "?",
+  ...NON_LATIN_SENTENCE_ENDING_PUNCTUATION,
+]);
+// Unlike the other non-Latin enders, the fullwidth full stop also serves as
+// a decimal point in fullwidth numbers (３．１４), so it is not a boundary
+// when a digit follows. The ideographic full stop 。 never marks decimals
+// and stays unconditional.
+const FULLWIDTH_FULL_STOP = "．";
 const TRAILING_SENTENCE_PUNCTUATION = new Set(['"', "'", ")", "]"]);
-const EAGER_CLAUSE_PUNCTUATION = new Set([",", ";", ":"]);
+// After a non-Latin ender, locale closers (CJK corner brackets, curly
+// quotes, guillemets) belong to the sentence they follow, exactly like the
+// ASCII closers above; without this, 「こんにちは。」 would orphan 」 into
+// the next segment. Superset of the ASCII closers so mixed-script text still
+// consumes those too.
+const NON_LATIN_TRAILING_SENTENCE_PUNCTUATION = new Set([
+  ...TRAILING_SENTENCE_PUNCTUATION,
+  "」",
+  "｣", // halfwidth corner bracket closer
+  "』",
+  "）",
+  "】",
+  "〕",
+  "〙",
+  "〗",
+  "〛",
+  "》",
+  "〉",
+  "］",
+  "｝",
+  "”",
+  "’",
+  "»",
+]);
+// Clause punctuation in whitespace-free scripts is likewise a valid eager
+// boundary regardless of what follows.
+const NON_LATIN_EAGER_CLAUSE_PUNCTUATION = new Set([
+  "、", // ideographic comma
+  "､", // halfwidth ideographic comma
+  "，", // fullwidth comma
+  "；", // fullwidth semicolon
+  "،", // Arabic comma
+]);
+const EAGER_CLAUSE_PUNCTUATION = new Set([
+  ",",
+  ";",
+  ":",
+  ...NON_LATIN_EAGER_CLAUSE_PUNCTUATION,
+]);
+// The fullwidth comma also serves as a thousands separator in fullwidth
+// numbers (１，０００円), so it is not a clause boundary inside a number. The
+// ideographic comma 、 never appears in numbers and needs no guard.
+const FULLWIDTH_COMMA = "，";
 // A clause boundary only counts in eager mode once this much text precedes
 // it — "Sure, " alone would be a one-word blip.
 const EAGER_MIN_CLAUSE_PREFIX_CHARS = 24;
 
 export interface ExtractSpeakableSegmentsOptions {
   /**
-   * Trade segment quality for onset latency: clause punctuation (`,` `;` `:`)
-   * followed by whitespace also ends a segment once at least
+   * Trade segment quality for onset latency: clause punctuation (`,` `;` `:`
+   * followed by whitespace, or a non-Latin clause mark like `、` `，` `،`
+   * regardless of what follows) also ends a segment once at least
    * ~24 chars precede it, and the max buffered length before a forced split
    * drops from 180 to 60 chars. Applies only until the
    * first segment of the call is found — later segments keep full-sentence
@@ -79,6 +149,27 @@ function findSpeakableBoundary(
   let inItalic = false;
   let inUnderscore = false;
   let skipSpanChar = false;
+  // Inside a Markdown link URL (`](...)`), non-Latin terminators are path
+  // characters, not sentence enders: splitting there leaves both fragments
+  // unmatchable by the sanitizer's complete-link pattern, so markup or the
+  // raw URL gets spoken. Tracked as a paren depth, not a boolean, because
+  // the sanitizer's link pattern accepts balanced parens inside the URL
+  // (https://example.com/a_(b)/c) and the suppression must cover the same
+  // links. 0 = not inside a URL.
+  let linkUrlDepth = 0;
+  // Inside a potential Markdown link label (`[label](url)`, `![alt](url)`),
+  // between `[` and `](`. Terminators there are label text, and a boundary
+  // splits the link the same way a URL split does. The state resolves
+  // deterministically so a stray `[` in prose cannot suppress boundaries
+  // forever: `](` hands off to the URL state; `]` without `(` or a newline
+  // cancels the label, splitting at the first boundary it suppressed; a
+  // buffer-final `]` or an unterminated label keeps buffering (the rest may
+  // arrive in the next delta), bounded by the force flush and the length
+  // cap.
+  let inLinkLabel = false;
+  // First boundary suppressed while the current label is open; returned when
+  // the label turns out not to be a link.
+  let suppressedLabelBoundary: number | null = null;
 
   for (let index = 0; index < text.length; index += 1) {
     const char = text[index];
@@ -87,8 +178,54 @@ function findSpeakableBoundary(
     }
     const inOpenSpan = inBacktick || inBold || inItalic || inUnderscore;
 
-    if (!inOpenSpan && char === "\n") {
-      return index + 1;
+    if (char === "\n") {
+      // A URL never spans a line: an unclosed link stops suppressing here so
+      // malformed markup cannot defer the newline split. Labels do not span
+      // lines either; a suppressed boundary reopens ahead of the newline.
+      linkUrlDepth = 0;
+      inLinkLabel = false;
+      if (suppressedLabelBoundary !== null) {
+        return suppressedLabelBoundary;
+      }
+      if (!inOpenSpan) {
+        return index + 1;
+      }
+    }
+
+    if (linkUrlDepth > 0) {
+      if (char === "(") {
+        linkUrlDepth += 1;
+      } else if (char === ")") {
+        linkUrlDepth -= 1;
+      }
+      continue;
+    }
+    if (char === "]" && inLinkLabel && text[index + 1] !== "(") {
+      if (index + 1 === text.length) {
+        // A buffer-final `]` is ambiguous while streaming: the `(` may open
+        // the next delta. Keep buffering; force and the length cap below
+        // still flush.
+        break;
+      }
+      // Not a link after all: the label text is plain prose, so the first
+      // boundary it suppressed becomes the split point again.
+      inLinkLabel = false;
+      if (suppressedLabelBoundary !== null) {
+        return suppressedLabelBoundary;
+      }
+    }
+    if (char === "]" && text[index + 1] === "(") {
+      inLinkLabel = false;
+      suppressedLabelBoundary = null;
+      linkUrlDepth = 1;
+      // Skip the opening paren so it does not double-count the depth.
+      index += 1;
+      continue;
+    }
+    // A `[` inside an open inline span is literal text (e.g. inline code
+    // containing an array literal), never a link-label opener.
+    if (char === "[" && !inLinkLabel && !inOpenSpan) {
+      inLinkLabel = true;
     }
 
     if (
@@ -96,21 +233,75 @@ function findSpeakableBoundary(
       eager &&
       EAGER_CLAUSE_PUNCTUATION.has(char) &&
       index >= EAGER_MIN_CLAUSE_PREFIX_CHARS &&
-      isWhitespace(text[index + 1] ?? "")
+      (NON_LATIN_EAGER_CLAUSE_PUNCTUATION.has(char) ||
+        isWhitespace(text[index + 1] ?? "")) &&
+      !isFullwidthGroupingComma(text, index)
     ) {
-      return index + 1;
+      if (inLinkLabel) {
+        suppressedLabelBoundary ??= index + 1;
+      } else {
+        return index + 1;
+      }
     }
 
     if (!inOpenSpan && SENTENCE_ENDING_PUNCTUATION.has(char)) {
-      let boundary = index + 1;
-      while (
-        boundary < text.length &&
-        TRAILING_SENTENCE_PUNCTUATION.has(text[boundary] ?? "")
-      ) {
-        boundary += 1;
+      const isNonLatinEnder = NON_LATIN_SENTENCE_ENDING_PUNCTUATION.has(char);
+      // The fullwidth full stop doubles as a decimal point in fullwidth
+      // numbers (３．１４です), so a following digit suppresses the boundary.
+      // It also appears inside Latin filename-style tokens (report．pdf):
+      // ASCII word chars on both sides mark it as part of the token, not a
+      // sentence end. CJK neighbors keep the boundary, since Japanese
+      // sentences legitimately end in ． between CJK words.
+      const isAmbiguousDecimalPoint =
+        char === FULLWIDTH_FULL_STOP &&
+        (isDecimalDigit(text[index + 1]) ||
+          (isAsciiWordChar(text[index - 1]) &&
+            isAsciiWordChar(text[index + 1])));
+      if (isNonLatinEnder && !isAmbiguousDecimalPoint) {
+        // Non-Latin enders are valid boundaries regardless of what follows;
+        // consume adjacent enders (本当！？) so combined punctuation keeps
+        // its prosody and never becomes a punctuation-only segment, then
+        // consume locale closers so they stay with their sentence.
+        let boundary = index + 1;
+        while (
+          boundary < text.length &&
+          (NON_LATIN_SENTENCE_ENDING_PUNCTUATION.has(text[boundary] ?? "") ||
+            NON_LATIN_TRAILING_SENTENCE_PUNCTUATION.has(text[boundary] ?? ""))
+        ) {
+          boundary += 1;
+        }
+        // A buffer-final ender is ambiguous while streaming: the next delta
+        // may open with closers that belong to this sentence (`「こんにちは。`
+        // then `」`) or, for ．, with decimal digits (`３．` then `１４です`).
+        // Keep buffering; the `force` flush path still emits a buffer-final
+        // sentence at end of turn, and the length cap below still bounds the
+        // buffer. ASCII enders keep their end-of-text boundary: whitespace
+        // scripts do not attach closers across deltas the same way.
+        if (boundary < text.length) {
+          if (inLinkLabel) {
+            suppressedLabelBoundary ??= boundary;
+          } else {
+            return boundary;
+          }
+        }
       }
-      if (boundary === text.length || isWhitespace(text[boundary] ?? "")) {
-        return boundary;
+      if (!isNonLatinEnder) {
+        // ASCII enders require following whitespace so decimals (3.14) and
+        // file names don't split.
+        let boundary = index + 1;
+        while (
+          boundary < text.length &&
+          TRAILING_SENTENCE_PUNCTUATION.has(text[boundary] ?? "")
+        ) {
+          boundary += 1;
+        }
+        if (boundary === text.length || isWhitespace(text[boundary] ?? "")) {
+          if (inLinkLabel) {
+            suppressedLabelBoundary ??= boundary;
+          } else {
+            return boundary;
+          }
+        }
       }
     }
 
@@ -119,8 +310,8 @@ function findSpeakableBoundary(
     } else if (char === "`") {
       inBacktick = !inBacktick;
     } else if (char === "*") {
-      const prev = text[index - 1];
-      const next = text[index + 1];
+      const prev = codePointBefore(text, index);
+      const next = codePointAfter(text, index);
       if (next === "*") {
         inBold = !inBold;
         skipSpanChar = true;
@@ -141,8 +332,8 @@ function findSpeakableBoundary(
     } else if (char === "_") {
       // Same word-boundary rule as `*`. Since `_` neighbors in identifiers
       // like `my_var` are word chars, they never open or close a span.
-      const prev = text[index - 1];
-      const next = text[index + 1];
+      const prev = codePointBefore(text, index);
+      const next = codePointAfter(text, index);
       if (inUnderscore) {
         if (prev !== undefined && !isWhitespace(prev) && !isWordChar(next)) {
           inUnderscore = false;
@@ -161,14 +352,93 @@ function findSpeakableBoundary(
     return null;
   }
 
+  // The length cap bounds how long an open label can defer: past it, the
+  // first boundary the label suppressed beats an arbitrary mid-text split.
+  if (suppressedLabelBoundary !== null) {
+    return suppressedLabelBoundary;
+  }
+
   // Length-threshold splits are a hard cap and must flush even inside an
-  // open span — unbalanced markers there are an accepted edge.
+  // open span: unbalanced markers there are an accepted edge.
   const preferredBoundary = findLastWhitespaceBoundary(text, charThreshold);
-  return preferredBoundary ?? charThreshold;
+  if (preferredBoundary !== null) {
+    return preferredBoundary;
+  }
+  // Never split a UTF-16 surrogate pair: if the cap lands mid-pair, step
+  // back one unit so both halves stay in the same segment.
+  if (isHighSurrogate(text.charCodeAt(charThreshold - 1))) {
+    return charThreshold - 1;
+  }
+  return charThreshold;
 }
 
+/**
+ * Accepts a full code point (one or two UTF-16 units) so non-BMP letters
+ * like 𠮟 count as word chars, matching the sanitizer's Unicode lookarounds
+ * in calls/tts-text-sanitizer.ts.
+ */
 function isWordChar(value: string | undefined): boolean {
-  return value !== undefined && /\w/.test(value);
+  return value !== undefined && /[\p{L}\p{N}_]/u.test(value);
+}
+
+function isDecimalDigit(value: string | undefined): boolean {
+  return value !== undefined && /[0-9０-９]/.test(value);
+}
+
+// ASCII-only on purpose: filename-style tokens (report．pdf) are Latin
+// script, while CJK neighbors around ． mark real sentence text.
+function isAsciiWordChar(value: string | undefined): boolean {
+  return value !== undefined && /[A-Za-z0-9_]/.test(value);
+}
+
+/**
+ * A fullwidth comma acting as a thousands separator (１，０００円): a digit
+ * precedes it and a digit follows, or the buffer ends right after it while
+ * streaming (the digits may arrive in the next delta, so keep buffering).
+ */
+function isFullwidthGroupingComma(text: string, index: number): boolean {
+  if (text[index] !== FULLWIDTH_COMMA || !isDecimalDigit(text[index - 1])) {
+    return false;
+  }
+  const next = text[index + 1];
+  return next === undefined || isDecimalDigit(next);
+}
+
+/**
+ * The full code point ending just before `index`. When the preceding unit is
+ * a low surrogate, includes its high surrogate so `isWordChar` sees the
+ * whole character instead of half a pair.
+ */
+function codePointBefore(text: string, index: number): string | undefined {
+  const unit = text[index - 1];
+  if (unit === undefined) {
+    return undefined;
+  }
+  if (isLowSurrogate(unit.charCodeAt(0))) {
+    const high = text[index - 2];
+    if (high !== undefined && isHighSurrogate(high.charCodeAt(0))) {
+      return high + unit;
+    }
+  }
+  return unit;
+}
+
+/**
+ * The full code point starting just after `index`. When the following unit
+ * is a high surrogate, includes its low surrogate.
+ */
+function codePointAfter(text: string, index: number): string | undefined {
+  const unit = text[index + 1];
+  if (unit === undefined) {
+    return undefined;
+  }
+  if (isHighSurrogate(unit.charCodeAt(0))) {
+    const low = text[index + 2];
+    if (low !== undefined && isLowSurrogate(low.charCodeAt(0))) {
+      return unit + low;
+    }
+  }
+  return unit;
 }
 
 function findLastWhitespaceBoundary(

package/src/tts/synthesis-stream.ts

@@ -56,6 +56,12 @@ export interface SynthesisEmitOptions {
   /** Preferred PCM sample rate in Hz, forwarded on the provider request. */
   sampleRateHz?: number;
 
+  /**
+   * Language hint forwarded on the provider request
+   * (see {@link TtsSynthesisRequest.language}).
+   */
+  language?: string;
+
   /** Abort signal forwarded to the provider and checked before each emit. */
   signal?: AbortSignal;
 
@@ -106,6 +112,7 @@ export async function synthesizeAndEmit(
     voiceId: options.voiceId,
     outputFormat: options.outputFormat,
     sampleRateHz: options.sampleRateHz,
+    language: options.language,
     signal,
   };
 

package/src/tts/types.ts

@@ -113,6 +113,13 @@ export interface TtsSynthesisRequest {
    * on provider-documented behavior (see each provider's format mapping).
    */
   sampleRateHz?: number;
+
+  /**
+   * Optional lowercase BCP-47 base subtag (e.g. `"es"`, `"hi"`) hinting the
+   * language of `text`. Providers that support language enforcement apply
+   * it; providers that cannot use it must ignore it.
+   */
+  language?: string;
 }
 
 /** Output of a completed TTS synthesis call. */

package/src/usage/__tests__/subagent-attribution.test.ts

@@ -0,0 +1,109 @@
+import { beforeEach, describe, expect, test } from "bun:test";
+
+import { getDb } from "../../persistence/db-connection.js";
+import { initializeDb } from "../../persistence/db-init.js";
+import {
+  resetSubagentAttributionCacheForTests,
+  resolveSubagentAttribution,
+} from "../subagent-attribution.js";
+
+await initializeDb();
+
+let counter = 0;
+
+function insertConversation(opts: {
+  subagentRole?: string;
+  subagentSpawnMode?: string;
+}): string {
+  counter += 1;
+  const id = `conv-attr-${counter}`;
+  const db = getDb();
+  db.run(
+    `INSERT INTO conversations (id, conversation_type, created_at, updated_at, subagent_role, subagent_spawn_mode)
+     VALUES ('${id}', 'background', 1000, 1000, ${
+       opts.subagentRole === undefined ? "NULL" : `'${opts.subagentRole}'`
+     }, ${
+       opts.subagentSpawnMode === undefined
+         ? "NULL"
+         : `'${opts.subagentSpawnMode}'`
+     })`,
+  );
+  return id;
+}
+
+describe("resolveSubagentAttribution", () => {
+  beforeEach(() => {
+    resetSubagentAttributionCacheForTests();
+  });
+
+  test("returns the stamped role and spawn mode for a subagent conversation", () => {
+    const id = insertConversation({
+      subagentRole: "advisor",
+      subagentSpawnMode: "advisor_consult",
+    });
+
+    expect(resolveSubagentAttribution(id)).toEqual({
+      subagentRole: "advisor",
+      subagentSpawnMode: "advisor_consult",
+    });
+  });
+
+  test("returns nulls for an ordinary conversation", () => {
+    const id = insertConversation({});
+
+    expect(resolveSubagentAttribution(id)).toEqual({
+      subagentRole: null,
+      subagentSpawnMode: null,
+    });
+  });
+
+  test("returns nulls without a conversation id", () => {
+    expect(resolveSubagentAttribution(undefined)).toEqual({
+      subagentRole: null,
+      subagentSpawnMode: null,
+    });
+    expect(resolveSubagentAttribution("")).toEqual({
+      subagentRole: null,
+      subagentSpawnMode: null,
+    });
+  });
+
+  test("returns nulls for an unknown conversation and does not cache the miss", () => {
+    // The columns are stamped at conversation creation, so a miss means the
+    // row has not landed yet, and caching it would pin an empty result.
+    expect(resolveSubagentAttribution("conv-not-yet")).toEqual({
+      subagentRole: null,
+      subagentSpawnMode: null,
+    });
+
+    const db = getDb();
+    db.run(
+      `INSERT INTO conversations (id, conversation_type, created_at, updated_at, subagent_role, subagent_spawn_mode)
+       VALUES ('conv-not-yet', 'background', 1000, 1000, 'coder', 'regular')`,
+    );
+
+    expect(resolveSubagentAttribution("conv-not-yet")).toEqual({
+      subagentRole: "coder",
+      subagentSpawnMode: "regular",
+    });
+  });
+
+  test("memoizes a resolved conversation", () => {
+    const id = insertConversation({
+      subagentRole: "general",
+      subagentSpawnMode: "fork",
+    });
+    expect(resolveSubagentAttribution(id).subagentSpawnMode).toBe("fork");
+
+    // The columns are immutable after creation, so a later write must not be
+    // observable, proving the second call did not hit the database.
+    const db = getDb();
+    db.run(
+      `UPDATE conversations SET subagent_spawn_mode = 'regular' WHERE id = '${id}'`,
+    );
+
+    expect(resolveSubagentAttribution(id).subagentSpawnMode).toBe("fork");
+    resetSubagentAttributionCacheForTests();
+    expect(resolveSubagentAttribution(id).subagentSpawnMode).toBe("regular");
+  });
+});

package/src/usage/subagent-attribution.ts

@@ -0,0 +1,108 @@
+import { rawGet } from "../persistence/raw-query.js";
+import { getLogger } from "../util/logger.js";
+
+const log = getLogger("subagent-attribution");
+
+/**
+ * The two delegated-work attribution dimensions carried on a conversation:
+ * the subagent role it was spawned with, and how it was spawned. Both null
+ * for a conversation that is not a subagent.
+ */
+export interface SubagentAttribution {
+  subagentRole: string | null;
+  subagentSpawnMode: string | null;
+}
+
+const NOT_A_SUBAGENT: SubagentAttribution = {
+  subagentRole: null,
+  subagentSpawnMode: null,
+};
+
+/**
+ * Bound on the memo below. Both columns are stamped once at conversation
+ * creation and never updated, so a hit is always correct and the only cost of
+ * a bounded cache is an occasional re-read. Sized to comfortably cover the
+ * conversations resident in one process without becoming a memory concern.
+ */
+const MAX_CACHED_CONVERSATIONS = 2048;
+
+/**
+ * Insertion-ordered memo. `Map` iteration order is insertion order, so
+ * dropping the first key evicts the oldest entry, sufficient for a lookup
+ * whose values are immutable and whose miss cost is a single primary-key read.
+ */
+const cache = new Map<string, SubagentAttribution>();
+
+/**
+ * Resolve the delegated-work attribution for a conversation, for forwarding on
+ * the runtime proxy's `X-Vellum-Subagent-*` headers.
+ *
+ * Reads the columns stamped at spawn time (migration 362) rather than the
+ * `subagents` table: `subagents` rows are deleted on dispose, and the billing
+ * path must not depend on a row that may already be gone.
+ *
+ * Never throws. This runs on the provider dispatch hot path and attribution
+ * must never be able to take a model request down. Any failure degrades to
+ * "no attribution", which is exactly how the platform treats a missing header.
+ */
+export function resolveSubagentAttribution(
+  conversationId: string | undefined,
+): SubagentAttribution {
+  if (conversationId === undefined || conversationId.length === 0) {
+    return NOT_A_SUBAGENT;
+  }
+  const cached = cache.get(conversationId);
+  if (cached !== undefined) {
+    return cached;
+  }
+
+  const row = readRow(conversationId);
+  if (!row) {
+    // Either no row yet, or the read failed. Deliberately NOT cached: caching
+    // a miss would pin an empty result for a conversation whose row lands
+    // moments later, or for the whole process after one transient DB error.
+    return NOT_A_SUBAGENT;
+  }
+
+  const attribution: SubagentAttribution = {
+    subagentRole: row.subagent_role,
+    subagentSpawnMode: row.subagent_spawn_mode,
+  };
+  if (cache.size >= MAX_CACHED_CONVERSATIONS) {
+    const oldest = cache.keys().next();
+    if (!oldest.done) {
+      cache.delete(oldest.value);
+    }
+  }
+  cache.set(conversationId, attribution);
+  return attribution;
+}
+
+interface ConversationSubagentRow {
+  subagent_role: string | null;
+  subagent_spawn_mode: string | null;
+}
+
+/**
+ * Read the two columns, or `null` when the conversation has no row or the
+ * read fails. Failure covers the pre-migration case (columns absent) and any
+ * DB-unavailable context, such as a unit test exercising the provider stack
+ * alone. Attribution must never be able to take a model request down.
+ */
+function readRow(conversationId: string): ConversationSubagentRow | null {
+  try {
+    return rawGet<ConversationSubagentRow>(
+      "usage:subagentAttribution",
+      `SELECT subagent_role, subagent_spawn_mode FROM conversations WHERE id = ?`,
+      conversationId,
+    );
+  } catch (err) {
+    log.debug({ err, conversationId }, "Subagent attribution lookup failed");
+    return null;
+  }
+}
+
+/** Test seam: drops every memoized entry. */
+export function resetSubagentAttributionCacheForTests(): void {
+  cache.clear();
+}