@vellumai/assistant 0.12.5 → 0.12.6-dev.202609292116.41f7caa

package/src/calls/media-stream-stt-session.ts

@@ -9,8 +9,10 @@
  *   inbound mu-law audio is decoded to 16 kHz PCM16 and fed to a
  *   {@link StreamingTranscriber} resolved for the configured
  *   `services.stt` provider. Replies trigger on the provider's
- *   utterance-boundary `final` events; barge-in (`onSpeechStart`) always
- *   fires from the local energy VAD, never from transcriber partials.
+ *   utterance-boundary `final` events; interim `partial` events are
+ *   forwarded as the freshest words spoken since that final but start no
+ *   turn of their own, and barge-in (`onSpeechStart`) always fires from
+ *   the local energy VAD, never from transcriber partials.
  *   Frames arriving while the provider session is still starting are
  *   held in a bounded buffer and flushed on start (overflow drops the
  *   oldest frames and is counted + logged at teardown).
@@ -20,10 +22,11 @@
  *   transcriber is available for the configured provider, or the
  *   provider closed the streaming session unexpectedly mid-call.
  *
- * This module is **transport-neutral** — it exposes callback hooks
- * (`onSpeechStart`, `onTranscriptFinal`, `onDtmf`, `onStop`) rather than
- * driving any call flow itself; `media-stream-server.ts` instantiates and
- * connects it to the media-stream WebSocket ingress.
+ * This module is **transport-neutral**: it exposes callback hooks
+ * (`onSpeechStart`, `onTranscriptFinal`, `onTranscriptPartial`, `onDtmf`,
+ * `onStop`) rather than driving any call flow itself;
+ * `media-stream-server.ts` instantiates and connects it to the
+ * media-stream WebSocket ingress.
  *
  * Error handling:
  * - When the telephony resolver returns a non-supported status, the
@@ -142,6 +145,19 @@ export interface MediaStreamSttSessionCallbacks {
    */
   onTranscriptFinal?: (text: string, durationMs: number) => void;
 
+  /**
+   * Called for each interim transcript the streaming provider revises on
+   * its way to the next final. The text is the provider's running guess at
+   * the words spoken since that final, so each call supersedes the last and
+   * the run resets when {@link onTranscriptFinal} commits it. Empty
+   * partials are suppressed, and batch mode never emits one.
+   *
+   * Turn taking still runs off the finals: this is the freshest text a
+   * boundary the front door held mid-thought can be re-judged on, not a
+   * boundary of its own.
+   */
+  onTranscriptPartial?: (text: string) => void;
+
   /**
    * Called when a DTMF digit is received from Twilio.
    */
@@ -542,8 +558,8 @@ export class MediaStreamSttSession {
   /**
    * Map streaming transcriber events onto the session callbacks.
    *
-   * Partials are ignored for turn-taking: replies trigger only on the
-   * provider's utterance-boundary finals, and barge-in comes from the
+   * Partials are forwarded but own no turn taking: replies trigger only on
+   * the provider's utterance-boundary finals, and barge-in comes from the
    * local VAD.
    */
   private handleStreamingEvent(event: SttStreamServerEvent): void {
@@ -552,8 +568,13 @@ export class MediaStreamSttSession {
     }
 
     switch (event.type) {
-      case "partial":
+      case "partial": {
+        const text = event.text.trim();
+        if (text.length > 0) {
+          this.callbacks.onTranscriptPartial?.(text);
+        }
         return;
+      }
       case "finalized":
         return;
       case "turn-start":

package/src/calls/twilio-provider.ts

@@ -23,27 +23,10 @@ const log = getLogger("twilio-provider");
 export class TwilioVoiceProvider implements VoiceProvider {
   readonly name = "twilio";
 
-  // ── Credential helpers ──────────────────────────────────────────────
-
-  private async getCredentials(): Promise<{
-    accountSid: string;
-    authToken: string;
-  }> {
-    return await getTwilioCredentials();
-  }
-
-  private authHeader(accountSid: string, authToken: string): string {
-    return twilioAuthHeader(accountSid, authToken);
-  }
-
-  private baseUrl(accountSid: string): string {
-    return twilioBaseUrl(accountSid);
-  }
-
   // ── VoiceProvider interface ─────────────────────────────────────────
 
   async initiateCall(opts: InitiateCallOptions): Promise<{ callSid: string }> {
-    const { accountSid, authToken } = await this.getCredentials();
+    const { accountSid, authToken } = await getTwilioCredentials();
 
     const body = new URLSearchParams({
       From: opts.from,
@@ -80,10 +63,10 @@ export class TwilioVoiceProvider implements VoiceProvider {
 
     log.info({ from: opts.from, to: opts.to }, "Initiating Twilio call");
 
-    const res = await fetch(`${this.baseUrl(accountSid)}/Calls.json`, {
+    const res = await fetch(`${twilioBaseUrl(accountSid)}/Calls.json`, {
       method: "POST",
       headers: {
-        Authorization: this.authHeader(accountSid, authToken),
+        Authorization: twilioAuthHeader(accountSid, authToken),
         "Content-Type": "application/x-www-form-urlencoded",
       },
       body: body.toString(),
@@ -108,18 +91,18 @@ export class TwilioVoiceProvider implements VoiceProvider {
   }
 
   async endCall(callSid: string): Promise<void> {
-    const { accountSid, authToken } = await this.getCredentials();
+    const { accountSid, authToken } = await getTwilioCredentials();
 
     log.info({ callSid }, "Ending Twilio call");
 
     const body = new URLSearchParams({ Status: "completed" });
 
     const res = await fetch(
-      `${this.baseUrl(accountSid)}/Calls/${callSid}.json`,
+      `${twilioBaseUrl(accountSid)}/Calls/${callSid}.json`,
       {
         method: "POST",
         headers: {
-          Authorization: this.authHeader(accountSid, authToken),
+          Authorization: twilioAuthHeader(accountSid, authToken),
           "Content-Type": "application/x-www-form-urlencoded",
         },
         body: body.toString(),
@@ -143,14 +126,14 @@ export class TwilioVoiceProvider implements VoiceProvider {
   }
 
   async getCallStatus(callSid: string): Promise<string> {
-    const { accountSid, authToken } = await this.getCredentials();
+    const { accountSid, authToken } = await getTwilioCredentials();
 
     const res = await fetch(
-      `${this.baseUrl(accountSid)}/Calls/${callSid}.json`,
+      `${twilioBaseUrl(accountSid)}/Calls/${callSid}.json`,
       {
         method: "GET",
         headers: {
-          Authorization: this.authHeader(accountSid, authToken),
+          Authorization: twilioAuthHeader(accountSid, authToken),
         },
       },
     );
@@ -183,7 +166,7 @@ export class TwilioVoiceProvider implements VoiceProvider {
   async checkCallerIdEligibility(
     phoneNumber: string,
   ): Promise<{ eligible: boolean; reason?: string }> {
-    const { accountSid, authToken } = await this.getCredentials();
+    const { accountSid, authToken } = await getTwilioCredentials();
     const encodedNumber = encodeURIComponent(phoneNumber);
 
     let incomingOk = false;
@@ -191,13 +174,13 @@ export class TwilioVoiceProvider implements VoiceProvider {
 
     // Check incoming phone numbers (owned by this account)
     const incomingRes = await fetch(
-      `${this.baseUrl(
+      `${twilioBaseUrl(
         accountSid,
       )}/IncomingPhoneNumbers.json?PhoneNumber=${encodedNumber}`,
       {
         method: "GET",
         headers: {
-          Authorization: this.authHeader(accountSid, authToken),
+          Authorization: twilioAuthHeader(accountSid, authToken),
         },
       },
     );
@@ -223,13 +206,13 @@ export class TwilioVoiceProvider implements VoiceProvider {
 
     // Check outgoing caller IDs (verified with this account)
     const outgoingRes = await fetch(
-      `${this.baseUrl(
+      `${twilioBaseUrl(
         accountSid,
       )}/OutgoingCallerIds.json?PhoneNumber=${encodedNumber}`,
       {
         method: "GET",
         headers: {
-          Authorization: this.authHeader(accountSid, authToken),
+          Authorization: twilioAuthHeader(accountSid, authToken),
         },
       },
     );

package/src/calls/voice-session-bridge.ts

@@ -41,6 +41,10 @@ import {
   pendingStandaloneImagePersist,
   SIGHT_FRAME_TURN_HOLD_MS,
 } from "../live-voice/live-voice-photo.js";
+import {
+  formatShareTargetsForPrompt,
+  type ShareTargetSnapshot,
+} from "../live-voice/share-targets.js";
 import { resolveAttachmentsForPersist } from "../persistence/attachments-store.js";
 import {
   deleteMessageById,
@@ -462,6 +466,13 @@ export interface VoiceTurnOptions {
   macosDesktopSession?: boolean;
   /** The desktop client currently shares a surface with this voice session. */
   screenSharing?: boolean;
+  /**
+   * The controls the shared surface offers to be pointed at, from the
+   * client's newest read of its accessibility tree. Offered alongside the
+   * screen-annotation instructions so the first `screen_point_at` names a
+   * control the lookup can resolve.
+   */
+  shareTargets?: ShareTargetSnapshot;
   /** Front-door verdict: this action needs only the shared screen and current context. */
   screenAction?: boolean;
   /** Whether this is an inbound call (no outbound task). */
@@ -2145,8 +2156,14 @@ export async function startVoiceTurn(
             signal: opts.signal,
           });
           if (annotation !== null) {
+            const controls =
+              opts.shareTargets !== undefined
+                ? formatShareTargetsForPrompt(opts.shareTargets)
+                : null;
             conversation.setVoiceCallControlPrompt(
-              [voiceCallControlPrompt, annotation].filter(Boolean).join("\n\n"),
+              [voiceCallControlPrompt, annotation, controls]
+                .filter(Boolean)
+                .join("\n\n"),
             );
           }
         }

package/src/cli/commands/browser.ts

@@ -214,8 +214,7 @@ function attachOperationAction(
       opts.output &&
       result.screenshots?.length
     ) {
-      const screenshot = result.screenshots[0];
-      const buffer = Buffer.from(screenshot.data, "base64");
+      const buffer = Buffer.from(result.screenshots[0].data, "base64");
       try {
         writeFileSync(String(opts.output), buffer);
       } catch (err) {

package/src/cli/lib/__tests__/plugin-artifact.test.ts

@@ -123,6 +123,23 @@ describe("parsePluginArtifact", () => {
     });
   });
 
+  test("ignores a malformed icon beside a valid artifact", () => {
+    expect(
+      parsePluginArtifact({
+        vellum: {
+          artifact: {
+            url: "https://example.com/App.dmg",
+            sha256: VALID_SHA,
+          },
+          icon: 42,
+        },
+      }),
+    ).toEqual({
+      url: "https://example.com/App.dmg",
+      sha256: VALID_SHA,
+    });
+  });
+
   test.each([
     ["no vellum block", { name: "x" }],
     ["no artifact block", { vellum: {} }],
@@ -202,6 +219,12 @@ describe("parsePluginIcon", () => {
     expect(parsePluginIcon({ vellum: { icon: sixteen } })).toBe(sixteen);
   });
 
+  test("ignores a malformed artifact beside a valid icon", () => {
+    expect(
+      parsePluginIcon({ vellum: { artifact: "App.dmg", icon: "🚀" } }),
+    ).toBe("🚀");
+  });
+
   test.each([
     ["no vellum block", { name: "x" }],
     ["no icon field", { vellum: {} }],

package/src/cli/lib/__tests__/plugin-catalog-visibility.test.ts

@@ -33,6 +33,7 @@ describe("filterPluginCatalogByFeatureFlags", () => {
       "ramp",
       "semrush",
       "typeform",
+      "upwork",
       "notion",
     ]);
 
@@ -56,6 +57,7 @@ describe("filterPluginCatalogByFeatureFlags", () => {
       "ramp",
       "semrush",
       "typeform",
+      "upwork",
     ]);
 
     const result = filterPluginCatalogByFeatureFlags(input, () => true);

package/src/cli/lib/__tests__/plugin-details.test.ts

@@ -309,6 +309,22 @@ describe("getPluginDetails (bundled catalog, offline)", () => {
     expect(details.icon).toBeNull();
   });
 
+  test("reads a legacy object-form license from an installed package.json", async () => {
+    const target = join(workspace, "caveman");
+    mkdirSync(target, { recursive: true });
+    writeFileSync(
+      join(target, "package.json"),
+      JSON.stringify({ version: "2.0.0", license: { type: "MIT" } }),
+    );
+
+    const details = await getPluginDetails(
+      { name: "caveman" },
+      { fetch: makeFetch({}), workspacePluginsDir: workspace },
+    );
+
+    expect(details.license).toBe("MIT");
+  });
+
   test("reads metadata from an installed standard-only plugin.json", async () => {
     const target = join(workspace, "caveman");
     mkdirSync(target, { recursive: true });

package/src/cli/lib/bundled-marketplace.json

@@ -135,6 +135,32 @@
       "homepage": "https://github.com/topoteretes/cognee-integrations/tree/main/integrations/vellum-assistant",
       "icon": "\ud83d\udd78\ufe0f"
     },
+    {
+      "name": "contractor-missed-call-dispatcher",
+      "description": "Captures missed trade contractor inbound calls, fires an instant 5-second SMS qualification text-back, triages emergencies, and stages draft jobs in Jobber or ServiceTitan.",
+      "category": "commerce",
+      "icon": "📲",
+      "license": "MIT",
+      "homepage": "https://github.com/ZeebBoyBlue/contractor-missed-call-dispatcher",
+      "source": {
+        "source": "github",
+        "repo": "ZeebBoyBlue/contractor-missed-call-dispatcher",
+        "ref": "2d293cdcb68af03d446b61a9915a0ee3dc842d4e"
+      }
+    },
+    {
+      "name": "contractor-photo-to-quote",
+      "description": "Turns job-site and equipment photos into standardized trade estimates and quotes in Jobber or QuickBooks with multi-tier Good/Better/Best options in 5 minutes.",
+      "category": "commerce",
+      "icon": "📸",
+      "license": "MIT",
+      "homepage": "https://github.com/ZeebBoyBlue/contractor-photo-to-quote",
+      "source": {
+        "source": "github",
+        "repo": "ZeebBoyBlue/contractor-photo-to-quote",
+        "ref": "6c136186faaf03bcf125d202f0b1f51f97d94544"
+      }
+    },
     {
       "name": "contractor-voice-invoicer",
       "description": "Converts 20-second truck voice memos and job photos into itemized client invoices with custom labor rates, material markups, and one-click accounting staging.",
@@ -461,6 +487,19 @@
       "license": "MIT",
       "icon": "\ud83d\udcac"
     },
+    {
+      "name": "quickbooks-expense-categorizer",
+      "description": "Scans uncategorized QuickBooks Online expenses and bank feeds, matches receipts, and stages categorized entries with confidence scoring for 1-click batch review.",
+      "category": "commerce",
+      "icon": "🧾",
+      "license": "MIT",
+      "homepage": "https://github.com/ZeebBoyBlue/quickbooks-expense-categorizer",
+      "source": {
+        "source": "github",
+        "repo": "ZeebBoyBlue/quickbooks-expense-categorizer",
+        "ref": "120776619698cc3e9490caf5c756465f5caaa6c2"
+      }
+    },
     {
       "name": "reading-pal",
       "source": {
@@ -487,6 +526,19 @@
         "ref": "f1905e639941e98172c5032b6d722f668882d75d"
       }
     },
+    {
+      "name": "shopify-payout-reconciler",
+      "description": "Unbundles lump-sum Shopify merchant bank deposits into gross sales, processor fees, customer refunds, shipping labels, and sales tax for penny-accurate QuickBooks reconciliation.",
+      "category": "commerce",
+      "icon": "🛍️",
+      "license": "MIT",
+      "homepage": "https://github.com/ZeebBoyBlue/shopify-payout-reconciler",
+      "source": {
+        "source": "github",
+        "repo": "ZeebBoyBlue/shopify-payout-reconciler",
+        "ref": "079fe0d86629686243a588cf3c4867d38d6ab8b8"
+      }
+    },
     {
       "name": "simple-memory",
       "source": {
@@ -1368,9 +1420,9 @@
       "source": {
         "source": "github",
         "repo": "vellum-ai/runway-plugin",
-        "ref": "2a642f14ee8ca5032efefae06b6a610dec13e1a1"
+        "ref": "bb017a1e018c7be140c02c8463273c2d85f3b9d9"
       },
-      "description": "Collaboratively direct, approve, create, track, cancel, and download Runway Gen-4.5 videos.",
+      "description": "Securely set up, collaboratively direct, approve, create, track, cancel, and download Runway Gen-4.5 videos.",
       "category": "other",
       "submittedBy": "marina"
     },

package/src/cli/lib/inspect-plugin.ts

@@ -16,6 +16,8 @@
  * is a thin wrapper that supplies production deps and formats the result.
  */
 
+import { z } from "zod";
+
 import { PRESERVED_ENTRIES } from "../../plugins/plugin-tree-walk.js";
 import type { FetchLike } from "./fetch-like.js";
 import {
@@ -50,6 +52,14 @@ import type { PluginCatalog, PluginSearchMatch } from "./search-plugins.js";
 /** Full commit SHA (40 hex SHA-1 or 64 hex SHA-256). */
 const FULL_SHA_RE = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/i;
 
+const GitHubCommitResponseSchema = z.object({
+  commit: z.object({
+    committer: z.object({
+      date: z.string(),
+    }),
+  }),
+});
+
 /**
  * Drift classification between the installed copy and the marketplace pin.
  *
@@ -294,23 +304,13 @@ export async function fetchCommitDate(
     if (!res.ok) {
       return null;
     }
-    const json: unknown = JSON.parse(await res.text());
-    if (typeof json !== "object" || json === null) {
-      return null;
-    }
-    const commit = (json as Record<string, unknown>).commit;
-    if (typeof commit !== "object" || commit === null) {
-      return null;
-    }
-    const committer = (commit as Record<string, unknown>).committer;
-    if (typeof committer !== "object" || committer === null) {
-      return null;
-    }
-    const date = (committer as Record<string, unknown>).date;
-    if (typeof date !== "string") {
+    const parsed = GitHubCommitResponseSchema.safeParse(
+      JSON.parse(await res.text()),
+    );
+    if (!parsed.success) {
       return null;
     }
-    const ms = Date.parse(date);
+    const ms = Date.parse(parsed.data.commit.committer.date);
     return Number.isFinite(ms) ? new Date(ms).toISOString() : null;
   } catch {
     return null;

package/src/cli/lib/list-installed-plugins.ts

@@ -16,12 +16,14 @@ import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 
+import { z } from "zod";
+
 import { getWorkspacePluginsDir } from "../../util/platform.js";
 import {
   LEGACY_PLUGIN_MANIFEST,
   readPluginManifest,
 } from "../../util/plugin-manifest.js";
-import { parsePluginIcon } from "./plugin-artifact.js";
+import { PluginPackageJsonSchema } from "./plugin-artifact.js";
 import { readValidatedPluginIcon } from "./plugin-icon-file.js";
 
 /**
@@ -39,6 +41,15 @@ const DEFAULT_PLUGINS_DIR = join(
   "defaults",
 );
 
+const DefaultPluginManifestSchema = z.object({
+  name: z.string(),
+  version: z.string().optional().catch(undefined),
+});
+
+const InstallMetadataSchema = z.object({
+  installedAt: z.string(),
+});
+
 /** Minimal manifest fields surfaced to the CLI. */
 export interface PluginPackageMetadata {
   readonly name?: string;
@@ -163,23 +174,17 @@ function readPluginEntry(
     issues.push(err instanceof Error ? err.message : String(err));
     return { name, target, packageJson: null, issues, ...iconFields };
   }
-  const meta = manifest.raw;
-  const icon =
-    manifest.source === LEGACY_PLUGIN_MANIFEST
-      ? parsePluginIcon(meta)
-      : undefined;
+  const parsedPackageJson = PluginPackageJsonSchema.safeParse(manifest.raw);
+  const legacyMetadata =
+    manifest.source === LEGACY_PLUGIN_MANIFEST && parsedPackageJson.success
+      ? parsedPackageJson.data
+      : null;
   const packageJson: PluginPackageMetadata = {
     name: manifest.name,
     version: manifest.version,
     description: manifest.description,
-    peerDependencies:
-      manifest.source === LEGACY_PLUGIN_MANIFEST &&
-      typeof meta.peerDependencies === "object" &&
-      meta.peerDependencies !== null &&
-      !Array.isArray(meta.peerDependencies)
-        ? (meta.peerDependencies as Record<string, string>)
-        : undefined,
-    ...(icon ? { icon } : {}),
+    peerDependencies: legacyMetadata?.peerDependencies,
+    ...(legacyMetadata?.icon ? { icon: legacyMetadata.icon } : {}),
   };
 
   return { name, target, packageJson, issues, ...iconFields };
@@ -319,14 +324,11 @@ function readDefaultPluginManifests(): readonly DefaultPluginManifest[] {
       continue;
     }
     try {
-      const raw = readFileSync(pkgJsonPath, "utf8");
-      const parsed = JSON.parse(raw) as Record<string, unknown>;
-      if (typeof parsed.name === "string") {
-        manifests.push({
-          name: parsed.name,
-          version:
-            typeof parsed.version === "string" ? parsed.version : undefined,
-        });
+      const parsed = DefaultPluginManifestSchema.safeParse(
+        JSON.parse(readFileSync(pkgJsonPath, "utf8")),
+      );
+      if (parsed.success) {
+        manifests.push(parsed.data);
       }
     } catch {
       // Skip malformed entries — lenient like listInstalledPlugins.
@@ -345,12 +347,11 @@ function getPluginInstallDate(plugin: AllPluginInfo): number {
   const metaPath = join(plugin.target, "install-meta.json");
   try {
     if (existsSync(metaPath)) {
-      const raw = JSON.parse(readFileSync(metaPath, "utf8")) as Record<
-        string,
-        unknown
-      >;
-      if (typeof raw.installedAt === "string") {
-        const ms = Date.parse(raw.installedAt);
+      const parsed = InstallMetadataSchema.safeParse(
+        JSON.parse(readFileSync(metaPath, "utf8")),
+      );
+      if (parsed.success) {
+        const ms = Date.parse(parsed.data.installedAt);
         if (Number.isFinite(ms)) {
           return ms;
         }