nexus-ai-pro 2.20.2 → 2.22.0

package/dist-cjs/telephony/providers/telnyx.js

@@ -65,9 +65,6 @@ class TelnyxTelephonyProvider {
         if (request.redirectUrl) {
             throw new errors_js_1.TelephonyProviderError('Telnyx calls are controlled by command, not redirected to a webhook', 'telnyx');
         }
-        if (request.gather?.input && !request.gather.input.includes('dtmf')) {
-            throw new errors_js_1.TelephonyProviderError('Telnyx gathers keypad digits, not speech', 'telnyx');
-        }
         const sent = [];
         const run = async (action, body) => {
             sent.push({ action, body });
@@ -86,15 +83,27 @@ class TelnyxTelephonyProvider {
         }
         if (request.gather) {
             const gather = request.gather;
+            const input = gather.input ?? ['dtmf'];
+            if (input.includes('speech')) {
+                await run('transcription_start', {
+                    language: voice.language,
+                    transcription_tracks: 'inbound',
+                    ...(this.config.transcriptionEngine ? { transcription_engine: this.config.transcriptionEngine } : {}),
+                });
+            }
             const digits = {
                 ...(gather.numDigits !== undefined ? { minimum_digits: 1, maximum_digits: gather.numDigits } : {}),
                 ...(gather.timeoutSeconds !== undefined ? { timeout_millis: gather.timeoutSeconds * 1_000 } : {}),
                 ...(gather.finishOnKey ? { terminating_digit: gather.finishOnKey } : {}),
             };
-            if (gather.prompts?.length)
-                await run('gather_using_speak', { payload: gather.prompts.join(' '), ...voice, ...digits });
-            else
-                await run('gather', digits);
+            if (input.includes('dtmf')) {
+                if (gather.prompts?.length)
+                    await run('gather_using_speak', { payload: gather.prompts.join(' '), ...voice, ...digits });
+                else
+                    await run('gather', digits);
+            }
+            else if (gather.prompts?.length)
+                await run('speak', { payload: gather.prompts.join(' '), ...voice });
         }
         if (request.dial?.conference) {
             await this.joinConference({
@@ -410,6 +419,34 @@ class TelnyxTelephonyProvider {
     async stopRecording(request, context) {
         await this.command(request.callId, 'record_stop', {}, request.signal, context);
     }
+    parseGatherResult(body) {
+        const event = webhookEvent(body);
+        if (!event)
+            return undefined;
+        const { type, payload } = event;
+        const callId = stringValue(payload.call_control_id) ?? '';
+        if (type === 'call.gather.ended') {
+            return { providerUsed: 'telnyx', callId, digits: stringValue(payload.digits) ?? '', final: true, raw: event.raw };
+        }
+        if (type !== 'call.transcription')
+            return undefined;
+        const data = objectValue(payload.transcription_data);
+        const speech = stringValue(data.transcript);
+        if (speech === undefined)
+            return undefined;
+        const confidence = numberValue(data.confidence);
+        return {
+            providerUsed: 'telnyx',
+            callId,
+            speech,
+            ...(confidence !== undefined ? { confidence } : {}),
+            final: data.is_final === true,
+            raw: event.raw,
+        };
+    }
+    async endGather(request, context) {
+        await this.command(request.callId, 'transcription_stop', {}, request.signal, context);
+    }
     parseRecordingCallback(body) {
         const event = webhookEvent(body);
         if (event?.type !== 'call.recording.saved')

package/dist-cjs/telephony/providers/twilio.d.ts

@@ -1,5 +1,5 @@
 import type { ProviderCallContext } from '../../types/lifecycle.js';
-import type { CreateCallRequest, CreateCallResponse, DeleteRecordingRequest, DownloadRecordingRequest, EndCallRequest, GetCallRequest, JoinConferenceRequest, ListPhoneNumbersRequest, SendMessageRequest, StartRecordingRequest, StopRecordingRequest, TelephonyCallDetails, TelephonyConferenceJoin, TelephonyMessage, TelephonyRecording, TelephonyRecordingAudio, TelephonyRecordingStarted, TelephonyTransfer, TransferCallRequest, TelephonyMediaStreamEvent, TelephonyOutboundAudioMessage, TelephonyPhoneNumber, TelephonyProvider, TelephonyProviderInfo, TelephonyResponseRequest, TelephonyStatusCallback, TelephonyWebhookResponse, TelephonyWebhookValidationRequest, UpdatePhoneNumberRequest } from '../../types/telephony.js';
+import type { CreateCallRequest, CreateCallResponse, DeleteRecordingRequest, DownloadRecordingRequest, EndCallRequest, GetCallRequest, JoinConferenceRequest, ListPhoneNumbersRequest, SendMessageRequest, StartRecordingRequest, StopRecordingRequest, TelephonyCallDetails, TelephonyConferenceJoin, TelephonyMessage, TelephonyRecording, TelephonyRecordingAudio, TelephonyRecordingStarted, TelephonyTransfer, TransferCallRequest, TelephonyMediaStreamEvent, TelephonyOutboundAudioMessage, TelephonyPhoneNumber, TelephonyProvider, TelephonyProviderInfo, TelephonyResponseRequest, TelephonyStatusCallback, TelephonyWebhookResponse, TelephonyWebhookValidationRequest, UpdatePhoneNumberRequest, TelephonyGatherResult } from '../../types/telephony.js';
 /** Options for the Twilio telephony provider. */
 export interface TwilioTelephonyProviderConfig {
     /** Twilio account SID. Required for REST calls. */
@@ -34,6 +34,8 @@ export declare class TwilioTelephonyProvider implements TelephonyProvider {
     endCall(request: EndCallRequest, context?: ProviderCallContext): Promise<TelephonyCallDetails>;
     /** Parses a status callback into neutral call details. */
     parseStatusCallback(body: string | URLSearchParams | Record<string, string | number | boolean | undefined>): TelephonyStatusCallback | undefined;
+    /** Parses what a `<Gather>` posted to its action: `SpeechResult` with its `Confidence`, and `Digits`. */
+    parseGatherResult(body: string | URLSearchParams | Record<string, unknown>): TelephonyGatherResult | undefined;
     /** Lists the account's phone numbers. */
     listPhoneNumbers(request?: ListPhoneNumbersRequest, context?: ProviderCallContext): Promise<TelephonyPhoneNumber[]>;
     /** Changes where a phone number sends its calls and status callbacks. */

package/dist-cjs/telephony/providers/twilio.js

@@ -137,6 +137,22 @@ class TwilioTelephonyProvider {
             raw: params,
         };
     }
+    parseGatherResult(body) {
+        const params = toParamRecord(body);
+        const callId = params.CallSid;
+        if (!callId || (params.SpeechResult === undefined && params.Digits === undefined))
+            return undefined;
+        const confidence = numberValue(params.Confidence);
+        return {
+            providerUsed: 'twilio',
+            callId,
+            ...(params.SpeechResult !== undefined ? { speech: params.SpeechResult } : {}),
+            ...(confidence !== undefined ? { confidence } : {}),
+            ...(params.Digits !== undefined ? { digits: params.Digits } : {}),
+            final: true,
+            raw: params,
+        };
+    }
     async listPhoneNumbers(request = {}, context) {
         const query = new URLSearchParams();
         if (request.phoneNumber)

package/dist-cjs/tracing/stores.d.ts

@@ -13,6 +13,8 @@ export interface MemoryTraceStoreOptions {
  * runs. Bounded, like every other in-memory store here.
  */
 export declare class MemoryTraceStore implements TraceStore {
+    /** Not durable: its runs live only in this process. */
+    readonly durable = false;
     private readonly runs;
     private readonly maxRuns;
     constructor(options?: MemoryTraceStoreOptions);

package/dist-cjs/tracing/stores.js

@@ -14,6 +14,7 @@ Object.defineProperty(exports, "assembleTree", { enumerable: true, get: function
 Object.defineProperty(exports, "isAfterCursor", { enumerable: true, get: function () { return query_js_1.isAfterCursor; } });
 Object.defineProperty(exports, "runCursor", { enumerable: true, get: function () { return query_js_1.runCursor; } });
 class MemoryTraceStore {
+    durable = false;
     runs = new Map();
     maxRuns;
     constructor(options = {}) {

package/dist-cjs/tracing/tracer.d.ts

@@ -33,7 +33,8 @@ export interface TracerOptions {
      * run stays in the store with every run that finished, and a run in progress can be watched.
      * Sampling is then decided when the root starts. A trace not sampled is still held in memory and
      * written at the end when tail sampling keeps it, so `keepErrors` still shows every failure.
-     * Defaults to false, which writes at the end, as before.
+     * Defaults to false, which writes at the end, as before; under 3.0's defaults, `useDefaults('3.0')`,
+     * to true on a durable store, one whose `durable` is not `false`.
      */
     incremental?: boolean;
     /**
@@ -139,10 +140,12 @@ export declare class Tracer {
     /** Runs that finished, whose events are closed. Weak, so a run never finished is never held. */
     private readonly closed;
     private readonly now;
+    /** Whether runs are written as they start and finish: as asked, or as the defaults in effect say. */
+    private readonly writesAsItRuns;
     constructor(options: TracerOptions);
     /** Where runs are written. */
     get store(): TraceStore;
-    /** Whether runs are written as they start and finish, as `incremental` asks. */
+    /** Whether runs are written as they start and finish, as `incremental` asks, or 3.0's defaults do. */
     get incremental(): boolean;
     /** The run currently in scope, if any. */
     current(): {

package/dist-cjs/tracing/tracer.js

@@ -4,6 +4,7 @@ exports.Tracer = void 0;
 exports.w3cTraceId = w3cTraceId;
 exports.w3cSpanId = w3cSpanId;
 exports.stripFields = stripFields;
+const defaults_js_1 = require("../core/defaults.js");
 const ids_js_1 = require("../utils/ids.js");
 const node_async_hooks_1 = require("node:async_hooks");
 const DEFAULT_MAX_EVENTS = 128;
@@ -15,15 +16,17 @@ class Tracer {
     scopes = new WeakMap();
     closed = new WeakSet();
     now;
+    writesAsItRuns;
     constructor(options) {
         this.options = options;
         this.now = options.now ?? (() => new Date());
+        this.writesAsItRuns = options.incremental ?? ((0, defaults_js_1.nexusDefaults)() === '3.0' && options.store?.durable !== false);
     }
     get store() {
         return this.options.store;
     }
     get incremental() {
-        return this.options.incremental === true;
+        return this.writesAsItRuns;
     }
     current() {
         const active = this.context.getStore();
@@ -59,7 +62,7 @@ class Tracer {
             metadata: { ...this.options.metadata, ...options.metadata },
             ...(options.links?.length ? { links: linksOf(options.links) } : {}),
         };
-        if (this.options.incremental && sampled) {
+        if (this.writesAsItRuns && sampled) {
             this.enqueue(run.traceId, () => this.options.store.save(this.prepare(run)));
         }
         else {
@@ -152,7 +155,7 @@ class Tracer {
                 ...(options.provider ? { provider: options.provider } : {}),
                 metadata: { ...run.metadata, ...options.metadata },
             };
-            if (this.options.incremental && sampled) {
+            if (this.writesAsItRuns && sampled) {
                 await this.enqueue(run.traceId, async () => {
                     const prepared = this.prepare(finished);
                     await this.options.store.save(prepared);

package/dist-cjs/types/embeddings.d.ts

@@ -2,7 +2,7 @@ import type { ProviderCallContext } from './lifecycle.js';
 import type { CacheConfig, RateLimitConfig, RetryConfig } from './config.js';
 import type { ModelStatus } from './providers.js';
 import type { ResponseCost, TokenUsage } from './response.js';
-import type { AudioContent, DocumentContent, ImageContent, TextContent } from './messages.js';
+import type { AudioContent, BinaryBuffer, DocumentContent, ImageContent, TextContent } from './messages.js';
 /**
  * What the vector will be used for.
  *
@@ -16,12 +16,33 @@ export type EmbeddingEncodingFormat = 'float' | 'base64';
 /** How an input longer than the model's limit is handled. */
 export type EmbeddingTruncateMode = 'none' | 'start' | 'end';
 /** What an embedding input can hold besides text. */
-export type EmbeddingModality = 'text' | 'image' | 'audio' | 'document';
+export type EmbeddingModality = 'text' | 'image' | 'audio' | 'video' | 'document';
 /**
- * A part of an embedding input: the same text, image, audio, and document parts a message takes.
- * Media must hold its content, as bytes, base64, or a URL; a path or a transcript is refused.
+ * A video to embed, by its content. A message's video part names a file on disk, which a provider
+ * cannot be sent, so an embedding takes the video itself: its bytes or base64, with its container,
+ * such as `mp4` or `mov`, or a URL a provider reads, such as a Files API URI Google embeds from.
  */
-export type EmbeddingPart = TextContent | ImageContent | AudioContent | DocumentContent;
+export interface EmbeddingVideoPart {
+    /** Discriminates this part. */
+    type: 'video';
+    /** The video. */
+    source: {
+        buffer: BinaryBuffer;
+        format?: string;
+    } | {
+        base64: string;
+        format?: string;
+    } | {
+        url: string;
+        format?: string;
+    };
+}
+/**
+ * A part of an embedding input: the same text, image, audio, and document parts a message takes, and
+ * a video by its content. Media must hold its content, as bytes, base64, or a URL; a path or a
+ * transcript is refused.
+ */
+export type EmbeddingPart = TextContent | ImageContent | AudioContent | DocumentContent | EmbeddingVideoPart;
 /**
  * One input made of parts, such as an image with its caption, embedded as one vector in the same
  * space as text. Only a model that takes each part's kind is sent it; any other is refused.
@@ -132,6 +153,11 @@ export interface EmbeddingMeta {
     usage: TokenUsage;
     /** What it cost, from the model's price. */
     cost: ResponseCost;
+    /**
+     * Audio and video parts whose length could not be read from their bytes, such as one given by URL,
+     * and so are not in `cost` for a model that prices them by length.
+     */
+    unpricedMedia?: number;
     /** True when every input was served from cache and no provider call was made. */
     cacheHit: boolean;
     /** Inputs served from cache when only part of the batch was. */
@@ -308,6 +334,22 @@ export interface EmbeddingModelCapabilities {
     costPer1kInput: number;
     /** Price per image embedded, in US dollars, for a model that takes images. */
     costPerImage?: number;
+    /** Price per second of audio embedded, in US dollars, its length read from the audio's own headers. */
+    costPerAudioSecond?: number;
+    /** Price per frame the model takes from a video, in US dollars. */
+    costPerVideoFrame?: number;
+    /**
+     * How many frames the model takes from a video, which `costPerVideoFrame` prices: `perSecond`, and
+     * at most `max`. Defaults to one a second, with no most.
+     */
+    videoFrames?: {
+        perSecond: number;
+        max?: number;
+    };
+    /** The longest audio part it takes, in seconds. A longer one is refused before it is sent. */
+    maxAudioSeconds?: number;
+    /** The longest video part it takes, in seconds. A longer one is refused before it is sent. */
+    maxVideoSeconds?: number;
     /** What its inputs can hold. Text only when absent. */
     inputModalities?: readonly EmbeddingModality[];
     /** True when the model already returns unit-length vectors. */

package/dist-cjs/types/messages.d.ts

@@ -1,3 +1,4 @@
+import type { McpCallerRequests } from '../mcp/protocol.js';
 import type { CacheTtl, ReasoningEffort } from './providers.js';
 import type { GraphEvent } from './graph.js';
 import type { AssetInput } from './images.js';
@@ -244,6 +245,12 @@ export interface ToolContext {
      * subgraph events, so a tool calls it as `context.forward?.()`.
      */
     forward?: (event: GraphEvent, namespace: readonly string[]) => void;
+    /**
+     * When an MCP server runs the tool, what it may ask the client that called it: a completion from
+     * its model, an answer from its person, and its roots. Absent elsewhere, so a tool calls it as
+     * `context.mcp?.createMessage()`.
+     */
+    mcp?: McpCallerRequests;
 }
 /** What a tool returned, correlated with the call that asked for it. */
 export interface ToolCallResult {

package/dist-cjs/types/principal.d.ts

@@ -14,7 +14,7 @@ export interface Principal {
     scopes?: readonly string[];
     /** The caller's roles, for policies and tools that decide by role rather than by scope. */
     roles?: readonly string[];
-    /** How the caller was authenticated: `jwt`, `api-key`, `proxy`, or a name of your own. */
+    /** How the caller was authenticated: `jwt`, `api-key`, `session`, `proxy`, or a name of your own. */
     method?: string;
     /** The verified claims or attributes the principal was built from, for anything the fields above leave out. */
     claims?: Readonly<Record<string, unknown>>;

package/dist-cjs/types/response.d.ts

@@ -193,6 +193,11 @@ export interface StreamChunk {
     meta?: Partial<ResponseMeta>;
     /** What went wrong, for an `error` chunk. */
     error?: string;
+    /**
+     * For a `text` chunk, its tokens' log probabilities, in order, when the request asked for
+     * `logprobs` and the provider streams them: the chunks' together are the response's.
+     */
+    logprobs?: TokenLogprob[];
 }
 /** A streamed completion: iterate it for chunks, or abort it. */
 export interface NexusStream extends AsyncIterable<StreamChunk> {

package/dist-cjs/types/telephony.d.ts

@@ -76,6 +76,37 @@ export interface TelephonyGatherConfig {
     /** Text spoken to the caller while listening. */
     prompts?: string[];
 }
+/** What a caller gave a gather: what they said, the keys they pressed, or both. */
+export interface TelephonyGatherResult {
+    /** Provider that collected it. */
+    providerUsed: string;
+    /** The provider's id for the call. */
+    callId: string;
+    /** What the caller said, as transcribed. */
+    speech?: string;
+    /** How sure the transcription is, from 0 to 1, when the provider says. */
+    confidence?: number;
+    /** The keys the caller pressed. */
+    digits?: string;
+    /**
+     * False for speech still being transcribed, which a later result replaces; true for what the
+     * caller finished saying, and for keys.
+     */
+    final: boolean;
+    /** The webhook, unmodified. */
+    raw?: unknown;
+}
+/** Stops a gather still listening: on Telnyx, the transcription a speech gather started. */
+export interface EndGatherRequest {
+    /** Provider the call belongs to. */
+    provider?: string;
+    /** The provider's id for the call. */
+    callId: string;
+    /** The tenant it is for: its credentials, regions, and budget apply. */
+    tenantId?: string;
+    /** Aborts the request. */
+    signal?: AbortSignal;
+}
 /** A conference a call joins, created by the first call to join it. */
 export interface TelephonyConferenceConfig {
     /** The conference's name: every call that joins the same name meets in it. */
@@ -747,6 +778,13 @@ export interface TelephonyProvider {
     downloadRecording?(request: DownloadRecordingRequest, context?: ProviderCallContext): Promise<TelephonyRecordingAudio>;
     /** Deletes a recording at the provider. */
     deleteRecording?(request: DeleteRecordingRequest, context?: ProviderCallContext): Promise<void>;
+    /**
+     * Parses what a gather collected, from the webhook that carries it, or returns `undefined` for a
+     * body that is not one.
+     */
+    parseGatherResult?(body: string | URLSearchParams | Record<string, unknown>): TelephonyGatherResult | undefined;
+    /** Stops a gather still listening, on a provider whose gather does not end by itself. */
+    endGather?(request: EndGatherRequest, context?: ProviderCallContext): Promise<void>;
     /** Sends a text message. */
     sendMessage?(request: SendMessageRequest, context?: ProviderCallContext): Promise<TelephonyMessage>;
     /**

package/dist-cjs/types/tracing.d.ts

@@ -176,6 +176,12 @@ export interface TraceStore {
     addFeedback?(runId: string, feedback: RunFeedback): Promise<void> | void;
     /** Removes runs older than a cutoff, returning how many went. */
     prune?(before: string): Promise<number> | number;
+    /**
+     * `false` for a store whose runs live only in this process, such as `MemoryTraceStore`, where a
+     * trace written as it runs outlives nothing. Any other store is durable, which 3.0's defaults write to
+     * incrementally.
+     */
+    readonly durable?: boolean;
 }
 /**
  * Receives every finished run a tracer keeps, after redaction, to send somewhere else: an

package/dist-cjs/version.d.ts

@@ -2,4 +2,4 @@
  * This package's version, recorded where provenance matters, such as on every experiment. A test
  * keeps it equal to `package.json`.
  */
-export declare const NEXUS_VERSION = "2.20.2";
+export declare const NEXUS_VERSION = "2.22.0";

package/dist-cjs/version.js

@@ -1,4 +1,4 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.NEXUS_VERSION = void 0;
-exports.NEXUS_VERSION = '2.20.2';
+exports.NEXUS_VERSION = '2.22.0';

package/dist-cjs/voice/providers/google.d.ts

@@ -0,0 +1,44 @@
+import type { ProviderCallContext } from '../../types/lifecycle.js';
+import type { SpeechRequest, SpeechResponse, TranscriptionRequest, TranscriptionResponse, VoiceProvider, VoiceProviderInfo } from '../../types/voice.js';
+/** Options for the Gemini voice provider. */
+export interface GoogleVoiceProviderConfig {
+    /** Gemini API key, sent as `x-goog-api-key`. */
+    apiKey: string;
+    /** API base URL. Defaults to `https://generativelanguage.googleapis.com/v1beta`. */
+    baseUrl?: string;
+    /** The model that transcribes. Defaults to `gemini-3.8-flash`. */
+    transcriptionModel?: string;
+    /** The model that speaks. Defaults to `gemini-3.8-flash-tts`. */
+    speechModel?: string;
+    /** Voice used when a request names none, one of Gemini's prebuilt voices. Defaults to `Kore`. */
+    defaultVoice?: string;
+    /** Headers added to every request. */
+    headers?: Record<string, string>;
+    /** Replaces the global `fetch`. */
+    fetch?: typeof fetch;
+}
+/**
+ * Gemini speech-to-text and text-to-speech, through `generateContent`: a text model transcribes audio
+ * it hears, and a speech model speaks text in one of Gemini's voices, as WAV or raw PCM. Experimental
+ * until a live run is recorded.
+ */
+export declare class GoogleVoiceProvider implements VoiceProvider {
+    private readonly config;
+    /** Always `google`, with transcription and speech. */
+    readonly info: VoiceProviderInfo;
+    constructor(config: GoogleVoiceProviderConfig);
+    /**
+     * Transcribes audio: its words, in the language spoken, and nothing else. `language` and `prompt`
+     * guide the model; a request for timings is refused, since Gemini's transcript has none.
+     */
+    transcribe(request: TranscriptionRequest, context?: ProviderCallContext): Promise<TranscriptionResponse>;
+    /**
+     * Speaks text in one of Gemini's voices, as WAV, the default, or raw 16-bit PCM. `instructions`, and a
+     * `speed` other than 1, direct the delivery as a system instruction, never spoken.
+     */
+    speak(request: SpeechRequest, context?: ProviderCallContext): Promise<SpeechResponse>;
+    /** The audio as a part of the request: a Files API URI by reference, anything else read and sent inline. */
+    private audioPart;
+    private read;
+    private generate;
+}