@juspay/neurolink 9.64.0 → 9.65.0

package/dist/providers/xai.js

@@ -0,0 +1,171 @@
+import { createOpenAI } from "@ai-sdk/openai";
+import { stepCountIs, streamText } from "ai";
+import { XaiModels } from "../constants/enums.js";
+import { BaseProvider } from "../core/baseProvider.js";
+import { DEFAULT_MAX_STEPS } from "../core/constants.js";
+import { streamAnalyticsCollector } from "../core/streamAnalytics.js";
+import { isNeuroLink } from "../neurolink.js";
+import { createLoggingFetch } from "../utils/loggingFetch.js";
+import { tracers, ATTR, withClientStreamSpan } from "../telemetry/index.js";
+import { AuthenticationError, InvalidModelError, NetworkError, ProviderError, RateLimitError, } from "../types/index.js";
+import { logger } from "../utils/logger.js";
+import { createXaiConfig, getProviderModel, validateApiKey, } from "../utils/providerConfig.js";
+import { composeAbortSignals, createTimeoutController, TimeoutError, } from "../utils/timeout.js";
+import { emitToolEndFromStepFinish } from "../utils/toolEndEmitter.js";
+import { resolveToolChoice } from "../utils/toolChoice.js";
+import { toAnalyticsStreamResult } from "./providerTypeUtils.js";
+/**
+ * Logging fetch wrapper — masks the proxy URL on non-2xx responses so
+ * stack traces and log lines don't leak the upstream's internal token /
+ * tenant id (mirrors the deepseek/groq/etc. providers).
+ */
+const XAI_DEFAULT_BASE_URL = "https://api.x.ai/v1";
+const getXaiApiKey = () => validateApiKey(createXaiConfig());
+const getDefaultXaiModel = () => getProviderModel("XAI_MODEL", XaiModels.GROK_3);
+/**
+ * xAI Grok Provider
+ *
+ * OpenAI-compatible chat completions at api.x.ai/v1. Supports the Grok
+ * family: grok-2, grok-3, grok-3-mini, grok-2-vision-latest (multimodal),
+ * and grok-beta. Streaming and tool calling supported.
+ *
+ * @see https://docs.x.ai/api
+ */
+export class XaiProvider extends BaseProvider {
+    model;
+    apiKey;
+    baseURL;
+    constructor(modelName, sdk, _region, credentials) {
+        const validatedNeurolink = isNeuroLink(sdk) ? sdk : undefined;
+        super(modelName, "xai", validatedNeurolink);
+        const overrideApiKey = credentials?.apiKey?.trim();
+        this.apiKey =
+            overrideApiKey && overrideApiKey.length > 0
+                ? overrideApiKey
+                : getXaiApiKey();
+        this.baseURL =
+            credentials?.baseURL ?? process.env.XAI_BASE_URL ?? XAI_DEFAULT_BASE_URL;
+        const xai = createOpenAI({
+            apiKey: this.apiKey,
+            baseURL: this.baseURL,
+            fetch: createLoggingFetch("xai"),
+        });
+        this.model = xai.chat(this.modelName);
+        logger.debug("xAI Provider initialized", {
+            modelName: this.modelName,
+            providerName: this.providerName,
+            baseURL: this.baseURL,
+        });
+    }
+    async executeStream(options, _analysisSchema) {
+        return withClientStreamSpan({
+            name: "neurolink.provider.stream",
+            tracer: tracers.provider,
+            attributes: {
+                [ATTR.GEN_AI_SYSTEM]: "xai",
+                [ATTR.GEN_AI_MODEL]: this.modelName,
+                [ATTR.GEN_AI_OPERATION]: "stream",
+                [ATTR.NL_STREAM_MODE]: true,
+            },
+        }, async () => this.executeStreamInner(options), (r) => r.stream, (r, wrapped) => ({ ...r, stream: wrapped }));
+    }
+    async executeStreamInner(options) {
+        this.validateStreamOptions(options);
+        const startTime = Date.now();
+        const timeout = this.getTimeout(options);
+        const timeoutController = createTimeoutController(timeout, this.providerName, "stream");
+        try {
+            const shouldUseTools = !options.disableTools && this.supportsTools();
+            const tools = shouldUseTools
+                ? options.tools || (await this.getAllTools())
+                : {};
+            const messages = await this.buildMessagesForStream(options);
+            const model = await this.getAISDKModelWithMiddleware(options);
+            const result = await streamText({
+                model,
+                messages,
+                temperature: options.temperature,
+                maxOutputTokens: options.maxTokens,
+                tools,
+                stopWhen: stepCountIs(options.maxSteps || DEFAULT_MAX_STEPS),
+                toolChoice: resolveToolChoice(options, tools, shouldUseTools),
+                abortSignal: composeAbortSignals(options.abortSignal, timeoutController?.controller.signal),
+                experimental_telemetry: this.telemetryHandler.getTelemetryConfig(options),
+                experimental_repairToolCall: this.getToolCallRepairFn(options),
+                onStepFinish: ({ toolCalls, toolResults }) => {
+                    emitToolEndFromStepFinish(this.neurolink?.getEventEmitter(), toolResults);
+                    this.handleToolExecutionStorage(toolCalls, toolResults, options, new Date()).catch((error) => {
+                        logger.warn("[XaiProvider] Failed to store tool executions", {
+                            provider: this.providerName,
+                            error: error instanceof Error ? error.message : String(error),
+                        });
+                    });
+                },
+            });
+            timeoutController?.cleanup();
+            const transformedStream = this.createTextStream(result);
+            const analyticsPromise = streamAnalyticsCollector.createAnalytics(this.providerName, this.modelName, toAnalyticsStreamResult(result), Date.now() - startTime, {
+                requestId: `xai-stream-${Date.now()}`,
+                streamingMode: true,
+            });
+            return {
+                stream: transformedStream,
+                provider: this.providerName,
+                model: this.modelName,
+                analytics: analyticsPromise,
+                metadata: { startTime, streamId: `xai-${Date.now()}` },
+            };
+        }
+        catch (error) {
+            timeoutController?.cleanup();
+            throw this.handleProviderError(error);
+        }
+    }
+    getProviderName() {
+        return this.providerName;
+    }
+    getDefaultModel() {
+        return getDefaultXaiModel();
+    }
+    getAISDKModel() {
+        return this.model;
+    }
+    formatProviderError(error) {
+        if (error instanceof TimeoutError) {
+            return new NetworkError(`Request timed out: ${error.message}`, "xai");
+        }
+        const errorRecord = error;
+        const message = typeof errorRecord?.message === "string"
+            ? errorRecord.message
+            : "Unknown error";
+        if (message.includes("Invalid API key") ||
+            message.includes("Authentication") ||
+            message.includes("401") ||
+            message.includes("invalid_api_key")) {
+            return new AuthenticationError("Invalid xAI API key. Please check your XAI_API_KEY environment variable. Get one at https://console.x.ai/", "xai");
+        }
+        if (message.includes("rate limit") || message.includes("429")) {
+            return new RateLimitError("xAI rate limit exceeded. Back off and retry.", "xai");
+        }
+        if (message.includes("model_not_found") || message.includes("404")) {
+            return new InvalidModelError(`xAI model '${this.modelName}' not found. Use grok-2-latest, grok-3, grok-3-mini, grok-2-vision-latest, or grok-beta.`, "xai");
+        }
+        if (message.includes("insufficient_quota") ||
+            message.includes("quota exceeded")) {
+            return new ProviderError("xAI account has insufficient quota. Top up at https://console.x.ai/", "xai");
+        }
+        return new ProviderError(`xAI error: ${message}`, "xai");
+    }
+    async validateConfiguration() {
+        return typeof this.apiKey === "string" && this.apiKey.trim().length > 0;
+    }
+    getConfiguration() {
+        return {
+            provider: this.providerName,
+            model: this.modelName,
+            defaultModel: getDefaultXaiModel(),
+            baseURL: this.baseURL,
+        };
+    }
+}
+export default XaiProvider;

package/dist/telemetry/index.d.ts

@@ -1,6 +1,6 @@
 export { TelemetryService } from "./telemetryService.js";
 export { tracers } from "./tracers.js";
-export { withSpan, withClientSpan } from "./withSpan.js";
+export { withSpan, withClientSpan, withStreamSpan, withClientStreamSpan, } from "./withSpan.js";
 export { ATTR } from "./attributes.js";
 /**
  * Initialize telemetry for NeuroLink

package/dist/telemetry/index.js

@@ -1,7 +1,7 @@
 // Optional Telemetry Infrastructure (Phase 2)
 export { TelemetryService } from "./telemetryService.js";
 export { tracers } from "./tracers.js";
-export { withSpan, withClientSpan } from "./withSpan.js";
+export { withSpan, withClientSpan, withStreamSpan, withClientStreamSpan, } from "./withSpan.js";
 export { ATTR } from "./attributes.js";
 import { logger } from "../utils/logger.js";
 /**

package/dist/telemetry/tracers.d.ts

@@ -1,3 +1,22 @@
+/**
+ * Pre-resolved tracers cached at module load.
+ *
+ * Module-load timing is intentional and safe: `trace.getTracer(name)` from
+ * `@opentelemetry/api` always returns a proxy that defers to whatever
+ * `TracerProvider` is registered globally at the moment a span is created.
+ * If the OTel SDK initializes *after* this module is imported (the common
+ * case — telemetry setup typically runs once at app boot, after lazy
+ * imports have already pulled this file), the tracer reference is still
+ * valid; spans created via it correctly route to the SDK once registered.
+ *
+ * Pre-SDK calls produce a noop span — `span.end()` etc. are valid; nothing
+ * is exported. This is OTel's documented contract:
+ * https://opentelemetry.io/docs/specs/otel/trace/api/#tracerprovider
+ *
+ * If a future refactor moves to a non-proxy TracerProvider API (e.g. a
+ * custom provider where `getTracer` returns the live instance), revisit
+ * this file and switch to lazy accessors.
+ */
 export declare const tracers: {
     readonly sdk: import("@opentelemetry/api").Tracer;
     readonly provider: import("@opentelemetry/api").Tracer;

package/dist/telemetry/tracers.js

@@ -1,4 +1,23 @@
 import { trace } from "@opentelemetry/api";
+/**
+ * Pre-resolved tracers cached at module load.
+ *
+ * Module-load timing is intentional and safe: `trace.getTracer(name)` from
+ * `@opentelemetry/api` always returns a proxy that defers to whatever
+ * `TracerProvider` is registered globally at the moment a span is created.
+ * If the OTel SDK initializes *after* this module is imported (the common
+ * case — telemetry setup typically runs once at app boot, after lazy
+ * imports have already pulled this file), the tracer reference is still
+ * valid; spans created via it correctly route to the SDK once registered.
+ *
+ * Pre-SDK calls produce a noop span — `span.end()` etc. are valid; nothing
+ * is exported. This is OTel's documented contract:
+ * https://opentelemetry.io/docs/specs/otel/trace/api/#tracerprovider
+ *
+ * If a future refactor moves to a non-proxy TracerProvider API (e.g. a
+ * custom provider where `getTracer` returns the live instance), revisit
+ * this file and switch to lazy accessors.
+ */
 export const tracers = {
     sdk: trace.getTracer("neurolink"),
     provider: trace.getTracer("neurolink.provider"),

package/dist/telemetry/withSpan.d.ts

@@ -2,3 +2,38 @@ import { type Span } from "@opentelemetry/api";
 import type { SpanOptions } from "../types/index.js";
 export declare function withSpan<T>(options: SpanOptions, fn: (span: Span) => Promise<T>): Promise<T>;
 export declare function withClientSpan<T>(options: Omit<SpanOptions, "kind">, fn: (span: Span) => Promise<T>): Promise<T>;
+/**
+ * Span wrapper for streaming operations.
+ *
+ * Unlike {@link withSpan}, which ends the span when `fn` resolves, this
+ * helper extends the span's lifetime until the **consumer** of the returned
+ * iterable reaches end-of-stream (success), throws (error), or aborts.
+ *
+ * Required for any operation whose result is a producer (`StreamResult`,
+ * `AsyncIterable`-returning function, etc.) — ending the span on `fn`
+ * resolution would capture only the setup phase and report zero tokens /
+ * meaningless duration.
+ *
+ * @param options       Span options (name, tracer, attributes, kind).
+ * @param fn            Callback that produces the result. Must NOT depend on
+ *                      the iterable being consumed before returning.
+ * @param getIterable   Selector that picks the `AsyncIterable` out of the
+ *                      result for wrapping.
+ * @param setIterable   Setter that returns a new result with the iterable
+ *                      replaced by the wrapped (span-lifetime-extending)
+ *                      iterable. Should be a pure function — clone the
+ *                      result rather than mutating in place.
+ *
+ * @example
+ * ```ts
+ * return withStreamSpan(
+ *   { name: "neurolink.provider.stream", tracer, attributes: {...} },
+ *   async () => this.executeStreamInner(options),
+ *   (r) => r.stream,
+ *   (r, wrapped) => ({ ...r, stream: wrapped }),
+ * );
+ * ```
+ */
+export declare function withStreamSpan<TResult, TChunk>(options: SpanOptions, fn: (span: Span) => Promise<TResult>, getIterable: (result: TResult) => AsyncIterable<TChunk>, setIterable: (result: TResult, wrapped: AsyncGenerator<TChunk>) => TResult): Promise<TResult>;
+/** Convenience CLIENT-kind alias matching `withClientSpan`. */
+export declare function withClientStreamSpan<TResult, TChunk>(options: Omit<SpanOptions, "kind">, fn: (span: Span) => Promise<TResult>, getIterable: (result: TResult) => AsyncIterable<TChunk>, setIterable: (result: TResult, wrapped: AsyncGenerator<TChunk>) => TResult): Promise<TResult>;

package/dist/telemetry/withSpan.js

@@ -32,3 +32,106 @@ export async function withSpan(options, fn) {
 export async function withClientSpan(options, fn) {
     return withSpan({ ...options, kind: SpanKind.CLIENT }, fn);
 }
+/**
+ * Span wrapper for streaming operations.
+ *
+ * Unlike {@link withSpan}, which ends the span when `fn` resolves, this
+ * helper extends the span's lifetime until the **consumer** of the returned
+ * iterable reaches end-of-stream (success), throws (error), or aborts.
+ *
+ * Required for any operation whose result is a producer (`StreamResult`,
+ * `AsyncIterable`-returning function, etc.) — ending the span on `fn`
+ * resolution would capture only the setup phase and report zero tokens /
+ * meaningless duration.
+ *
+ * @param options       Span options (name, tracer, attributes, kind).
+ * @param fn            Callback that produces the result. Must NOT depend on
+ *                      the iterable being consumed before returning.
+ * @param getIterable   Selector that picks the `AsyncIterable` out of the
+ *                      result for wrapping.
+ * @param setIterable   Setter that returns a new result with the iterable
+ *                      replaced by the wrapped (span-lifetime-extending)
+ *                      iterable. Should be a pure function — clone the
+ *                      result rather than mutating in place.
+ *
+ * @example
+ * ```ts
+ * return withStreamSpan(
+ *   { name: "neurolink.provider.stream", tracer, attributes: {...} },
+ *   async () => this.executeStreamInner(options),
+ *   (r) => r.stream,
+ *   (r, wrapped) => ({ ...r, stream: wrapped }),
+ * );
+ * ```
+ */
+export async function withStreamSpan(options, fn, getIterable, setIterable) {
+    const { name, tracer, kind = SpanKind.CLIENT, attributes } = options;
+    const span = tracer.startSpan(name, { kind });
+    if (attributes) {
+        for (const [key, value] of Object.entries(attributes)) {
+            if (value !== undefined) {
+                span.setAttribute(key, value);
+            }
+        }
+    }
+    let result;
+    try {
+        result = await fn(span);
+    }
+    catch (error) {
+        span.setStatus({
+            code: SpanStatusCode.ERROR,
+            message: error instanceof Error ? error.message : String(error),
+        });
+        if (error instanceof Error) {
+            span.recordException(error);
+        }
+        span.end();
+        throw error;
+    }
+    // Setup succeeded. Now wrap the iterable so the span ends with the stream.
+    const original = getIterable(result);
+    let ended = false;
+    const endOnce = (status) => {
+        if (ended) {
+            return;
+        }
+        ended = true;
+        if (status) {
+            span.setStatus(status);
+        }
+        else {
+            span.setStatus({ code: SpanStatusCode.OK });
+        }
+        span.end();
+    };
+    // Build an AsyncGenerator (not just an AsyncIterable) so the wrapped
+    // value is structurally assignable wherever the original stream type
+    // already expected `AsyncGenerator<X>` (e.g. Ollama's executeStream
+    // returns `AsyncGenerator<{ content: string }>`).
+    const wrapped = (async function* () {
+        try {
+            for await (const chunk of original) {
+                yield chunk;
+            }
+            endOnce();
+        }
+        catch (err) {
+            // recordException must come BEFORE span.end() — OTel ignores events
+            // on ended spans, so flipping the order silently drops the exception.
+            if (err instanceof Error) {
+                span.recordException(err);
+            }
+            endOnce({
+                code: SpanStatusCode.ERROR,
+                message: err instanceof Error ? err.message : String(err),
+            });
+            throw err;
+        }
+    })();
+    return setIterable(result, wrapped);
+}
+/** Convenience CLIENT-kind alias matching `withClientSpan`. */
+export async function withClientStreamSpan(options, fn, getIterable, setIterable) {
+    return withStreamSpan({ ...options, kind: SpanKind.CLIENT }, fn, getIterable, setIterable);
+}

package/dist/types/avatar.d.ts

@@ -0,0 +1,143 @@
+/**
+ * Avatar / Lip-sync Type Definitions
+ *
+ * Types for generating talking-head videos by combining a portrait image
+ * with narration audio (or text routed through a TTS provider first).
+ *
+ * Avatar is a separate modality from Video — Video is image-to-video
+ * (motion synthesis); Avatar is image + audio → lip-synced video.
+ *
+ * @module types/avatar
+ */
+/**
+ * Output formats for avatar videos.
+ */
+export type AvatarVideoFormat = "mp4" | "webm" | "mov";
+/**
+ * Quality presets for avatar generation. Provider-specific mappings:
+ *   - D-ID: "standard" → 720p, "hd" → 1080p
+ *   - HeyGen: "standard" → 720p, "hd" → 1080p with enhancement
+ *   - MuseTalk (Replicate): single quality only; "hd" is no-op
+ */
+export type AvatarQuality = "standard" | "hd";
+/**
+ * Options for avatar video generation.
+ */
+export type AvatarOptions = {
+    /** Source portrait image (Buffer, file path, or HTTPS URL). */
+    image: Buffer | string;
+    /**
+     * Audio source — direct lip-sync.
+     * Either provide `audio` OR `text` (with optional `ttsProvider` / `voice`).
+     */
+    audio?: Buffer | string;
+    /**
+     * Text for the avatar to speak. When provided without `audio`, the
+     * NeuroLink dispatcher first runs TTS (`ttsProvider`) to produce audio,
+     * then passes the audio to the avatar handler.
+     */
+    text?: string;
+    /** TTS provider for text → audio when `text` is used. Default: "openai-tts". */
+    ttsProvider?: string;
+    /** Voice id passed through to the TTS provider when `text` is used. */
+    voice?: string;
+    /** Avatar provider override (e.g. "d-id", "heygen", "replicate"). */
+    provider?: string;
+    /** Output quality preset. */
+    quality?: AvatarQuality;
+    /** Output format (default: "mp4"). */
+    format?: AvatarVideoFormat;
+    /** Output file path (optional — buffer is always returned in the result). */
+    output?: string;
+    /** Per-call timeout in ms (default: 5 minutes). */
+    timeout?: number;
+    /** Provider-specific additional options. */
+    [key: string]: unknown;
+};
+/**
+ * Result of an avatar generation request.
+ */
+export type AvatarResult = {
+    /** Generated video buffer. */
+    buffer: Buffer;
+    /** Output format. */
+    format: AvatarVideoFormat;
+    /** File size in bytes. */
+    size: number;
+    /** Duration in seconds (when reported by the provider). */
+    duration?: number;
+    /** Provider used. */
+    provider?: string;
+    /** Performance / request metadata. */
+    metadata?: {
+        /** Request latency in milliseconds. */
+        latency: number;
+        /** Provider name. */
+        provider?: string;
+        /** Model variant used (when applicable). */
+        model?: string;
+        /** Job / talk identifier from the upstream. */
+        jobId?: string;
+        /** Any additional provider-specific metadata. */
+        [key: string]: unknown;
+    };
+};
+/**
+ * Handler contract for avatar / lip-sync providers.
+ *
+ * Implementations enforce their own timeouts. Recommended:
+ *   - Per-request fetch timeout: 30 seconds
+ *   - Total job-completion timeout: 5 minutes
+ */
+export type AvatarHandler = {
+    /**
+     * Generate a talking-head video from an image + audio (or pre-rendered text).
+     */
+    generate(options: AvatarOptions): Promise<AvatarResult>;
+    /** Validate the provider is configured (auth, base URL, etc.). */
+    isConfigured(): boolean;
+    /** Maximum supported audio length in seconds (provider-specific). */
+    readonly maxAudioDurationSeconds?: number;
+    /** Output formats supported by this handler. */
+    readonly supportedFormats?: readonly AvatarVideoFormat[];
+};
+/**
+ * Valid avatar video formats — runtime validation array.
+ */
+export declare const VALID_AVATAR_FORMATS: readonly AvatarVideoFormat[];
+/**
+ * D-ID `/talks` API response shape.
+ *
+ * Used by `DIDAvatar` handler to type-check upstream responses. Lives here
+ * (in `src/lib/types/`) per CLAUDE.md rule 2; the handler imports it via
+ * the types barrel.
+ */
+export type DIDTalkResponse = {
+    id: string;
+    status?: string;
+    result_url?: string;
+    error?: {
+        kind?: string;
+        description?: string;
+    };
+    duration?: number;
+};
+/**
+ * HeyGen `/v1/video_status.get` response shape.
+ */
+export type HeyGenVideoStatusResponse = {
+    code?: number;
+    data?: {
+        id?: string;
+        status?: "pending" | "processing" | "completed" | "failed";
+        video_url?: string;
+        thumbnail_url?: string;
+        duration?: number;
+        error?: {
+            code?: string;
+            message?: string;
+            detail?: string;
+        };
+    };
+    message?: string;
+};

package/dist/types/avatar.js

@@ -0,0 +1,19 @@
+/**
+ * Avatar / Lip-sync Type Definitions
+ *
+ * Types for generating talking-head videos by combining a portrait image
+ * with narration audio (or text routed through a TTS provider first).
+ *
+ * Avatar is a separate modality from Video — Video is image-to-video
+ * (motion synthesis); Avatar is image + audio → lip-synced video.
+ *
+ * @module types/avatar
+ */
+/**
+ * Valid avatar video formats — runtime validation array.
+ */
+export const VALID_AVATAR_FORMATS = [
+    "mp4",
+    "webm",
+    "mov",
+];

package/dist/types/cli.d.ts

@@ -8,6 +8,8 @@ import type { ToolCall, ToolResult } from "./tools.js";
 import type { TTSResult } from "./tts.js";
 import type { VideoGenerationResult } from "./multimodal.js";
 import type { PPTGenerationResult } from "./ppt.js";
+import type { AvatarResult } from "./avatar.js";
+import type { MusicResult } from "./music.js";
 import type { OAuthTokens } from "./auth.js";
 import type { ClaudeSubscriptionTier } from "./subscription.js";
 import type { ServerFramework } from "./server.js";
@@ -393,6 +395,10 @@ export type CliGenerateResult = CommandResult & {
     audio?: TTSResult;
     /** Video generation result when video mode is enabled */
     video?: VideoGenerationResult;
+    /** Avatar (talking-head) generation result when avatar mode is enabled */
+    avatar?: AvatarResult;
+    /** Music generation result when music mode is enabled */
+    music?: MusicResult;
     /** PPT generation result when ppt mode is enabled */
     ppt?: PPTGenerationResult;
     imageOutput?: {