@builder.io/ai-utils 0.81.3 → 0.83.0

package/src/realtime.js

@@ -0,0 +1,158 @@
+import { z } from "zod";
+export const BUILDER_REALTIME_MODEL = "gpt-realtime-2.1";
+export const BUILDER_REALTIME_MAX_SDP_LENGTH = 256000;
+export const BUILDER_REALTIME_MAX_SESSION_BYTES = 64000;
+const utf8ByteLength = (value) => new TextEncoder().encode(value).length;
+const boundedRecord = (maxBytes, message) => z.record(z.string(), z.unknown()).refine((value) => {
+    try {
+        return utf8ByteLength(JSON.stringify(value)) <= maxBytes;
+    }
+    catch (_a) {
+        return false;
+    }
+}, { message });
+const RealtimeAudioFormatSchema = z.union([
+    z.string().min(1).max(128),
+    boundedRecord(2048, "Audio format configuration is too large"),
+]);
+const RealtimeInputAudioSchema = z.object({
+    format: RealtimeAudioFormatSchema.optional(),
+    noise_reduction: z
+        .object({
+        type: z.enum(["near_field", "far_field"]),
+    })
+        .nullable()
+        .optional(),
+    transcription: z
+        .object({
+        model: z.literal("gpt-4o-mini-transcribe"),
+        language: z.string().min(1).max(32).optional(),
+        prompt: z.string().max(4096).optional(),
+    })
+        .nullable()
+        .optional(),
+    turn_detection: z
+        .object({
+        type: z.enum(["server_vad", "semantic_vad"]),
+        threshold: z.number().min(0).max(1).optional(),
+        prefix_padding_ms: z.number().int().min(0).max(10000).optional(),
+        silence_duration_ms: z.number().int().min(0).max(60000).optional(),
+        idle_timeout_ms: z.number().int().min(0).max(3600000).optional(),
+        eagerness: z.enum(["low", "medium", "high", "auto"]).optional(),
+        create_response: z.boolean().optional(),
+        interrupt_response: z.boolean().optional(),
+    })
+        .nullable()
+        .optional(),
+});
+const RealtimeOutputAudioSchema = z.object({
+    format: RealtimeAudioFormatSchema.optional(),
+    voice: z
+        .union([
+        z.string().min(1).max(128),
+        z.object({ id: z.string().min(1).max(256) }),
+    ])
+        .optional(),
+    speed: z.number().positive().max(4).optional(),
+});
+export const OpenAIRealtimeFunctionToolSchema = z.object({
+    type: z.literal("function"),
+    name: z
+        .string()
+        .min(1)
+        .max(64)
+        .regex(/^[a-zA-Z0-9_-]+$/),
+    description: z.string().max(4096).optional(),
+    parameters: boundedRecord(32000, "Realtime tool parameter schema is too large").optional(),
+    strict: z.boolean().optional(),
+});
+export const OpenAIRealtimeMcpToolSchema = z.object({
+    type: z.literal("mcp"),
+    server_label: z.string().min(1).max(128),
+    server_url: z.url().max(2048).optional(),
+    server_description: z.string().max(4096).optional(),
+    authorization: z.string().min(1).max(8192).optional(),
+    allowed_tools: z
+        .union([
+        z.array(z.string().min(1).max(128)).max(64),
+        boundedRecord(8000, "MCP allowed-tools configuration is too large"),
+    ])
+        .optional(),
+    require_approval: z
+        .union([
+        z.enum(["always", "never"]),
+        boundedRecord(8000, "MCP approval configuration is too large"),
+    ])
+        .optional(),
+});
+export const OpenAIRealtimeToolSchema = z.discriminatedUnion("type", [
+    OpenAIRealtimeFunctionToolSchema,
+    OpenAIRealtimeMcpToolSchema,
+]);
+const OpenAIRealtimeToolChoiceSchema = z.union([
+    z.enum(["auto", "none", "required"]),
+    z.object({
+        type: z.literal("function"),
+        name: z.string().min(1).max(64),
+    }),
+    z.object({
+        type: z.literal("mcp"),
+        server_label: z.string().min(1).max(128),
+        name: z.string().min(1).max(128).optional(),
+    }),
+]);
+/**
+ * The subset of OpenAI's Realtime session configuration accepted by Builder
+ * Connect. Zod objects intentionally strip unknown keys before this config is
+ * forwarded to OpenAI.
+ */
+export const OpenAIRealtimeSessionConfigSchema = z
+    .object({
+    type: z.literal("realtime"),
+    model: z.literal(BUILDER_REALTIME_MODEL),
+    output_modalities: z.tuple([z.literal("audio")]),
+    instructions: z.string().max(32000).optional(),
+    audio: z
+        .object({
+        input: RealtimeInputAudioSchema.optional(),
+        output: RealtimeOutputAudioSchema.optional(),
+    })
+        .optional(),
+    tools: z.array(OpenAIRealtimeToolSchema).max(32).optional(),
+    tool_choice: OpenAIRealtimeToolChoiceSchema.optional(),
+    max_output_tokens: z
+        .union([z.literal("inf"), z.number().int().min(1).max(4096)])
+        .optional(),
+    prompt: z
+        .object({
+        id: z.string().min(1).max(256),
+        version: z.string().min(1).max(128).optional(),
+        variables: boundedRecord(16000, "Realtime prompt variables are too large").optional(),
+    })
+        .nullable()
+        .optional(),
+    tracing: z
+        .union([
+        z.literal("auto"),
+        boundedRecord(8000, "Realtime tracing configuration is too large"),
+    ])
+        .nullable()
+        .optional(),
+    truncation: z
+        .union([
+        z.enum(["auto", "disabled"]),
+        boundedRecord(2048, "Realtime truncation configuration is too large"),
+    ])
+        .optional(),
+    include: z.array(z.string().min(1).max(256)).max(16).optional(),
+})
+    .refine((session) => utf8ByteLength(JSON.stringify(session)) <=
+    BUILDER_REALTIME_MAX_SESSION_BYTES, { message: "Realtime session configuration is too large" });
+export const BuilderRealtimeSessionRequestSchema = z.object({
+    sdp: z
+        .string()
+        .min(1)
+        .max(BUILDER_REALTIME_MAX_SDP_LENGTH)
+        .refine((sdp) => sdp.trim().length > 0, { message: "SDP cannot be blank" }),
+    session: OpenAIRealtimeSessionConfigSchema,
+});

package/src/realtime.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/src/realtime.spec.js

@@ -0,0 +1,122 @@
+import { describe, expect, it } from "vitest";
+import { BUILDER_REALTIME_MAX_SDP_LENGTH, BuilderRealtimeSessionRequestSchema, OpenAIRealtimeSessionConfigSchema, } from "./realtime.js";
+const validSession = {
+    type: "realtime",
+    model: "gpt-realtime-2.1",
+    output_modalities: ["audio"],
+    instructions: "Help the user operate their Builder app.",
+};
+describe("BuilderRealtimeSessionRequestSchema", () => {
+    it("accepts a bounded audio session with tools", () => {
+        var _a;
+        const result = BuilderRealtimeSessionRequestSchema.parse({
+            sdp: "v=0\r\no=- 1 1 IN IP4 127.0.0.1",
+            session: {
+                ...validSession,
+                audio: {
+                    input: {
+                        turn_detection: {
+                            type: "semantic_vad",
+                            eagerness: "auto",
+                            create_response: true,
+                            interrupt_response: true,
+                        },
+                    },
+                    output: { voice: "marin" },
+                },
+                tools: [
+                    {
+                        type: "function",
+                        name: "navigate",
+                        description: "Navigate within the app",
+                        parameters: {
+                            type: "object",
+                            properties: { path: { type: "string" } },
+                            required: ["path"],
+                        },
+                    },
+                ],
+            },
+        });
+        expect(result.session.model).toBe("gpt-realtime-2.1");
+        expect((_a = result.session.tools) === null || _a === void 0 ? void 0 : _a[0]).toMatchObject({
+            type: "function",
+            name: "navigate",
+        });
+    });
+    it("strips unknown request, session, and known nested config keys", () => {
+        const sdp = " v=0\r\n";
+        const result = BuilderRealtimeSessionRequestSchema.parse({
+            sdp,
+            ignoredRequestValue: "secret",
+            session: {
+                ...validSession,
+                ignoredSessionValue: "secret",
+                audio: {
+                    output: {
+                        voice: "marin",
+                        ignoredAudioValue: "secret",
+                    },
+                },
+            },
+        });
+        expect(result).toEqual({
+            sdp,
+            session: {
+                ...validSession,
+                audio: { output: { voice: "marin" } },
+            },
+        });
+    });
+    it.each([
+        [{ ...validSession, type: "transcription" }],
+        [{ ...validSession, model: "gpt-realtime" }],
+        [{ ...validSession, output_modalities: ["text"] }],
+        [{ ...validSession, output_modalities: ["audio", "text"] }],
+        [{ ...validSession, output_modalities: [] }],
+    ])("rejects unsupported sessions", (session) => {
+        expect(BuilderRealtimeSessionRequestSchema.safeParse({ sdp: "v=0", session })
+            .success).toBe(false);
+    });
+    it("rejects transcription models that do not match the metered rate", () => {
+        expect(OpenAIRealtimeSessionConfigSchema.safeParse({
+            ...validSession,
+            audio: {
+                input: { transcription: { model: "gpt-4o-transcribe" } },
+            },
+        }).success).toBe(false);
+    });
+    it("rejects empty and oversized SDP offers", () => {
+        expect(BuilderRealtimeSessionRequestSchema.safeParse({
+            sdp: "   ",
+            session: validSession,
+        }).success).toBe(false);
+        expect(BuilderRealtimeSessionRequestSchema.safeParse({
+            sdp: "x".repeat(BUILDER_REALTIME_MAX_SDP_LENGTH + 1),
+            session: validSession,
+        }).success).toBe(false);
+    });
+    it("bounds instructions, tool count, and tool parameter schemas", () => {
+        expect(OpenAIRealtimeSessionConfigSchema.safeParse({
+            ...validSession,
+            instructions: "x".repeat(32001),
+        }).success).toBe(false);
+        expect(OpenAIRealtimeSessionConfigSchema.safeParse({
+            ...validSession,
+            tools: Array.from({ length: 33 }, (_, index) => ({
+                type: "function",
+                name: `tool_${index}`,
+            })),
+        }).success).toBe(false);
+        expect(OpenAIRealtimeSessionConfigSchema.safeParse({
+            ...validSession,
+            tools: [
+                {
+                    type: "function",
+                    name: "large_tool",
+                    parameters: { description: "x".repeat(32001) },
+                },
+            ],
+        }).success).toBe(false);
+    });
+});