@engenty/wizards-runtime 0.0.0-stage → 0.7.5

package/apps/runtime/dist/media/conversation.js

@@ -0,0 +1,167 @@
+import { langNameEn } from "@engenty-wizards/shared/definition";
+import { NotedError } from "../model-errors.js";
+import { conversationAccess } from "../models.js";
+/**
+ * A live conversation with a wizard: the browser talks to a realtime speech model directly
+ * (audio both ways over WebRTC); this runtime only mints the short-lived secret the browser
+ * connects with, and writes the instructions and the tools the model drives the run with. The
+ * tools run in the browser against the run's own routes, so audio never touches the runtime.
+ */
+const SECRETS_URL = "https://api.openai.com/v1/realtime/client_secrets";
+const CALLS_URL = "https://api.openai.com/v1/realtime/calls";
+/** How long a minted secret opens a session; the session itself goes on until it ends. */
+const SECRET_TTL_SECONDS = 600;
+/** The voice the wizard speaks with: one that speaks every language engenty does. */
+const VOICE = "marin";
+/** What the model may do: one tool per command of the run page, run by the browser. */
+export const CONVERSATION_TOOLS = [
+    {
+        type: "function",
+        name: "get_state",
+        description: "What the run waits for right now: the page with its fields (which are answered, which are still open, which need the screen), the review, or what the wizard is working on. Call it when unsure, and after the state changed.",
+        parameters: { type: "object", properties: {}, additionalProperties: false },
+    },
+    {
+        type: "function",
+        name: "set_field",
+        description: "Keeps the person's answer to one field of the current page. Call it once per field as soon as the person answered; for a choice give one of the options exactly.",
+        parameters: {
+            type: "object",
+            properties: {
+                field: { type: "string", description: "The field's id from the state." },
+                value: {
+                    description: "The answer: a string, a number, true/false for a toggle, or an array of options for a multiple choice.",
+                },
+            },
+            required: ["field", "value"],
+            additionalProperties: false,
+        },
+    },
+    {
+        type: "function",
+        name: "submit_page",
+        description: "Sends the current page once every required field that can be answered by voice has a value. Fields that need the screen are filled there by the person; say so and wait.",
+        parameters: { type: "object", properties: {}, additionalProperties: false },
+    },
+    {
+        type: "function",
+        name: "accept_review",
+        description: "Accepts what the wizard made, when the person is happy with it.",
+        parameters: { type: "object", properties: {}, additionalProperties: false },
+    },
+    {
+        type: "function",
+        name: "regenerate",
+        description: "Asks the wizard to make one of the shown outputs again, with what should change.",
+        parameters: {
+            type: "object",
+            properties: {
+                target: { type: "string", description: "The step id of the output, from the state." },
+                note: { type: "string", description: "What should change, in the person's words." },
+            },
+            required: ["target", "note"],
+            additionalProperties: false,
+        },
+    },
+    {
+        type: "function",
+        name: "answer_ask",
+        description: "Answers the question a running step asks before it changes something in a connected account: allow it, or skip it.",
+        parameters: {
+            type: "object",
+            properties: { allow: { type: "boolean" } },
+            required: ["allow"],
+            additionalProperties: false,
+        },
+    },
+    {
+        type: "function",
+        name: "look",
+        description: "Takes a look through the person's camera right now: a picture of what they are showing arrives as the next message. Only while the camera is on; otherwise ask them to switch it on.",
+        parameters: { type: "object", properties: {}, additionalProperties: false },
+    },
+    {
+        type: "function",
+        name: "take_photo",
+        description: "Takes a photo with the person's camera and puts it into a photo field of the current page. Say 'hold it still' first. Only while the camera is on.",
+        parameters: {
+            type: "object",
+            properties: { field: { type: "string", description: "The id of the image field." } },
+            required: ["field"],
+            additionalProperties: false,
+        },
+    },
+    {
+        type: "function",
+        name: "go_back",
+        description: "Goes back to the page before, when the person wants to change an earlier answer.",
+        parameters: { type: "object", properties: {}, additionalProperties: false },
+    },
+];
+/** The rules the model speaks by: written in English, it speaks the wizard's language. */
+export function conversationInstructions(view) {
+    const about = [`You are ${view.wizard.title}.`, view.wizard.description, view.wizard.intro]
+        .filter(Boolean)
+        .join(" ");
+    const rules = [
+        "You hold a conversation that fills in a wizard step by step. The person also sees the wizard on the screen.",
+        `Speak ${langNameEn(view.lang)} (${view.lang}), short and friendly, as on the phone. One sentence, then the next question.`,
+        "Ask one field at a time, in the order of the state. After each answer call set_field. When every field you can fill by voice is answered, call submit_page.",
+        "For a choice, name the options (at most four at a time) and pass exactly one of them on.",
+        "Read e-mail addresses, links and numbers back before passing them on.",
+        "A field that needs the screen (a photo, a file, a signature, a connection) is filled there by the person: say so in one sentence and wait for the next state.",
+        "While the wizard works, say in one sentence what it is doing and wait. Never make anything up.",
+        "At a review, sum the text up in a few words and ask whether it fits. Then accept_review, or regenerate with what the person wants changed.",
+        "At the end, say what was made and that it is on the screen.",
+        "With the camera on you can see what the person shows with look, and put a photo into a photo field with take_photo; ask for a photo field like this: 'Hold it to the camera, I will take the picture.' With the camera off, ask the person to switch it on, or leave the field to the screen.",
+    ];
+    return `${about}\n\n${rules.join("\n")}`;
+}
+/** Mints the secret a browser opens a session with; a failure carries a note for the run's language. */
+export async function mintConversation(view) {
+    const { apiKey, model } = await conversationAccess();
+    const res = await fetch(SECRETS_URL, {
+        method: "POST",
+        headers: { authorization: `Bearer ${apiKey}`, "content-type": "application/json" },
+        body: JSON.stringify({
+            expires_after: { anchor: "created_at", seconds: SECRET_TTL_SECONDS },
+            session: {
+                type: "realtime",
+                model,
+                instructions: conversationInstructions(view),
+                tools: CONVERSATION_TOOLS,
+                tool_choice: "auto",
+                audio: {
+                    // What the person said, written down: the call screen shows it as captions.
+                    input: {
+                        transcription: { model: "gpt-4o-mini-transcribe" },
+                        // Phones and laptops play the wizard's voice through their own speakers, which the mic
+                        // picks up: far-field filtering and a higher bar for speech keep it from interrupting itself.
+                        noise_reduction: { type: "far_field" },
+                        turn_detection: { type: "server_vad", threshold: 0.7, silence_duration_ms: 600 },
+                    },
+                    output: { voice: VOICE },
+                },
+            },
+        }),
+        signal: AbortSignal.timeout(15_000),
+    });
+    if (!res.ok) {
+        // The provider's answer goes to the log; the page hears only that it did not work, and why in a word.
+        const detail = (await res.text()).slice(0, 500);
+        console.error(`[talk] the provider refused the session (${res.status}):`, detail);
+        throw new NotedError(res.status === 401 || res.status === 403
+            ? { code: "talkKeyRefused" }
+            : { code: "talkFailed", params: { status: res.status } });
+    }
+    const body = (await res.json());
+    if (!body.value) {
+        throw new NotedError({ code: "talkNoSecret" });
+    }
+    return {
+        clientSecret: body.value,
+        expiresAt: new Date((body.expires_at ?? Date.now() / 1000 + SECRET_TTL_SECONDS) * 1000).toISOString(),
+        model,
+        endpoint: `${CALLS_URL}?model=${encodeURIComponent(model)}`,
+    };
+}

package/apps/runtime/dist/media/ffmpeg.js

@@ -0,0 +1,121 @@
+import { spawn } from "node:child_process";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { env } from "../env.js";
+let ffmpegCheck = null;
+export function hasFfmpeg() {
+    ffmpegCheck ??= new Promise((resolve) => {
+        const p = spawn(env.ffmpegPath, ["-version"], { stdio: "ignore" });
+        p.on("error", () => resolve(false));
+        p.on("exit", (code) => resolve(code === 0));
+    });
+    return ffmpegCheck;
+}
+/** Runs ffmpeg to its end; what it wrote to stderr comes back either way. */
+export function ffmpeg(args) {
+    return new Promise((resolve, reject) => {
+        const p = spawn(env.ffmpegPath, ["-hide_banner", ...args], {
+            stdio: ["ignore", "ignore", "pipe"],
+        });
+        let stderr = "";
+        p.stderr.on("data", (d) => {
+            stderr = (stderr + String(d)).slice(-20_000);
+        });
+        p.on("error", reject);
+        p.on("exit", (code) => resolve({ code: code ?? 1, stderr }));
+    });
+}
+/** A folder for one job's files, removed when the job is over. */
+export async function withTempDir(use) {
+    const dir = await mkdtemp(join(tmpdir(), "wizard-media-"));
+    try {
+        return await use(dir);
+    }
+    finally {
+        await rm(dir, { recursive: true, force: true });
+    }
+}
+/** What ffmpeg says about a media file: how long it is, whether it carries sound, its picture size. */
+export async function probeMedia(path) {
+    // Without an output ffmpeg prints the streams and exits with an error; the print is the answer.
+    const { stderr } = await ffmpeg(["-i", path]);
+    const m = stderr.match(/Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/);
+    const seconds = m ? Number(m[1]) * 3600 + Number(m[2]) * 60 + Number(m[3]) : 0;
+    const size = stderr.match(/Stream #\d+:\d+.*Video:.*?, (\d{2,5})x(\d{2,5})/);
+    return {
+        seconds,
+        audio: /Stream #\d+:\d+.*Audio:/.test(stderr),
+        width: size ? Number(size[1]) : 0,
+        height: size ? Number(size[2]) : 0,
+    };
+}
+/**
+ * A video no larger than `short` pixels on its short side (720 for 720p), sound kept. One that
+ * fits already comes back as it is; without ffmpeg too.
+ */
+export async function capVideo(bytes, short) {
+    if (!(await hasFfmpeg())) {
+        return bytes;
+    }
+    return withTempDir(async (dir) => {
+        const input = join(dir, "in.mp4");
+        const out = join(dir, "out.mp4");
+        await writeFile(input, bytes);
+        const { width, height } = await probeMedia(input);
+        if (!(width && height) || Math.min(width, height) <= short) {
+            return bytes;
+        }
+        const scale = width >= height ? `scale=-2:${short}` : `scale=${short}:-2`;
+        const { code } = await ffmpeg([
+            "-y",
+            "-loglevel",
+            "error",
+            "-i",
+            input,
+            "-vf",
+            scale,
+            "-c:v",
+            "libx264",
+            "-preset",
+            "veryfast",
+            "-crf",
+            "21",
+            "-pix_fmt",
+            "yuv420p",
+            "-c:a",
+            "copy",
+            "-movflags",
+            "+faststart",
+            out,
+        ]);
+        return code === 0 ? new Uint8Array(await readFile(out)) : bytes;
+    });
+}
+/**
+ * Any audio as MP3 — what every phone and browser plays — with `tags` written into the file
+ * (ID3). An MP3 is not encoded again, only tagged. `null` without ffmpeg.
+ */
+export async function toMp3(bytes, ext, tags = {}) {
+    if (!(await hasFfmpeg())) {
+        return null;
+    }
+    return withTempDir(async (dir) => {
+        const input = join(dir, `in.${ext}`);
+        const out = join(dir, "out.mp3");
+        await writeFile(input, bytes);
+        const { code } = await ffmpeg([
+            "-y",
+            "-loglevel",
+            "error",
+            "-i",
+            input,
+            ...(ext === "mp3" ? ["-codec:a", "copy"] : ["-codec:a", "libmp3lame", "-q:a", "3"]),
+            "-id3v2_version",
+            "3",
+            ...Object.entries(tags).flatMap(([key, value]) => ["-metadata", `${key}=${value}`]),
+            out,
+        ]);
+        return code === 0 ? new Uint8Array(await readFile(out)) : null;
+    });
+}

package/apps/runtime/dist/media/generate.js

@@ -0,0 +1,120 @@
+import { experimental_generateVideo, generateImage, generateSpeech, generateText } from "ai";
+import { extFor } from "../files/storage.js";
+import { imageCostUsd, imageModel, isHarnessVendor, speechCostUsd, speechModel, textModel, videoCostUsd, videoModel, } from "../models.js";
+import { capVideo, toMp3 } from "./ffmpeg.js";
+import { speechTags } from "./marking.js";
+/** One image. Gemini image models answer through chat with an image file; the rest through generateImage. */
+export async function generateImageMedia(input) {
+    const { prompt, reference, abortSignal } = input;
+    const aspect = (input.aspectRatio ?? "1:1");
+    const image = await imageModel(input.call);
+    // The gateway books its own calls; a client's images are part of the person's subscription.
+    const costUsd = image.metered || isHarnessVendor(image.vendor) ? 0 : imageCostUsd(image.ref);
+    if (image.chat) {
+        const chat = await textModel("image", input.call);
+        const text = `${prompt}\n\nAspect ratio: ${aspect}. Return exactly one image.`;
+        const result = await generateText({
+            model: chat.model,
+            abortSignal,
+            providerOptions: {
+                google: { responseModalities: ["IMAGE", "TEXT"], imageConfig: { aspectRatio: aspect } },
+            },
+            messages: [
+                {
+                    role: "user",
+                    content: reference
+                        ? [
+                            { type: "text", text },
+                            { type: "image", image: reference.bytes, mediaType: reference.mediaType },
+                        ]
+                        : text,
+                },
+            ],
+        });
+        const file = result.files?.find((f) => (f.mediaType ?? "").startsWith("image/"));
+        if (!file?.uint8Array?.byteLength) {
+            throw new Error("The image model returned no image.");
+        }
+        return {
+            bytes: file.uint8Array,
+            mime: file.mediaType ?? "image/png",
+            costUsd,
+            system: image.ref,
+        };
+    }
+    const result = await generateImage({
+        model: image.model,
+        n: 1,
+        abortSignal,
+        prompt: reference ? { images: [reference.bytes], text: prompt } : prompt,
+        aspectRatio: aspect,
+    });
+    const img = result.image ?? result.images?.[0];
+    if (!img?.uint8Array?.byteLength) {
+        throw new Error("The image model returned no image.");
+    }
+    return {
+        bytes: img.uint8Array,
+        mime: img.mediaType ?? "image/png",
+        costUsd,
+        system: image.ref,
+    };
+}
+/** The short side of a video in pixels, per resolution. */
+const VIDEO_SHORT_SIDE = { "720p": 720, "480p": 480 };
+/** The pixel size a model is asked for: the short side as the resolution says, the long one as the aspect. */
+function videoSize(aspect, resolution) {
+    const short = VIDEO_SHORT_SIDE[resolution];
+    const [w, h] = aspect.split(":").map(Number);
+    const long = Math.round((short * Math.max(w, h)) / Math.min(w, h) / 2) * 2;
+    return w >= h ? `${long}x${short}` : `${short}x${long}`;
+}
+/**
+ * One video clip. Text-to-video, or image-to-video when a reference is given. Rendered at 720p,
+ * or 480p when the step asks: a model that gives more anyway is scaled down to it.
+ */
+export async function generateVideoMedia(input) {
+    const duration = Math.min(Math.max(Math.round(input.duration ?? 8), 4), 10);
+    const aspect = (input.aspectRatio === "1:1" ? "16:9" : (input.aspectRatio ?? "16:9"));
+    const resolution = input.resolution ?? "720p";
+    const videoClass = await videoModel(input.call);
+    const result = await experimental_generateVideo({
+        model: videoClass.model,
+        prompt: input.reference ? { image: input.reference.bytes, text: input.prompt } : input.prompt,
+        aspectRatio: aspect,
+        resolution: videoSize(aspect, resolution),
+        duration,
+        abortSignal: input.abortSignal,
+    });
+    const video = result.video ?? result.videos?.[0];
+    if (!video?.uint8Array?.byteLength) {
+        throw new Error("The video model returned no video.");
+    }
+    return {
+        bytes: await capVideo(video.uint8Array, VIDEO_SHORT_SIDE[resolution]),
+        mime: video.mediaType ?? "video/mp4",
+        costUsd: videoClass.metered ? 0 : videoCostUsd(videoClass.ref, duration, resolution),
+        system: videoClass.ref,
+    };
+}
+/** A text read aloud, as MP3 where ffmpeg is at hand (else as the model gave it). */
+export async function generateSpeechMedia(input) {
+    const speech = await speechModel(input.call);
+    const result = await generateSpeech({
+        model: speech.model,
+        text: input.text,
+        instructions: input.style || undefined,
+        abortSignal: input.abortSignal,
+    });
+    const audio = result.audio;
+    if (!audio?.uint8Array?.byteLength) {
+        throw new Error("The speech model returned no audio.");
+    }
+    const mime = audio.mediaType ?? "audio/mpeg";
+    const costUsd = speech.metered ? 0 : speechCostUsd(speech.ref, input.text.length);
+    // As MP3, and saying in its tags that a model speaks here.
+    const mp3 = await toMp3(audio.uint8Array, extFor(mime) === "bin" ? "wav" : extFor(mime), speechTags({ origin: "generated", system: speech.ref }));
+    return mp3
+        ? { bytes: mp3, mime: "audio/mpeg", costUsd, system: speech.ref }
+        : { bytes: audio.uint8Array, mime, costUsd, system: speech.ref };
+}

package/apps/runtime/dist/media/marking.js

@@ -0,0 +1,121 @@
+import { crc32 } from "node:zlib";
+const SOURCE_TYPE = {
+    generated: "http://cv.iptc.org/newscodes/digitalsourcetype/trainedAlgorithmicMedia",
+    edited: "http://cv.iptc.org/newscodes/digitalsourcetype/compositeWithTrainedAlgorithmicMedia",
+};
+function xml(s) {
+    return s
+        .replace(/&/g, "&")
+        .replace(/</g, "&lt;")
+        .replace(/>/g, "&gt;")
+        .replace(/"/g, "&quot;");
+}
+/** The XMP packet that says how the file came to be. */
+export function xmpPacket(marking) {
+    const system = marking.system ? `\n   Iptc4xmpExt:AISystemUsed="${xml(marking.system)}"` : "";
+    return `<?xpacket begin="" id="W5M0MpCehiHzreSzNTczkc9d"?>
+<x:xmpmeta xmlns:x="adobe:ns:meta/">
+ <rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+  <rdf:Description rdf:about=""
+   xmlns:Iptc4xmpExt="http://iptc.org/std/Iptc4xmpExt/2008-02-29/"
+   xmlns:xmp="http://ns.adobe.com/xap/1.0/"
+   Iptc4xmpExt:DigitalSourceType="${SOURCE_TYPE[marking.origin]}"${system}
+   xmp:CreatorTool="engenty wizards"/>
+ </rdf:RDF>
+</x:xmpmeta>
+<?xpacket end="w"?>`;
+}
+/** Whether the file already says it is AI media: signed Content Credentials, or a source type in its XMP. */
+export function isMarked(bytes) {
+    const data = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    return data.includes("c2pa") || data.includes("DigitalSourceType");
+}
+const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
+/** PNG: an iTXt chunk "XML:com.adobe.xmp" right after the header chunk. */
+function markPng(data, xmp) {
+    if (!data.subarray(0, 8).equals(PNG_SIGNATURE) || data.toString("latin1", 12, 16) !== "IHDR") {
+        return null;
+    }
+    const headerEnd = 8 + 12 + data.readUInt32BE(8);
+    const body = Buffer.concat([
+        Buffer.from("XML:com.adobe.xmp\0\0\0\0\0", "latin1"),
+        Buffer.from(xmp, "utf8"),
+    ]);
+    const type = Buffer.from("iTXt", "latin1");
+    const length = Buffer.alloc(4);
+    length.writeUInt32BE(body.length);
+    const check = Buffer.alloc(4);
+    check.writeUInt32BE(crc32(Buffer.concat([type, body])) >>> 0);
+    return Buffer.concat([
+        data.subarray(0, headerEnd),
+        length,
+        type,
+        body,
+        check,
+        data.subarray(headerEnd),
+    ]);
+}
+/** JPEG: an APP1 segment with the XMP, after the start marker and the JFIF/Exif segments. */
+function markJpeg(data, xmp) {
+    if (data[0] !== 0xff || data[1] !== 0xd8) {
+        return null;
+    }
+    let at = 2;
+    // Skip APP0 (JFIF) and an APP1 that holds Exif: readers expect them first.
+    while (at + 4 <= data.length &&
+        data[at] === 0xff &&
+        (data[at + 1] === 0xe0 || data[at + 1] === 0xe1)) {
+        at += 2 + data.readUInt16BE(at + 2);
+    }
+    const body = Buffer.concat([
+        Buffer.from("http://ns.adobe.com/xap/1.0/\0", "latin1"),
+        Buffer.from(xmp, "utf8"),
+    ]);
+    if (body.length + 2 > 0xffff) {
+        return null;
+    }
+    const head = Buffer.from([0xff, 0xe1, 0, 0]);
+    head.writeUInt16BE(body.length + 2, 2);
+    return Buffer.concat([data.subarray(0, at), head, body, data.subarray(at)]);
+}
+/** The id of the box that holds XMP in an MP4 or QuickTime file. */
+const XMP_BOX = Buffer.from("be7acfcb97a942e89c71999491e3afac", "hex");
+/** MP4 / MOV: a top-level `uuid` box with the XMP, at the end of the file. */
+function markMp4(data, xmp) {
+    if (data.toString("latin1", 4, 8) !== "ftyp") {
+        return null;
+    }
+    const body = Buffer.from(xmp, "utf8");
+    const head = Buffer.alloc(8);
+    head.writeUInt32BE(8 + XMP_BOX.length + body.length);
+    head.write("uuid", 4, "latin1");
+    return Buffer.concat([data, head, XMP_BOX, body]);
+}
+/**
+ * The file with its marking. Returned unchanged when it is already marked, or of a kind this
+ * cannot write (audio is tagged where it is encoded, see `speechTags`).
+ */
+export function markMedia(bytes, mime, marking) {
+    if (isMarked(bytes)) {
+        return bytes;
+    }
+    const data = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    const xmp = xmpPacket(marking);
+    const marked = mime === "image/png"
+        ? markPng(data, xmp)
+        : mime === "image/jpeg"
+            ? markJpeg(data, xmp)
+            : mime === "video/mp4" || mime === "video/quicktime"
+                ? markMp4(data, xmp)
+                : null;
+    return marked ? new Uint8Array(marked) : bytes;
+}
+/** What an audio file says about itself, as the tags ffmpeg writes (ID3 for MP3). */
+export function speechTags(marking) {
+    return {
+        comment: "AI-generated voice",
+        DigitalSourceType: SOURCE_TYPE[marking.origin],
+        ...(marking.system ? { AISystemUsed: marking.system } : {}),
+        encoded_by: "engenty wizards",
+    };
+}