@nylorun/runtime 0.24.0-beta → 0.26.0-beta

package/dist/model/models.d.ts

@@ -8,6 +8,8 @@ export type Selection = Readonly<{
         contextWindow?: number;
         maxTokens?: number;
         reasoning?: boolean;
+        /** What the endpoint reads (protocol 12); text only unless it says so. */
+        inputs?: readonly string[];
         compat?: Readonly<Record<string, unknown>>;
     }>;
 }>;

package/dist/model/models.js

@@ -25,7 +25,7 @@ export function modelsFor(selection, credentials, options = {}) {
         provider: "custom",
         baseUrl: selection.custom.baseUrl,
         reasoning: selection.custom.reasoning ?? false,
-        input: ["text", "image"],
+        input: selection.custom.inputs?.includes("image") ? ["text", "image"] : ["text"],
         cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
         contextWindow: selection.custom.contextWindow ?? CUSTOM_CONTEXT_WINDOW,
         maxTokens: selection.custom.maxTokens ?? CUSTOM_MAX_TOKENS,

package/dist/model/native-blocks.d.ts

@@ -0,0 +1,35 @@
+import type { TextContent } from "@earendil-works/pi-ai";
+/** The kinds of file a native block carries; images are pi-ai's own. */
+export type NativeFileKind = "pdf" | "office" | "audio" | "video";
+/** A file for one model call: its kind, media type, name and base64 bytes. */
+export interface NativeFile {
+    readonly kind: NativeFileKind;
+    readonly mediaType: string;
+    readonly name: string;
+    /** The bytes, base64-encoded. */
+    readonly data: string;
+}
+/**
+ * The kinds each pi-ai API has a native block for here: the model inputs an operator may add
+ * (`NATIVE_INPUTS`), images aside, so the two tables never disagree.
+ */
+export declare const NATIVE_FILE_KINDS: Readonly<Record<string, readonly NativeFileKind[]>>;
+/** The kinds of file `api` carries in a native block; none for an API without a rewriter. */
+export declare function nativeKindsOf(api: string): readonly NativeFileKind[];
+export declare class NativeBlocks {
+    private readonly files;
+    /** Why the last `rewrite` could not carry every file; undefined when it did. */
+    error?: string;
+    /** How many files this call carries. */
+    get size(): number;
+    /** A text part standing for `file` in pi-ai's context, replaced by `rewrite`. */
+    add(file: NativeFile): TextContent;
+    /**
+     * `payload` (an `onPayload` request body for `api`) with each marker block replaced by its
+     * native block. Rewrites in place and returns the same payload; sets `error` when a file
+     * could not be carried.
+     */
+    rewrite(payload: unknown, api: string): unknown;
+    /** The native block for a marker block of ours; undefined for any other block. */
+    private replace;
+}

package/dist/model/native-blocks.js

@@ -0,0 +1,170 @@
+/**
+ * Native file blocks (multimodal messages, M4): a PDF, an office document, audio or video the
+ * model reads, carried to the provider in its own block. pi-ai's messages hold only text and
+ * images, so a file goes in as a marker text part (`add`), which pi-ai never merges or escapes,
+ * and the provider payload pi-ai builds is rewritten before it is sent (`rewrite`, from the
+ * call's `onPayload`): each marker block becomes the API's native block. Images never come
+ * here: pi-ai carries them itself.
+ *
+ * One instance serves one model call. pi-ai builds the payload again on every attempt, so
+ * `rewrite` can run more than once; each run sets `error` afresh. A file whose kind or format
+ * the API has no block for, an API with no rewriter, or a marker left in the payload (pi-ai
+ * folded it into a longer string, or dropped it) is `error`: the call fails with it rather than
+ * sending a marker to the model.
+ */
+import { randomUUID } from "node:crypto";
+import { essence } from "../artifacts/media-types.js";
+import { NATIVE_INPUTS } from "./capabilities.js";
+/**
+ * The kinds each pi-ai API has a native block for here: the model inputs an operator may add
+ * (`NATIVE_INPUTS`), images aside, so the two tables never disagree.
+ */
+export const NATIVE_FILE_KINDS = Object.freeze(Object.fromEntries(Object.entries(NATIVE_INPUTS).map(([api, inputs]) => [
+    api,
+    Object.freeze(inputs.filter((input) => input !== "text" && input !== "image")),
+])));
+/** The kinds of file `api` carries in a native block; none for an API without a rewriter. */
+export function nativeKindsOf(api) {
+    return Object.hasOwn(NATIVE_FILE_KINDS, api) ? NATIVE_FILE_KINDS[api] : [];
+}
+const MARKER = /^\[\[nylorun-file:([0-9a-f-]{36})\]\]$/;
+const MARKER_ANYWHERE = /\[\[nylorun-file:[0-9a-f-]{36}\]\]/;
+const dataUrl = (file) => `data:${essence(file.mediaType)};base64,${file.data}`;
+/** `cache_control` the marker block carried (pi-ai marks the last block of a request). */
+const cacheOf = (block) => block.cache_control === undefined ? {} : { cache_control: block.cache_control };
+const textOfType = (type) => (block) => block.type === type ? block.text : undefined;
+/** `input_audio.format` for a media type: OpenAI Chat Completions takes WAV and MP3 only. */
+function audioFormat(mediaType) {
+    switch (essence(mediaType)) {
+        case "audio/wav":
+        case "audio/x-wav":
+        case "audio/wave":
+            return "wav";
+        case "audio/mpeg":
+        case "audio/mp3":
+            return "mp3";
+        default:
+            return undefined;
+    }
+}
+const anthropic = (file, block) => ({
+    type: "document",
+    source: { type: "base64", media_type: essence(file.mediaType), data: file.data },
+    title: file.name,
+    ...cacheOf(block),
+});
+const responses = (file) => ({
+    type: "input_file",
+    filename: file.name,
+    file_data: dataUrl(file),
+});
+const completions = (file, block) => {
+    switch (file.kind) {
+        case "pdf":
+            return { type: "file", file: { filename: file.name, file_data: dataUrl(file) }, ...cacheOf(block) };
+        case "audio": {
+            const format = audioFormat(file.mediaType);
+            if (!format)
+                return "this API reads WAV and MP3 audio only";
+            return { type: "input_audio", input_audio: { data: file.data, format }, ...cacheOf(block) };
+        }
+        default:
+            return { type: "video_url", video_url: { url: dataUrl(file) }, ...cacheOf(block) };
+    }
+};
+const google = (file) => ({
+    inlineData: { mimeType: essence(file.mediaType), data: file.data },
+});
+const REWRITERS = {
+    "anthropic-messages": { marker: textOfType("text"), native: anthropic },
+    "openai-responses": { marker: textOfType("input_text"), native: responses },
+    "azure-openai-responses": { marker: textOfType("input_text"), native: responses },
+    "openai-completions": { marker: textOfType("text"), native: completions },
+    "google-generative-ai": { marker: (block) => (block.type === undefined ? block.text : undefined), native: google },
+    "google-vertex": { marker: (block) => (block.type === undefined ? block.text : undefined), native: google },
+};
+const isBlock = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+export class NativeBlocks {
+    files = new Map();
+    /** Why the last `rewrite` could not carry every file; undefined when it did. */
+    error;
+    /** How many files this call carries. */
+    get size() {
+        return this.files.size;
+    }
+    /** A text part standing for `file` in pi-ai's context, replaced by `rewrite`. */
+    add(file) {
+        const id = randomUUID();
+        this.files.set(id, file);
+        return { type: "text", text: `[[nylorun-file:${id}]]` };
+    }
+    /**
+     * `payload` (an `onPayload` request body for `api`) with each marker block replaced by its
+     * native block. Rewrites in place and returns the same payload; sets `error` when a file
+     * could not be carried.
+     */
+    rewrite(payload, api) {
+        this.error = undefined;
+        if (this.files.size === 0)
+            return payload;
+        const rewriter = Object.hasOwn(REWRITERS, api) ? REWRITERS[api] : undefined;
+        if (!rewriter) {
+            this.error = `The model's API (${api}) cannot carry files other than images`;
+            return payload;
+        }
+        const kinds = nativeKindsOf(api);
+        const placed = new Set();
+        let leftover = false;
+        const visit = (value) => {
+            if (Array.isArray(value)) {
+                for (let index = 0; index < value.length; index++) {
+                    const item = value[index];
+                    const replaced = isBlock(item) ? this.replace(item, rewriter, kinds, api) : undefined;
+                    if (replaced) {
+                        value[index] = replaced.block;
+                        placed.add(replaced.id);
+                    }
+                    else
+                        visit(item);
+                }
+            }
+            else if (isBlock(value)) {
+                for (const item of Object.values(value))
+                    visit(item);
+            }
+            else if (typeof value === "string" && MARKER_ANYWHERE.test(value)) {
+                leftover = true;
+            }
+        };
+        visit(payload);
+        if (this.error)
+            return payload;
+        if (leftover || placed.size < this.files.size) {
+            const missing = [...this.files].find(([id]) => !placed.has(id))?.[1];
+            this.error = missing
+                ? `File "${missing.name}" did not reach the ${api} request as its own block`
+                : `A file marker was left in the ${api} request`;
+        }
+        return payload;
+    }
+    /** The native block for a marker block of ours; undefined for any other block. */
+    replace(block, rewriter, kinds, api) {
+        const text = rewriter.marker(block);
+        if (typeof text !== "string")
+            return undefined;
+        const id = MARKER.exec(text)?.[1];
+        const file = id === undefined ? undefined : this.files.get(id);
+        if (!file)
+            return undefined;
+        const refuse = (reason) => {
+            this.error ??= `File "${file.name}" (${essence(file.mediaType)}) cannot be sent to ${api}: ${reason}`;
+            return undefined;
+        };
+        if (!kinds.includes(file.kind))
+            return refuse(`this API has no block for ${file.kind} files`);
+        const native = rewriter.native(file, block);
+        if (typeof native === "string")
+            return refuse(native);
+        return { id: id, block: native };
+    }
+}

package/dist/model/pi-model.js

@@ -1,9 +1,12 @@
 import { parseJsonWithRepair, retryAssistantCall, } from "@earendil-works/pi-ai";
+import { mediaReferenceOf } from "@nylorun/core/define";
 import { FileUnavailableError, } from "../artifacts/files.js";
-import { essence, isModelImage, isText } from "../artifacts/media-types.js";
+import { essence, inputLabel, isModelImage, isText, modelInputOf, unreadableReason, } from "../artifacts/media-types.js";
 import { scrub } from "../redact.js";
+import { capabilitiesOf } from "./capabilities.js";
 import { classifyAssistantError, failure } from "./classify.js";
 import { modelsFor } from "./models.js";
+import { NativeBlocks, nativeKindsOf } from "./native-blocks.js";
 import { projectSecrets } from "./settings.js";
 const OPENAI_COMPATIBLE_REASONING = /^reasoning(_content|_text)?$/;
 const emptyUsage = () => ({
@@ -23,8 +26,6 @@ const noAmbientAuth = {
         return false;
     },
 };
-class InvalidRequest extends Error {
-}
 /**
  * Node model adapter. Local provider configuration is read only when invoked.
  * A provider failure is returned as a failure outcome; only an abort is thrown.
@@ -44,7 +45,7 @@ export function piModel(options = {}) {
         const root = options.root ?? "";
         const stored = await options.readHostModel?.();
         if (!stored)
-            return failure("auth", "Model provider is not configured. Set it in Studio, or run nylo configure (npx @nylorun/cli configure).", false);
+            return failure("auth", "Model provider is not configured. Set it in Studio, or run nylorun tenant models set.", false);
         const secrets = [
             ...projectSecrets(root),
             ...credentialSecrets(stored.credential),
@@ -64,18 +65,22 @@ export function piModel(options = {}) {
                 : {}),
         };
         const registry = modelsFor(selection, hostCredentialStore(stored, options.writeHostCredential), { environment: false, authContext: noAmbientAuth });
-        const selected = registry.getModel(selection.provider, selection.model);
-        if (!selected)
-            return failure("invalid_request", "Unknown model. Run nylo configure (npx @nylorun/cli configure).", false);
-        let request;
-        try {
-            request = await buildContext(call, selected, options);
-        }
-        catch (error) {
-            if (error instanceof InvalidRequest)
-                return failure("invalid_request", error.message, false);
-            throw error;
-        }
+        const found = registry.getModel(selection.provider, selection.model);
+        if (!found)
+            return failure("invalid_request", "Unknown model. Run nylorun tenant models set.", false);
+        // What the model reads (protocol 12): its catalog's inputs and the operator's additions.
+        // pi-ai drops an image the model's `input` lacks, so the call's model carries an added one.
+        const { inputs } = capabilitiesOf(found, selection.custom
+            ? stored.settings?.inputs
+            : stored.addedInputs?.[`${selection.provider}/${selection.model}`]);
+        const selected = {
+            ...found,
+            input: inputs.includes("image") ? ["text", "image"] : ["text"],
+        };
+        // PDFs, Office documents, audio and video go in the API's own blocks: pi-ai builds the
+        // request with a marker in each one's place and `onPayload` swaps it for the bytes.
+        const natives = new NativeBlocks();
+        const request = await buildContext(call, selected, inputs, options, natives);
         // Publish only portable references, never the materialized provider image bytes.
         context.reportPreparedCall?.({
             adapter: "runtime.pi-ai",
@@ -93,7 +98,20 @@ export function piModel(options = {}) {
             timeoutMs: settings.requestTimeoutMs,
             maxRetries: 2,
             maxRetryDelayMs: 60_000,
+            ...(natives.size > 0
+                ? {
+                    // A file that cannot be carried stops the request before it is sent.
+                    onPayload: (payload) => {
+                        const rewritten = natives.rewrite(payload, selected.api);
+                        if (natives.error)
+                            throw new Error(natives.error);
+                        return rewritten;
+                    },
+                }
+                : {}),
         };
+        // A file the request could not carry fails the call at once: no message, so never retried.
+        const nativeFailure = () => failedMessage(selected);
         const attempt = async () => {
             const idle = new AbortController();
             const signal = AbortSignal.any([context.signal, idle.signal]);
@@ -126,6 +144,8 @@ export function piModel(options = {}) {
                     }
                 }
                 const response = await stream.result();
+                if (natives.error)
+                    return nativeFailure();
                 if (idle.signal.aborted && !context.signal.aborted)
                     return idleTimeout(selected, settings.idleTimeoutMs);
                 return response;
@@ -133,6 +153,8 @@ export function piModel(options = {}) {
             catch (error) {
                 if (context.signal.aborted)
                     throw error;
+                if (natives.error)
+                    return nativeFailure();
                 if (idle.signal.aborted)
                     return idleTimeout(selected, settings.idleTimeoutMs);
                 return {
@@ -151,6 +173,8 @@ export function piModel(options = {}) {
         }, context.signal);
         // An abort is never a failure outcome: the host decides what it means.
         context.signal.throwIfAborted();
+        if (natives.error)
+            return redacted(failure("invalid_request", natives.error, false));
         if (response.stopReason === "aborted")
             return redacted(failure("transient", "The provider stream was aborted.", true));
         if (response.stopReason === "error") {
@@ -158,7 +182,7 @@ export function piModel(options = {}) {
             return redacted(outcome.code === "auth"
                 ? {
                     ...outcome,
-                    message: `${outcome.message} Update the provider credential in Studio's Model Settings, or run nylo configure.`,
+                    message: `${outcome.message} Update the provider credential in Studio's Model Settings, or run nylorun tenant models set.`,
                 }
                 : outcome);
         }
@@ -231,87 +255,74 @@ export function piModel(options = {}) {
         };
     };
 }
-async function buildContext(call, selected, options) {
+async function buildContext(call, selected, inputs, options, natives) {
     const messages = [];
     const instructions = [];
-    const content = async (parts, toolResult = false) => {
+    /**
+     * A file a media part names (protocol 6, 12), as the model gets it: a text file as a `<file>`
+     * block, an image the model reads as an image, a PDF, Office document, audio or video the model
+     * reads as a marker for its native block (`natives`), and anything else, or a file that cannot
+     * be read, as a note. Its bytes go only into the provider request built here, never into the
+     * transcript or the journal, and no file fails the call here.
+     */
+    const materialize = async (part) => {
+        let name = mediaReferenceOf(part.reference)?.name ?? "file";
+        let type = essence(part.mediaType);
+        const note = (why) => ({
+            type: "text",
+            text: `(File ${JSON.stringify(name)}, ${type}, is not shown: ${why}.)`,
+        });
+        const refused = unreadableReason(part.mediaType, inputs);
+        if (refused)
+            return note(refused);
+        if (!options.files)
+            return note("files are unavailable");
+        let file;
+        try {
+            file = await options.files(part.reference);
+        }
+        catch (error) {
+            if (!(error instanceof FileUnavailableError))
+                throw error;
+            return note(error.message.replace(/\.$/u, ""));
+        }
+        name = file.name;
+        type = essence(file.mediaType);
+        const stored = unreadableReason(file.mediaType, inputs);
+        if (stored)
+            return note(stored);
+        if (isModelImage(file.mediaType))
+            return { type: "image", data: Buffer.from(file.bytes).toString("base64"), mimeType: type };
+        if (isText(file.mediaType)) {
+            let text;
+            try {
+                text = new TextDecoder("utf-8", { fatal: true }).decode(file.bytes);
+            }
+            catch {
+                return note("it is not UTF-8 text");
+            }
+            return {
+                type: "text",
+                text: `<file name=${JSON.stringify(file.name)} type=${JSON.stringify(type)}>\n${text}\n</file>`,
+            };
+        }
+        const kind = modelInputOf(file.mediaType);
+        if (isNativeKind(kind) && nativeKindsOf(selected.api).includes(kind))
+            return natives.add({
+                kind,
+                mediaType: type,
+                name: file.name,
+                data: Buffer.from(file.bytes).toString("base64"),
+            });
+        return note(`the Runtime does not send ${inputLabel(kind)} to this model's API (${selected.api})`);
+    };
+    const content = async (parts) => {
         const result = [];
         for (const part of parts) {
             if (part.type === "text")
                 result.push({ type: "text", text: part.text });
-            else if (part.type === "media" && toolResult) {
-                // An image a tool returned, stored as an artifact (R2b C11): shown to a model that
-                // reads images. Otherwise, or when it cannot be read, a note: the call still goes.
-                const note = (why) => ({
-                    type: "text",
-                    text: `(The tool returned an image, ${essence(part.mediaType)}, that is not shown: ${why}.)`,
-                });
-                if (!selected.input.includes("image") || !isModelImage(part.mediaType)) {
-                    result.push(note(selected.input.includes("image")
-                        ? "the model reads PNG, JPEG, GIF and WebP images only"
-                        : "this model does not accept images"));
-                    continue;
-                }
-                if (!options.files) {
-                    result.push(note("files are unavailable"));
-                    continue;
-                }
-                try {
-                    const file = await options.files(part.reference);
-                    if (!isModelImage(file.mediaType))
-                        result.push(note(`it is ${essence(file.mediaType)}`));
-                    else
-                        result.push({
-                            type: "image",
-                            data: Buffer.from(file.bytes).toString("base64"),
-                            mimeType: essence(file.mediaType),
-                        });
-                }
-                catch (error) {
-                    if (!(error instanceof FileUnavailableError))
-                        throw error;
-                    result.push(note(error.message.replace(/\.$/u, "")));
-                }
-            }
-            else if (part.type === "media") {
-                // A file the message named by artifact (protocol 6): its bytes go only into the
-                // provider request built here, never into the transcript or the journal.
-                if (!options.files)
-                    throw new InvalidRequest("Files are unavailable; pass a file resolver to piModel({ files }).");
-                let file;
-                try {
-                    file = await options.files(part.reference);
-                }
-                catch (error) {
-                    if (error instanceof FileUnavailableError)
-                        throw new InvalidRequest(error.message);
-                    throw error;
-                }
-                if (isModelImage(file.mediaType)) {
-                    if (!selected.input.includes("image"))
-                        throw new InvalidRequest("The configured model does not accept images.");
-                    result.push({
-                        type: "image",
-                        data: Buffer.from(file.bytes).toString("base64"),
-                        mimeType: essence(file.mediaType),
-                    });
-                }
-                else if (isText(file.mediaType)) {
-                    let text;
-                    try {
-                        text = new TextDecoder("utf-8", { fatal: true }).decode(file.bytes);
-                    }
-                    catch {
-                        throw new InvalidRequest(`${file.name} is not UTF-8 text.`);
-                    }
-                    result.push({
-                        type: "text",
-                        text: `<file name=${JSON.stringify(file.name)} type=${JSON.stringify(essence(file.mediaType))}>\n${text}\n</file>`,
-                    });
-                }
-                else
-                    throw new InvalidRequest(`The model reads images and text files; ${file.name} is ${essence(file.mediaType)}.`);
-            }
+            else if (part.type === "media")
+                result.push(await materialize(part));
         }
         return result;
     };
@@ -326,7 +337,7 @@ async function buildContext(call, selected, options) {
                 toolName: item.toolName,
                 isError: item.status !== "completed",
                 timestamp: Date.now(),
-                content: await content(item.content, true),
+                content: await content(item.content),
             });
         }
         else if (item.kind === "message" && item.role === "assistant") {
@@ -357,6 +368,7 @@ async function buildContext(call, selected, options) {
         tools,
     };
 }
+const isNativeKind = (input) => input === "pdf" || input === "office" || input === "audio" || input === "video";
 /**
  * A replayed assistant message keeps the model that produced it, so pi-ai converts
  * history correctly after a model switch (tool-call ids, foreign reasoning). Messages
@@ -501,7 +513,7 @@ function hostCredentialStore(stored, write) {
             return next;
         },
         async delete() {
-            throw new Error("Replace the model provider in Studio, or with nylo configure.");
+            throw new Error("Replace the model provider in Studio, or with nylorun tenant models set.");
         },
     };
 }