@nylorun/runtime 0.24.0-beta → 0.26.0-beta

package/dist/api/ag-ui/history.js

@@ -1,10 +1,44 @@
 import { isEmptyAssistantMessage, parseTranscriptEvent, } from "@nylorun/core/contracts";
 import { outputText, toolResultOf } from "./translate.js";
+/** The `provider` of a file source that names one of the Tenant's artifacts. */
+export const NYLORUN_FILE_PROVIDER = "nylorun";
+/** A file part as an AG-UI media part, its kind from its media type when the log has it. */
+function filePart(artifactId, mediaType) {
+    const top = mediaType?.split("/")[0];
+    const type = top === "image" || top === "audio" || top === "video" ? top : "document";
+    return {
+        type,
+        source: {
+            type: "file",
+            value: artifactId,
+            provider: NYLORUN_FILE_PROVIDER,
+            ...(mediaType ? { mimeType: mediaType } : {}),
+        },
+    };
+}
+/** A message's parts (protocol 6): its text, or text and media parts when it has files. */
+function partsContent(parts, mediaTypes) {
+    if (!parts.some((part) => part.type === "file"))
+        return parts.flatMap((part) => (typeof part.text === "string" ? [part.text] : [])).join("\n");
+    return parts.flatMap((part) => part.type === "file" && typeof part.artifactId === "string"
+        ? [filePart(part.artifactId, mediaTypes.get(part.artifactId))]
+        : typeof part.text === "string"
+            ? [{ type: "text", text: part.text }]
+            : []);
+}
 export function messagesFromEvents(items) {
     const messages = [];
     const results = new Set();
+    // Each artifact's media type, from its events, for the files a message names.
+    const mediaTypes = new Map();
     let sawAssistant = false;
     for (const raw of items) {
+        if (raw.type === "artifact.created" || raw.type === "artifact.version.created") {
+            const p = (raw.payload ?? {});
+            if (typeof p.artifactId === "string" && typeof p.contentType === "string")
+                mediaTypes.set(p.artifactId, p.contentType);
+            continue;
+        }
         if (raw.type === "command.message") {
             const p = (raw.payload ?? {});
             messages.push({
@@ -14,8 +48,7 @@ export function messagesFromEvents(items) {
                 content: typeof p.content === "string"
                     ? p.content
                     : Array.isArray(p.parts)
-                        ? // A message with parts (protocol 6): its text; files are artifacts, not text.
-                            p.parts.flatMap((part) => (typeof part.text === "string" ? [part.text] : [])).join("\n")
+                        ? partsContent(p.parts, mediaTypes)
                         : JSON.stringify(p.data),
             });
             sawAssistant = false;

package/dist/api/ag-ui/routes.js

@@ -12,7 +12,9 @@
  * - `POST …/threads/:thread/cancel` — cancel the running turn.
  *
  * Each thread maps to one session per subject, agent and thread. The message id is the
- * idempotency key, approvals become interrupts resumed through `resume`, and the last AG-UI
+ * idempotency key, a message's media parts become file parts (bytes sent inline are stored as
+ * artifacts of the session, `../inbound-files.ts`; a file the model does not read is a
+ * `RUN_ERROR`), approvals become interrupts resumed through `resume`, and the last AG-UI
  * event made from each Runtime event carries its cursor as the SSE id. A run's session is
  * created on its first run with the options in `forwardedProps.nylorun.session` and never
  * changed afterwards. A stream opened with an issuer token ends at the token's expiry with
@@ -22,7 +24,8 @@ import { randomUUID } from "node:crypto";
 import { EventType, } from "@ag-ui/core";
 import { RunAgentInputSchema } from "@ag-ui/core/schemas";
 import { PutSessionRequestSchema, } from "@nylorun/core/contracts";
-import { messagesFromEvents } from "./history.js";
+import { messagesFromEvents, NYLORUN_FILE_PROVIDER } from "./history.js";
+import { discardFiles, inlineFileName, messageParts, refuseUrl, } from "../inbound-files.js";
 import { sessionIdFor } from "./session-id.js";
 import { SSE_CONTENT_TYPE, SSE_HEARTBEAT, sseFrame } from "./sse.js";
 import { RunTranslator } from "./translate.js";
@@ -60,11 +63,38 @@ function runError(error) {
         : error.message;
     return { type: EventType.RUN_ERROR, message, code };
 }
-/** The trailing user message: the Runtime keeps the transcript, AG-UI resends all of it. */
+/** A media part's file: bytes sent inline, or an artifact the Runtime already holds. */
+function inboundFile(part, index) {
+    const { source } = part;
+    if (source.type === "url")
+        return refuseUrl();
+    if (source.type === "file") {
+        if (source.provider !== NYLORUN_FILE_PROVIDER)
+            return fail(400, `A file source names an artifact: provider "${NYLORUN_FILE_PROVIDER}" and its artifact id`, { code: "invalid_request" });
+        return { type: "artifact", artifactId: source.value };
+    }
+    const metadata = (part.metadata ?? {});
+    const named = [metadata.filename, metadata.name].find((value) => typeof value === "string" && value !== "");
+    return {
+        type: "inline",
+        name: named ?? inlineFileName(source.mimeType, index),
+        mediaType: source.mimeType,
+        data: source.value,
+    };
+}
+/**
+ * The trailing user message: the Runtime keeps the transcript, AG-UI resends all of it. Its
+ * text, or its parts when it carries a media part.
+ */
 function lastUserMessage(messages) {
     const last = messages.at(-1);
     if (!last || last.role !== "user")
         return fail(400, "The run's last message must be a user message");
+    if (typeof last.content !== "string" && last.content.some((part) => part.type !== "text")) {
+        let files = 0;
+        const parts = last.content.map((part) => part.type === "text" ? { type: "text", text: part.text } : inboundFile(part, files++));
+        return { id: last.id, text: "", parts };
+    }
     const text = typeof last.content === "string"
         ? last.content
         : last.content
@@ -138,12 +168,23 @@ async function submit(ctx, id, input, scope, trace) {
     const message = lastUserMessage(input.messages);
     // The AG-UI message id is the idempotency key: a retried run replays the same turn, and
     // history gives the message back its client id.
-    return accepted(ctx, id, {
-        type: "message",
-        requestId: randomUUID(),
-        idempotencyKey: message.id,
-        content: message.text,
-    }, scope, trace);
+    if (!message.parts)
+        return accepted(ctx, id, {
+            type: "message",
+            requestId: randomUUID(),
+            idempotencyKey: message.id,
+            content: message.text,
+        }, scope, trace);
+    const access = accessOf(scope);
+    const { parts, stored } = await messageParts(ctx, id, message.id, message.parts, access);
+    try {
+        return await accepted(ctx, id, { type: "message", requestId: randomUUID(), idempotencyKey: message.id, parts }, scope, trace);
+    }
+    catch (error) {
+        // A refused message leaves no files behind.
+        await discardFiles(ctx, stored, access);
+        throw error;
+    }
 }
 /**
  * Streams translated events until the run ends. Closing the connection stops the read, never
@@ -254,8 +295,9 @@ trace) {
         cursor = await submit(ctx, id, input, scope, trace);
     }
     catch (error) {
-        // A busy session or a limit is a run that could not start, not a transport failure.
-        if (error instanceof HttpError && (error.status === 409 || error.status === 429))
+        // A busy session, a limit or a file the model does not read is a run that could not start,
+        // not a transport failure.
+        if (error instanceof HttpError && [409, 422, 429].includes(error.status))
             return stream(response, translator, undefined, [runError(error)]);
         throw error;
     }