@springbrand/agent-runtime 0.2.0-alpha.45 → 0.2.0-alpha.47

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@springbrand/agent-runtime",
-  "version": "0.2.0-alpha.45",
+  "version": "0.2.0-alpha.47",
   "type": "module",
   "files": [
     "src",

package/src/kernel/bindings.ts

@@ -905,6 +905,13 @@ export type RuntimeModelProtocol =
   | "google-generative-ai"
   | "openai-codex-responses";
 
+export type RuntimeModelInputModality =
+  | "text"
+  | "image"
+  | "file"
+  | "audio"
+  | "video";
+
 /**
  * 描述一个已解析凭据的模型 API 端点。
  *
@@ -921,6 +928,10 @@ export interface RuntimeModelEndpoint {
   headers?: Readonly<Record<string, string>>;
   baseURL: string;
   models: readonly string[];
+  /** Explicit model input capabilities keyed by configured model ID. */
+  modelInputModalities?: Readonly<
+    Record<string, readonly RuntimeModelInputModality[]>
+  >;
   /** OpenRouter upstream provider pins keyed by model ID; pinned requests never fall back. */
   openRouterProviderPins?: Readonly<Record<string, string>>;
 }

package/src/pi/message/conversion.ts

@@ -21,7 +21,16 @@ function attachmentText(
   part: Extract<UIMessage["parts"][number], { type: "file" }>,
 ): string {
   const name = part.filename?.trim() || "attachment";
-  return `[Attachment: ${name} (${part.mediaType}) path=${part.url}]`;
+  let attachmentId: string | undefined;
+  try {
+    attachmentId = /^\/attachments\/([^/]+)$/u
+      .exec(new URL(part.url).pathname)?.[1];
+  } catch {
+    // Non-URL file handles keep their original model-visible path.
+  }
+  return `[Attachment: ${name} (${part.mediaType}) ${
+    attachmentId ? `id=${attachmentId}` : `path=${part.url}`
+  }]`;
 }
 
 function userContent(
@@ -43,7 +52,15 @@ function userContent(
         data: inlineImage[2]!,
       });
     } else {
-      content.push({ type: "text", text: attachmentText(part) });
+      content.push(
+        { type: "text", text: attachmentText(part) },
+        {
+          type: "file",
+          url: part.url,
+          mimeType: part.mediaType,
+          ...(part.filename ? { filename: part.filename } : {}),
+        },
+      );
     }
   }
   if (content.length === 0) throw new Error("User message content is required");

package/src/pi/message/model-input.ts

@@ -0,0 +1,43 @@
+import type { Message, UserMessage } from "@earendil-works/pi-ai";
+import type { RuntimeModelInputModality } from "../../kernel/bindings";
+
+const UNSUPPORTED_FILE_PLACEHOLDER =
+  "(file omitted: model API does not support this file type)";
+
+/** Project canonical hosted files against the selected model and Adapter capabilities. */
+export function modelInputMessages(
+  messages: readonly Message[],
+  modelInputModalities: readonly RuntimeModelInputModality[],
+  adapterInputModalities: readonly RuntimeModelInputModality[],
+): Message[] {
+  return messages.map((message) => {
+    if (message.role !== "user" || !Array.isArray(message.content)) return message;
+    const content: Exclude<UserMessage["content"], string> = [];
+    for (const block of message.content) {
+      const modality = block.type === "file"
+        ? inputModalityOf(block.mimeType)
+        : null;
+      const supported = block.type !== "file" || Boolean(
+        modality && modelInputModalities.includes(modality) &&
+          adapterInputModalities.includes(modality),
+      );
+      if (supported) {
+        content.push(block);
+      } else {
+        const last = content.at(-1);
+        if (last?.type !== "text" || last.text !== UNSUPPORTED_FILE_PLACEHOLDER) {
+          content.push({ type: "text", text: UNSUPPORTED_FILE_PLACEHOLDER });
+        }
+      }
+    }
+    return { ...message, content };
+  });
+}
+
+function inputModalityOf(mediaType: string): RuntimeModelInputModality | null {
+  if (mediaType.startsWith("image/")) return "image";
+  if (mediaType === "application/pdf") return "file";
+  if (mediaType.startsWith("audio/")) return "audio";
+  if (mediaType.startsWith("video/")) return "video";
+  return null;
+}

package/src/pi/runtime-adapter/models.ts

@@ -7,8 +7,10 @@ import {
   type Api,
   type AssistantMessage,
   type AssistantMessageEvent,
+  type Context,
   type Model,
   type MutableModels,
+  type ProviderStreams,
 } from "@earendil-works/pi-ai";
 import { anthropicMessagesApi } from "@earendil-works/pi-ai/api/anthropic-messages.lazy";
 import {
@@ -43,9 +45,11 @@ import {
 } from "agents/chat";
 import type {
   RuntimeModelEndpoint,
+  RuntimeModelInputModality,
   RuntimeModelProtocol,
   RuntimeProviderPort,
 } from "../../kernel/bindings";
+import { modelInputMessages } from "../message/model-input";
 import { openRouterMessagesApi } from "./openrouter-messages";
 
 const CATALOGS = {
@@ -57,6 +61,17 @@ const CATALOGS = {
 } satisfies Record<RuntimeModelProtocol, readonly Model<Api>[]>;
 
 const PROVIDER_MAX_RETRIES = 2;
+
+const MESSAGES_HOSTED_INPUT_MODALITIES = ["image", "file"] as const;
+const NO_HOSTED_INPUT_MODALITIES: readonly RuntimeModelInputModality[] = [];
+
+function adapterHostedInputModalities(
+  protocol: RuntimeModelProtocol,
+): readonly RuntimeModelInputModality[] {
+  return protocol === "openrouter-messages" || protocol === "anthropic-messages"
+    ? MESSAGES_HOSTED_INPUT_MODALITIES
+    : NO_HOSTED_INPUT_MODALITIES;
+}
 // 看门狗要抓的是「连接死了」,不是「模型想得慢」——这两件事在流上分不开:
 // 推理模型在中转后面是「闷头想完再吐」,静默期一个字节都没有,也没有 keepalive。
 //
@@ -602,19 +617,36 @@ function apiFor(protocol: RuntimeModelProtocol): Api {
   }
 }
 
-function piApiFor(protocol: RuntimeModelProtocol) {
-  switch (protocol) {
-    case "openai-chat":
-      return openAICompletionsApi();
-    case "openrouter-messages":
-      return openRouterMessagesApi();
-    case "anthropic-messages":
-      return anthropicMessagesApi();
-    case "google-generative-ai":
-      return googleGenerativeAIApi();
-    case "openai-codex-responses":
-      return openAICodexResponsesApi();
-  }
+function piApiFor(endpoint: RuntimeModelEndpoint): ProviderStreams {
+  const protocol = endpoint.protocol;
+  const api = (() => {
+    switch (protocol) {
+      case "openai-chat":
+        return openAICompletionsApi();
+      case "openrouter-messages":
+        return openRouterMessagesApi();
+      case "anthropic-messages":
+        return anthropicMessagesApi();
+      case "google-generative-ai":
+        return googleGenerativeAIApi();
+      case "openai-codex-responses":
+        return openAICodexResponsesApi();
+    }
+  })();
+  const contextFor = (model: Model<Api>, context: Context): Context => ({
+    ...context,
+    messages: modelInputMessages(
+      context.messages,
+      endpoint.modelInputModalities?.[model.id] ?? model.input,
+      adapterHostedInputModalities(protocol),
+    ),
+  });
+  return {
+    stream: (model, context, options) =>
+      api.stream(model, contextFor(model, context), options),
+    streamSimple: (model, context, options) =>
+      api.streamSimple(model, contextFor(model, context), options),
+  };
 }
 
 function anthropicMessagesModelForOpenRouter(
@@ -653,6 +685,7 @@ function configuredModel(
     baseUrl: _baseUrl,
     compat: catalogCompat,
     headers: catalogHeaders,
+    input: catalogInput,
     ...metadata
   } = catalogModel;
   const headers = {
@@ -661,6 +694,12 @@ function configuredModel(
   };
   const openRouterProviderPin = endpoint.openRouterProviderPins?.[modelId];
   const configuredApi = apiFor(endpoint.protocol);
+  const runtimeInputModalities =
+    endpoint.modelInputModalities?.[modelId] ?? catalogInput;
+  const input = runtimeInputModalities.filter(
+    (modality): modality is "text" | "image" =>
+      modality === "text" || modality === "image",
+  );
   // OpenRouter uses dotted versions and optional transport suffixes while the
   // Anthropic catalog uses dashed base model ids.
   const anthropicMessagesCompat = endpoint.protocol === "openrouter-messages"
@@ -668,6 +707,7 @@ function configuredModel(
     : undefined;
   return {
     ...metadata,
+    input,
     api: configuredApi,
     provider: providerId(endpoint, index),
     baseUrl: endpoint.baseURL,
@@ -751,7 +791,7 @@ export function configurePiModels(
       ),
       // cloudflareStreams 在运行时把 baseURL 里的 {CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}
       // 占位符替换成真实值；如果 baseURL 不含占位符则直接透传，不影响现有端点。
-      api: cloudflareStreams(piApiFor(endpoint.protocol)),
+      api: cloudflareStreams(piApiFor(endpoint)),
     })
   );
   models.clearProviders();

package/src/runtime-assembler.ts

@@ -568,6 +568,15 @@ class RuntimeBuilder {
             ? { headers: Object.freeze({ ...endpoint.headers }) }
             : {}),
           models: Object.freeze([...endpoint.models]),
+          ...(endpoint.modelInputModalities
+            ? {
+                modelInputModalities: Object.freeze(Object.fromEntries(
+                  Object.entries(endpoint.modelInputModalities).map(
+                    ([model, modalities]) => [model, Object.freeze([...modalities])],
+                  ),
+                )),
+              }
+            : {}),
         }),
       )),
     });