@qjp88995/agent-runtime 0.0.0-stage → 0.1.0

package/src/images/delivery.ts

@@ -0,0 +1,508 @@
+import {
+  type LanguageModel,
+  type LanguageModelMiddleware,
+  wrapLanguageModel,
+} from 'ai';
+
+import { InMemoryVendorFileCache } from '../in-process/vendor-file-cache.js';
+import type { ImageSource } from '../ports/image-source.js';
+import { type Logger, silentLogger } from '../ports/logger.js';
+import type {
+  VendorFileCache,
+  VendorFileRecord,
+} from '../ports/vendor-file-cache.js';
+import type { LlmHandles, LlmProviderOptions } from '../turn/turn-run.js';
+import {
+  DEFAULT_BYTE_CACHE_LIMIT,
+  ImageByteCache,
+  type LoadedImage,
+} from './byte-cache.js';
+import {
+  collectPendingImages,
+  DEFAULT_DELIVERY_TEXTS,
+  type DeliveredImage,
+  deliverImages,
+  type DeliveryTexts,
+  isPinnedCredentialUnavailable,
+  type PendingImage,
+  rejectedVendorFiles,
+  urlSupported,
+  type V4Message,
+} from './prompt.js';
+import {
+  DIRECT_ACCOUNT,
+  VendorFiles,
+  type VendorFileStoreLike,
+} from './vendor-files.js';
+
+/**
+ * 递送要的那一点模型句柄（provider-client `ModelHandle` 的结构子集；runtime 不依赖那个包，
+ * 兼容性由 `tools/live-images` 的类型检查钉住）。
+ */
+export interface ModelHandleLike {
+  model: LanguageModel;
+  providerOptions?: LlmProviderOptions | undefined;
+  replayedMetadataKeys?: readonly string[];
+  /** 厂商 Files API；没有为 null（只走公网 URL / base64）。 */
+  fileStore: VendorFileStoreLike | null;
+}
+
+export interface ResolveModelOptions {
+  reasoningEffort?: string;
+  affinity?: string;
+  pin?: string;
+}
+
+/** provider-client `ModelResolver` 的结构子集（`ProviderClient` / `DirectModelResolver` 都满足）。 */
+export interface ModelResolverLike<H extends ModelHandleLike> {
+  resolve(
+    ref: { vendor: string; modelId: string },
+    options?: ResolveModelOptions
+  ): Promise<H>;
+}
+
+export interface ImageDeliveryOptions {
+  /** 字节从哪来。 */
+  source: ImageSource;
+  /** Files API 的上传缓存。缺省进程内（`InMemoryVendorFileCache`），多实例换宿主的库实现。 */
+  cache?: VendorFileCache;
+  /** 上传时的 TTL（秒）；省略 = 厂商默认。每次上传读一次，宿主可接系统参数。 */
+  ttlSeconds?: () => number | undefined;
+  /** 进程内字节缓存上限（缺省 256 MiB）。 */
+  byteCacheLimit?: number;
+  texts?: Partial<DeliveryTexts>;
+  logger?: Logger;
+  now?: () => number;
+}
+
+/** 一次请求的递送结果：改写好的 prompt、用的哪个句柄、哪个账号、交出去了哪些 file id。 */
+interface Attempt {
+  prompt: V4Message[];
+  handle: ModelHandleLike;
+  /** 这次请求钉住的账号（只有中心模式的真 ref 才算；直连 / 未定为 undefined）。 */
+  pinned: string | undefined;
+  /** 交出去的 file id 落在哪个账号（直连是 `DIRECT_ACCOUNT`）。 */
+  account: string | undefined;
+  fileIds: string[];
+}
+
+type CallParams = Parameters<
+  NonNullable<LanguageModelMiddleware['wrapStream']>
+>[0]['params'];
+interface CallableModel {
+  supportedUrls?:
+    PromiseLike<Record<string, RegExp[]>> | Record<string, RegExp[]>;
+  doGenerate(params: CallParams): PromiseLike<unknown>;
+  doStream(params: CallParams): PromiseLike<unknown>;
+}
+
+function callable(model: LanguageModel): CallableModel {
+  if (typeof model === 'string') {
+    throw new Error(
+      'image delivery needs a model object, not a gateway model id string'
+    );
+  }
+  return model as unknown as CallableModel;
+}
+
+function basenameOf(path: string): string {
+  const i = path.lastIndexOf('/');
+  return i >= 0 ? path.slice(i + 1) : path;
+}
+
+/**
+ * 图片递送（agent-base `llm/image-delivery` + `ImageDeliveryService`）。进程级单例：字节缓存与
+ * 上传合流跨会话共享；每一轮用 `wrap` 把那一轮的模型句柄包起来。
+ *
+ * ## 三档（用户拍板，2026-08-27）
+ *
+ * 1. **厂商 Files API**（句柄有 `fileStore` 时；按 `(厂商, 账号, 字节)` 缓存 file id）；
+ * 2. **公网 URL**：`ImageSource.publicUrl` 给了、且承载包 `supportedUrls` 认这个地址与媒体类型；
+ * 3. **base64**——全线公约数。
+ *
+ * 每一档失败都退到下一档，**绝不抛**：一张图递不出去要变成一句给模型看的话，不能让这个会话从此
+ * 发不出请求。
+ *
+ * ## pin（中心模式，2026-11-11 定）
+ *
+ * 厂商文件绑账号，中心又会替我们换 key，所以引用旧文件的请求得 pin 到文件所在的那把。判据在
+ * **每次请求时**按这次 prompt 里的图定（prompt 就是完整历史）：
+ *
+ * - 历史里的图在某个账号上有新鲜的 file id → 选覆盖张数最多的那个账号（平手取最近用过的），
+ *   用 `resolve(…, { pin })` 造一个钉住它的句柄发这次请求；缺的图也传到这个账号上（上传同样 pin）。
+ * - 一个都没有 → 先不 pin 地传第一张（落在中心按粘性选的那把上），拿到 `credentialRef` 后**再 pin
+ *   到它**发请求——这样上传与请求一定在同一个账号上，不靠粘性碰运气。
+ * - 直连模式（`credentialRef` 为 null）只有一把 key，不 pin。
+ *
+ * 两种失败各重做一次递送：厂商说 file id 不存在（错误里点名了这次的 id）→ 忘掉它们、重传；
+ * 中心说钉住的 key 不可用（409 `pinned_credential_unavailable`）→ 这次请求排除那个账号再挑
+ * （多半落到「一个都没有」，重传到新 key 上）。
+ */
+export class ImageDelivery {
+  readonly files: VendorFiles;
+  private readonly bytes: ImageByteCache;
+  private readonly texts: DeliveryTexts;
+  private readonly logger: Logger;
+
+  constructor(private readonly options: ImageDeliveryOptions) {
+    this.logger = options.logger ?? silentLogger('ImageDelivery');
+    this.files = new VendorFiles({
+      cache: options.cache ?? new InMemoryVendorFileCache(),
+      ttlSeconds: options.ttlSeconds,
+      logger: this.logger,
+      now: options.now,
+    });
+    this.bytes = new ImageByteCache(
+      options.source,
+      options.byteCacheLimit ?? DEFAULT_BYTE_CACHE_LIMIT
+    );
+    this.texts = { ...DEFAULT_DELIVERY_TEXTS, ...options.texts };
+  }
+
+  /**
+   * 把一轮的模型句柄包成会递送图片的模型。
+   *
+   * @param vendor 厂商 slug（缓存键的一段）。
+   * @param base 不 pin 解析出的句柄（带会话粘性）。没有图的请求原样走它，一个对象都不重建。
+   * @param pinned 按 ref 造钉住的句柄（同一个 wrap 内按 ref 记忆）。缺席 = 不 pin（直连）。
+   */
+  wrap(args: {
+    vendor: string;
+    base: ModelHandleLike;
+    pinned?: (credentialRef: string) => Promise<ModelHandleLike>;
+  }): LanguageModel {
+    const memo = new Map<string, Promise<ModelHandleLike>>();
+    const pinnedHandle = args.pinned
+      ? (ref: string) => {
+          let p = memo.get(ref);
+          if (!p) {
+            p = args.pinned!(ref);
+            memo.set(ref, p);
+            p.catch(() => memo.delete(ref));
+          }
+          return p;
+        }
+      : undefined;
+    const run = <T>(
+      params: CallParams,
+      passthrough: () => PromiseLike<T>,
+      call: (model: CallableModel, params: CallParams) => PromiseLike<T>
+    ) =>
+      this.run(args.vendor, args.base, pinnedHandle, params, passthrough, call);
+
+    const middleware: LanguageModelMiddleware = {
+      specificationVersion: 'v4',
+      wrapGenerate: ({ doGenerate, params }) =>
+        run(params, doGenerate, (m, p) => m.doGenerate(p)) as ReturnType<
+          typeof doGenerate
+        >,
+      wrapStream: ({ doStream, params }) =>
+        run(params, doStream, (m, p) => m.doStream(p)) as ReturnType<
+          typeof doStream
+        >,
+    };
+    return wrapLanguageModel({
+      model: callable(args.base.model) as unknown as Exclude<
+        LanguageModel,
+        string
+      >,
+      middleware,
+    });
+  }
+
+  private async run<T>(
+    vendor: string,
+    base: ModelHandleLike,
+    pinnedHandle: ((ref: string) => Promise<ModelHandleLike>) | undefined,
+    params: CallParams,
+    passthrough: () => PromiseLike<T>,
+    call: (model: CallableModel, params: CallParams) => PromiseLike<T>
+  ): Promise<T> {
+    const prompt = params.prompt as unknown as readonly V4Message[];
+    const pending = collectPendingImages(prompt);
+    // 快路径：一张我们的图都没有就原样放行。
+    if (pending.length === 0) return passthrough();
+
+    const excluded = new Set<string>();
+    let staleRetried = false;
+    let pinRetried = false;
+    for (;;) {
+      const attempt = await this.prepare(
+        vendor,
+        base,
+        pinnedHandle,
+        prompt,
+        pending,
+        excluded
+      );
+      try {
+        return await call(callable(attempt.handle.model), {
+          ...params,
+          prompt: attempt.prompt as unknown as CallParams['prompt'],
+        });
+      } catch (err) {
+        const rejected = rejectedVendorFiles(err, attempt.fileIds);
+        if (
+          !staleRetried &&
+          attempt.account !== undefined &&
+          rejected.length > 0
+        ) {
+          staleRetried = true;
+          await this.files.forget(vendor, attempt.account, rejected);
+          continue;
+        }
+        if (
+          !pinRetried &&
+          attempt.pinned !== undefined &&
+          isPinnedCredentialUnavailable(err)
+        ) {
+          pinRetried = true;
+          this.logger.warn(
+            `pinned credential ${attempt.pinned} of ${vendor} unavailable; re-delivering without it`
+          );
+          excluded.add(attempt.pinned);
+          continue;
+        }
+        throw err;
+      }
+    }
+  }
+
+  private async prepare(
+    vendor: string,
+    base: ModelHandleLike,
+    pinnedHandle: ((ref: string) => Promise<ModelHandleLike>) | undefined,
+    prompt: readonly V4Message[],
+    pending: readonly PendingImage[],
+    excluded: ReadonlySet<string>
+  ): Promise<Attempt> {
+    const loaded = new Map<string, LoadedImage | null>();
+    for (const { src } of pending) {
+      if (loaded.has(src)) continue;
+      try {
+        loaded.set(src, await this.bytes.load(src));
+      } catch (err) {
+        // 对象被删 / 存储不可达：给模型一句话而不是抛——抛了这个会话每次发言都会死在这里。
+        this.logger.warn(
+          `image ${src} unavailable for delivery: ${String(err)}`
+        );
+        loaded.set(src, null);
+      }
+    }
+
+    const state: Attempt = {
+      prompt: [],
+      handle: base,
+      pinned: undefined,
+      account: undefined,
+      fileIds: [],
+    };
+    const usePin = async (account: string) => {
+      state.account = account;
+      if (account === DIRECT_ACCOUNT || !pinnedHandle) return;
+      try {
+        state.handle = await pinnedHandle(account);
+        state.pinned = account;
+      } catch (err) {
+        // 造不出钉住的句柄（目录刷新失败之类）：这次不 pin，交给粘性。
+        this.logger.warn(`cannot pin ${vendor} to ${account}: ${String(err)}`);
+      }
+    };
+
+    let records: VendorFileRecord[] = [];
+    if (base.fileStore) {
+      const hashes = [
+        ...new Set(
+          [...loaded.values()].flatMap(l => (l ? [l.contentHash] : []))
+        ),
+      ];
+      records = (await this.files.lookup(vendor, hashes)).filter(
+        r => !excluded.has(r.account)
+      );
+      const account = pickAccount(records.filter(r => this.files.isFresh(r)));
+      if (account !== undefined) await usePin(account);
+    }
+
+    const resolve = async (img: PendingImage): Promise<DeliveredImage> => {
+      const bytes = loaded.get(img.src);
+      if (!bytes) {
+        return { kind: 'unavailable', reason: this.texts.unavailable };
+      }
+      const mediaType = img.mediaType || bytes.contentType;
+
+      // 1. 厂商 Files API
+      const store = state.handle.fileStore;
+      if (store) {
+        const fileId = await this.vendorFile(
+          vendor,
+          store,
+          img,
+          bytes,
+          mediaType,
+          records,
+          state,
+          usePin
+        );
+        if (fileId) {
+          state.fileIds.push(fileId);
+          return {
+            kind: 'file',
+            vendorFileId: fileId,
+            part: {
+              type: 'file',
+              mediaType,
+              data: {
+                type: 'reference',
+                reference: { [store.referenceKey]: fileId },
+              },
+            },
+          };
+        }
+      }
+
+      // 2. 公网 URL
+      const url = this.options.source.publicUrl?.(img.src) ?? null;
+      if (url && /^https?:\/\//.test(url)) {
+        const supported =
+          (await callable(state.handle.model).supportedUrls) ?? {};
+        if (urlSupported(supported, mediaType, url)) {
+          return {
+            kind: 'file',
+            part: {
+              type: 'file',
+              mediaType,
+              data: { type: 'url', url: new URL(url) },
+            },
+          };
+        }
+      }
+
+      // 3. base64
+      return {
+        kind: 'file',
+        part: {
+          type: 'file',
+          mediaType,
+          data: { type: 'data', data: bytes.data },
+        },
+      };
+    };
+
+    state.prompt = await deliverImages(prompt, resolve, this.texts);
+    return state;
+  }
+
+  /** 第一档：在本次请求的账号上取（或传）这张图，拿不到给 null。 */
+  private async vendorFile(
+    vendor: string,
+    store: VendorFileStoreLike,
+    img: PendingImage,
+    bytes: LoadedImage,
+    mediaType: string,
+    records: VendorFileRecord[],
+    state: Attempt,
+    usePin: (account: string) => Promise<void>
+  ): Promise<string | null> {
+    const same = (r: VendorFileRecord) =>
+      r.account === state.account && r.contentHash === bytes.contentHash;
+    if (state.account !== undefined) {
+      const hit = records.find(r => same(r) && this.files.isFresh(r));
+      if (hit) {
+        this.files.touch(hit);
+        return hit.fileId;
+      }
+    }
+    const uploaded = await this.files.upload(
+      {
+        vendor,
+        store,
+        contentHash: bytes.contentHash,
+        src: img.src,
+        data: bytes.data,
+        mediaType,
+        ...(img.path ? { filename: basenameOf(img.path) } : {}),
+        previous: records.filter(r => r.contentHash === bytes.contentHash),
+      },
+      state.pinned
+    );
+    if (!uploaded) return null;
+    if (state.account === undefined) {
+      // 第一张上传定账号：之后的图与这次请求都钉到它上面。
+      await usePin(uploaded.account);
+    } else if (uploaded.account !== state.account) {
+      // 钉住了却落到别的账号（不该发生）：这个 id 这次请求用不了，记下的账留给以后。
+      this.logger.warn(
+        `upload for ${vendor} landed on ${uploaded.account}, expected ${state.account}`
+      );
+      records.push(uploaded);
+      return null;
+    }
+    records.push(uploaded);
+    return uploaded.fileId;
+  }
+}
+
+/** 覆盖张数最多的账号；平手取最近用过的。 */
+function pickAccount(records: readonly VendorFileRecord[]): string | undefined {
+  const score = new Map<string, { hashes: Set<string>; last: number }>();
+  for (const r of records) {
+    const s = score.get(r.account) ?? { hashes: new Set<string>(), last: 0 };
+    s.hashes.add(r.contentHash);
+    s.last = Math.max(s.last, r.lastUsedAt.getTime());
+    score.set(r.account, s);
+  }
+  let best: string | undefined;
+  let bestScore: { hashes: Set<string>; last: number } | undefined;
+  for (const [account, s] of score) {
+    if (
+      !bestScore ||
+      s.hashes.size > bestScore.hashes.size ||
+      (s.hashes.size === bestScore.hashes.size && s.last > bestScore.last)
+    ) {
+      best = account;
+      bestScore = s;
+    }
+  }
+  return best;
+}
+
+/**
+ * 解析一轮要用的模型并接上图片递送。`affinity` 传**根会话 id**（子 Agent 与根共用一把 key：
+ * 前缀缓存与厂商文件都绑账号）。`images` 缺席 = 不递送（prompt 里出现图片引用时承载包会响亮地抛）。
+ *
+ * 返回解析出的完整句柄（宿主要它的能力、上限、单价）与交给 `runTurn` 的 `llm`。
+ */
+export async function resolveTurnModel<H extends ModelHandleLike>(args: {
+  resolver: ModelResolverLike<H>;
+  ref: { vendor: string; modelId: string };
+  reasoningEffort?: string;
+  affinity?: string;
+  images?: ImageDelivery | null;
+}): Promise<{ handle: H; llm: LlmHandles }> {
+  const options: ResolveModelOptions = {
+    ...(args.reasoningEffort !== undefined
+      ? { reasoningEffort: args.reasoningEffort }
+      : {}),
+    ...(args.affinity !== undefined ? { affinity: args.affinity } : {}),
+  };
+  const handle = await args.resolver.resolve(args.ref, options);
+  const model = args.images
+    ? args.images.wrap({
+        vendor: args.ref.vendor,
+        base: handle,
+        pinned: pin => args.resolver.resolve(args.ref, { ...options, pin }),
+      })
+    : handle.model;
+  return {
+    handle,
+    llm: {
+      model,
+      ...(handle.providerOptions !== undefined
+        ? { providerOptions: handle.providerOptions }
+        : {}),
+      ...(handle.replayedMetadataKeys !== undefined
+        ? { replayedMetadataKeys: handle.replayedMetadataKeys }
+        : {}),
+    },
+  };
+}

package/src/images/index.ts

@@ -0,0 +1,35 @@
+export {
+  DEFAULT_BYTE_CACHE_LIMIT,
+  ImageByteCache,
+  type LoadedImage,
+} from './byte-cache.js';
+export {
+  ImageDelivery,
+  type ImageDeliveryOptions,
+  type ModelHandleLike,
+  type ModelResolverLike,
+  type ResolveModelOptions,
+  resolveTurnModel,
+} from './delivery.js';
+export {
+  collectPendingImages,
+  DEFAULT_DELIVERY_TEXTS,
+  type DeliveredImage,
+  deliverImages,
+  type DeliveryTexts,
+  errorText,
+  type ImageResolver,
+  isPinnedCredentialUnavailable,
+  mentionsVendorFile,
+  type PendingImage,
+  rejectedVendorFiles,
+  urlSupported,
+} from './prompt.js';
+export {
+  DIRECT_ACCOUNT,
+  MIN_VENDOR_FILE_TTL_SECONDS,
+  VENDOR_FILE_EXPIRY_MARGIN_MS,
+  VendorFiles,
+  type VendorFilesOptions,
+  type VendorFileStoreLike,
+} from './vendor-files.js';