@downcity/agent 1.1.86 → 1.1.92

package/src/executor/messages/AssistantFileResource.ts

@@ -0,0 +1,155 @@
+/**
+ * AssistantFileResource：assistant file part 的本地资源落盘工具。
+ *
+ * 关键点（中文）
+ * - 只处理运行期产生的 assistant file part，不参与 user 附件注入。
+ * - 将 `data:*;base64,...` 写入 `.downcity/resources`，历史中只保留 `file://` 绝对 URL。
+ * - 资源文件按内容 hash 命名，天然去重并避免重复写入大文件。
+ */
+
+import crypto from "node:crypto";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import fs from "fs-extra";
+import type { FileUIPart } from "ai";
+import { getDowncityResourcesDirPath } from "@/config/Paths.js";
+
+type ParsedDataUrl = {
+  /**
+   * data URL 声明的媒体类型。
+   */
+  media_type: string;
+  /**
+   * data URL 解码后的二进制内容。
+   */
+  bytes: Buffer;
+};
+
+/**
+ * assistant file part 资源落盘参数。
+ */
+export interface MaterializeAssistantFilePartsParams {
+  /**
+   * 当前项目根目录。
+   *
+   * 关键点（中文）
+   * - 正常 session run 会显式传入 projectRoot。
+   * - 旧入口未传时回退到 `process.cwd()`，保证行为可用。
+   */
+  projectRoot?: string;
+
+  /**
+   * 待处理的 assistant file parts。
+   */
+  parts: FileUIPart[];
+}
+
+function resolve_project_root(projectRoot: string | undefined): string {
+  const raw = String(projectRoot || "").trim();
+  return path.resolve(raw || process.cwd());
+}
+
+function parse_data_url(url: string): ParsedDataUrl | null {
+  const raw = String(url || "").trim();
+  if (!raw.startsWith("data:")) return null;
+  const comma_index = raw.indexOf(",");
+  if (comma_index < 0) return null;
+
+  const header = raw.slice(5, comma_index);
+  const body = raw.slice(comma_index + 1);
+  const header_parts = header.split(";").filter(Boolean);
+  const media_type =
+    header_parts.find((item) => item.includes("/")) || "application/octet-stream";
+  const is_base64 = header_parts.some((item) => item.toLowerCase() === "base64");
+
+  try {
+    const bytes = is_base64
+      ? Buffer.from(body, "base64")
+      : Buffer.from(decodeURIComponent(body), "utf8");
+    if (bytes.length === 0) return null;
+    return {
+      media_type,
+      bytes,
+    };
+  } catch {
+    return null;
+  }
+}
+
+function extension_from_media_type(mediaType: string): string {
+  const value = String(mediaType || "").toLowerCase();
+  if (value === "image/png") return ".png";
+  if (value === "image/jpeg" || value === "image/jpg") return ".jpg";
+  if (value === "image/webp") return ".webp";
+  if (value === "image/gif") return ".gif";
+  if (value === "application/pdf") return ".pdf";
+  return ".bin";
+}
+
+function extension_from_filename(filename: string | undefined): string {
+  const ext = path.extname(String(filename || "").trim()).toLowerCase();
+  if (!ext || ext.length > 12) return "";
+  return /^[.][a-z0-9]+$/u.test(ext) ? ext : "";
+}
+
+async function write_resource_file(params: {
+  projectRoot: string;
+  mediaType: string;
+  filename?: string;
+  bytes: Buffer;
+}): Promise<string> {
+  const hash = crypto.createHash("sha256").update(params.bytes).digest("hex");
+  const ext =
+    extension_from_filename(params.filename) ||
+    extension_from_media_type(params.mediaType);
+  const file_name = `${hash}${ext}`;
+  const resources_dir = getDowncityResourcesDirPath(params.projectRoot);
+  const file_path = path.join(resources_dir, file_name);
+
+  await fs.ensureDir(resources_dir);
+  try {
+    await fs.writeFile(file_path, params.bytes, { flag: "wx" });
+  } catch (error) {
+    const code = (error as NodeJS.ErrnoException).code;
+    if (code !== "EEXIST") throw error;
+  }
+
+  return file_path;
+}
+
+/**
+ * 将 assistant file part 中的 data URL 资源落盘为 `file://` 绝对 URL。
+ */
+export async function materializeAssistantFileParts(
+  params: MaterializeAssistantFilePartsParams,
+): Promise<FileUIPart[]> {
+  const parts = Array.isArray(params.parts) ? params.parts : [];
+  if (parts.length === 0) return [];
+
+  const project_root = resolve_project_root(params.projectRoot);
+  const out: FileUIPart[] = [];
+
+  for (const part of parts) {
+    const parsed = parse_data_url(String(part.url || ""));
+    if (!parsed) {
+      out.push(part);
+      continue;
+    }
+
+    const media_type = String(part.mediaType || parsed.media_type || "").trim();
+    const file_path = await write_resource_file({
+      projectRoot: project_root,
+      mediaType: media_type || parsed.media_type,
+      filename: typeof part.filename === "string" ? part.filename : undefined,
+      bytes: parsed.bytes,
+    });
+
+    out.push({
+      ...part,
+      mediaType: media_type || parsed.media_type,
+      url: pathToFileURL(file_path).href,
+    });
+  }
+
+  return out;
+}

package/src/executor/messages/SessionAttachmentMapper.ts

@@ -9,6 +9,7 @@
 
 import fs from "fs-extra";
 import path from "node:path";
+import { fileURLToPath } from "node:url";
 import {
   isFileUIPart,
   isTextUIPart,
@@ -61,6 +62,64 @@ function buildDataUrl(mediaType: string, buffer: Buffer): string {
   return `data:${safeType};base64,${base64}`;
 }
 
+async function hydrateFileUrlPart(part: FileUIPart): Promise<FileUIPart> {
+  const url = String(part.url || "").trim();
+  if (!url.startsWith("file://")) return part;
+  try {
+    const absPath = fileURLToPath(url);
+    const buffer = await fs.readFile(absPath);
+    const mediaType =
+      String(part.mediaType || "").trim() ||
+      guessAttachmentMediaTypeFromPath(absPath) ||
+      "application/octet-stream";
+    return {
+      ...part,
+      mediaType,
+      url: buildDataUrl(mediaType, buffer),
+    };
+  } catch {
+    return part;
+  }
+}
+
+/**
+ * 将历史中的 `file://` file part 临时转换为模型可消费的 data URL。
+ *
+ * 关键点（中文）
+ * - 该函数只修改本轮内存消息，不回写历史。
+ * - 持久化层继续保留轻量 `file://` 绝对 URL，避免 JSONL 存储 base64。
+ */
+export async function hydrateFileUrlPartsForModel(
+  messages: SessionMessageV1[],
+): Promise<SessionMessageV1[]> {
+  if (!Array.isArray(messages) || messages.length === 0) return messages;
+
+  const out: SessionMessageV1[] = [];
+  for (const message of messages) {
+    const parts = Array.isArray(message?.parts) ? message.parts : [];
+    if (!parts.some((part) => isFileUIPart(part as FileUIPart))) {
+      out.push(message);
+      continue;
+    }
+
+    const nextParts: SessionMessageV1["parts"] = [];
+    let changed = false;
+    for (const part of parts) {
+      if (!isFileUIPart(part as FileUIPart)) {
+        nextParts.push(part);
+        continue;
+      }
+      const nextPart = await hydrateFileUrlPart(part as FileUIPart);
+      if (nextPart !== part) changed = true;
+      nextParts.push(nextPart as SessionMessageV1["parts"][number]);
+    }
+
+    out.push(changed ? { ...message, parts: nextParts } : message);
+  }
+
+  return out;
+}
+
 /**
  * 在 user 消息上注入 FileUIPart，以便多模态模型直接消费本地附件。
  */

package/src/executor/messages/SessionMessageCodec.ts

@@ -15,7 +15,10 @@ import {
   type ToolSet,
 } from "ai";
 import type { SessionMessageV1 } from "@/executor/types/SessionMessages.js";
-import { injectFilePartsFromAttachments } from "@executor/messages/SessionAttachmentMapper.js";
+import {
+  hydrateFileUrlPartsForModel,
+  injectFilePartsFromAttachments,
+} from "@executor/messages/SessionAttachmentMapper.js";
 
 /**
  * 过滤回调返回值中的 user 文本消息。
@@ -76,8 +79,11 @@ export async function toModelMessages(
   // 第一步（中文）：在 user 消息上注入 file parts（多模态附件）。
   const enrichedMessages = await injectFilePartsFromAttachments(messages);
 
-  // 第二步（中文）：转换前先剔除 UI 层 id 字段，仅保留模型需要的数据结构。
-  const input = enrichedMessages.map((message) => {
+  // 第二步（中文）：把历史里的 file:// 资源在内存中 hydrate 成模型可消费的 data URL。
+  const hydratedMessages = await hydrateFileUrlPartsForModel(enrichedMessages);
+
+  // 第三步（中文）：转换前先剔除 UI 层 id 字段，仅保留模型需要的数据结构。
+  const input = hydratedMessages.map((message) => {
     // 解构去掉 id。
     const { id: _id, ...rest } = message;
 

package/src/executor/tools/plugin/PluginToolBridge.ts

@@ -14,7 +14,11 @@ import type {
   PluginCallInput,
   PluginCallToolResult,
 } from "@/executor/tools/plugin/types/PluginTool.js";
-import { enqueueAssistantFileParts } from "@executor/SessionRunScope.js";
+import { materializeAssistantFileParts } from "@executor/messages/AssistantFileResource.js";
+import {
+  enqueueAssistantFileParts,
+  getSessionRunContext,
+} from "@executor/SessionRunScope.js";
 
 let plugin_tool_runtime: PluginPort | null = null;
 
@@ -131,9 +135,16 @@ export async function invokePluginCallTool(
       action,
       payload,
     });
-    const file_parts = result.success
+    const raw_file_parts = result.success
       ? extract_assistant_file_parts(result.data)
       : [];
+    const file_parts =
+      raw_file_parts.length > 0
+        ? await materializeAssistantFileParts({
+            projectRoot: getSessionRunContext()?.projectRoot,
+            parts: raw_file_parts,
+          })
+        : [];
     if (file_parts.length > 0) {
       enqueueAssistantFileParts(file_parts);
     }

package/src/index.ts

@@ -229,6 +229,10 @@ export type {
   ImagePluginContent,
   ImagePluginFileContent,
   ImagePluginInput,
+  ImagePluginJobCreateResult,
+  ImagePluginJobResult,
+  ImagePluginJobStatus,
+  ImagePluginJobStatusResult,
   ImagePluginMessage,
   ImagePluginOptions,
   ImagePluginResult,

package/src/plugin/core/ImagePlugin.ts

@@ -7,10 +7,14 @@
  * - action 返回 AI SDK UIMessage，后续由 plugin tool bridge 抽取 file parts 写回 assistant 消息。
  */
 
+import crypto from "node:crypto";
 import type { AgentContext } from "@/types/runtime/agent/AgentContext.js";
 import type { JsonObject, JsonValue } from "@/types/common/Json.js";
 import type {
   ImagePluginInput,
+  ImagePluginJobCreateResult,
+  ImagePluginJobResult,
+  ImagePluginJobStatusResult,
   ImagePluginOptions,
   ImagePluginResult,
 } from "@/types/plugin/ImagePlugin.js";
@@ -20,6 +24,39 @@ const DEFAULT_IMAGE_PLUGIN_NAME = "image";
 const DEFAULT_IMAGE_PLUGIN_TITLE = "Image";
 const DEFAULT_IMAGE_PLUGIN_DESCRIPTION =
   "Generate images and return them as assistant file parts.";
+const DEFAULT_WAIT_TIMEOUT_MS = 60_000;
+const DEFAULT_POLL_INTERVAL_MS = 3_000;
+
+type LocalImageJobRecord = {
+  /**
+   * 图片任务唯一 ID。
+   */
+  job_id: string;
+  /**
+   * 当前任务状态。
+   */
+  status: "queued" | "running" | "succeeded" | "failed";
+  /**
+   * 成功时的图片结果。
+   */
+  result?: ImagePluginResult;
+  /**
+   * 失败时的错误信息。
+   */
+  error?: string;
+  /**
+   * 人类可读状态说明。
+   */
+  message?: string;
+  /**
+   * 任务创建时间。
+   */
+  created_at: string;
+  /**
+   * 任务更新时间。
+   */
+  updated_at: string;
+};
 
 /**
  * 判断值是否为普通对象。
@@ -40,6 +77,15 @@ function normalize_image_payload(payload: JsonValue | undefined): ImagePluginInp
   return { ...record } as ImagePluginInput;
 }
 
+function normalize_job_id_payload(payload: JsonValue | undefined): { job_id: string } {
+  const record = to_record(payload ?? {});
+  const job_id = String(record?.job_id || "").trim();
+  if (!job_id) {
+    throw new TypeError("ImagePlugin job action requires job_id");
+  }
+  return { job_id };
+}
+
 /**
  * 校验 image 函数返回的 UIMessage。
  */
@@ -51,6 +97,25 @@ function normalize_image_result(result: ImagePluginResult): ImagePluginResult {
   return result;
 }
 
+/**
+ * 归一化任务状态查询结果，确保 status action 不携带图片结果。
+ */
+function normalize_job_status_result(
+  result: ImagePluginJobStatusResult,
+): ImagePluginJobStatusResult {
+  return {
+    job_id: result.job_id,
+    status: result.status,
+    ...(result.message ? { message: result.message } : {}),
+    ...(result.error ? { error: result.error } : {}),
+    ...(typeof result.poll_after_ms === "number"
+      ? { poll_after_ms: result.poll_after_ms }
+      : {}),
+    ...(result.created_at ? { created_at: result.created_at } : {}),
+    ...(result.updated_at ? { updated_at: result.updated_at } : {}),
+  };
+}
+
 /**
  * Agent 图片生成插件。
  */
@@ -71,6 +136,12 @@ export class ImagePlugin extends BasePlugin {
   readonly description: string;
 
   private readonly image: ImagePluginOptions["image"];
+  private readonly create_job?: ImagePluginOptions["create"];
+  private readonly read_job_status?: ImagePluginOptions["status"];
+  private readonly read_job_result?: ImagePluginOptions["result"];
+  private readonly wait_timeout_ms: number;
+  private readonly poll_interval_ms: number;
+  private readonly local_jobs = new Map<string, LocalImageJobRecord>();
 
   constructor(options: ImagePluginOptions) {
     super();
@@ -78,8 +149,23 @@ export class ImagePlugin extends BasePlugin {
     if (!name) {
       throw new Error("ImagePlugin requires a non-empty name");
     }
-    if (typeof options.image !== "function") {
-      throw new Error("ImagePlugin requires an image(input) function");
+    const has_custom_job_api = Boolean(
+      options.create || options.status || options.result,
+    );
+    if (
+      has_custom_job_api &&
+      (typeof options.create !== "function" ||
+        typeof options.status !== "function" ||
+        typeof options.result !== "function")
+    ) {
+      throw new Error(
+        "ImagePlugin custom job API requires create, status, and result functions",
+      );
+    }
+    if (!has_custom_job_api && typeof options.image !== "function") {
+      throw new Error(
+        "ImagePlugin requires either image(input) or create/status/result functions",
+      );
     }
     this.name = name;
     this.title = String(options.title || DEFAULT_IMAGE_PLUGIN_TITLE).trim();
@@ -87,6 +173,17 @@ export class ImagePlugin extends BasePlugin {
       options.description || DEFAULT_IMAGE_PLUGIN_DESCRIPTION,
     ).trim();
     this.image = options.image;
+    this.create_job = options.create;
+    this.read_job_status = options.status;
+    this.read_job_result = options.result;
+    this.wait_timeout_ms =
+      typeof options.wait_timeout_ms === "number" && options.wait_timeout_ms > 0
+        ? options.wait_timeout_ms
+        : DEFAULT_WAIT_TIMEOUT_MS;
+    this.poll_interval_ms =
+      typeof options.poll_interval_ms === "number" && options.poll_interval_ms > 0
+        ? options.poll_interval_ms
+        : DEFAULT_POLL_INTERVAL_MS;
   }
 
   /**
@@ -94,22 +191,202 @@ export class ImagePlugin extends BasePlugin {
    */
   system(_context: AgentContext): string {
     return [
-      "Image generation is available through the plugin_call tool.",
-      `Call plugin "${this.name}" action "generate" when the user asks to create, render, draw, or edit an image.`,
-      "Pass a JSON payload with prompt, optional size/aspect_ratio/quality/n, and optional provider_options.",
-      "The generated image files will be attached to the final assistant message automatically.",
+      "Image generation is available through the plugin_call tool as an observable job workflow.",
+      `Call plugin "${this.name}" action "create" when the user asks to create, render, draw, or edit an image.`,
+      `Then call plugin "${this.name}" action "status" with { job_id } to inspect progress.`,
+      `When status is succeeded, call plugin "${this.name}" action "result" with { job_id } to attach the generated files.`,
+      "Use action \"generate\" only as a compatibility shortcut when you explicitly need to wait for completion.",
+      "Pass a JSON payload with prompt, optional size/aspect_ratio/quality/n, and optional provider_options to create/generate.",
     ].join("\n");
   }
 
+  private create_local_job(input: ImagePluginInput): ImagePluginJobCreateResult {
+    if (typeof this.image !== "function") {
+      throw new Error("ImagePlugin local image job requires image(input)");
+    }
+    const now = new Date().toISOString();
+    const job_id = `img_${crypto.randomUUID()}`;
+    const record: LocalImageJobRecord = {
+      job_id,
+      status: "running",
+      message: "image job is running",
+      created_at: now,
+      updated_at: now,
+    };
+    this.local_jobs.set(job_id, record);
+
+    void this.run_local_job(record, input, this.image);
+
+    return {
+      job_id,
+      status: record.status,
+      message: record.message,
+      poll_after_ms: this.poll_interval_ms,
+      created_at: record.created_at,
+      updated_at: record.updated_at,
+    };
+  }
+
+  private async run_local_job(
+    record: LocalImageJobRecord,
+    input: ImagePluginInput,
+    image: NonNullable<ImagePluginOptions["image"]>,
+  ): Promise<void> {
+    try {
+      const message = normalize_image_result(await image(input));
+      record.status = "succeeded";
+      record.result = message;
+      record.message = "image job succeeded";
+      record.updated_at = new Date().toISOString();
+    } catch (error) {
+      record.status = "failed";
+      record.error = String(error);
+      record.message = "image job failed";
+      record.updated_at = new Date().toISOString();
+    }
+  }
+
+  private read_local_job(job_id: string): LocalImageJobRecord {
+    const record = this.local_jobs.get(job_id);
+    if (!record) {
+      throw new Error(`Unknown image job: ${job_id}`);
+    }
+    return record;
+  }
+
+  private serialize_local_status(record: LocalImageJobRecord): ImagePluginJobStatusResult {
+    return {
+      job_id: record.job_id,
+      status: record.status,
+      ...(record.message ? { message: record.message } : {}),
+      ...(record.error ? { error: record.error } : {}),
+      ...(record.status === "running" || record.status === "queued"
+        ? { poll_after_ms: this.poll_interval_ms }
+        : {}),
+      created_at: record.created_at,
+      updated_at: record.updated_at,
+    };
+  }
+
+  private serialize_local_result(record: LocalImageJobRecord): ImagePluginJobResult {
+    return {
+      job_id: record.job_id,
+      status: record.status,
+      ...(record.result ? { result: record.result } : {}),
+      ...(record.error ? { error: record.error } : {}),
+      ...(record.message ? { message: record.message } : {}),
+      created_at: record.created_at,
+      updated_at: record.updated_at,
+    };
+  }
+
+  private async wait_for_job(job_id: string): Promise<ImagePluginResult> {
+    const deadline = Date.now() + this.wait_timeout_ms;
+    while (Date.now() <= deadline) {
+      const result = this.read_job_result
+        ? await this.read_job_result({ job_id })
+        : this.serialize_local_result(this.read_local_job(job_id));
+      if (result.status === "succeeded" && result.result) {
+        return normalize_image_result(result.result);
+      }
+      if (result.status === "failed") {
+        throw new Error(result.error || result.message || "image job failed");
+      }
+      await new Promise((resolve) => setTimeout(resolve, this.poll_interval_ms));
+    }
+    throw new Error(`image job timed out: ${job_id}`);
+  }
+
   /**
    * 显式 action 集合。
    */
   readonly actions = {
+    create: {
+      execute: async ({ payload }: { payload: JsonValue }) => {
+        try {
+          const input = normalize_image_payload(payload);
+          const result = this.create_job
+            ? await this.create_job(input)
+            : this.create_local_job(input);
+          return {
+            success: true,
+            data: result as unknown as JsonObject,
+            message: result.message || "image job created",
+          };
+        } catch (error) {
+          return {
+            success: false,
+            error: String(error),
+            message: String(error),
+          };
+        }
+      },
+    },
+    status: {
+      execute: async ({ payload }: { payload: JsonValue }) => {
+        try {
+          const input = normalize_job_id_payload(payload);
+          const result = this.read_job_status
+            ? normalize_job_status_result(await this.read_job_status(input))
+            : this.serialize_local_status(this.read_local_job(input.job_id));
+          return {
+            success: true,
+            data: result as unknown as JsonObject,
+            message: result.message || `image job ${result.status}`,
+          };
+        } catch (error) {
+          return {
+            success: false,
+            error: String(error),
+            message: String(error),
+          };
+        }
+      },
+    },
+    result: {
+      execute: async ({ payload }: { payload: JsonValue }) => {
+        try {
+          const input = normalize_job_id_payload(payload);
+          const result = this.read_job_result
+            ? await this.read_job_result(input)
+            : this.serialize_local_result(this.read_local_job(input.job_id));
+          if (result.status === "succeeded" && result.result) {
+            return {
+              success: true,
+              data: result.result as unknown as JsonObject,
+              message: result.message || "image job succeeded",
+            };
+          }
+          if (result.status === "failed") {
+            return {
+              success: false,
+              data: result as unknown as JsonObject,
+              error: result.error || result.message || "image job failed",
+              message: result.message || "image job failed",
+            };
+          }
+          return {
+            success: true,
+            data: result as unknown as JsonObject,
+            message: result.message || `image job ${result.status}`,
+          };
+        } catch (error) {
+          return {
+            success: false,
+            error: String(error),
+            message: String(error),
+          };
+        }
+      },
+    },
     generate: {
       execute: async ({ payload }: { payload: JsonValue }) => {
         try {
           const input = normalize_image_payload(payload);
-          const message = normalize_image_result(await this.image(input));
+          const job = this.create_job
+            ? await this.create_job(input)
+            : this.create_local_job(input);
+          const message = await this.wait_for_job(job.job_id);
           return {
             success: true,
             data: message as unknown as JsonObject,