nolo-cli 0.1.40 → 0.1.42

package/ai/tools/openaiImageTool.ts

@@ -0,0 +1,267 @@
+import { callToolApi } from "./toolApiClient";
+import { selectCurrentSpaceId } from "../../create/space/spaceSlice";
+import { selectUserId } from "../../auth/authSlice";
+import { selectCurrentServer } from "../../app/settings/settingSlice";
+import { selectCurrentDialogKey } from "../../chat/dialog/dialogSlice";
+import { addContentAction } from "../../create/space/content/addContentAction";
+import { ContentType } from "../../app/types";
+import { fileKey } from "../../database/keys";
+import { buildDatabaseFileContentUrl } from "../../database/fileUrl";
+import { extractCustomId } from "../../core/prefix";
+
+const DEFAULT_OPENAI_IMAGE_MODEL = "gpt-image-1.5" as const;
+const DEFAULT_OPENAI_IMAGE_MODEL_V2 = "gpt-image-2" as const;
+
+type OpenAIImageInput = {
+  data: string;
+  mimeType?: string;
+};
+
+type OpenAIImageArgs = {
+  prompt: string;
+  operation?: "generate" | "edit";
+  images?: OpenAIImageInput[];
+  mask?: OpenAIImageInput;
+  size?: "1024x1024" | "1024x1536" | "1536x1024" | "auto";
+  quality?: "low" | "medium" | "high" | "auto";
+  background?: "transparent" | "opaque" | "auto";
+  outputFormat?: "png" | "jpeg" | "webp";
+  outputCompression?: number;
+  moderation?: "low" | "auto";
+  n?: number;
+};
+
+const buildOpenAIImageLlmContext = ({
+  result,
+  userId,
+  currentServer,
+}: {
+  result: any;
+  userId?: string | null;
+  currentServer?: string | null;
+}): string | undefined => {
+  const files = Array.isArray(result?.files) ? result.files : [];
+  if (files.length === 0) return undefined;
+
+  const lines = [
+    "The image generation tool produced the following reusable images.",
+    "If you mention, embed, or tabulate these images later, reuse these exact references and never invent placeholder URLs.",
+  ];
+
+  for (let index = 0; index < files.length; index += 1) {
+    const file = files[index];
+    const bareFileId =
+      typeof file?.fileId === "string" && file.fileId.trim()
+        ? file.fileId.trim()
+        : "";
+    const fileDbKey =
+      userId && bareFileId ? fileKey.single(userId, bareFileId) : "";
+    const fileUrl = buildDatabaseFileContentUrl(
+      currentServer,
+      fileDbKey || bareFileId
+    );
+    const originalName =
+      typeof file?.metadata?.originalName === "string"
+        ? file.metadata.originalName.trim()
+        : "";
+
+    lines.push(`image ${index + 1}:`);
+    if (bareFileId) lines.push(`- fileId: ${bareFileId}`);
+    if (fileDbKey) lines.push(`- fileDbKey: ${fileDbKey}`);
+    if (fileUrl) lines.push(`- url: ${fileUrl}`);
+    if (originalName) lines.push(`- name: ${originalName}`);
+  }
+
+  return lines.join("\n");
+};
+
+export const openAIGptImageFunctionSchema = {
+  name: "openAIGptImage",
+  description: [
+    "使用 OpenAI GPT Image 1.5 生成或编辑图片。",
+    "",
+    "使用建议（面向模型）：",
+    "1. 当用户要为当前项目生成插画、封面、配图、海报、Logo 草图、背景素材时，优先使用本工具。",
+    "2. 如果用户提供了现有图片并要求改图、延展、换背景、加元素，也使用本工具，并在 images 中显式传入相关图片。",
+    "3. 这是单轮图片生成/编辑工具；如果只是分析图片内容，不要调用它。",
+    "",
+    "关于 images：",
+    "- 不传 images：文生图。",
+    "- 传 images：走图片编辑接口。",
+    "- images[*].data 支持 Base64、data URL、或 http(s) URL。",
+  ].join("\n"),
+  parameters: {
+    type: "object",
+    properties: {
+      prompt: {
+        type: "string",
+        description: "生成或编辑图片的提示词。",
+      },
+      images: {
+        type: "array",
+        description: "可选输入图片。传入后会按图片编辑模式处理。",
+        items: {
+          type: "object",
+          properties: {
+            data: {
+              type: "string",
+              description: "图片数据，可为 Base64、data URL、或 http(s) URL。",
+            },
+            mimeType: {
+              type: "string",
+              description: "可选 MIME 类型，例如 image/png 或 image/jpeg。",
+            },
+          },
+          required: ["data"],
+        },
+      },
+      size: {
+        type: "string",
+        description:
+          '输出尺寸，可选 "1024x1024"、"1024x1536"、"1536x1024"、"auto"。',
+      },
+      quality: {
+        type: "string",
+        description:
+          '输出质量，可选 "low"、"medium"、"high"、"auto"。',
+      },
+      background: {
+        type: "string",
+        description:
+          '背景，可选 "transparent"、"opaque"、"auto"。',
+      },
+      outputFormat: {
+        type: "string",
+        description: '输出格式，可选 "png"、"jpeg"、"webp"。',
+      },
+      outputCompression: {
+        type: "number",
+        description: "JPEG/WebP 输出压缩率，0-100。",
+      },
+      moderation: {
+        type: "string",
+        description: '内容过滤级别，可选 "low" 或 "auto"。',
+      },
+      n: {
+        type: "number",
+        description: "一次生成的图片数量。",
+      },
+    },
+    required: ["prompt"],
+  },
+};
+
+export const openAIGptImageGenerateFunctionSchema = {
+  ...openAIGptImageFunctionSchema,
+  name: "openAIGptImageGenerate",
+  description:
+    "使用 OpenAI GPT Image 2 生成新图片。适合文生图，以及把参考图作为灵感来源生成新画面。GPT Image 2 当前不支持 transparent background，默认传 background: \"opaque\"。如果 outputFormat 是 png（或未显式改成 jpeg/webp），不要传 outputCompression。",
+};
+
+export const openAIGptImageEditFunctionSchema = {
+  ...openAIGptImageFunctionSchema,
+  name: "openAIGptImageEdit",
+  description:
+    "使用 OpenAI GPT Image 2 编辑现有图片。适合单图编辑、多图参考合成，以及带 mask 的精确修改。优先复用用户已上传的原始 http(s) 图片 URL，不要自己手写 data URL 或重编码缩略图。GPT Image 2 当前不支持 transparent background，默认传 background: \"opaque\"。如果 outputFormat 是 png（或未显式改成 jpeg/webp），不要传 outputCompression。",
+};
+
+const runOpenAIImageTool = async (
+  args: OpenAIImageArgs,
+  thunkApi: any,
+  model: string
+): Promise<{ rawData: any; displayData: string; llmContext?: string }> => {
+  const trimmedPrompt = args.prompt?.trim();
+  if (!trimmedPrompt) {
+    throw new Error("prompt 不能为空");
+  }
+
+  const state = thunkApi.getState();
+  const currentDialogKey = selectCurrentDialogKey(state);
+  const dialogId = currentDialogKey
+    ? extractCustomId(currentDialogKey)
+    : undefined;
+
+  const result = await callToolApi(
+    thunkApi,
+    "/api/openai-image",
+    {
+      prompt: trimmedPrompt,
+      operation: args.operation,
+      images: Array.isArray(args.images) ? args.images : [],
+      mask: args.mask,
+      size: args.size,
+      quality: args.quality,
+      background: args.background,
+      outputFormat: args.outputFormat,
+      outputCompression: args.outputCompression,
+      moderation: args.moderation,
+      n: args.n,
+      model,
+      dialogId,
+    },
+    { withAuth: true }
+  );
+
+  const spaceId = selectCurrentSpaceId(state);
+  const userId = selectUserId(state);
+  const currentServer = selectCurrentServer(state);
+
+  if (spaceId && userId && result?.files?.length) {
+    for (let i = 0; i < result.files.length; i += 1) {
+      const file = result.files[i];
+      if (!file?.fileId) continue;
+      const contentKey = fileKey.single(userId, file.fileId);
+      const title = file.metadata?.originalName || `OpenAI Image ${i + 1}`;
+      try {
+        await addContentAction(
+          { spaceId, contentKey, title, type: ContentType.IMAGE },
+          { dispatch: thunkApi.dispatch, getState: thunkApi.getState }
+        );
+      } catch (error) {
+        console.warn("[openAIGptImageFunc] Failed to add image to space:", error);
+      }
+    }
+  }
+
+  const llmContext = buildOpenAIImageLlmContext({
+    result,
+    userId,
+    currentServer,
+  });
+
+  return {
+    rawData: result,
+    displayData:
+      result?.text || "已使用 OpenAI GPT Image 生成新的图片资源。",
+    llmContext,
+  };
+};
+
+export const openAIGptImageGenerateFunc = async (
+  args: OpenAIImageArgs,
+  thunkApi: any
+): Promise<{ rawData: any; displayData: string; llmContext?: string }> => {
+  return runOpenAIImageTool(
+    { ...args, operation: "generate" },
+    thunkApi,
+    DEFAULT_OPENAI_IMAGE_MODEL_V2
+  );
+};
+
+export const openAIGptImageEditFunc = async (
+  args: OpenAIImageArgs,
+  thunkApi: any
+): Promise<{ rawData: any; displayData: string; llmContext?: string }> => {
+  return runOpenAIImageTool(
+    { ...args, operation: "edit" },
+    thunkApi,
+    DEFAULT_OPENAI_IMAGE_MODEL_V2
+  );
+};
+
+export const openAIGptImageFunc = async (
+  args: OpenAIImageArgs,
+  thunkApi: any
+): Promise<{ rawData: any; displayData: string; llmContext?: string }> => {
+  return runOpenAIImageTool(args, thunkApi, DEFAULT_OPENAI_IMAGE_MODEL);
+};

package/ai/tools/prepareTools.ts

@@ -0,0 +1,193 @@
+import i18n from "../../app/i18n";
+import {
+  toolRegistry,
+  toolDefinitionsByName,
+  findToolExecutor,
+  ToolBehavior,
+} from ".";
+import { canonicalizeToolNames } from "./toolNameAliases";
+import { sanitizeToolForProvider } from "./toolSchemaCompatibility";
+
+function getTranslation(keys: string[], defaultVal: string): string {
+  for (const key of keys) {
+    if (i18n.exists(key)) {
+      return i18n.t(key);
+    }
+    const aiKey = key.startsWith("ai:") ? key : `ai:${key}`;
+    if (i18n.exists(aiKey)) {
+      return i18n.t(aiKey);
+    }
+  }
+  return defaultVal;
+}
+
+export function translateSchema(schema: any): any {
+  if (!schema || typeof schema !== "object") return schema;
+
+  const nextSchema = JSON.parse(JSON.stringify(schema));
+  const toolName = nextSchema.name;
+  if (!toolName) return nextSchema;
+
+  const walk = (node: any, path: string) => {
+    if (!node || typeof node !== "object") return;
+
+    if (typeof node.description === "string") {
+      const keysToTry = [
+        `${path}.description`,
+        `${path}.desc`,
+        path,
+        node.description,
+      ];
+      node.description = getTranslation(keysToTry, node.description);
+    }
+
+    if (node.parameters && typeof node.parameters === "object") {
+      walk(node.parameters, path);
+    }
+
+    if (node.properties && typeof node.properties === "object") {
+      for (const [key, child] of Object.entries(node.properties)) {
+        const nextPath = path.endsWith(".params") ? `${path}.${key}` : `${path}.params.${key}`;
+        walk(child, nextPath);
+      }
+    }
+
+    if (node.items && typeof node.items === "object") {
+      walk(node.items, path);
+    }
+
+    const compositions = ["anyOf", "oneOf", "allOf"] as const;
+    for (const comp of compositions) {
+      if (Array.isArray(node[comp])) {
+        node[comp].forEach((child: any, idx: number) => {
+          const suffix = child.type === "string" ? "stringDesc" : child.type === "object" ? "objectDesc" : `${idx}`;
+          walk(child, `${path}.${suffix}`);
+        });
+      }
+    }
+  };
+
+  walk(nextSchema, `tools.${toolName}`);
+  return nextSchema;
+}
+
+function buildActivitySchema() {
+  const refs = {
+    type: "array",
+    items: {
+      type: "object",
+      properties: {
+        type: { type: "string", enum: ["file", "terminal", "url"] },
+        path: { type: "string" },
+        id: { type: "string" },
+        label: { type: "string" },
+        url: { type: "string" },
+      },
+    },
+  };
+  const action = {
+    type: "object",
+    properties: {
+      title: { type: "string" },
+      kind: {
+        type: "string",
+        enum: ["read", "write", "edit", "search", "terminal", "version", "test", "build", "preview", "other"],
+      },
+      detail: { type: "string" },
+      refs,
+    },
+    required: ["title"],
+  };
+  const planPhase = {
+    type: "object",
+    properties: {
+      id: { type: "string" },
+      title: { type: "string" },
+      index: { type: "number" },
+      status: { type: "string", enum: ["pending", "running", "success", "failed"] },
+    },
+    required: ["id", "title"],
+  };
+  return {
+    type: "object",
+    description:
+      "Optional human-readable activity metadata for UI progress display only. Use plan for the visible task skeleton, phase for the current task step, and action for this specific tool call.",
+    properties: {
+      plan: {
+        type: "object",
+        properties: {
+          title: { type: "string" },
+          phases: {
+            type: "array",
+            items: planPhase,
+          },
+        },
+        required: ["phases"],
+      },
+      phase: {
+        type: "object",
+        properties: {
+          id: { type: "string" },
+          title: { type: "string" },
+          index: { type: "number" },
+          total: { type: "number" },
+          status: { type: "string", enum: ["pending", "running", "success", "failed"] },
+        },
+        required: ["id", "title"],
+      },
+      action,
+      title: { type: "string" },
+      kind: action.properties.kind,
+      detail: { type: "string" },
+      refs,
+    },
+  };
+}
+
+function addActivityMetadataToToolSchema(tool: any) {
+  if (!tool?.function?.parameters || typeof tool.function.parameters !== "object") {
+    return tool;
+  }
+  const parameters = tool.function.parameters;
+  if (parameters.type !== "object" || !parameters.properties || typeof parameters.properties !== "object") {
+    return tool;
+  }
+  if (parameters.properties._activity) return tool;
+
+  return {
+    ...tool,
+    function: {
+      ...tool.function,
+      parameters: {
+        ...parameters,
+        properties: {
+          ...parameters.properties,
+          _activity: buildActivitySchema(),
+        },
+      },
+    },
+  };
+}
+
+export const prepareTools = (
+  toolNames: string[],
+  options?: { provider?: string; disabledToolNames?: string[] }
+) => {
+  // logic moved to mergeAgentToolsWithRuntime for better tiered capabilities
+  const disabledToolNames = new Set(
+    canonicalizeToolNames(options?.disabledToolNames ?? [])
+  );
+  return canonicalizeToolNames(toolNames)
+    .filter((toolName) => !disabledToolNames.has(toolName))
+    .map((toolName: string) => {
+      const regTool = toolRegistry[toolName];
+      if (!regTool) return regTool;
+      return {
+        ...regTool,
+        function: translateSchema(regTool.function),
+      };
+    })
+    .map(addActivityMetadataToToolSchema)
+    .map((tool) => sanitizeToolForProvider(tool, options?.provider))
+    .filter(Boolean); // 过滤掉未找到的工具
+};

package/ai/tools/readDocTool.ts

@@ -0,0 +1,88 @@
+// 文件路径: ai/tools/readDocTool.ts
+
+import { slateToSimplifiedMarkdown } from "../../create/editor/transforms/slateToSimplifiedMarkdown";
+import type { PageData } from "../../render/page/types";
+
+export interface ReadPageToolArgs {
+    id?: string; // 对应 dbKey，例如 "page-xxx"
+    doc?: string;
+    docKey?: string;
+    pageKey?: string;
+    key?: string;
+}
+
+/**
+ * [Schema] 定义了 'readDoc' 工具的结构。
+ */
+export const readDocFunctionSchema = {
+    name: "readDoc",
+    description: [
+        "读取指定页面的内容，并将结构化的数据转换为 Markdown 格式返回。",
+        "如果你拿到了页面的 dbKey（如 page-xxx），请使用此工具查看页面内容。",
+    ].join("\n"),
+    parameters: {
+        type: "object",
+        properties: {
+            id: {
+                type: "string",
+                description: "页面/文档的数据库键（dbKey），例如 page-xxx。",
+            },
+        },
+        required: ["id"],
+    },
+} as const;
+
+export const readPageFunctionSchema = {
+    ...readDocFunctionSchema,
+    name: "readPage",
+} as const;
+
+export const buildReadDocResult = (
+    pageData: PageData
+): { rawData: unknown; displayData: string } => {
+    const markdownContent = slateToSimplifiedMarkdown(pageData.slateData || []);
+
+    const rawData = {
+        success: true,
+        id: pageData.dbKey,
+        title: pageData.title,
+        content: markdownContent,
+        metadata: {
+            spaceId: pageData.spaceId,
+            created: pageData.created,
+        },
+    };
+
+    const displayData = `已成功读取页面《${pageData.title}》。\n\n内容如下：\n\n${markdownContent}`;
+    return { rawData, displayData };
+};
+
+/**
+ * [Executor] 'readDoc' 工具的执行函数。
+ */
+export async function readDocFunc(
+    args: ReadPageToolArgs,
+    thunkApi: any
+): Promise<{ rawData: unknown; displayData: string }> {
+    const id = args.id ?? args.doc ?? args.docKey ?? args.pageKey ?? args.key;
+
+    if (!id || !id.toLowerCase().startsWith("page-")) {
+        throw new Error(`无效的页面 ID: ${id}。页面 ID 通常以 "page-" 开头。`);
+    }
+
+    try {
+        // 1. 从数据库读取原始数据
+        const { readAction } = await import("../../database/actions/read");
+        const pageData = (await readAction({ dbKey: id }, thunkApi)) as PageData;
+
+        if (!pageData) {
+            throw new Error(`未找到 ID 为 ${id} 的页面。`);
+        }
+
+        return buildReadDocResult(pageData);
+    } catch (error: any) {
+        throw new Error(`读取页面时出错: ${error.message || String(error)}`);
+    }
+}
+
+export const readPageFunc = readDocFunc;