nolo-cli 0.1.26 → 0.1.29

package/server/handlers/openaiImageHandler.ts

@@ -0,0 +1,500 @@
+import { getNoloKey } from "../../ai/llm/getNoloKey";
+import { authenticateRequest, CORS_HEADERS_AUTH } from "../../auth/utils";
+import { getUser } from "../../auth/server/getUser";
+import { chargeExternalTool } from "../../ai/token/externalToolCost";
+import {
+  saveBufferAsFile,
+  type FileMetadata,
+} from "../../database/server/fileService";
+import { checkBasicBalance, checkMultipleImagesBalance } from "../../auth/billing";
+import { userActor } from "../../auth/actor";
+import { fetchConvMsgs } from "../api/methods";
+
+type OpenAIImageInput = {
+  data: string;
+  mimeType?: string;
+};
+
+type OpenAIImageParams = {
+  prompt: string;
+  operation?: "generate" | "edit";
+  images?: OpenAIImageInput[];
+  mask?: OpenAIImageInput;
+  size?: "1024x1024" | "1024x1536" | "1536x1024" | "auto";
+  quality?: "low" | "medium" | "high" | "auto";
+  background?: "transparent" | "opaque" | "auto";
+  outputFormat?: "png" | "jpeg" | "webp";
+  outputCompression?: number;
+  moderation?: "low" | "auto";
+  n?: number;
+  model?: "gpt-image-1.5" | "gpt-image-2";
+  dialogId?: string;
+};
+
+type NormalizedInputImage = {
+  mimeType: string;
+  buffer: Buffer;
+  filename: string;
+};
+
+type OpenAIResultImage = {
+  mimeType: string;
+  data: string;
+  revisedPrompt?: string;
+};
+
+type SavedFile = {
+  fileId: string;
+  metadata: FileMetadata;
+};
+
+const OPENAI_IMAGE_API_BASE = "https://api.openai.com/v1/images";
+const DEFAULT_MODEL = "gpt-image-1.5" as const;
+const MODEL_V2 = "gpt-image-2" as const;
+
+const jsonResponse = (body: unknown, status = 200): Response =>
+  new Response(JSON.stringify(body), {
+    status,
+    headers: { "Content-Type": "application/json", ...CORS_HEADERS_AUTH },
+  });
+
+const errorResponse = (
+  status: number,
+  message: string,
+  code: string,
+  details?: unknown
+) =>
+  jsonResponse(
+    {
+      error: {
+        message,
+        code,
+        details,
+      },
+    },
+    status
+  );
+
+const isHttpUrl = (value: string): boolean => /^https?:\/\//i.test(value);
+
+const isDataUrl = (value: string): boolean =>
+  typeof value === "string" && value.startsWith("data:");
+
+const collectLatestImageUrlsFromMessages = (messages: unknown): string[] => {
+  if (!Array.isArray(messages)) return [];
+  for (let index = messages.length - 1; index >= 0; index -= 1) {
+    const content = (messages[index] as any)?.content;
+    if (!Array.isArray(content)) continue;
+    const urls = content
+      .map((part) =>
+        part?.type === "image_url" && typeof part.image_url?.url === "string"
+          ? part.image_url.url.trim()
+          : ""
+      )
+      .filter((url): url is string => isHttpUrl(url));
+    if (urls.length > 0) return [...new Set(urls)];
+  }
+  return [];
+};
+
+const replaceEditImagesWithDialogUrls = async (
+  params: OpenAIImageParams,
+  userId: string
+): Promise<OpenAIImageParams> => {
+  const images = Array.isArray(params.images) ? params.images : [];
+  const hasHttpImage = images.some((image) => typeof image?.data === "string" && isHttpUrl(image.data));
+  const shouldEdit =
+    params.operation === "edit" || (params.operation == null && (images.length > 0 || params.mask));
+  if (!shouldEdit || hasHttpImage || !params.dialogId) return params;
+
+  const actor = userActor(userId);
+  if (!actor) return params;
+  const messages = await fetchConvMsgs(
+    { dialogId: params.dialogId, limit: 80 },
+    { userId, actor }
+  ).catch(() => []);
+  const urls = collectLatestImageUrlsFromMessages(messages);
+  if (urls.length === 0) return params;
+
+  const desiredCount = Math.max(1, images.length);
+  return {
+    ...params,
+    images: urls.slice(0, desiredCount).map((url) => ({ data: url })),
+  };
+};
+
+const parseDataUrl = (
+  dataUrl: string
+): { mimeType: string; data: string } | null => {
+  const match = /^data:(.+?);base64,(.+)$/.exec(dataUrl);
+  if (!match) return null;
+  const [, mimeType, data] = match;
+  return { mimeType, data };
+};
+
+const guessExtension = (mimeType: string): string => {
+  const normalized = mimeType.toLowerCase();
+  if (normalized === "image/png") return "png";
+  if (normalized === "image/webp") return "webp";
+  if (normalized === "image/jpeg" || normalized === "image/jpg") return "jpg";
+  return "bin";
+};
+
+const fetchImageFromUrl = async (url: string): Promise<NormalizedInputImage> => {
+  const res = await fetch(url);
+  if (!res.ok) {
+    const text = await res.text().catch(() => "");
+    throw new Error(`拉取图片失败: ${res.status} ${res.statusText} ${text}`.trim());
+  }
+  const arrayBuffer = await res.arrayBuffer();
+  const mimeType = res.headers.get("content-type") || "image/png";
+  return {
+    mimeType,
+    buffer: Buffer.from(arrayBuffer),
+    filename: `input.${guessExtension(mimeType)}`,
+  };
+};
+
+const normalizeInputImages = async (
+  images: OpenAIImageInput[] | undefined,
+  mask?: OpenAIImageInput
+): Promise<{ images: NormalizedInputImage[]; maskImage: NormalizedInputImage | null }> => {
+  if (!Array.isArray(images) || images.length === 0) {
+    const maskImage = mask ? await normalizeSingleImage(mask, 0) : null;
+    return { images: [], maskImage };
+  }
+
+  const normalized = await Promise.all(
+    images.map(async (image, index) => {
+      return normalizeSingleImage(image, index + 1);
+    })
+  );
+
+  const maskImage = mask ? await normalizeSingleImage(mask, 0) : null;
+
+  return {
+    images: normalized.filter(
+      (item): item is NormalizedInputImage => item !== null
+    ),
+    maskImage,
+  };
+};
+
+const normalizeSingleImage = async (
+  image: OpenAIImageInput,
+  index: number
+): Promise<NormalizedInputImage | null> => {
+  if (!image?.data) return null;
+  if (isHttpUrl(image.data)) {
+    return fetchImageFromUrl(image.data);
+  }
+  if (isDataUrl(image.data)) {
+    const parsed = parseDataUrl(image.data);
+    if (!parsed) return null;
+    return {
+      mimeType: parsed.mimeType,
+      buffer: Buffer.from(parsed.data, "base64"),
+      filename: `input-${index}.${guessExtension(parsed.mimeType)}`,
+    };
+  }
+  const mimeType = image.mimeType ?? "image/png";
+  return {
+    mimeType,
+    buffer: Buffer.from(image.data, "base64"),
+    filename: `input-${index}.${guessExtension(mimeType)}`,
+  };
+};
+
+const parseOutputMimeType = (outputFormat?: string | null) => {
+  switch ((outputFormat || "").toLowerCase()) {
+    case "jpeg":
+    case "jpg":
+      return "image/jpeg";
+    case "webp":
+      return "image/webp";
+    default:
+      return "image/png";
+  }
+};
+
+const shouldSendOutputCompression = (
+  outputFormat?: string | null,
+  outputCompression?: number
+) => {
+  if (typeof outputCompression !== "number") return false;
+  const normalized = (outputFormat || "").toLowerCase();
+  return normalized === "jpeg" || normalized === "jpg" || normalized === "webp";
+};
+
+const saveGeneratedImages = async (params: {
+  userId: string;
+  prompt: string;
+  model: string;
+  images: OpenAIResultImage[];
+}): Promise<SavedFile[]> => {
+  const { userId, prompt, model, images } = params;
+  const tenantId = userId || "default";
+  const ts = Date.now();
+  const files: SavedFile[] = [];
+
+  for (let i = 0; i < images.length; i += 1) {
+    const image = images[i];
+    const ext = guessExtension(image.mimeType);
+    const originalName = `${model}-${ts}-${i + 1}.${ext}`;
+    const buffer = Buffer.from(image.data, "base64");
+
+    const { fileId, metadata } = await saveBufferAsFile(buffer, {
+      tenantId,
+      originalName,
+      mimeType: image.mimeType,
+      ownerType: "user",
+      ownerId: userId,
+      uploadedBy: userId,
+      ownerDbKey: undefined,
+      source: "ai-generated",
+      model,
+      prompt,
+      tags: ["ai-generated", "openai", "image"],
+      clientMetadata: null,
+      clientProvidedId: undefined,
+    });
+
+    files.push({ fileId, metadata });
+  }
+
+  return files;
+};
+
+const callOpenAIImageApi = async (
+  params: OpenAIImageParams,
+  apiKey: string
+): Promise<{
+  images: OpenAIResultImage[];
+  raw: any;
+}> => {
+  const { images: normalizedImages, maskImage } = await normalizeInputImages(
+    params.images,
+    params.mask
+  );
+  const model = params.model || DEFAULT_MODEL;
+  const imageCount = normalizedImages.length + (maskImage ? 1 : 0);
+  const operation = params.operation ?? (imageCount > 0 ? "edit" : "generate");
+  const endpoint = operation === "edit" ? "edits" : "generations";
+  const url = `${OPENAI_IMAGE_API_BASE}/${endpoint}`;
+
+  const headers: Record<string, string> = {
+    Authorization: `Bearer ${apiKey}`,
+  };
+
+  let response: Response;
+  if (operation === "edit") {
+    const form = new FormData();
+    form.set("model", model);
+    form.set("prompt", params.prompt);
+    if (params.size) form.set("size", params.size);
+    if (params.quality) form.set("quality", params.quality);
+    if (params.background) form.set("background", params.background);
+    if (params.outputFormat) form.set("output_format", params.outputFormat);
+    if (shouldSendOutputCompression(params.outputFormat, params.outputCompression)) {
+      form.set("output_compression", String(params.outputCompression));
+    }
+    if (params.moderation) form.set("moderation", params.moderation);
+    if (typeof params.n === "number") form.set("n", String(params.n));
+
+    for (const image of normalizedImages) {
+      const file = new File([image.buffer], image.filename, {
+        type: image.mimeType,
+      });
+      form.append("image[]", file);
+    }
+
+    if (maskImage) {
+      const maskFile = new File([maskImage.buffer], maskImage.filename, {
+        type: maskImage.mimeType,
+      });
+      form.set("mask", maskFile);
+    }
+
+    response = await fetch(url, {
+      method: "POST",
+      headers,
+      body: form,
+    });
+  } else {
+    headers["Content-Type"] = "application/json";
+    response = await fetch(url, {
+      method: "POST",
+      headers,
+      body: JSON.stringify({
+        model,
+        prompt: params.prompt,
+        size: params.size,
+        quality: params.quality,
+        background: params.background,
+        output_format: params.outputFormat,
+        ...(shouldSendOutputCompression(params.outputFormat, params.outputCompression)
+          ? { output_compression: params.outputCompression }
+          : {}),
+        moderation: params.moderation,
+        n: params.n,
+      }),
+    });
+  }
+
+  const rawText = await response.text().catch(() => "");
+  if (!response.ok) {
+    throw new Error(
+      `OpenAI 图片接口请求失败: ${response.status} ${response.statusText} ${rawText}`
+    );
+  }
+
+  let json: any;
+  try {
+    json = rawText ? JSON.parse(rawText) : {};
+  } catch (error) {
+    throw new Error(
+      `OpenAI 图片接口响应解析失败: ${
+        error instanceof Error ? error.message : String(error)
+      }`
+    );
+  }
+
+  const data = Array.isArray(json?.data) ? json.data : [];
+  const fallbackMimeType = parseOutputMimeType(params.outputFormat);
+  const images = data
+    .map((item: any) => {
+      const b64 = typeof item?.b64_json === "string" ? item.b64_json : "";
+      if (!b64) return null;
+      return {
+        mimeType: fallbackMimeType,
+        data: b64,
+        revisedPrompt:
+          typeof item?.revised_prompt === "string" ? item.revised_prompt : undefined,
+      };
+    })
+    .filter((item: OpenAIResultImage | null): item is OpenAIResultImage => !!item);
+
+  return { images, raw: json };
+};
+
+export const handleOpenAIImage = async (req: Request): Promise<Response> => {
+  try {
+    const auth = await authenticateRequest(req);
+    if (auth instanceof Response) return auth;
+
+    const { userId } = auth;
+    const { balance = 0 } = await getUser(userId);
+    const body = (await req.json().catch(() => null)) as OpenAIImageParams | null;
+
+    if (!body || typeof body !== "object") {
+      return errorResponse(400, "请求体必须是 JSON 对象。", "INVALID_BODY");
+    }
+
+    const prompt = typeof body.prompt === "string" ? body.prompt.trim() : "";
+    if (!prompt) {
+      return errorResponse(400, "prompt 不能为空。", "INVALID_PARAMS");
+    }
+
+    const imageCount = Array.isArray(body.images) ? body.images.length : 0;
+    const maskImage = body.mask ? [body.mask] : [];
+    const allInputImages = imageCount + maskImage.length;
+    
+    // Determine model: explicit operation uses v2, legacy path uses v1.5
+    const hasExplicitOperation = body.operation === "generate" || body.operation === "edit";
+    const model = body.model || (hasExplicitOperation ? MODEL_V2 : DEFAULT_MODEL);
+
+    // Validate gpt-image-2 constraints
+    if (model === MODEL_V2 && body.background === "transparent") {
+      return errorResponse(
+        400,
+        "gpt-image-2 暂不支持 transparent background。",
+        "UNSUPPORTED_BACKGROUND"
+      );
+    }
+
+    const basic = checkBasicBalance(balance);
+    if (!basic.allowed && basic.error) {
+      return errorResponse(
+        basic.error.status,
+        basic.error.message,
+        basic.error.code,
+        basic.error.details
+      );
+    }
+
+    const multi = checkMultipleImagesBalance(balance, allInputImages);
+    if (!multi.allowed && multi.error) {
+      return errorResponse(
+        multi.error.status,
+        multi.error.message,
+        multi.error.code,
+        multi.error.details
+      );
+    }
+
+    const apiKey = getNoloKey("openai");
+    if (!apiKey) {
+      return errorResponse(
+        500,
+        "OPENAI_KEY / OPENAI_API_KEY 未配置。",
+        "MISSING_OPENAI_KEY"
+      );
+    }
+
+    const resolvedBody = await replaceEditImagesWithDialogUrls(
+      {
+        ...body,
+        prompt,
+        model,
+      },
+      userId
+    );
+
+    const result = await callOpenAIImageApi(resolvedBody, apiKey);
+
+    const files = await saveGeneratedImages({
+      userId,
+      prompt,
+      model,
+      images: result.images,
+    });
+    if (files.length > 0) {
+      await chargeExternalTool({
+        userId,
+        toolId: model,
+        dialogId: body.dialogId,
+        count: files.length,
+        reason: `openai_image:${model}`,
+      });
+    }
+
+    const revisedPrompts = result.images
+      .map((image) => image.revisedPrompt)
+      .filter((value): value is string => typeof value === "string" && value.trim().length > 0);
+
+    return jsonResponse({
+      text:
+        files.length > 0
+          ? `已生成 ${files.length} 张图片。`
+          : "OpenAI 已完成图片处理，但没有返回可保存的图片。",
+      imageCount: files.length,
+      revisedPrompts,
+      files: files.map((file) => ({
+        fileId: file.fileId,
+        metadata: {
+          ...file.metadata,
+          prompt,
+          model,
+        },
+      })),
+      raw: result.raw,
+    });
+  } catch (error) {
+    console.error("[OpenAI Image] handler failed:", error);
+    return errorResponse(
+      500,
+      error instanceof Error ? error.message : "OpenAI 图片生成失败",
+      "OPENAI_IMAGE_FAILED"
+    );
+  }
+};

package/server/handlers/providerHealthHandler.ts

@@ -0,0 +1,36 @@
+import { checkAdminPermission, createErrorResponse, createSuccessResponse, handleOptionsRequest } from "../../auth/server/shared";
+import { authenticateRequest } from "../../auth/utils";
+import { runProviderHealthProbe } from "../providerHealthProbe";
+
+const parseAttempts = (value: string | null) => {
+  if (!value) return 2;
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) return 2;
+  return Math.max(1, Math.min(10, Math.floor(parsed)));
+};
+
+export async function handleProviderHealth(req: Request) {
+  if (req.method === "OPTIONS") return handleOptionsRequest();
+
+  const auth = await authenticateRequest(req);
+  if (auth instanceof Response) return auth;
+
+  const permissionError = checkAdminPermission(auth.userId);
+  if (permissionError) return permissionError;
+
+  try {
+    const url = new URL(req.url);
+    const attempts = parseAttempts(url.searchParams.get("attempts"));
+    const providers = await runProviderHealthProbe({ attempts });
+    return createSuccessResponse({
+      generatedAt: Date.now(),
+      attempts,
+      providers,
+    });
+  } catch (error) {
+    return createErrorResponse(
+      error instanceof Error ? error.message : String(error),
+      500
+    );
+  }
+}