nolo-cli 0.1.30 → 0.1.32

package/server/handlers/cloudflareBrowserHandler.ts

@@ -1,275 +0,0 @@
-// packages/server/handlers/cloudflareBrowserHandler.ts
-// Cloudflare Browser Rendering REST API handlers
-// Docs: https://developers.cloudflare.com/browser-rendering/rest-api/
-
-import { authenticateRequest, CORS_HEADERS_AUTH } from "../../auth/utils";
-import { getUser } from "../../auth/server/getUser";
-import { checkBasicBalance } from "../../auth/billing";
-import { chargeExternalTool } from "../../ai/token/externalToolCost";
-
-const CF_API_TOKEN = process.env.CLOUDFLARE_API_TOKEN;
-const CF_ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID;
-const CF_BASE = `https://api.cloudflare.com/client/v4/accounts`;
-
-function jsonError(
-  status: number,
-  message: string,
-  code?: string
-): Response {
-  return new Response(
-    JSON.stringify({ error: { message, code } }),
-    { status, headers: { "Content-Type": "application/json", ...CORS_HEADERS_AUTH } }
-  );
-}
-
-function jsonOk(data: any): Response {
-  return new Response(
-    JSON.stringify(data),
-    { status: 200, headers: { "Content-Type": "application/json", ...CORS_HEADERS_AUTH } }
-  );
-}
-
-async function checkAuthAndConfig(req: Request) {
-  if (!CF_API_TOKEN || !CF_ACCOUNT_ID) {
-    return jsonError(503, "服务未配置：缺少 CLOUDFLARE_API_TOKEN 或 CLOUDFLARE_ACCOUNT_ID", "CF_NOT_CONFIGURED");
-  }
-  const auth = await authenticateRequest(req);
-  if (auth instanceof Response) return auth;
-  const { balance = 0 } = await getUser(auth.userId);
-  const balanceCheck = checkBasicBalance(balance);
-  if (!balanceCheck.allowed && balanceCheck.error) {
-    return jsonError(balanceCheck.error.status, balanceCheck.error.message, balanceCheck.error.code);
-  }
-  return auth;
-}
-
-// ──────────────────────────────────────────────────────────
-// POST /api/cf-screenshot
-// ──────────────────────────────────────────────────────────
-export async function handleCfScreenshot(req: Request): Promise<Response> {
-  if (req.method === "OPTIONS") return new Response(null, { status: 204, headers: CORS_HEADERS_AUTH });
-
-  const authOrError = await checkAuthAndConfig(req);
-  if (authOrError instanceof Response) return authOrError;
-
-  let body: any;
-  try { body = await req.json(); }
-  catch { return jsonError(400, "JSON 解析失败", "INVALID_BODY"); }
-
-  const { url, html, fullPage, viewport, gotoOptions, dialogId } = body;
-  if (!url && !html) return jsonError(400, "必须提供 url 或 html 参数", "MISSING_INPUT");
-  if (url && typeof url !== "string") return jsonError(400, "url 必须是字符串", "INVALID_URL");
-
-  const cfBody: Record<string, any> = {};
-  if (url) cfBody.url = url;
-  if (html) cfBody.html = html;
-  if (viewport) cfBody.viewport = viewport;
-  if (gotoOptions) cfBody.gotoOptions = gotoOptions;
-  if (fullPage) cfBody.screenshotOptions = { fullPage: true };
-
-  const resp = await fetch(`${CF_BASE}/${CF_ACCOUNT_ID}/browser-rendering/screenshot`, {
-    method: "POST",
-    headers: { Authorization: `Bearer ${CF_API_TOKEN}`, "Content-Type": "application/json" },
-    body: JSON.stringify(cfBody),
-  });
-
-  if (!resp.ok) {
-    const err = await resp.text();
-    return jsonError(502, `Cloudflare API 错误: ${err}`, "CF_ERROR");
-  }
-
-  const imageBuffer = await resp.arrayBuffer();
-  const base64 = Buffer.from(imageBuffer).toString("base64");
-  const browserMs = resp.headers.get("X-Browser-Ms-Used");
-  const browserMsUsed = browserMs ? parseInt(browserMs) : undefined;
-
-  if (browserMsUsed && browserMsUsed > 0) {
-    await chargeExternalTool({
-      userId: authOrError.userId,
-      toolId: "cf_browser_rendering",
-      dialogId,
-      durationMinutes: browserMsUsed / 60_000,
-    });
-  }
-
-  return jsonOk({
-    dataUrl: `data:image/png;base64,${base64}`,
-    mimeType: "image/png",
-    browserMsUsed,
-    source: url ?? "html",
-  });
-}
-
-// ──────────────────────────────────────────────────────────
-// POST /api/cf-markdown
-// ──────────────────────────────────────────────────────────
-export async function handleCfMarkdown(req: Request): Promise<Response> {
-  if (req.method === "OPTIONS") return new Response(null, { status: 204, headers: CORS_HEADERS_AUTH });
-
-  const authOrError = await checkAuthAndConfig(req);
-  if (authOrError instanceof Response) return authOrError;
-
-  let body: any;
-  try { body = await req.json(); }
-  catch { return jsonError(400, "JSON 解析失败", "INVALID_BODY"); }
-
-  const { url, html, rejectRequestPattern, gotoOptions, dialogId } = body;
-  if (!url && !html) return jsonError(400, "必须提供 url 或 html 参数", "MISSING_INPUT");
-  if (url && typeof url !== "string") return jsonError(400, "url 必须是字符串", "INVALID_URL");
-
-  const cfBody: Record<string, any> = {};
-  if (url) cfBody.url = url;
-  if (html) cfBody.html = html;
-  if (rejectRequestPattern) cfBody.rejectRequestPattern = rejectRequestPattern;
-  if (gotoOptions) cfBody.gotoOptions = gotoOptions;
-
-  const resp = await fetch(`${CF_BASE}/${CF_ACCOUNT_ID}/browser-rendering/markdown`, {
-    method: "POST",
-    headers: { Authorization: `Bearer ${CF_API_TOKEN}`, "Content-Type": "application/json" },
-    body: JSON.stringify(cfBody),
-  });
-
-  if (!resp.ok) {
-    const err = await resp.text();
-    return jsonError(502, `Cloudflare API 错误: ${err}`, "CF_ERROR");
-  }
-
-  const data = await resp.json() as { success: boolean; result: string };
-  const browserMs = resp.headers.get("X-Browser-Ms-Used");
-  const browserMsUsed = browserMs ? parseInt(browserMs) : undefined;
-
-  if (browserMsUsed && browserMsUsed > 0) {
-    await chargeExternalTool({
-      userId: authOrError.userId,
-      toolId: "cf_browser_rendering",
-      dialogId,
-      durationMinutes: browserMsUsed / 60_000,
-    });
-  }
-
-  return jsonOk({
-    markdown: data.result,
-    success: data.success,
-    browserMsUsed,
-    source: url ?? "html",
-  });
-}
-
-// ──────────────────────────────────────────────────────────
-// POST /api/cf-pdf
-// ──────────────────────────────────────────────────────────
-export async function handleCfPdf(req: Request): Promise<Response> {
-  if (req.method === "OPTIONS") return new Response(null, { status: 204, headers: CORS_HEADERS_AUTH });
-
-  const authOrError = await checkAuthAndConfig(req);
-  if (authOrError instanceof Response) return authOrError;
-
-  let body: any;
-  try { body = await req.json(); }
-  catch { return jsonError(400, "JSON 解析失败", "INVALID_BODY"); }
-
-  const { url, html, pdfOptions, gotoOptions, dialogId } = body;
-  if (!url && !html) return jsonError(400, "必须提供 url 或 html 参数", "MISSING_INPUT");
-  if (url && typeof url !== "string") return jsonError(400, "url 必须是字符串", "INVALID_URL");
-
-  const cfBody: Record<string, any> = {};
-  if (url) cfBody.url = url;
-  if (html) cfBody.html = html;
-  if (pdfOptions) cfBody.pdfOptions = pdfOptions;
-  if (gotoOptions) cfBody.gotoOptions = gotoOptions;
-
-  const resp = await fetch(`${CF_BASE}/${CF_ACCOUNT_ID}/browser-rendering/pdf`, {
-    method: "POST",
-    headers: { Authorization: `Bearer ${CF_API_TOKEN}`, "Content-Type": "application/json" },
-    body: JSON.stringify(cfBody),
-  });
-
-  if (!resp.ok) {
-    const err = await resp.text();
-    return jsonError(502, `Cloudflare API 错误: ${err}`, "CF_ERROR");
-  }
-
-  const pdfBuffer = await resp.arrayBuffer();
-  const base64 = Buffer.from(pdfBuffer).toString("base64");
-  const browserMs = resp.headers.get("X-Browser-Ms-Used");
-  const browserMsUsed = browserMs ? parseInt(browserMs) : undefined;
-
-  let filename = "document.pdf";
-  if (url) {
-    try { filename = new URL(url).hostname.replace(/^www\./, "") + ".pdf"; } catch {}
-  }
-
-  if (browserMsUsed && browserMsUsed > 0) {
-    await chargeExternalTool({
-      userId: authOrError.userId,
-      toolId: "cf_browser_rendering",
-      dialogId,
-      durationMinutes: browserMsUsed / 60_000,
-    });
-  }
-
-  return jsonOk({
-    dataUrl: `data:application/pdf;base64,${base64}`,
-    mimeType: "application/pdf",
-    filename,
-    browserMsUsed,
-    source: url ?? "html",
-  });
-}
-
-// ──────────────────────────────────────────────────────────
-// POST /api/cf-json
-// ──────────────────────────────────────────────────────────
-export async function handleCfJson(req: Request): Promise<Response> {
-  if (req.method === "OPTIONS") return new Response(null, { status: 204, headers: CORS_HEADERS_AUTH });
-
-  const authOrError = await checkAuthAndConfig(req);
-  if (authOrError instanceof Response) return authOrError;
-
-  let body: any;
-  try { body = await req.json(); }
-  catch { return jsonError(400, "JSON 解析失败", "INVALID_BODY"); }
-
-  const { url, html, prompt, response_format, dialogId } = body;
-  if (!url && !html) return jsonError(400, "必须提供 url 或 html 参数", "MISSING_INPUT");
-  if (!prompt && !response_format) return jsonError(400, "必须提供 prompt 或 response_format", "MISSING_PROMPT");
-  if (url && typeof url !== "string") return jsonError(400, "url 必须是字符串", "INVALID_URL");
-  if (prompt && typeof prompt !== "string") return jsonError(400, "prompt 必须是字符串", "INVALID_PROMPT");
-
-  const cfBody: Record<string, any> = {};
-  if (url) cfBody.url = url;
-  if (html) cfBody.html = html;
-  if (prompt) cfBody.prompt = prompt;
-  if (response_format) cfBody.response_format = response_format;
-
-  const resp = await fetch(`${CF_BASE}/${CF_ACCOUNT_ID}/browser-rendering/json`, {
-    method: "POST",
-    headers: { Authorization: `Bearer ${CF_API_TOKEN}`, "Content-Type": "application/json" },
-    body: JSON.stringify(cfBody),
-  });
-
-  if (!resp.ok) {
-    const err = await resp.text();
-    return jsonError(502, `Cloudflare API 错误: ${err}`, "CF_ERROR");
-  }
-
-  const data = await resp.json() as { success: boolean; result: unknown };
-  const browserMs = resp.headers.get("X-Browser-Ms-Used");
-  const browserMsUsed = browserMs ? parseInt(browserMs) : undefined;
-
-  if (browserMsUsed && browserMsUsed > 0) {
-    await chargeExternalTool({
-      userId: authOrError.userId,
-      toolId: "cf_browser_rendering",
-      dialogId,
-      durationMinutes: browserMsUsed / 60_000,
-    });
-  }
-
-  return jsonOk({
-    result: data.result,
-    success: data.success,
-    browserMsUsed,
-    source: url ?? "html",
-  });
-}

package/server/handlers/cloudflareCrawlHandler.ts

@@ -1,260 +0,0 @@
-// packages/server/handlers/cloudflareCrawlHandler.ts
-// Cloudflare Browser Rendering /crawl endpoint proxy
-// Docs: https://developers.cloudflare.com/browser-rendering/rest-api/crawl-endpoint/
-
-import { authenticateRequest, CORS_HEADERS_AUTH } from "../../auth/utils";
-import { getUser } from "../../auth/server/getUser";
-import { checkBasicBalance } from "../../auth/billing";
-
-const CF_API_TOKEN = process.env.CLOUDFLARE_API_TOKEN;
-const CF_ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID;
-const CF_BASE = `https://api.cloudflare.com/client/v4/accounts`;
-
-const POLL_INTERVAL_MS = 3_000;
-const MAX_WAIT_MS = 60_000; // 最多等 60s，避免请求超时
-
-// Allowed values for the `formats` parameter (Cloudflare Browser Rendering docs)
-const ALLOWED_FORMATS = new Set(["markdown", "html", "text", "screenshot", "links"]);
-
-// jobId from Cloudflare is a UUID-like string; reject anything that isn't safe for a URL segment
-const JOB_ID_RE = /^[a-zA-Z0-9_\-]{1,128}$/;
-
-function validateJobId(id: string | undefined): id is string {
-  return typeof id === "string" && JOB_ID_RE.test(id);
-}
-
-function isPlainObject(v: unknown): v is Record<string, unknown> {
-  return v !== null && typeof v === "object" && !Array.isArray(v) &&
-    Object.getPrototypeOf(v) === Object.prototype;
-}
-
-function jsonError(status: number, message: string, code = "CF_CRAWL_ERROR", detail?: any): Response {
-  return new Response(
-    JSON.stringify({ error: { message, code, detail } }),
-    { status, headers: { "Content-Type": "application/json", ...CORS_HEADERS_AUTH } }
-  );
-}
-
-function jsonOk(data: any): Response {
-  return new Response(
-    JSON.stringify(data),
-    { status: 200, headers: { "Content-Type": "application/json", ...CORS_HEADERS_AUTH } }
-  );
-}
-
-// ──────────────────────────────────────────────────────────
-// POST /api/cloudflare-crawl  → 启动爬取（可选等待结果）
-// ──────────────────────────────────────────────────────────
-export async function handleCloudflareCrawlStart(req: Request): Promise<Response> {
-  if (req.method === "OPTIONS") return new Response(null, { status: 204, headers: CORS_HEADERS_AUTH });
-
-  if (!CF_API_TOKEN || !CF_ACCOUNT_ID) {
-    return jsonError(503, "服务未配置：缺少 CLOUDFLARE_API_TOKEN 或 CLOUDFLARE_ACCOUNT_ID", "CF_NOT_CONFIGURED");
-  }
-
-  const auth = await authenticateRequest(req);
-  if (auth instanceof Response) return auth;
-
-  const { userId } = auth;
-  const { balance = 0 } = await getUser(userId);
-  const balanceCheck = checkBasicBalance(balance);
-  if (!balanceCheck.allowed && balanceCheck.error) {
-    return jsonError(balanceCheck.error.status, balanceCheck.error.message, balanceCheck.error.code);
-  }
-
-  let body: any;
-  try {
-    body = await req.json();
-  } catch {
-    return jsonError(400, "请求体必须是合法 JSON", "INVALID_BODY");
-  }
-
-  const {
-    url,
-    limit = 10,
-    depth,
-    formats = ["markdown"],
-    render = true,
-    source,
-    maxAge,
-    modifiedSince,
-    options: crawlOptions,
-    wait = true, // wait=true 时服务端轮询直到完成
-  } = body;
-
-  if (!url || typeof url !== "string") {
-    return jsonError(400, "url 参数是必需的", "MISSING_URL");
-  }
-  try {
-    const u = new URL(url);
-    if (u.protocol !== "http:" && u.protocol !== "https:") throw new Error("协议不支持");
-  } catch {
-    return jsonError(400, "url 必须是合法的 http/https 地址", "INVALID_URL");
-  }
-
-  // 参数范围校验：limit / depth 不允许负数、0 或超出上游限制
-  const validatedLimit = typeof limit === "number" && Number.isFinite(limit)
-    ? Math.min(Math.max(Math.floor(limit), 1), 100_000)
-    : 10;
-  if (depth !== undefined) {
-    if (typeof depth !== "number" || !Number.isFinite(depth) || depth < 1) {
-      return jsonError(400, "depth 必须是大于 0 的整数", "INVALID_DEPTH");
-    }
-  }
-  const validatedDepth = depth !== undefined ? Math.min(Math.floor(depth), 100_000) : undefined;
-
-  // formats: must be array of allowed strings, default ["markdown"]
-  const rawFormats = Array.isArray(formats) ? formats : ["markdown"];
-  const validatedFormats = rawFormats.filter((f: unknown) => typeof f === "string" && ALLOWED_FORMATS.has(f));
-  if (validatedFormats.length === 0) {
-    return jsonError(400, `formats 不合法，允许值: ${[...ALLOWED_FORMATS].join(", ")}`, "INVALID_FORMATS");
-  }
-
-  // render: must be boolean
-  if (render !== undefined && typeof render !== "boolean") {
-    return jsonError(400, "render 必须是布尔值", "INVALID_RENDER");
-  }
-
-  // source: must be string if present
-  if (source !== undefined && typeof source !== "string") {
-    return jsonError(400, "source 必须是字符串", "INVALID_SOURCE");
-  }
-
-  // maxAge: must be non-negative finite number if present
-  if (maxAge !== undefined) {
-    if (typeof maxAge !== "number" || !Number.isFinite(maxAge) || maxAge < 0) {
-      return jsonError(400, "maxAge 必须是非负数", "INVALID_MAX_AGE");
-    }
-  }
-
-  // modifiedSince: must be a valid ISO date string if present
-  if (modifiedSince !== undefined) {
-    if (typeof modifiedSince !== "string" || isNaN(Date.parse(modifiedSince))) {
-      return jsonError(400, "modifiedSince 必须是合法的日期字符串", "INVALID_MODIFIED_SINCE");
-    }
-  }
-
-  // options (crawlOptions): must be a plain object if present
-  if (crawlOptions !== undefined && !isPlainObject(crawlOptions)) {
-    return jsonError(400, "options 必须是普通对象", "INVALID_OPTIONS");
-  }
-
-  // 1. 启动爬取任务
-  const crawlBody: Record<string, any> = { url, limit: validatedLimit, formats: validatedFormats, render: render ?? true };
-  if (validatedDepth !== undefined) crawlBody.depth = validatedDepth;
-  if (source !== undefined) crawlBody.source = source;
-  if (maxAge !== undefined) crawlBody.maxAge = maxAge;
-  if (modifiedSince !== undefined) crawlBody.modifiedSince = modifiedSince;
-  if (crawlOptions !== undefined) crawlBody.options = crawlOptions;
-
-  const startResp = await fetch(`${CF_BASE}/${CF_ACCOUNT_ID}/browser-rendering/crawl`, {
-    method: "POST",
-    headers: {
-      Authorization: `Bearer ${CF_API_TOKEN}`,
-      "Content-Type": "application/json",
-    },
-    body: JSON.stringify(crawlBody),
-  });
-
-  const startJson = await startResp.json() as any;
-  if (!startResp.ok || !startJson.success) {
-    console.error("Cloudflare crawl start failed:", startJson);
-    return jsonError(502, "启动 Cloudflare 爬取任务失败", "CF_START_FAILED", startJson);
-  }
-
-  const jobId: string = startJson.result;
-
-  // 2. 不等待时直接返回 jobId
-  if (!wait) {
-    return jsonOk({ jobId, status: "running" });
-  }
-
-  // 3. 轮询直到任务完成（最多 MAX_WAIT_MS）
-  const deadline = Date.now() + MAX_WAIT_MS;
-  while (Date.now() < deadline) {
-    await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
-
-    const statusResp = await fetch(
-      `${CF_BASE}/${CF_ACCOUNT_ID}/browser-rendering/crawl/${jobId}?limit=1`,
-      { headers: { Authorization: `Bearer ${CF_API_TOKEN}` } }
-    );
-
-    // 状态查询本身失败时继续等待，不能误判为已结束
-    if (!statusResp.ok) continue;
-    const statusJson = await statusResp.json() as any;
-    if (!statusJson?.success) continue;
-
-    const status: string = statusJson?.result?.status;
-    if (status && status !== "running") break;
-  }
-
-  // 4. 获取完整结果
-  const resultResp = await fetch(
-    `${CF_BASE}/${CF_ACCOUNT_ID}/browser-rendering/crawl/${jobId}`,
-    { headers: { Authorization: `Bearer ${CF_API_TOKEN}` } }
-  );
-  const resultJson = await resultResp.json() as any;
-
-  if (!resultResp.ok) {
-    return jsonError(502, "获取爬取结果失败", "CF_RESULT_FAILED", resultJson);
-  }
-
-  return jsonOk({ jobId, ...resultJson.result });
-}
-
-// ──────────────────────────────────────────────────────────
-// GET /api/cloudflare-crawl/:jobId  → 查询任务状态/结果
-// ──────────────────────────────────────────────────────────
-export async function handleCloudflareCrawlStatus(req: Request): Promise<Response> {
-  if (req.method === "OPTIONS") return new Response(null, { status: 204, headers: CORS_HEADERS_AUTH });
-
-  if (!CF_API_TOKEN || !CF_ACCOUNT_ID) {
-    return jsonError(503, "服务未配置", "CF_NOT_CONFIGURED");
-  }
-
-  const auth = await authenticateRequest(req);
-  if (auth instanceof Response) return auth;
-
-  // Reject path traversal before URL normalization resolves it away
-  const rawPath = req.url.split("?")[0];
-  if (rawPath.includes("..")) {
-    return jsonError(400, "jobId 格式不合法", "INVALID_JOB_ID");
-  }
-
-  const url = new URL(req.url);
-  // Guard: path traversal (after normalization) may resolve route segments away
-  if (!url.pathname.startsWith("/api/cloudflare-crawl/")) {
-    return jsonError(400, "jobId 格式不合法", "INVALID_JOB_ID");
-  }
-  const rawJobId = url.pathname.split("/").pop();
-  if (!validateJobId(rawJobId)) {
-    return jsonError(400, "jobId 格式不合法", "INVALID_JOB_ID");
-  }
-  const jobId = rawJobId;
-
-  const rawLimit = url.searchParams.get("limit");
-  const status = url.searchParams.get("status");
-  const cursor = url.searchParams.get("cursor");
-
-  const params = new URLSearchParams();
-  if (rawLimit !== null) {
-    const n = parseInt(rawLimit, 10);
-    if (isNaN(n) || n < 1) return jsonError(400, "limit 必须是正整数", "INVALID_LIMIT");
-    params.set("limit", String(Math.min(n, 100_000)));
-  }
-  if (status) params.set("status", status);
-  if (cursor) params.set("cursor", cursor);
-  const qs = params.toString() ? `?${params}` : "";
-
-  const cfResp = await fetch(
-    `${CF_BASE}/${CF_ACCOUNT_ID}/browser-rendering/crawl/${jobId}${qs}`,
-    { headers: { Authorization: `Bearer ${CF_API_TOKEN}` } }
-  );
-  const cfJson = await cfResp.json() as any;
-
-  if (!cfResp.ok) {
-    return jsonError(502, "查询爬取任务失败", "CF_STATUS_FAILED", cfJson);
-  }
-
-  return jsonOk(cfJson.result);
-}