@fudrouter/fsrouter 0.6.64 → 0.6.66

package/open-sse/handlers/videoProviders/weavy.js

@@ -0,0 +1,198 @@
+import path from "path";
+import { execFile } from "child_process";
+import { promisify } from "util";
+import { updateProviderConnection } from '../../dist/lib/localDb.js';
+
+// 10 MB buffer — video polling can produce large stderr logs
+const MAX_BUFFER = 10 * 1024 * 1024;
+
+// Timeout slightly longer than Python's own internal timeout (600 s for video)
+// to let Python emit its own error rather than getting killed by Node.
+const EXEC_TIMEOUT_MS = 660_000; // 11 minutes
+
+export default {
+  async generate(credentials, prompt, body, log) {
+    const email = credentials.email || credentials.name || "";
+    const token = credentials.accessToken || credentials.apiKey || "";
+    const model = body.model || "";
+    const modelId = model.replace(/^weavy-/, "");
+    const size = body.aspect_ratio || body.size || "16:9";
+    const duration = body.duration ? parseInt(body.duration) : null;
+    const imageUrl = body.image_url || body.imageReference || body.start_frame || body.startFrame || "";
+    const endImageUrl = body.end_image_url || body.endFrame || body.end_frame || "";
+    const videoUrl = body.video_url || body.videoReference || "";
+    const negativePrompt = body.negative_prompt || "";
+
+    log?.debug?.(
+      "WEAVY_VIDEO",
+      `Running video generation via python: email=${email}, model=${modelId}, prompt="${prompt.slice(0, 30)}..." dur=${duration} img=${imageUrl ? "yes" : "no"} endImg=${endImageUrl ? "yes" : "no"} vid=${videoUrl ? "yes" : "no"} negPrompt=${negativePrompt ? "yes" : "no"}`
+    );
+
+    const venvPython = path.resolve(process.cwd(), ".venv/bin/python");
+    const scriptPath = path.resolve(process.cwd(), "src/automation/weavy_generate.py");
+    const profilesDir = path.resolve(process.cwd(), "profiles/weavy");
+
+    const execFileAsync = promisify(execFile);
+    let stdout;
+    let stderr;
+
+    try {
+      const args = [
+        scriptPath,
+        "--email", email,
+        "--prompt", prompt,
+        "--aspect-ratio", size,
+        "--model-type", "video",
+        "--model", modelId,
+        "--profiles-dir", profilesDir,
+        "--token", token,
+      ];
+      if (duration !== null && !isNaN(duration)) {
+        args.push("--duration", duration.toString());
+      }
+      if (imageUrl) {
+        args.push("--image-url", imageUrl);
+      }
+      if (endImageUrl) {
+        args.push("--end-image-url", endImageUrl);
+      }
+      if (videoUrl) {
+        args.push("--video-url", videoUrl);
+      }
+      if (negativePrompt) {
+        args.push("--negative-prompt", negativePrompt);
+      }
+
+      const res = await execFileAsync(venvPython, args, {
+        maxBuffer: MAX_BUFFER,
+        timeout: EXEC_TIMEOUT_MS,
+      });
+      stdout = res.stdout;
+      stderr = res.stderr;
+    } catch (err) {
+      // Parse python JSON error from stdout if available
+      if (err.stdout) {
+        try {
+          const parsed = JSON.parse(err.stdout.trim());
+          if (parsed.message) {
+            const cleanedMessage = token ? parsed.message.split(token).join("[REDACTED_JWT_TOKEN]") : parsed.message;
+            const customErr = new Error(cleanedMessage);
+            const statusMatch = cleanedMessage.match(/(?:Status|HTTP) (\d+)/i);
+            if (statusMatch) {
+              customErr.status = parseInt(statusMatch[1], 10);
+            }
+            if (customErr.stack && token) {
+              customErr.stack = customErr.stack.split(token).join("[REDACTED_JWT_TOKEN]");
+            }
+            throw customErr;
+          }
+        } catch (parseErr) {
+          if (!(parseErr instanceof SyntaxError)) {
+            if (parseErr.message && token) {
+              parseErr.message = parseErr.message.split(token).join("[REDACTED_JWT_TOKEN]");
+            }
+            if (parseErr.stack && token) {
+              parseErr.stack = parseErr.stack.split(token).join("[REDACTED_JWT_TOKEN]");
+            }
+            throw parseErr;
+          }
+        }
+      }
+
+      // Redact token from error message, stack, and stderr before logging / re-throwing
+      if (err.message) {
+        const statusMatch = err.message.match(/(?:Status|HTTP) (\d+)/i);
+        if (statusMatch) {
+          err.status = parseInt(statusMatch[1], 10);
+        }
+        if (token) {
+          err.message = err.message.split(token).join("[REDACTED_JWT_TOKEN]");
+        }
+      }
+      if (err.stack && token) {
+        err.stack = err.stack.split(token).join("[REDACTED_JWT_TOKEN]");
+      }
+      if (err.stderr && token) {
+        err.stderr = err.stderr.split(token).join("[REDACTED_JWT_TOKEN]");
+      }
+
+      // Log stderr so the caller can see what Python printed
+      if (err.stderr) {
+        log?.debug?.("WEAVY_VIDEO", `stderr: ${err.stderr.slice(0, 500)}`);
+      }
+      // killed === true means Node killed the process (timeout or OOM)
+      if (err.killed || err.code === "ETIMEDOUT") {
+        const timeoutErr = new Error("Weavy video generation timed out (Node exec limit)");
+        if (timeoutErr.stack && token) {
+          timeoutErr.stack = timeoutErr.stack.split(token).join("[REDACTED_JWT_TOKEN]");
+        }
+        throw timeoutErr;
+      }
+      throw err;
+    }
+
+    // Surface Python stderr for debugging (trimmed to avoid log spam)
+    if (stderr) {
+      const cleanedStderr = token ? stderr.split(token).join("[REDACTED_JWT_TOKEN]") : stderr;
+      log?.debug?.("WEAVY_VIDEO", `python stderr: ${cleanedStderr.slice(0, 500)}`);
+    }
+
+    let parsed;
+    try {
+      parsed = JSON.parse(stdout.trim());
+    } catch (parseErr) {
+      const cleanedStdout = token ? stdout.slice(0, 200).split(token).join("[REDACTED_JWT_TOKEN]") : stdout.slice(0, 200);
+      log?.debug?.("WEAVY_VIDEO", `JSON parse failed. stdout="${cleanedStdout}"`);
+      const customErr = new Error(`Weavy video returned non-JSON output: ${cleanedStdout}`);
+      if (customErr.stack && token) {
+        customErr.stack = customErr.stack.split(token).join("[REDACTED_JWT_TOKEN]");
+      }
+      throw customErr;
+    }
+
+    if (parsed.status === "success" && Array.isArray(parsed.urls)) {
+      // If Python refreshed the token, propagate it so the caller can persist it
+      const result = { urls: parsed.urls };
+      const connId = credentials.connectionId || credentials.id;
+      if (connId) {
+        const updates = {};
+        if (parsed.balance !== undefined) {
+          updates.last_balance = parsed.balance;
+        }
+        if (parsed.refreshed_token) {
+          result.refreshedToken = parsed.refreshed_token;
+          updates.accessToken = parsed.refreshed_token;
+          updates.cached_jwt = parsed.refreshed_token;
+          updates.testStatus = "active";
+          try {
+            const payload = JSON.parse(Buffer.from(parsed.refreshed_token.split('.')[1], 'base64').toString());
+            if (payload && payload.exp) {
+              updates.jwt_expires_at = payload.exp;
+              updates.expiresAt = new Date(payload.exp * 1000).toISOString();
+            }
+          } catch (e) {
+            const exp = Math.floor(Date.now() / 1000) + 3600;
+            updates.jwt_expires_at = exp;
+            updates.expiresAt = new Date(exp * 1000).toISOString();
+          }
+        }
+        if (Object.keys(updates).length > 0) {
+          updateProviderConnection(connId, updates).catch(() => {});
+        }
+      }
+      return result;
+    } else {
+      const errMsg = parsed.message || "Weavy video generation failed";
+      const cleanedMsg = token ? errMsg.split(token).join("[REDACTED_JWT_TOKEN]") : errMsg;
+      const apiErr = new Error(cleanedMsg);
+      const statusMatch = cleanedMsg.match(/(?:Status|HTTP) (\d+)/i);
+      if (statusMatch) {
+        apiErr.status = parseInt(statusMatch[1], 10);
+      }
+      if (apiErr.stack && token) {
+        apiErr.stack = apiErr.stack.split(token).join("[REDACTED_JWT_TOKEN]");
+      }
+      throw apiErr;
+    }
+  },
+};

package/open-sse/handlers/webFetch.js

@@ -0,0 +1,200 @@
+/**
+ * Web Fetch Handler
+ *
+ * Handles POST /v1/web/fetch requests.
+ * Dispatches to a web-fetch provider executor (Firecrawl, Jina Reader, Tavily, or TinyFish).
+ *
+ * Request format:
+ * {
+ *   "url": "https://example.com",
+ *   "provider": "firecrawl" | "jina-reader" | "tavily-search" | "tinyfish",  // optional
+ *   "format": "markdown" | "html" | "links" | "screenshot",
+ *   "depth": 0 | 1 | 2,
+ *   "wait_for_selector": "main",
+ *   "include_metadata": true
+ * }
+ */
+
+import { buildErrorBody } from "../utils/error.js";
+
+// ── Inline sanitizeErrorMessage stub ─────────────────────────────────
+function sanitizeErrorMessage(message) {
+  if (typeof message !== "string") return "An error occurred";
+  return message.replace(/\/[^\s]+\/node_modules/g, "[node_modules]");
+}
+
+// ── Provider executor stubs ──────────────────────────────────────────
+// These are placeholders; real implementations should be wired up from
+// the executors/ directory when available.
+
+async function firecrawlFetch({ url, format, depth, waitForSelector, includeMetadata, credentials }) {
+  const apiKey = credentials?.apiKey;
+  if (!apiKey) return { success: false, status: 401, error: "Firecrawl API key required" };
+
+  const res = await fetch("https://api.firecrawl.dev/v1/scrape", {
+    method: "POST",
+    headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` },
+    body: JSON.stringify({ url, formats: [format], waitForSelector, depth }),
+  });
+
+  if (!res.ok) {
+    const errText = await res.text();
+    return { success: false, status: res.status, error: sanitizeErrorMessage(errText) };
+  }
+
+  const data = await res.json();
+  const content = data?.data?.markdown || data?.data?.html || "";
+  const metadata = includeMetadata
+    ? { title: data?.data?.metadata?.title || null, description: data?.data?.metadata?.description || null }
+    : null;
+
+  return {
+    success: true,
+    data: { provider: "firecrawl", url, content, links: [], metadata, screenshot_url: null },
+  };
+}
+
+async function jinaReaderFetch({ url, format, includeMetadata, credentials }) {
+  const headers = { Accept: "application/json" };
+  if (credentials?.apiKey) headers["Authorization"] = `Bearer ${credentials.apiKey}`;
+
+  const res = await fetch(`https://r.jina.ai/${url}`, { headers });
+  if (!res.ok) {
+    const errText = await res.text();
+    return { success: false, status: res.status, error: sanitizeErrorMessage(errText) };
+  }
+
+  const data = await res.json();
+  const content = data?.data?.content || data?.data?.markdown || "";
+  const metadata = includeMetadata
+    ? { title: data?.data?.title || null, description: data?.data?.description || null }
+    : null;
+
+  return {
+    success: true,
+    data: { provider: "jina-reader", url, content, links: [], metadata, screenshot_url: null },
+  };
+}
+
+async function tavilyFetch({ url, format, includeMetadata, credentials }) {
+  const apiKey = credentials?.apiKey;
+  if (!apiKey) return { success: false, status: 401, error: "Tavily API key required" };
+
+  const res = await fetch("https://api.tavily.com/extract", {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ api_key: apiKey, urls: [url] }),
+  });
+
+  if (!res.ok) {
+    const errText = await res.text();
+    return { success: false, status: res.status, error: sanitizeErrorMessage(errText) };
+  }
+
+  const data = await res.json();
+  const result = data?.results?.[0] || {};
+  const content = result.raw_content || result.content || "";
+  const metadata = includeMetadata
+    ? { title: result.title || null, description: null }
+    : null;
+
+  return {
+    success: true,
+    data: { provider: "tavily-search", url, content, links: [], metadata, screenshot_url: null },
+  };
+}
+
+async function tinyfishFetch({ url, format, includeMetadata, credentials }) {
+  const apiKey = credentials?.apiKey;
+  if (!apiKey) return { success: false, status: 401, error: "TinyFish API key required" };
+
+  const res = await fetch("https://api.tinyfish.io/v1/fetch", {
+    method: "POST",
+    headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` },
+    body: JSON.stringify({ url, format }),
+  });
+
+  if (!res.ok) {
+    const errText = await res.text();
+    return { success: false, status: res.status, error: sanitizeErrorMessage(errText) };
+  }
+
+  const data = await res.json();
+  const content = data?.content || "";
+  const metadata = includeMetadata
+    ? { title: data?.title || null, description: data?.description || null }
+    : null;
+
+  return {
+    success: true,
+    data: { provider: "tinyfish", url, content, links: data?.links || [], metadata, screenshot_url: null },
+  };
+}
+
+// ── Constants ────────────────────────────────────────────────────────
+const WEB_FETCH_PROVIDERS = ["firecrawl", "jina-reader", "tavily-search", "tinyfish"];
+
+/**
+ * Execute a web fetch request against the specified (or auto-selected) provider.
+ */
+export async function handleWebFetch(req, credentials, resolvedProvider) {
+  const provider = resolvedProvider ?? req.provider ?? "firecrawl";
+
+  const format = req.format ?? "markdown";
+  const includeMetadata = req.include_metadata ?? false;
+
+  try {
+    switch (provider) {
+      case "firecrawl":
+        return await firecrawlFetch({
+          url: req.url,
+          format,
+          depth: req.depth ?? 0,
+          waitForSelector: req.wait_for_selector,
+          includeMetadata,
+          credentials,
+        });
+
+      case "jina-reader":
+        return await jinaReaderFetch({
+          url: req.url,
+          format,
+          includeMetadata,
+          credentials,
+        });
+
+      case "tavily-search":
+        return await tavilyFetch({
+          url: req.url,
+          format,
+          includeMetadata,
+          credentials,
+        });
+
+      case "tinyfish":
+        return await tinyfishFetch({
+          url: req.url,
+          format,
+          includeMetadata,
+          credentials,
+        });
+
+      default: {
+        return {
+          success: false,
+          status: 400,
+          error: `Unknown web fetch provider: ${provider}`,
+        };
+      }
+    }
+  } catch (err) {
+    const msg =
+      err instanceof Error ? sanitizeErrorMessage(err.message) : sanitizeErrorMessage(String(err));
+    const body = buildErrorBody(502, msg);
+    return {
+      success: false,
+      status: 502,
+      error: body.error.message,
+    };
+  }
+}

package/open-sse/index.js

@@ -0,0 +1,79 @@
+// Patch global fetch with proxy support (must be first)
+import "./utils/proxyFetch.js";
+
+// Config
+export { PROVIDERS } from "./config/providers.js";
+export { OAUTH_ENDPOINTS, CLAUDE_SYSTEM_PROMPT } from "./config/appConstants.js";
+export { CACHE_TTL, DEFAULT_MAX_TOKENS, COOLDOWN_MS, BACKOFF_CONFIG } from "./config/runtimeConfig.js";
+export { 
+  PROVIDER_MODELS, 
+  getProviderModels,
+  getDefaultModel, 
+  isValidModel,
+  findModelName,
+  getModelTargetFormat,
+  PROVIDER_ID_TO_ALIAS,
+  getModelsByProviderId
+} from "./config/providerModels.js";
+
+// Translator
+export { FORMATS } from "./translator/formats.js";
+export { 
+  register, 
+  translateRequest, 
+  translateResponse, 
+  needsTranslation, 
+  initState, 
+  initTranslators 
+} from "./translator/index.js";
+
+// Services
+export { 
+  detectFormat, 
+  getTargetFormat 
+} from "./services/provider.js";
+
+export { parseModel, resolveModelAliasFromMap, getModelInfoCore } from "./services/model.js";
+
+export {
+  checkFallbackError,
+  isAccountUnavailable,
+  getUnavailableUntil,
+  filterAvailableAccounts
+} from "./services/accountFallback.js";
+
+export {
+  TOKEN_EXPIRY_BUFFER_MS,
+  refreshAccessToken,
+  refreshClaudeOAuthToken,
+  refreshGoogleToken,
+  refreshQwenToken,
+  refreshCodexToken,
+  refreshIflowToken,
+  refreshGitHubToken,
+  refreshCopilotToken,
+  getAccessToken,
+  refreshTokenByProvider
+} from "./services/tokenRefresh.js";
+
+export {
+  CODEX_MAX_REFRESH_AGE_MS,
+  shouldRefreshCredentials,
+  refreshProviderCredentials,
+  mergeRefreshedCredentials,
+  mergeProviderSpecificData,
+} from "./services/oauthCredentialManager.js";
+
+// Handlers
+export { handleChatCore, isTokenExpiringSoon } from "./handlers/chatCore.js";
+export { createStreamController, pipeWithDisconnect, createDisconnectAwareStream } from "./utils/streamHandler.js";
+
+// Executors
+export { getExecutor, hasSpecializedExecutor } from "./executors/index.js";
+
+// Utils
+export { errorResponse, formatProviderError } from "./utils/error.js";
+export { 
+  createSSETransformStreamWithLogger, 
+  createPassthroughStreamWithLogger 
+} from "./utils/stream.js";