@fudrouter/fsrouter 0.6.61 → 0.6.63

package/open-sse/services/rateLimitSemaphore.js

@@ -0,0 +1,112 @@
+const gates = /* @__PURE__ */ new Map();
+function getGate(modelStr, maxConcurrency = 3) {
+  if (!gates.has(modelStr)) {
+    gates.set(modelStr, {
+      running: 0,
+      max: maxConcurrency,
+      queue: [],
+      rateLimitedUntil: null
+    });
+  }
+  const gate = gates.get(modelStr);
+  gate.max = maxConcurrency;
+  return gate;
+}
+function isRateLimited(gate) {
+  if (!gate.rateLimitedUntil) return false;
+  if (Date.now() >= gate.rateLimitedUntil) {
+    gate.rateLimitedUntil = null;
+    return false;
+  }
+  return true;
+}
+function drainQueue(modelStr) {
+  const gate = gates.get(modelStr);
+  if (!gate) return;
+  while (gate.queue.length > 0 && gate.running < gate.max && !isRateLimited(gate)) {
+    const next = gate.queue.shift();
+    if (!next) break;
+    clearTimeout(next.timer);
+    gate.running++;
+    next.resolve(createReleaseFn(modelStr));
+  }
+  if (gate.running === 0 && gate.queue.length === 0) {
+    gates.delete(modelStr);
+  }
+}
+function createReleaseFn(modelStr) {
+  let released = false;
+  return () => {
+    if (released) return;
+    released = true;
+    const gate = gates.get(modelStr);
+    if (gate && gate.running > 0) {
+      gate.running--;
+      if (gate.running === 0 && gate.queue.length === 0) {
+        gates.delete(modelStr);
+        return;
+      }
+      drainQueue(modelStr);
+    }
+  };
+}
+function acquire(modelStr, { maxConcurrency = 3, timeoutMs = 3e4, maxQueueSize } = {}) {
+  const gate = getGate(modelStr, maxConcurrency);
+  if (gate.running < gate.max && !isRateLimited(gate)) {
+    gate.running++;
+    return Promise.resolve(createReleaseFn(modelStr));
+  }
+  if (typeof maxQueueSize === "number" && maxQueueSize >= 0 && gate.queue.length >= maxQueueSize) {
+    const err = new Error(`Semaphore queue full (${maxQueueSize}) for ${modelStr}`);
+    err.code = "SEMAPHORE_QUEUE_FULL";
+    if (gate.running === 0 && gate.queue.length === 0) gates.delete(modelStr);
+    return Promise.reject(err);
+  }
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      const idx = gate.queue.findIndex((item) => item.timer === timer);
+      if (idx !== -1) gate.queue.splice(idx, 1);
+      const err = new Error(`Semaphore timeout after ${timeoutMs}ms for ${modelStr}`);
+      err.code = "SEMAPHORE_TIMEOUT";
+      reject(err);
+    }, timeoutMs);
+    gate.queue.push({ resolve, reject, timer });
+  });
+}
+function markRateLimited(modelStr, cooldownMs) {
+  const gate = getGate(modelStr);
+  gate.rateLimitedUntil = Date.now() + cooldownMs;
+  setTimeout(() => {
+    if (gate.rateLimitedUntil && Date.now() >= gate.rateLimitedUntil) {
+      gate.rateLimitedUntil = null;
+      drainQueue(modelStr);
+    }
+  }, cooldownMs + 50);
+}
+function getStats() {
+  const stats = {};
+  for (const [model, gate] of gates) {
+    stats[model] = {
+      running: gate.running,
+      queued: gate.queue.length,
+      max: gate.max,
+      rateLimitedUntil: gate.rateLimitedUntil ? new Date(gate.rateLimitedUntil).toISOString() : null
+    };
+  }
+  return stats;
+}
+function resetAll() {
+  for (const [, gate] of gates) {
+    for (const item of gate.queue) {
+      clearTimeout(item.timer);
+      item.reject(new Error("Semaphore reset"));
+    }
+  }
+  gates.clear();
+}
+export {
+  acquire,
+  getStats,
+  markRateLimited,
+  resetAll
+};

package/open-sse/services/responsesInputSanitizer.js

@@ -0,0 +1,99 @@
+const INTERNAL_ASSISTANT_PHASES = /* @__PURE__ */ new Set(["commentary"]);
+const SERVER_ITEM_ID_PREFIX_BY_TYPE = {
+  function_call: "fc_",
+  message: "msg_",
+  reasoning: "rs_"
+};
+const SERVER_ITEM_ID_PATTERN = /^(fc|msg|rs|resp)_/;
+function toRecord(value) {
+  return value && typeof value === "object" && !Array.isArray(value) ? value : null;
+}
+function isResponsesMessageItem(record) {
+  return record.type === "message" || !record.type && typeof record.role === "string";
+}
+function isInternalAssistantMessage(record) {
+  if (!isResponsesMessageItem(record)) return false;
+  if (record.role !== "assistant") return false;
+  const phase = typeof record.phase === "string" ? record.phase.trim().toLowerCase() : "";
+  if (!phase) return false;
+  return INTERNAL_ASSISTANT_PHASES.has(phase);
+}
+function sanitizeFunctionName(name) {
+  return name.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 128);
+}
+function sanitizeInputItemId(record) {
+  if (typeof record.id !== "string") return record;
+  const type = typeof record.type === "string" ? record.type : "";
+  const expectedPrefix = SERVER_ITEM_ID_PREFIX_BY_TYPE[type];
+  const hasExpectedPrefix = expectedPrefix ? record.id.startsWith(expectedPrefix) : SERVER_ITEM_ID_PATTERN.test(record.id);
+  if (hasExpectedPrefix) return record;
+  const next = { ...record };
+  delete next.id;
+  return next;
+}
+function imageUrlToText(value) {
+  if (typeof value === "string") return value;
+  const record = toRecord(value);
+  return typeof record?.url === "string" ? record.url : "";
+}
+function sanitizeContentPart(part, role) {
+  const record = toRecord(part);
+  if (!record) return part;
+  if (record.type === "image_url") {
+    const url = imageUrlToText(record.image_url);
+    if (role === "user") {
+      const next = { type: "input_image", image_url: url };
+      const image = toRecord(record.image_url);
+      if (image?.detail !== void 0) next.detail = image.detail;
+      return next;
+    }
+    return { type: "output_text", text: url ? `[Image: ${url}]` : "[Image]" };
+  }
+  if (role === "assistant" && record.type === "input_image") {
+    const url = imageUrlToText(record.image_url);
+    return { type: "output_text", text: url ? `[Image: ${url}]` : "[Image]" };
+  }
+  return part;
+}
+function sanitizeMessageContent(record) {
+  if (!Array.isArray(record.content)) return record;
+  const role = typeof record.role === "string" ? record.role.toLowerCase() : "";
+  const content = record.content.map((part) => sanitizeContentPart(part, role));
+  return { ...record, content };
+}
+function sanitizeOutputContent(record) {
+  if (!Array.isArray(record.output)) return record;
+  const role = record.type === "function_call_output" ? "user" : "assistant";
+  const output = record.output.map((part) => sanitizeContentPart(part, role));
+  return { ...record, output };
+}
+function sanitizeInputItem(item) {
+  const record = toRecord(item);
+  if (!record) return item;
+  let next = sanitizeInputItemId(record);
+  if (isResponsesMessageItem(next)) {
+    next = sanitizeMessageContent(next);
+  }
+  next = sanitizeOutputContent(next);
+  if ((next.type === "function_call" || next.type === "function_call_output") && typeof next.name === "string" && !/^[a-zA-Z0-9_-]{1,128}$/.test(next.name)) {
+    next = { ...next, name: sanitizeFunctionName(next.name) };
+  }
+  return next;
+}
+function sanitizeResponsesInputItems(items, clone = true, options = {}) {
+  const dropInternalAssistantMessages = options.dropInternalAssistantMessages ?? true;
+  const sanitized = [];
+  for (const item of items) {
+    const record = toRecord(item);
+    if (dropInternalAssistantMessages && record && isInternalAssistantMessage(record)) {
+      continue;
+    }
+    const cloned = clone ? structuredClone(item) : item;
+    sanitized.push(sanitizeInputItem(cloned));
+  }
+  return sanitized;
+}
+export {
+  isInternalAssistantMessage,
+  sanitizeResponsesInputItems
+};

package/open-sse/services/slidingWindowLimiter.js

@@ -0,0 +1,44 @@
+const MAX_KEYS = 5e3;
+class SlidingWindowLimiter {
+  hits = /* @__PURE__ */ new Map();
+  now;
+  constructor(opts = {}) {
+    this.now = opts.now ?? Date.now;
+  }
+  /**
+   * Try to consume one slot for `key`. Records a timestamp and returns
+   * `{allowed:true}` when under the cap; returns `{allowed:false, retryAfterMs}`
+   * (without recording) when the trailing window is saturated.
+   */
+  tryAcquire(key, window) {
+    const { requests, windowMs } = window;
+    if (!(requests > 0) || !(windowMs > 0)) return { allowed: true, retryAfterMs: 0 };
+    const now = this.now();
+    const cutoff = now - windowMs;
+    const previous = this.hits.get(key);
+    const live = previous ? previous.filter((ts) => ts > cutoff) : [];
+    if (live.length >= requests) {
+      const retryAfterMs = Math.max(0, live[0] + windowMs - now);
+      this.hits.set(key, live);
+      return { allowed: false, retryAfterMs };
+    }
+    live.push(now);
+    this.set(key, live);
+    return { allowed: true, retryAfterMs: 0 };
+  }
+  /** Clear history for one key, or all keys when called with no argument. */
+  reset(key) {
+    if (key === void 0) this.hits.clear();
+    else this.hits.delete(key);
+  }
+  set(key, live) {
+    if (!this.hits.has(key) && this.hits.size >= MAX_KEYS) {
+      const oldest = this.hits.keys().next().value;
+      if (oldest !== void 0) this.hits.delete(oldest);
+    }
+    this.hits.set(key, live);
+  }
+}
+export {
+  SlidingWindowLimiter
+};

package/open-sse/services/streamRecovery.js

@@ -0,0 +1,355 @@
+import { STREAM_RECOVERY } from "../config/constants.js";
+class TruncatedStreamError extends Error {
+  constructor(message = "Provider stream ended without a terminal marker") {
+    super(message);
+    this.name = "TruncatedStreamError";
+  }
+}
+class HoldbackBuffer {
+  chunks = [];
+  bytes = 0;
+  startedAt = null;
+  holdbackMs;
+  maxBytes;
+  now;
+  committed = false;
+  constructor(options = {}) {
+    this.holdbackMs = options.holdbackMs ?? STREAM_RECOVERY.HOLDBACK_MS;
+    this.maxBytes = options.maxBytes ?? STREAM_RECOVERY.BUFFER_MAX_BYTES;
+    this.now = options.now ?? (() => Date.now());
+  }
+  /**
+   * Buffer `chunk` until the holdback window elapses or the byte cap is reached.
+   * Returns the chunks to emit downstream now: `[]` while still holding, or every
+   * buffered chunk (the just-pushed one included) at the moment of commit. After
+   * commit, chunks pass straight through.
+   */
+  push(chunk) {
+    if (this.committed) return [chunk];
+    if (this.startedAt === null) this.startedAt = this.now();
+    this.chunks.push(chunk);
+    this.bytes += chunk.byteLength;
+    if (this.bytes >= this.maxBytes || this.now() - this.startedAt >= this.holdbackMs) {
+      return this.flush();
+    }
+    return [];
+  }
+  /** Commit and return everything held so far. */
+  flush() {
+    if (this.committed) return [];
+    this.committed = true;
+    const out = this.chunks;
+    this.chunks = [];
+    this.bytes = 0;
+    this.startedAt = null;
+    return out;
+  }
+  /** Drop held chunks WITHOUT committing — used before a transparent retry. */
+  discard() {
+    this.chunks = [];
+    this.bytes = 0;
+    this.startedAt = null;
+  }
+  get hasBuffered() {
+    return this.chunks.length > 0;
+  }
+  /** Concatenated view of the currently-held (uncommitted) chunks, for inspection. */
+  peekBuffered() {
+    if (this.chunks.length === 0) return new Uint8Array(0);
+    if (this.chunks.length === 1) return this.chunks[0];
+    const out = new Uint8Array(this.bytes);
+    let offset = 0;
+    for (const chunk of this.chunks) {
+      out.set(chunk, offset);
+      offset += chunk.byteLength;
+    }
+    return out;
+  }
+}
+const RETRYABLE_TRANSPORT_CODES = /* @__PURE__ */ new Set([
+  "ECONNRESET",
+  "ECONNREFUSED",
+  "ETIMEDOUT",
+  "EPIPE",
+  "ENOTFOUND",
+  "ENETUNREACH",
+  "EHOSTUNREACH"
+]);
+const RETRYABLE_ERROR_NAMES = /* @__PURE__ */ new Set(["TimeoutError", "BodyTimeoutError"]);
+function isRetryableStreamError(error) {
+  if (error instanceof TruncatedStreamError) return true;
+  if (!error || typeof error !== "object") return false;
+  const name = error.name;
+  if (name === "AbortError" || name === "ResponseAborted") return false;
+  if (typeof name === "string" && RETRYABLE_ERROR_NAMES.has(name)) return true;
+  const code = error.code;
+  if (typeof code === "string") {
+    if (RETRYABLE_TRANSPORT_CODES.has(code)) return true;
+    if (code.startsWith("UND_ERR_")) return true;
+  }
+  const message = error.message;
+  if (typeof message === "string" && /terminated|socket hang up|econnreset/i.test(message)) {
+    return true;
+  }
+  return false;
+}
+const OPENAI_DONE_MARKER = "[DONE]";
+const ANTHROPIC_STOP_MARKER = "message_stop";
+function hasTerminalMarker(bytes) {
+  if (!bytes || bytes.byteLength === 0) return false;
+  const text = new TextDecoder().decode(bytes);
+  return text.includes(OPENAI_DONE_MARKER) || text.includes(ANTHROPIC_STOP_MARKER);
+}
+function scanOpenAiSseText(sse) {
+  let text = "";
+  let sawToolCall = false;
+  let terminal = false;
+  let parsedOpenAi = false;
+  if (typeof sse !== "string" || sse.length === 0) {
+    return { text, sawToolCall, terminal, parsedOpenAi };
+  }
+  for (const line of sse.split("\n")) {
+    const trimmed = line.trimStart();
+    if (!trimmed.startsWith("data:")) continue;
+    const payload = trimmed.slice(5).trim();
+    if (!payload) continue;
+    if (payload === "[DONE]") {
+      terminal = true;
+      continue;
+    }
+    let json;
+    try {
+      json = JSON.parse(payload);
+    } catch {
+      continue;
+    }
+    const choices = json?.choices;
+    if (!Array.isArray(choices)) continue;
+    for (const choice of choices) {
+      const delta = choice?.delta;
+      if (delta && typeof delta === "object") {
+        parsedOpenAi = true;
+        const content = delta.content;
+        if (typeof content === "string") text += content;
+        const toolCalls = delta.tool_calls;
+        if (Array.isArray(toolCalls) && toolCalls.length > 0) sawToolCall = true;
+      }
+      const finishReason = choice?.finish_reason;
+      if (finishReason != null) terminal = true;
+    }
+  }
+  return { text, sawToolCall, terminal, parsedOpenAi };
+}
+function makeContinuationBody(body, assistantSoFar) {
+  if (!body || typeof body !== "object") return null;
+  if (!Array.isArray(body.messages) || body.messages.length === 0) return null;
+  if (typeof assistantSoFar !== "string" || assistantSoFar.length === 0) return null;
+  return {
+    ...body,
+    messages: [...body.messages, { role: "assistant", content: assistantSoFar }],
+    stream: true
+  };
+}
+function trimContinuationOverlap(emitted, continuation) {
+  if (!continuation) return "";
+  if (!emitted) return continuation;
+  const max = Math.min(emitted.length, continuation.length, 512);
+  for (let k = max; k > 0; k--) {
+    if (emitted.endsWith(continuation.slice(0, k))) return continuation.slice(k);
+  }
+  return continuation;
+}
+function createRecoverableStream(initialStream, reopen, options) {
+  const maxRetries = options.maxEarlyRetries ?? STREAM_RECOVERY.EARLY_RETRY_MAX;
+  let reader = initialStream.getReader();
+  let holdback = new HoldbackBuffer({ now: options.now });
+  let retries = 0;
+  let finalized = false;
+  let cancelled = false;
+  const runFinalize = () => {
+    if (finalized) return;
+    finalized = true;
+    options.finalize();
+  };
+  const tryReopen = async (error) => {
+    if (cancelled || retries >= maxRetries) return false;
+    retries += 1;
+    options.onRetry?.(retries, error);
+    try {
+      await reader.cancel(error);
+    } catch {
+    }
+    let next = null;
+    try {
+      next = await reopen();
+    } catch {
+      next = null;
+    }
+    if (!next) return false;
+    reader = next.getReader();
+    holdback.discard();
+    return true;
+  };
+  const continueEnabled = typeof options.continueStream === "function";
+  const maxContinuations = options.maxContinuations ?? STREAM_RECOVERY.EARLY_RETRY_MAX;
+  const encoder = new TextEncoder();
+  const trackDecoder = new TextDecoder();
+  let continuations = 0;
+  let emittedTail = "";
+  let emittedText = "";
+  let emittedTerminal = false;
+  let emittedToolCall = false;
+  let emittedParsedOpenAi = false;
+  const emit = (controller, chunk) => {
+    controller.enqueue(chunk);
+    if (!continueEnabled) return;
+    emittedTail += trackDecoder.decode(chunk, { stream: true });
+    const boundary = emittedTail.lastIndexOf("\n\n");
+    if (boundary < 0) return;
+    const complete = emittedTail.slice(0, boundary + 2);
+    emittedTail = emittedTail.slice(boundary + 2);
+    const scan = scanOpenAiSseText(complete);
+    emittedText += scan.text;
+    if (scan.terminal) emittedTerminal = true;
+    if (scan.sawToolCall) emittedToolCall = true;
+    if (scan.parsedOpenAi) emittedParsedOpenAi = true;
+  };
+  const flushHeld = (controller) => {
+    for (const chunk of holdback.flush()) emit(controller, chunk);
+  };
+  const canContinue = () => continueEnabled && continuations < maxContinuations && emittedParsedOpenAi && !emittedToolCall && !emittedTerminal && emittedText.length > 0;
+  const emitCleanTerminal = (controller) => {
+    controller.enqueue(
+      encoder.encode('data: {"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n')
+    );
+    controller.enqueue(encoder.encode("data: [DONE]\n\n"));
+  };
+  const tryContinue = async (controller) => {
+    if (!canContinue()) return false;
+    continuations += 1;
+    options.onContinue?.(continuations, emittedText);
+    let contStream = null;
+    try {
+      contStream = await options.continueStream(emittedText);
+    } catch {
+      contStream = null;
+    }
+    if (!contStream) return false;
+    const contReader = contStream.getReader();
+    const contDecoder = new TextDecoder();
+    let raw = "";
+    for (; ; ) {
+      let r;
+      try {
+        r = await contReader.read();
+      } catch {
+        break;
+      }
+      if (r.done) break;
+      if (r.value) raw += contDecoder.decode(r.value, { stream: true });
+    }
+    const scan = scanOpenAiSseText(raw);
+    const suffix = trimContinuationOverlap(emittedText, scan.text);
+    if (suffix) {
+      emit(
+        controller,
+        encoder.encode(
+          `data: ${JSON.stringify({ choices: [{ index: 0, delta: { content: suffix } }] })}
+
+`
+        )
+      );
+    }
+    if (scan.terminal || scan.sawToolCall) {
+      emitCleanTerminal(controller);
+      return true;
+    }
+    if (await tryContinue(controller)) return true;
+    emitCleanTerminal(controller);
+    return true;
+  };
+  return new ReadableStream({
+    async pull(controller) {
+      for (; ; ) {
+        let result;
+        try {
+          result = await reader.read();
+        } catch (error) {
+          if (cancelled) return;
+          if (holdback.committed) {
+            if (isRetryableStreamError(error) && await tryContinue(controller)) {
+              runFinalize();
+              controller.close();
+              return;
+            }
+            runFinalize();
+            controller.error(error);
+            return;
+          }
+          if (isRetryableStreamError(error) && await tryReopen(error)) {
+            continue;
+          }
+          flushHeld(controller);
+          runFinalize();
+          controller.close();
+          return;
+        }
+        if (cancelled) return;
+        const { done, value } = result;
+        if (done) {
+          if (holdback.committed) {
+            if (!emittedTerminal && await tryContinue(controller)) {
+              runFinalize();
+              controller.close();
+              return;
+            }
+            runFinalize();
+            controller.close();
+            return;
+          }
+          if (hasTerminalMarker(holdback.peekBuffered())) {
+            flushHeld(controller);
+            runFinalize();
+            controller.close();
+            return;
+          }
+          if (await tryReopen(new TruncatedStreamError())) {
+            continue;
+          }
+          flushHeld(controller);
+          runFinalize();
+          controller.close();
+          return;
+        }
+        if (value === void 0) continue;
+        if (holdback.committed) {
+          emit(controller, value);
+          return;
+        }
+        const emitted = holdback.push(value);
+        if (emitted.length > 0) {
+          for (const chunk of emitted) emit(controller, chunk);
+          return;
+        }
+      }
+    },
+    async cancel(reason) {
+      cancelled = true;
+      runFinalize();
+      try {
+        await reader.cancel(reason);
+      } catch {
+      }
+    }
+  });
+}
+export {
+  HoldbackBuffer,
+  TruncatedStreamError,
+  createRecoverableStream,
+  hasTerminalMarker,
+  isRetryableStreamError,
+  makeContinuationBody,
+  scanOpenAiSseText,
+  trimContinuationOverlap
+};