mslxdff 0.1.39 → 0.1.42

package/src/routes/groups.js

@@ -0,0 +1,121 @@
+import { clientIp, json, readBody, errMsg } from "./helpers.js";
+
+export async function joinHandler({ req, res, groups, token, bans }) {
+  if (!groups) return json(res, 501, { error: "Groups service not configured" });
+  const ip = clientIp(req);
+  const banned = bans?.isBanned(ip);
+  if (banned) {
+    return json(res, 403, { error: `banned until ${new Date(banned.until).toISOString()}` });
+  }
+  let body;
+  try {
+    body = await readBody(req);
+  } catch {
+    return json(res, 400, { error: "Invalid JSON body" });
+  }
+  if (!body?.name) return json(res, 400, { error: "group name is required" });
+
+  const fail = (msg) => {
+    try {
+      if (bans) {
+        const b = bans.recordFailure(ip);
+        if (b) console.error(`${ip} banned (${bans.threshold} failed joins)`);
+      }
+    } catch {}
+    return json(res, 403, { error: msg });
+  };
+
+  if (!body.key) {
+    const auth = /^Bearer (.+)$/.exec(req.headers["authorization"] || "");
+    const hit = auth && groups.membersForToken(body.name, auth[1]);
+    if (!hit) return fail("invalid member token");
+    try {
+      const isBroadbandRe = String(body.url || hit.member?.url || "").startsWith("relay://") || body.kind === "broadband" || hit.member?.kind === "broadband";
+      const extra = {};
+      if (isBroadbandRe) {
+        extra.kind = "broadband";
+        extra.publicIp = ip;
+        extra.lastSeen = Date.now();
+        if (body.url) extra.url = String(body.url);
+      }
+      const refreshed = groups.upsertMember(body.name, {
+        memberName: body.memberName,
+        url: body.url || hit.member.url,
+        token: body.token || hit.member.token,
+        kind: extra.kind,
+        publicIp: extra.publicIp,
+        lastSeen: extra.lastSeen,
+      });
+      if (isBroadbandRe && refreshed) {
+        const targetId = Object.keys(refreshed).find((k) => refreshed[k].url === (body.url || hit.member.url));
+        if (targetId) {
+          refreshed[targetId].publicIp = ip;
+          refreshed[targetId].lastSeen = Date.now();
+          refreshed[targetId].kind = "broadband";
+        }
+      }
+      return json(res, 200, { object: "group", name: body.name, members: refreshed });
+    } catch (err) {
+      return json(res, 400, { error: errMsg(err) });
+    }
+  }
+
+  try {
+    const youPort = Number(body.myPort);
+    const youUrl = Number.isInteger(youPort) && youPort > 0 ? `http://${ip}:${youPort}` : "";
+    let memberUrl = String(body.url || youUrl);
+    if (!memberUrl) throw new Error("member url is required");
+    const isBroadband = String(memberUrl).startsWith("relay://") || body.kind === "broadband";
+    if (isBroadband) {
+      memberUrl = String(body.url || memberUrl);
+    }
+    const members = groups.addMember(body.name, {
+      key: body.key,
+      memberName: body.memberName,
+      url: memberUrl,
+      token: body.token,
+      kind: isBroadband ? "broadband" : "static",
+      publicIp: isBroadband ? ip : undefined,
+      lastSeen: isBroadband ? Date.now() : undefined,
+    });
+    if (bans) bans.clear(ip);
+    if (!members.leader) {
+      const leaderUrl = String(body.leaderUrl || "").replace(/\/+$/, "");
+      if (leaderUrl) {
+        groups.upsertMember(body.name, { memberName: "leader", url: leaderUrl, token });
+        Object.assign(members, { leader: { url: leaderUrl, token } });
+      }
+    }
+    json(res, 200, { object: "group", name: body.name, members, you: { url: memberUrl } });
+  } catch (err) {
+    return fail(errMsg(err));
+  }
+}
+
+export async function leaveHandler({ req, res, groups }) {
+  if (!groups) return json(res, 501, { error: "Groups service not configured" });
+  let body;
+  try {
+    body = await readBody(req);
+  } catch {
+    return json(res, 400, { error: "Invalid JSON body" });
+  }
+  if (!body?.name) return json(res, 400, { error: "group name is required" });
+  const auth = /^Bearer (.+)$/.exec(req.headers["authorization"] || "");
+  if (!auth) return json(res, 401, { error: "bearer token required" });
+  const group = groups.list()[body.name];
+  if (!group) return json(res, 404, { error: `group "${body.name}" not found` });
+  const hit = groups.membersForToken(body.name, auth[1]);
+  if (!hit) return json(res, 403, { error: "invalid member token" });
+  try {
+    const removed = groups.removeMember(body.name, { url: hit.member.url });
+    return json(res, 200, {
+      object: "group",
+      name: body.name,
+      removed: removed?.removed ?? null,
+      members: groups.list()[body.name]?.members ?? {},
+    });
+  } catch (err) {
+    return json(res, 400, { error: errMsg(err) });
+  }
+}

package/src/routes/helpers.js

@@ -0,0 +1,66 @@
+import { timingSafeEqual, createHash } from "node:crypto";
+
+export const errMsg = (err) => String(err?.message || err);
+
+export function clientIp(req) {
+  const fwd = req.headers["x-forwarded-for"];
+  const head = typeof fwd === "string" ? fwd.split(",")[0].trim() : "";
+  const raw = String(head || req.socket.remoteAddress || "");
+  return raw.replace(/^::ffff:/, "").replace(/^::1$/, "127.0.0.1") || null;
+}
+
+export function authorized(req, token) {
+  const header = req.headers["authorization"] || "";
+  const match = /^Bearer (.+)$/.exec(header);
+  if (!match) return false;
+  const digests = (s) => createHash("sha256").update(s).digest();
+  return timingSafeEqual(digests(match[1]), digests(token));
+}
+
+export function json(res, status, body) {
+  res.statusCode = status;
+  res.setHeader("Content-Type", "application/json");
+  res.end(JSON.stringify(body));
+}
+
+export function notFound(res) {
+  return json(res, 404, { error: "Not Found" });
+}
+
+export function readBody(req) {
+  return new Promise((resolve, reject) => {
+    let data = "";
+    req.on("data", (c) => (data += c));
+    req.on("end", () => {
+      try {
+        resolve(data ? JSON.parse(data) : {});
+      } catch (err) {
+        reject(err);
+      }
+    });
+    req.on("error", reject);
+  });
+}
+
+export function parseHops(header) {
+  const n = Number(header);
+  return Number.isInteger(n) && n >= 0 ? n : 0;
+}
+
+export const PROMPT_MAX_LEN = 160;
+
+export function summarizePrompt(body) {
+  const msgs = body?.messages;
+  if (!Array.isArray(msgs) || !msgs.length) return "";
+  const msg = msgs[msgs.length - 1];
+  const c = msg?.content;
+  let text = "";
+  if (typeof c === "string") text = c;
+  else if (Array.isArray(c)) {
+    text = c
+      .map((p) => (typeof p === "string" ? p : p && typeof p.text === "string" ? p.text : ""))
+      .join(" ");
+  }
+  text = String(text || "").replace(/\s+/g, " ").trim();
+  return text.length > PROMPT_MAX_LEN ? text.slice(0, PROMPT_MAX_LEN) + "…" : text;
+}

package/src/routes/index.js

@@ -0,0 +1,91 @@
+import { timingSafeEqual, createHash } from "node:crypto";
+import { DEFAULT_MAX_HOPS } from "../peers.js";
+import { json, notFound, authorized } from "./helpers.js";
+import { chatHandler } from "./chat.js";
+import { joinHandler, leaveHandler } from "./groups.js";
+import { heartbeatHandler, pollHandler, resultHandler, forwardHandler } from "./groups-relay.js";
+import { modelsHandler, modelsStatusHandler } from "./models-route.js";
+
+export function createRouter({ token, upstream, models, auto, logs, peers, maxHops = DEFAULT_MAX_HOPS, groups, bans, bus }) {
+  return async function router(req, res) {
+    const method = req.method || "GET";
+    const path = (req.url || "").split("?")[0];
+    const route = ROUTES.find((r) => r.method === method && r.path === path);
+    if (!route) return notFound(res);
+    if (route.requiresAuth && !authorized(req, token)) {
+      res.statusCode = 401;
+      res.setHeader("WWW-Authenticate", "Bearer");
+      return json(res, 401, { error: "Unauthorized" });
+    }
+    await route.handler({ req, res, upstream, models, auto, logs, peers, maxHops, groups, bans, token, bus });
+  };
+}
+
+const ROUTES = [
+  {
+    method: "GET",
+    path: "/health",
+    handler: ({ res }) => json(res, 200, { status: "ok" }),
+  },
+  {
+    method: "POST",
+    path: "/v1/chat/completions",
+    requiresAuth: true,
+    handler: chatHandler,
+  },
+  {
+    method: "POST",
+    path: "/v1/groups/join",
+    requiresAuth: false,
+    handler: joinHandler,
+  },
+  {
+    method: "POST",
+    path: "/v1/groups/leave",
+    requiresAuth: false,
+    handler: leaveHandler,
+  },
+  {
+    method: "POST",
+    path: "/v1/groups/relay/heartbeat",
+    requiresAuth: false,
+    handler: heartbeatHandler,
+  },
+  {
+    method: "POST",
+    path: "/v1/groups/relay/poll",
+    requiresAuth: false,
+    handler: pollHandler,
+  },
+  {
+    method: "POST",
+    path: "/v1/groups/relay/result",
+    requiresAuth: false,
+    handler: resultHandler,
+  },
+  {
+    method: "POST",
+    path: "/v1/groups/relay/forward",
+    requiresAuth: true,
+    handler: forwardHandler,
+  },
+  {
+    method: "GET",
+    path: "/v1/models",
+    requiresAuth: true,
+    handler: modelsHandler,
+  },
+  {
+    method: "GET",
+    path: "/v1/models/status",
+    requiresAuth: true,
+    handler: modelsStatusHandler,
+  },
+];
+
+// re-export for facade & tests
+export { errMsg, PROMPT_MAX_LEN, summarizePrompt, clientIp, authorized, json, notFound, readBody, parseHops } from "./helpers.js";
+export { buildFallbackInfo, applyFallbackHeaders, enrichNonStreamJson, enrichSseChunkText } from "./fallback.js";
+export { relay, SLOW_TOTAL_MS, STREAM_TIMEOUT_MS, STALL_TIMEOUT_MS, SCORE_STALL_MS, MAX_STREAM_MS } from "./stream.js";
+export { peerHealthyModels, racePeerCandidates, PEER_RACE_LIMIT } from "./peers.js";
+export { enqueueRelay, dequeueRelayForPoll, resolveRelay, tryBroadbandRelay } from "./relay-queue.js";

package/src/routes/models-route.js

@@ -0,0 +1,31 @@
+import { json, errMsg } from "./helpers.js";
+
+export async function modelsHandler({ res, models }) {
+  if (!models) return json(res, 501, { error: "Models service not configured" });
+  try {
+    const data = await models.get();
+    json(res, 200, data);
+  } catch (err) {
+    json(res, 502, { error: errMsg(err) });
+  }
+}
+
+export async function modelsStatusHandler({ res, models, auto }) {
+  const statuses = auto?.statuses?.() || {};
+  let ids = [];
+  try {
+    ids = (await models?.get?.())?.data?.map((m) => m.id) || [];
+  } catch {}
+  const seen = new Set();
+  const data = [];
+  for (const id of [...ids, ...Object.keys(statuses)]) {
+    if (seen.has(id)) continue;
+    seen.add(id);
+    const e = statuses[id];
+    const entry = typeof e === "number"
+      ? { id, status: "error", at: e }
+      : { id, status: e?.status || "normal", at: e?.at ?? null, code: e?.code ?? null };
+    data.push(entry);
+  }
+  json(res, 200, { object: "list", data });
+}

package/src/routes/peers.js

@@ -0,0 +1,125 @@
+import { performance } from "node:perf_hooks";
+import { isAutoModel } from "../auto.js";
+import { errMsg } from "./helpers.js";
+
+const PEER_TIMEOUT_MS = 30_000;
+const PEER_STATUS_TIMEOUT_MS = 2_000;
+
+export async function peerHealthyModels(peer, { timeoutMs = PEER_STATUS_TIMEOUT_MS, fetchImpl = fetch } = {}) {
+  try {
+    const res = await fetchImpl(`${peer.url}/v1/models/status`, {
+      headers: {
+        "Authorization": `Bearer ${peer.token || ""}`,
+        "Accept": "application/json",
+      },
+      signal: AbortSignal.timeout(timeoutMs),
+    });
+    if (!res.ok) return [];
+    const j = await res.json().catch(() => ({}));
+    return (j.data || [])
+      .filter((m) => m && typeof m.id === "string" && m.status === "normal")
+      .map((m) => m.id);
+  } catch {
+    return [];
+  }
+}
+
+async function forwardToPeer(peer, body, model, hops) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), PEER_TIMEOUT_MS);
+  try {
+    return await fetch(`${peer.url}/v1/chat/completions`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Authorization": `Bearer ${peer.token}`,
+        "x-mslxdff-hops": String(hops + 1),
+        "x-mslxdff-model-lock": model,
+        "Accept": "text/event-stream",
+      },
+      body: JSON.stringify({ ...body, model }),
+      signal: controller.signal,
+    });
+  } catch (err) {
+    return err;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+async function resolvePeerTarget(ctx, peer) {
+  const prevModel = ctx.peers.stat(peer.url)?.model;
+  const hot = ctx.peers.isHot(peer.url) && prevModel === ctx.model;
+  if (hot) return { peer, target: prevModel };
+  const isExplicit = !!ctx.model && !isAutoModel(ctx.model);
+  if (isExplicit) {
+    const healthy = await peerHealthyModels(peer);
+    if (!healthy.length) {
+      ctx.evt("peer-health", { peer: peer.url, healthy: [], count: 0, strict: true });
+      return { peer, target: ctx.model };
+    }
+    ctx.evt("peer-health", { peer: peer.url, healthy, count: healthy.length, strict: true });
+    return { peer, target: ctx.model };
+  }
+  const healthy = await peerHealthyModels(peer);
+  if (!healthy.length) {
+    await ctx.peers.recordError(peer.url);
+    ctx.logError(ctx.model, 0, `peer ${peer.url} has no healthy models`);
+    ctx.evt("peer-health", { peer: peer.url, healthy: [], count: 0 });
+    return null;
+  }
+  ctx.evt("peer-health", { peer: peer.url, healthy, count: healthy.length });
+  return { peer, target: healthy.includes(ctx.model) ? ctx.model : healthy[0] };
+}
+
+export const PEER_RACE_LIMIT = Number(process.env.MSLXDFF_PEER_RACE_LIMIT) > 0
+  ? Number(process.env.MSLXDFF_PEER_RACE_LIMIT)
+  : 3;
+
+export async function racePeerCandidates(candidates, ctx) {
+  for (let i = 0; i < candidates.length; i += PEER_RACE_LIMIT) {
+    const batch = candidates.slice(i, i + PEER_RACE_LIMIT);
+    const prepared = (await Promise.all(batch.map((peer) => resolvePeerTarget(ctx, peer)))).filter(Boolean);
+    if (!prepared.length) continue;
+    const completed = await new Promise((resolve) => {
+      const order = [];
+      const total = prepared.length;
+      for (const { peer, target } of prepared) {
+        ctx.evt("peer-request", { peer: peer.url, model: target, hops: ctx.hops + 1 });
+        const t0 = performance.now();
+        forwardToPeer(peer, ctx.body, target, ctx.hops).then((res) => {
+          const latencyMs = Math.round(performance.now() - t0);
+          const failed = res instanceof Error || res.status >= 400;
+          ctx.evt("peer-forward", { peer: peer.url, model: target, hops: ctx.hops + 1, latencyMs, ok: !failed });
+          if (failed) {
+            const status = res instanceof Error ? 502 : res.status;
+            ctx.logError(ctx.model, status, res instanceof Error ? errMsg(res) : `peer ${status}`);
+            ctx.evt("peer-error", { peer: peer.url, model: target, status, message: res instanceof Error ? errMsg(res) : null });
+            order.push({ ok: false, peer, target, res, status });
+          } else {
+            order.push({ ok: true, peer, target, res, latencyMs });
+          }
+          if (order.length === total) resolve(order);
+        });
+      }
+    });
+    const winner = completed.find((o) => o.ok);
+    if (winner) {
+      for (const o of completed) {
+        if (o === winner) continue;
+        if (!o.ok) {
+          await ctx.peers.recordError(o.peer.url);
+          await ctx.peers.recordResult(o.peer.url, { ok: false });
+        } else {
+          await ctx.peers.recordResult(o.peer.url, { ok: true, latencyMs: o.latencyMs, model: o.target });
+        }
+      }
+      return { peer: winner.peer, target: winner.target, res: winner.res, latencyMs: winner.latencyMs };
+    }
+    for (const o of completed) {
+      await ctx.peers.recordError(o.peer.url);
+      await ctx.peers.recordResult(o.peer.url, { ok: false });
+    }
+  }
+  return null;
+}

package/src/routes/relay-queue.js

@@ -0,0 +1,127 @@
+import { loadGroupsJoined } from "../state.js";
+
+const relayPending = new Map();
+const relayPendingByReqId = new Map();
+
+export function enqueueRelay({ group, target, reqId, body, hops }) {
+  const key = `${group}::${target}`;
+  const list = relayPending.get(key) || [];
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      const idx = list.findIndex((e) => e.reqId === reqId);
+      if (idx >= 0) list.splice(idx, 1);
+      relayPendingByReqId.delete(reqId);
+      reject(new Error("relay timeout"));
+    }, 30_000);
+    timer.unref?.();
+    const entry = { reqId, body, hops, resolve, reject, timer };
+    list.push(entry);
+    relayPending.set(key, list);
+    relayPendingByReqId.set(reqId, entry);
+  });
+}
+
+export function dequeueRelayForPoll({ group, target, limit = 10 }) {
+  const key = `${group}::${target}`;
+  const list = relayPending.get(key) || [];
+  const batch = list.splice(0, limit);
+  if (list.length) relayPending.set(key, list);
+  else relayPending.delete(key);
+  return batch.map((e) => ({ reqId: e.reqId, body: e.body, hops: e.hops }));
+}
+
+export function resolveRelay(reqId, result) {
+  const entry = relayPendingByReqId.get(reqId);
+  if (!entry) return false;
+  clearTimeout(entry.timer);
+  relayPendingByReqId.delete(reqId);
+  entry.resolve(result);
+  return true;
+}
+
+export function getRelayPending() { return relayPending; }
+export function getRelayPendingByReqId() { return relayPendingByReqId; }
+
+export async function tryBroadbandRelay({ groups, token: myToken, model, body, hops, bus, logs, reqId, evt, res, mark, perf0, stages }) {
+  try {
+    const joined = loadGroupsJoined();
+    const broadbandGroups = joined.filter((g) => g.kind === "broadband" || g.myUrl?.startsWith("relay://"));
+    const allCandidates = [];
+    if (groups) {
+      const localGroups = groups.list();
+      for (const [gName, g] of Object.entries(localGroups)) {
+        for (const [id, m] of Object.entries(g.members || {})) {
+          if (id === "leader") continue;
+          const isBb = m?.kind === "broadband" || String(m?.url || "").startsWith("relay://");
+          if (!isBb) continue;
+          const staleMs = Number(process.env.MSLXDFF_BROADBAND_STALE_MS) > 0 ? Number(process.env.MSLXDFF_BROADBAND_STALE_MS) : 90_000;
+          if (typeof m.lastSeen === "number" && Date.now() - m.lastSeen > staleMs) continue;
+          allCandidates.push({ group: gName, target: m.url, member: m, via: "local-leader", leaderUrl: null });
+        }
+      }
+    }
+    for (const g of joined) {
+      if (!g.leaderUrl) continue;
+      try {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), 5000);
+        const r = await fetch(`${g.leaderUrl}/v1/groups/join`, {
+          method: "POST",
+          headers: { "Content-Type": "application/json", "Authorization": `Bearer ${myToken}` },
+          body: JSON.stringify({ name: g.name, memberName: g.memberName, url: g.myUrl, token: myToken }),
+          signal: controller.signal,
+        });
+        clearTimeout(timer);
+        if (!r.ok) continue;
+        const data = await r.json().catch(() => ({}));
+        const members = data.members || {};
+        for (const [id, m] of Object.entries(members)) {
+          if (id === "leader") continue;
+          const isBb = m?.kind === "broadband" || String(m?.url || "").startsWith("relay://");
+          if (!isBb) continue;
+          if (m.url === g.myUrl) continue;
+          const staleMs = Number(process.env.MSLXDFF_BROADBAND_STALE_MS) > 0 ? Number(process.env.MSLXDFF_BROADBAND_STALE_MS) : 90_000;
+          if (typeof m.lastSeen === "number" && Date.now() - m.lastSeen > staleMs) continue;
+          allCandidates.push({ group: g.name, target: m.url, member: m, via: "via-leader", leaderUrl: g.leaderUrl });
+        }
+      } catch {}
+    }
+    if (!allCandidates.length) return null;
+    for (const cand of allCandidates) {
+      try {
+        evt?.("relay-try", { reqId, model, via: cand.via, target: cand.target, group: cand.group });
+        if (!cand.leaderUrl) {
+          const fwdBody = { model, ...body, model };
+          const reqIdLocal = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;
+          const promise = enqueueRelay({ group: cand.group, target: cand.target, reqId: reqIdLocal, body: fwdBody, hops });
+          const result = await promise;
+          if (result && result.status) {
+            return { via: "broadband-local", result, target: cand.target, group: cand.group };
+          }
+        } else {
+          const ctrl = new AbortController();
+          const t = setTimeout(() => ctrl.abort(), 35_000);
+          const r = await fetch(`${cand.leaderUrl}/v1/groups/relay/forward`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json", "Authorization": `Bearer ${myToken}`, "x-mslxdff-hops": String(hops + 1) },
+            body: JSON.stringify({ group: cand.group, target: cand.target, body: { ...body, model }, hops: hops + 1, reqId }),
+            signal: ctrl.signal,
+          });
+          clearTimeout(t);
+          if (!r.ok) {
+            const txt = await r.text().catch(() => "");
+            evt?.("relay-fail", { reqId, model, via: cand.via, target: cand.target, status: r.status, message: txt.slice(0, 200) });
+            continue;
+          }
+          return { via: "broadband-via-leader", result: r, target: cand.target, group: cand.group };
+        }
+      } catch (err) {
+        evt?.("relay-fail", { reqId, model, via: cand.via, target: cand.target, message: String(err?.message || err).slice(0, 200) });
+        continue;
+      }
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}