dsh-model-pool 0.0.0-stage → 0.1.0

package/lib/router.js

@@ -0,0 +1,584 @@
+/**
+ * The routing generator for `dsh-model-pool`.
+ *
+ * One `llm/stream` interception per pooled call. The listener in `index.js`
+ * hands every request naming the virtual pool route to `Router.route()`, which
+ * turns it into a sequence of concrete-provider attempts.
+ *
+ * ## Why a listener and not only the adapter
+ *
+ * `LlmRuntime.streamWithRegistration` runs the `llm/stream` waterfall *before*
+ * `adapterStream` projects files to text, images to placeholders, and developer
+ * messages to tool declarations (`dsh-llm/lib/index.js:2396-2398` vs
+ * `:2309-2349`). Intercepting at the waterfall therefore hands the inner
+ * `ctx.llm.stream()` the raw request, and the runtime performs exactly one
+ * correct projection for the real target route. Routing inside the adapter
+ * would project twice and could drop tool-addition history.
+ *
+ * ## Failover boundary
+ *
+ * An attempt may only be replaced while no content reached the consumer.
+ * Block/usage chunks are held in a small buffer until the first content-bearing
+ * chunk arrives, so an eagerly emitted `block-start` does not disable failover,
+ * while a stream that already produced text is never replayed.
+ *
+ * @module dsh-model-pool/lib/router
+ */
+
+import { order } from './strategies.js';
+import { OFF_PEAK, PEAK, periodAt, priorityRanking, rankOf } from './peak.js';
+import { canonicalEffortId, pickEffort } from './effort.js';
+import { filterByModalities, requiredModalities } from './capabilities.js';
+
+/**
+ * Display name advertised for one effort id in an effort list.
+ * @param {object[]|undefined} efforts - an advertised effort list.
+ * @param {string} id - the effort id to name.
+ * @returns {string|undefined} the advertised name, or undefined when the list has no such id.
+ */
+function effortDisplayName(efforts, id) {
+  const found = (Array.isArray(efforts) ? efforts : []).find(
+    (candidate) => candidate !== null && typeof candidate === "object" && candidate.id === id,
+  );
+  return found !== undefined && typeof found.name === "string" && found.name !== "" ? found.name : undefined;
+}
+
+/** Marks a failure that is safe to retry on the next candidate. */
+class AttemptFailure extends Error {
+  /**
+   * @param {{ message: string, code: string, [key: string]: unknown }} failure - provider-neutral facts.
+   */
+  constructor(failure) {
+    super(failure.message);
+    this.name = 'AttemptFailure';
+    this.failure = failure;
+  }
+}
+
+/**
+ * Whether one chunk carries model output that the consumer has already seen.
+ * `block-start` and `usage` carry no output, so they stay buffered.
+ * @param {object} chunk - one stream chunk.
+ * @returns {boolean} true once replaying the stream would duplicate output.
+ */
+function hasContent(chunk) {
+  switch (chunk.type) {
+    case 'text-delta':
+    case 'reasoning-delta':
+      return typeof chunk.text === 'string' && chunk.text !== '';
+    case 'tool-call-delta':
+      return (typeof chunk.argumentsDelta === 'string' && chunk.argumentsDelta !== '')
+        || typeof chunk.name === 'string';
+    case 'block-end':
+      return chunk.block !== undefined && chunk.block !== null;
+    default:
+      return false;
+  }
+}
+
+/**
+ * Build the terminal `finish` chunk for a routing failure.
+ * @param {{ message: string, code: string }} failure - provider-neutral facts.
+ * @returns {object} a terminal error chunk.
+ */
+function errorFinish(failure) {
+  return { type: 'finish', reason: { kind: 'error', failure } };
+}
+
+/**
+ * Resolve the peak/idle period and the provider ranking that applies to it.
+ *
+ * Separate from `periodOf` so `candidatesFor` can resolve it from the config it
+ * was already handed, instead of re-reading the live config halfway through a
+ * decision. `peakEnabled: false` short-circuits to `none` rather than to an
+ * empty ranking, so Settings can tell "switched off" from "no list configured".
+ *
+ * @param {object} cfg - current plain config.
+ * @param {number} now - current epoch milliseconds.
+ * @returns {{ period: string, ranking: string[]|undefined, rankingSource: string }} the resolved period facts.
+ */
+function periodFacts(cfg, now) {
+  if (cfg.peakEnabled === false) return { period: OFF_PEAK, ranking: undefined, rankingSource: 'none' };
+  const period = periodAt(now, cfg);
+  const ranking = priorityRanking(period, cfg);
+  // Only the period's own list can produce a ranking, so the label names it. With
+  // no such list the feature is inert and `providerOrder` governs instead — an
+  // empty `providerOrder` there simply leaves registration order in charge.
+  const rankingSource = ranking === undefined
+    ? 'providerOrder'
+    : period === PEAK ? 'peakProviderOrder' : 'offPeakProviderOrder';
+  return { period, ranking, rankingSource };
+}
+
+/**
+ * Stable re-order of candidates by their period rank.
+ *
+ * `Array#sort` is not required to be stable, and equal ranks must keep the
+ * `model.routes` order that the strategy treats as its own tie-break, so the
+ * input index is carried as the final key.
+ *
+ * @param {object[]} routes - candidates in their existing order.
+ * @param {string[]} ranking - effective provider ranking for the period.
+ * @returns {object[]} a new array, ranked first and otherwise untouched.
+ */
+function rankByPeriod(routes, ranking) {
+  return routes
+    .map((route, index) => ({ route, index }))
+    .sort((left, right) => rankOf(ranking, left.route.provider, left.route.priority)
+      - rankOf(ranking, right.route.provider, right.route.priority)
+      || left.index - right.index)
+    .map((wrapped) => wrapped.route);
+}
+
+/** How many routing decisions the state endpoint remembers. */
+const RECENT_LIMIT = 20;
+
+/** How many per-session route summaries are kept for the composer dock. */
+const SESSION_LIMIT = 50;
+
+/**
+ * One pool router bound to a live plugin instance.
+ */
+export class Router {
+  /**
+   * @param {object} options - router dependencies.
+   * @param {object} options.llm - the live `llm` service.
+   * @param {() => object} options.config - reads the current plain config.
+   * @param {() => object} options.snapshot - reads the current pool snapshot.
+   * @param {import('./health.js').HealthRegistry} options.health - live health registry.
+   * @param {object} [options.logger] - optional host logger.
+   */
+  constructor({ llm, config, snapshot, health, logger }) {
+    this.llm = llm;
+    this.config = config;
+    this.snapshot = snapshot;
+    this.health = health;
+    this.logger = logger;
+    /** Round-robin cursor, shared by every pool model. */
+    this.roundRobin = { cursor: 0 };
+    /** Most recent routing decisions, newest last. */
+    this.recent = [];
+    /**
+     * Peak/idle facts of the most recent `candidatesFor` call: `{ at, period,
+     * ranking, rankingSource }`, or undefined before the first decision — a
+     * Settings read then says "nothing routed yet" instead of guessing a period.
+     */
+    this.lastPeriod = undefined;
+    /**
+     * Last successful route per session, keyed by session id. The composer dock
+     * renders this so a user can see which concrete provider actually served
+     * their conversation, and the sticky strategy reads the same record to keep
+     * a conversation on one route. Bounded because sessions come and go.
+     */
+    this.sessions = new Map();
+  }
+
+  /**
+   * Record the route that actually served one session's most recent call.
+   * @param {string|undefined} sessionId - the calling session, when known.
+   * @param {object} model - the pool model that was selected.
+   * @param {object} route - the concrete route that produced content.
+   * @param {string|undefined} effort - the reasoning effort actually sent to the provider.
+   * @param {string|undefined} requestedEffort - the effort the composer asked for.
+   * @param {string|undefined} effortName - the provider's display name for `effort`.
+   * @returns {void}
+   */
+  recordSession(sessionId, model, route, effort, requestedEffort, effortName) {
+    if (typeof sessionId !== "string" || sessionId === "") return;
+    this.sessions.delete(sessionId);
+    const entry = {
+      at: Date.now(),
+      model: model.id,
+      modelName: model.name,
+      // Names first: the dock shows these, never the ids, so a surface that only
+      // reads the display fields stays correct without re-reading the catalog.
+      provider: route.provider,
+      providerName: route.providerName,
+      providerModel: route.model,
+      providerModelName: route.name,
+    };
+    // `effort` is what the provider received. `requestedEffort` is what the
+    // composer said, kept only when the two mean different tiers so the UI can
+    // say so. Compared on the canonical tier, because `off` and `disabled` are
+    // the same request and must not look like an adjustment.
+    if (effort !== undefined) entry.effort = effort;
+    if (typeof effortName === "string" && effortName !== "") entry.effortName = effortName;
+    const sameTier = typeof requestedEffort === "string" && canonicalEffortId(requestedEffort) === canonicalEffortId(effort);
+    if (typeof requestedEffort === "string" && requestedEffort !== "" && !sameTier) {
+      entry.requestedEffort = requestedEffort;
+      // Resolve the requested tier's display name from the list the composer showed
+      // (the pool's union) first: the target route need not advertise that tier, and
+      // the badge should then still say `Off` rather than a bare `off`.
+      const requestedName = effortDisplayName(model.reasoning?.efforts, requestedEffort)
+        ?? effortDisplayName(route.reasoning?.efforts, requestedEffort)
+        ?? requestedEffort;
+      entry.requestedEffortName = requestedName;
+    }
+    this.sessions.set(sessionId, entry);
+    // Insertion order doubles as recency, so the oldest key is the eviction.
+    while (this.sessions.size > SESSION_LIMIT) {
+      const oldest = this.sessions.keys().next();
+      if (oldest.done === true) break;
+      this.sessions.delete(oldest.value);
+    }
+  }
+
+  /**
+   * Read the last route a session used, or undefined when it never used the pool.
+   * @param {string|undefined} sessionId - session to look up.
+   * @returns {object|undefined} the recorded route summary.
+   */
+  sessionOf(sessionId) {
+    if (typeof sessionId !== "string" || sessionId === "") return undefined;
+    return this.sessions.get(sessionId);
+  }
+
+  /**
+   * Every live session summary, newest last.
+   * @returns {object[]} entries shaped as `{ sessionId, ...summary }`.
+   */
+  sessionEntries() {
+    return [...this.sessions].map(([sessionId, value]) => ({ sessionId, ...value }));
+  }
+
+  /**
+   * Record one routing decision for the Settings page.
+   * @param {object} entry - decision facts.
+   */
+  remember(entry) {
+    this.recent.push(entry);
+    if (this.recent.length > RECENT_LIMIT) this.recent.splice(0, this.recent.length - RECENT_LIMIT);
+  }
+
+  /**
+   * Order one pool model's candidates under the configured strategy and health.
+   * @param {object} cfg - current plain config.
+   * @param {object} model - the pool model.
+   * @param {number} now - current epoch milliseconds.
+   * @param {string|undefined} sessionId - the calling session, for sticky routing.
+   * @param {string[]} [required] - input modalities the request actually carries.
+   * @returns {object[]} candidate routes, best first.
+   */
+  candidatesFor(cfg, model, now, sessionId, required) {
+    const routes = model.routes.filter((route) => route.provider !== cfg.providerId);
+    // The pool advertises the UNION of its candidates' modalities, so the promise
+    // that an image is accepted is only kept by choosing an image-capable route.
+    // The filter runs first: narrowing after ordering could pick a body that the
+    // strategy ranked first but that cannot see the screenshot.
+    let usable = required === undefined ? routes : filterByModalities(routes, required);
+    if (cfg.skipUnavailable === true) {
+      // Narrow whatever the modality filter left, never the raw list: falling
+      // back to `routes` here would quietly restore a text-only route to an
+      // image call, which is the exact mismatch this filter exists to prevent.
+      const capable = usable;
+      const available = capable.filter((route) => this.health.isAvailable(route.provider, route.model, now, cfg.cooldownMs));
+      // A pool whose every candidate is cooling down still has to try: refusing
+      // outright would turn a transient outage into a hard failure.
+      if (available.length > 0) usable = available;
+    }
+    // The peak/idle overlay is applied to the candidate list *before* the
+    // strategy runs, so every strategy keeps its own key and inherits the period
+    // order as its stable tie-break; `priority` passes it straight through. The
+    // period is resolved once here, from the same `now` the rest of this call
+    // reasons about, so one decision is reproducible and cannot straddle a flip.
+    const { period, ranking, rankingSource } = periodFacts(cfg, now);
+    this.lastPeriod = { at: now, period, ranking, rankingSource };
+    if (ranking !== undefined) usable = rankByPeriod(usable, ranking);
+    const ordered = order(cfg.strategy, usable, {
+      now,
+      health: this.health,
+      roundRobin: this.roundRobin,
+      weights: cfg.weights,
+      costs: cfg.costs,
+      random: Math.random,
+    });
+
+    // Sticky routing: a session that already ran on one route keeps it, so the
+    // provider's prompt cache stays warm across turns. The binding is only a
+    // preference — it moves the remembered route to the front of the list that
+    // the strategy produced, so failover still walks the other candidates and a
+    // route that went unhealthy simply stops matching.
+    const bound = cfg.sticky === true ? this.stickyRouteFor(sessionId, model, ordered) : undefined;
+    const finalOrder = bound === undefined
+      ? ordered
+      : [bound, ...ordered.filter((route) => route !== bound)];
+
+    const limit = Math.max(1, Math.min(finalOrder.length, cfg.maxAttempts));
+    return finalOrder.slice(0, limit);
+  }
+
+  /**
+   * Resolve the period and provider ranking one instant would route under.
+   *
+   * Exposed for Settings: it answers "what would the pool do right now" without
+   * running a call, and it reads the live config so the answer tracks an edited
+   * policy. `candidatesFor` resolves the same facts from its own `cfg`/`now`, so
+   * the two can never disagree about an in-flight decision.
+   *
+   * @param {number} now - current epoch milliseconds.
+   * @returns {{ period: string, ranking: string[]|undefined, rankingSource: string }} the resolved period facts.
+   */
+  periodOf(now) {
+    return periodFacts(this.config(), now);
+  }
+
+  /**
+   * Find the remembered route for a session within this model's candidates.
+   * @param {string|undefined} sessionId - the calling session.
+   * @param {object} model - the pool model being routed.
+   * @param {object[]} candidates - the strategy-ordered candidates.
+   * @returns {object|undefined} the candidate to pin first, when the session already has one.
+   */
+  stickyRouteFor(sessionId, model, candidates) {
+    const entry = this.sessionOf(sessionId);
+    if (entry === undefined) return undefined;
+    // A pooled model that was re-selected with a different id must not inherit
+    // the previous model's route.
+    if (entry.model !== model.id) return undefined;
+    return candidates.find(
+      (route) => route.provider === entry.provider && route.model === entry.providerModel,
+    );
+  }
+
+  /**
+   * Run one concrete attempt, yielding its chunks.
+   *
+   * Throws `AttemptFailure` when the attempt failed before producing content,
+   * which is the only case the caller may retry.
+   *
+   * @param {object} options - the original pooled request.
+   * @param {object} route - the concrete candidate.
+   * @param {object} cfg - current plain config.
+   * @param {object} report - per-attempt facts; `commit` is invoked once, before the
+   * terminal chunk is yielded, so the caller can record the outcome without depending
+   * on the consumer draining this generator.
+   * @returns {AsyncGenerator<object>} the attempt's chunk stream.
+   */
+  async *attempt(options, route, cfg, report) {
+    // The pooled request carries the *selection* id. What reaches the provider is
+    // the tier this route actually supports, which may differ (a gateway without
+    // `off` gets `low`, `xhigh` collapses to `max`). Report the resolved value so
+    // the UI shows what really ran instead of what was asked for.
+    const effort = pickEffort(route.reasoning?.efforts, options.reasoningEffort);
+    if (report !== undefined && report !== null) {
+      report.effort = effort;
+      // The route advertises a display name per tier; carry it so the UI never
+      // has to show a raw id like `xhigh`.
+      const chosen = (Array.isArray(route.reasoning?.efforts) ? route.reasoning.efforts : []).find(
+        (candidate) => candidate !== null && typeof candidate === "object" && candidate.id === effort,
+      );
+      if (chosen !== undefined && typeof chosen.name === "string" && chosen.name !== "") {
+        report.effortName = chosen.name;
+      }
+    }
+    // Settle the route before the terminal chunk is handed to the consumer.
+    // An `llm/stream` wrapper above us (an answer-rewriting plugin, for one) may
+    // stop at `finish` and close the iterator instead of draining it, so nothing
+    // after our own last `yield` is guaranteed to run. Recording the session and
+    // the routing decision here makes the observation independent of whether the
+    // consumer asks for another chunk.
+    const settle = () => {
+      if (report === undefined || report === null || report.settled === true) return;
+      report.settled = true;
+      report.commit?.();
+    };
+    const { provider: _provider, model: _model, reasoningEffort: _effort, ...rest } = options;
+    const request = {
+      ...rest,
+      provider: route.provider,
+      model: route.model,
+      ...effort === undefined ? {} : { reasoningEffort: effort },
+    };
+
+    const startedAt = Date.now();
+    let firstChunkAt;
+    let committed = false;
+    const buffered = [];
+    let sawFinish = false;
+
+    try {
+      for await (const chunk of this.llm.stream(request)) {
+        if (firstChunkAt === undefined) firstChunkAt = Date.now();
+        if (chunk.type === 'finish') {
+          sawFinish = true;
+          const reason = chunk.reason;
+          if (reason?.kind === 'error' && !committed) {
+            this.health.recordFailure(route.provider, route.model, reason.failure?.code ?? 'ERROR');
+            throw new AttemptFailure(reason.failure ?? { message: 'provider failed', code: 'ERROR' });
+          }
+          if (reason?.kind === 'error') {
+            // Content already reached the consumer: forward the failure instead
+            // of replaying a partial answer.
+            this.health.recordFailure(route.provider, route.model, reason.failure?.code ?? 'ERROR');
+            settle();
+            for (const held of buffered) yield held;
+            yield chunk;
+            return;
+          }
+          if (reason?.kind === 'aborted') {
+            settle();
+            for (const held of buffered) yield held;
+            yield chunk;
+            return;
+          }
+          this.health.recordSuccess(route.provider, route.model, firstChunkAt === undefined ? 0 : firstChunkAt - startedAt);
+          settle();
+          for (const held of buffered) yield held;
+          yield chunk;
+          return;
+        }
+
+        if (committed) {
+          yield chunk;
+          continue;
+        }
+        buffered.push(chunk);
+        if (hasContent(chunk)) {
+          committed = true;
+          for (const held of buffered) yield held;
+          buffered.length = 0;
+        }
+      }
+    } catch (error) {
+      if (error instanceof AttemptFailure) throw error;
+      if (options.signal?.aborted) throw error;
+      this.health.recordFailure(route.provider, route.model, 'STREAM_FAILED');
+      throw new AttemptFailure({
+        message: error instanceof Error ? error.message : String(error),
+        code: 'STREAM_FAILED',
+      });
+    }
+
+    // A stream that ended without a terminal chunk is a provider bug. Treat it
+    // as a retryable failure when nothing was shown, and as a normal end when
+    // content was already delivered — but always close the protocol, because
+    // every consumer expects exactly one terminal `finish` chunk.
+    if (!sawFinish) {
+      if (committed) {
+        settle();
+        for (const held of buffered) yield held;
+        yield { type: 'finish', reason: { kind: 'stop' } };
+        return;
+      }
+      this.health.recordFailure(route.provider, route.model, 'NO_FINISH');
+      throw new AttemptFailure({ message: `provider "${route.provider}" ended its stream without a finish chunk`, code: 'NO_FINISH' });
+    }
+  }
+
+  /**
+   * Route one pooled request to a concrete provider.
+   * @param {object} options - the pooled `GenerateOptions`.
+   * @returns {AsyncGenerator<object>} the chunk stream the agent loop consumes.
+   */
+  async *route(options) {
+    // Diagnostic breadcrumb: the effort the runtime handed us, before any
+    // mapping. Without it a mismatch is indistinguishable from a bad selection.
+    // The carried modalities ride along so a route that was skipped (or a text-only
+    // route that was never eligible) is explainable from the Settings payload
+    // instead of looking like an arbitrary choice.
+    const carried = requiredModalities(options.messages);
+    this.lastInbound = {
+      at: Date.now(),
+      model: options.model,
+      reasoningEffort: options.reasoningEffort,
+      sessionId: options.sessionId,
+      purpose: options.purpose ?? null,
+      requiredModalities: carried,
+    };
+    const cfg = this.config();
+    const snapshot = this.snapshot();
+    const model = snapshot.models.find((candidate) => candidate.id === options.model);
+
+    if (model === undefined) {
+      this.remember({ at: Date.now(), model: options.model, outcome: 'unknown-model', provider: undefined });
+      yield errorFinish({
+        message: `the model pool has no model "${options.model}"; it may have been removed from every provider — refresh the pool or pick another model`,
+        code: 'NO_ADAPTER',
+      });
+      return;
+    }
+
+    // The modalities the request actually carries decide which candidates are
+    // eligible. A text-only call keeps every candidate; a screenshot must reach a
+    // route that can see it, because the pool told the user this model takes images.
+    const candidates = this.candidatesFor(cfg, model, Date.now(), options.sessionId, carried);
+    // Captured synchronously: `lastPeriod` is shared state, and a concurrent call
+    // may overwrite it before this one settles. The record must keep describing
+    // the period this decision was actually made under.
+    const { period, rankingSource } = this.lastPeriod;
+    if (candidates.length === 0) {
+      this.remember({ at: Date.now(), model: options.model, outcome: 'no-candidate', provider: undefined });
+      yield errorFinish({
+        message: `no provider currently serves "${options.model}" in the model pool`,
+        code: 'NO_ADAPTER',
+      });
+      return;
+    }
+
+    let lastFailure;
+    for (const route of candidates) {
+      if (options.signal?.aborted) break;
+      try {
+        const report = {};
+        // Auxiliary calls (session titles, compaction summaries) share the session's
+        // route but are not what the conversation ran on: they pick their own effort
+        // — usually `off`, which a route without an `off` tier maps to its cheapest
+        // tier. Letting one define the session record would report `Low` for a
+        // conversation that actually ran `High`, so they only reach `recent`.
+        const auxiliary = typeof options.purpose === 'string' && options.purpose !== '';
+        // Success is settled inside `attempt`, right before it yields the terminal
+        // chunk, rather than after `yield*` returns: a consumer above us may stop at
+        // `finish` and close the iterator, so the tail of this generator is not
+        // guaranteed to run. Settling first keeps the session dock and sticky
+        // routing correct under such a wrapper.
+        report.commit = () => {
+          this.remember({
+            at: Date.now(),
+            model: options.model,
+            outcome: 'routed',
+            provider: route.provider,
+            providerModel: route.model,
+            // Which period governed the ordering, and which list supplied the
+            // ranking, so Settings can explain the choice instead of implying it.
+            period,
+            rankingSource,
+            ...auxiliary ? { purpose: options.purpose } : {},
+          });
+          if (auxiliary) return;
+          // Record what this session actually ran on, so the composer dock can
+          // show the concrete provider behind the pool. `report.effort` is the tier
+          // that reached the provider, not the one the user selected.
+          this.recordSession(options.sessionId, model, route, report.effort, options.reasoningEffort, report.effortName);
+        };
+        yield* this.attempt(options, route, cfg, report);
+        // Reached only when the consumer drained the generator. A consumer that
+        // stopped at the terminal chunk already settled it inside `attempt`.
+        if (report.settled !== true) {
+          report.settled = true;
+          report.commit();
+        }
+        return;
+      } catch (error) {
+        if (!(error instanceof AttemptFailure)) throw error;
+        lastFailure = error.failure;
+        this.logger?.debug?.(
+          'model-pool: %s via %s failed (%s), trying the next candidate',
+          options.model,
+          route.provider,
+          error.failure.code,
+        );
+        if (cfg.fallback !== true) break;
+      }
+    }
+
+    const failure = lastFailure ?? { message: 'the model pool request was cancelled', code: 'ABORTED' };
+    this.remember({
+      at: Date.now(),
+      model: options.model,
+      outcome: options.signal?.aborted ? 'aborted' : 'failed',
+      provider: candidates.at(-1)?.provider,
+      reason: failure.code,
+    });
+    yield errorFinish(failure);
+  }
+}