@zenera/neo 1.1.10 → 1.1.12

package/dist/embeddings/limiter.js

@@ -0,0 +1,182 @@
+import { classify, retryAfterMs } from "./rate-limit.js";
+export const DEFAULT_MIN = 1;
+export const DEFAULT_START = 4;
+export const DEFAULT_MAX = 16;
+export const DEFAULT_STRIDE = 8;
+export const DEFAULT_INITIAL_BACKOFF_MS = 1_000;
+export const DEFAULT_MAX_BACKOFF_MS = 60_000;
+export const DEFAULT_MAX_RETRIES = 4;
+export class RateLimiter {
+    #min;
+    #max;
+    #stride;
+    #initial;
+    #maxBackoff;
+    #retries;
+    #limit;
+    #active = 0;
+    #successes = 0;
+    /** bumped on every decrease; a refusal from an older one is already answered */
+    #epoch = 0;
+    #penalty;
+    #pausedUntil = 0;
+    #timer;
+    #waiting = [];
+    constructor(options = {}) {
+        this.#min = Math.max(1, options.min ?? DEFAULT_MIN);
+        this.#max = Math.max(this.#min, options.max ?? DEFAULT_MAX);
+        this.#stride = Math.max(1, options.stride ?? DEFAULT_STRIDE);
+        this.#initial = Math.max(0, options.initialBackoffMs ?? DEFAULT_INITIAL_BACKOFF_MS);
+        this.#maxBackoff = Math.max(this.#initial, options.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS);
+        this.#retries = Math.max(0, options.maxRetries ?? DEFAULT_MAX_RETRIES);
+        this.#limit = clamp(options.start ?? DEFAULT_START, this.#min, this.#max);
+        this.#penalty = this.#initial;
+    }
+    /** How many may be in flight as things stand. Moves; read it for reporting only. */
+    get limit() {
+        return this.#limit;
+    }
+    get inFlight() {
+        return this.#active;
+    }
+    /**
+     * Runs the task when there is room, retries it when the failure allows, and
+     * gives up when it does not. The slot is released before any wait, so a
+     * task serving its backoff is not also holding the concurrency down.
+     */
+    async run(task, signal) {
+        let backoff = this.#initial;
+        for (let attempt = 0;; attempt++) {
+            const epoch = await this.#acquire(signal);
+            let failure = 'fatal';
+            let err;
+            try {
+                const value = await task();
+                this.#succeeded();
+                return value;
+            }
+            catch (thrown) {
+                err = thrown;
+                failure = classify(thrown);
+                // Before the slot is let go, so the pause is already in force
+                // when the next waiter is granted one.
+                if (failure === 'rate-limit') {
+                    this.#slowDown(epoch, retryAfterMs(thrown));
+                }
+            }
+            finally {
+                this.#release();
+            }
+            if (failure === 'fatal' || attempt >= this.#retries) {
+                throw err;
+            }
+            // A refusal is already being served by the shared pause, which the
+            // next acquire blocks on; waiting again here would double it.
+            if (failure === 'transient') {
+                await sleep(jitter(Math.min(backoff, this.#maxBackoff)), signal);
+                backoff *= 2;
+            }
+        }
+    }
+    #acquire(signal) {
+        if (signal?.aborted) {
+            return Promise.reject(signal.reason);
+        }
+        if (this.#free()) {
+            this.#active++;
+            return Promise.resolve(this.#epoch);
+        }
+        return new Promise((resolve, reject) => {
+            const waiter = {
+                grant: (epoch) => {
+                    signal?.removeEventListener('abort', abort);
+                    resolve(epoch);
+                },
+                fail: reject,
+            };
+            const abort = () => {
+                const at = this.#waiting.indexOf(waiter);
+                if (at >= 0) {
+                    this.#waiting.splice(at, 1);
+                }
+                reject(signal?.reason);
+            };
+            signal?.addEventListener('abort', abort, { once: true });
+            this.#waiting.push(waiter);
+        });
+    }
+    #free() {
+        return this.#active < this.#limit && Date.now() >= this.#pausedUntil;
+    }
+    #release() {
+        this.#active--;
+        this.#pump();
+    }
+    /**
+     * The slot is taken here rather than by the waiter, so several woken at
+     * once cannot each see the same free one before any of them claims it.
+     */
+    #pump() {
+        while (this.#waiting.length > 0 && this.#free()) {
+            this.#active++;
+            this.#waiting.shift().grant(this.#epoch);
+        }
+    }
+    #succeeded() {
+        this.#penalty = this.#initial;
+        // Only a limit that is actually the constraint has earned being raised;
+        // with nothing queued, more slots would go unused anyway.
+        if (this.#waiting.length === 0) {
+            return;
+        }
+        if (++this.#successes >= this.#stride) {
+            this.#successes = 0;
+            this.#limit = Math.min(this.#max, this.#limit + 1);
+            this.#pump();
+        }
+    }
+    #slowDown(epoch, retryAfter) {
+        if (epoch === this.#epoch) {
+            this.#epoch++;
+            this.#limit = Math.max(this.#min, Math.floor(this.#limit / 2));
+            this.#successes = 0;
+            this.#penalty = Math.min(this.#maxBackoff, this.#penalty * 2);
+        }
+        // Even a refusal that arrived too late to lower the limit still knows
+        // how long this provider wants to be left alone.
+        this.#pause(retryAfter ?? this.#penalty);
+    }
+    #pause(ms) {
+        const until = Date.now() + Math.max(0, ms);
+        if (until <= this.#pausedUntil) {
+            return;
+        }
+        this.#pausedUntil = until;
+        clearTimeout(this.#timer);
+        this.#timer = setTimeout(() => {
+            this.#timer = undefined;
+            this.#pump();
+        }, until - Date.now());
+        this.#timer.unref?.();
+    }
+}
+const clamp = (n, low, high) => Math.min(high, Math.max(low, n));
+/** Full jitter: without it, everything that backed off together returns together. */
+const jitter = (ms) => ms / 2 + Math.random() * (ms / 2);
+function sleep(ms, signal) {
+    if (signal?.aborted) {
+        return Promise.reject(signal.reason);
+    }
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => {
+            signal?.removeEventListener('abort', abort);
+            resolve();
+        }, ms);
+        const abort = () => {
+            clearTimeout(timer);
+            reject(signal?.reason);
+        };
+        signal?.addEventListener('abort', abort, { once: true });
+    });
+}
+//# sourceMappingURL=limiter.js.map

package/dist/embeddings/openai.d.ts

@@ -1,11 +1,12 @@
 import type OpenAI from 'openai';
-import { type Embedder, type EmbeddingRequest, type EmbeddingResponse } from '../embedding.ts';
+import type { Embedder, EmbeddingRequest, EmbeddingResponse } from '../embedding.ts';
+import { type BatchOptions } from './fanout.ts';
 /**
  * Provider-specific knobs. `taskType` has no counterpart here: OpenAI's models
  * are symmetric, so a query and a document are encoded identically and the
  * request's hint is simply not sent.
  */
-export interface OpenAIEmbedderOptions {
+export interface OpenAIEmbedderOptions extends BatchOptions {
     /** default width; a request may override it */
     dimensions?: number;
 }

package/dist/embeddings/openai.js

@@ -1,26 +1,67 @@
-import { embeddingResponse, } from "../embedding.js";
-import { zeroUsage } from "../types.js";
+import { callSites, called } from "../failure.js";
+import { fanout } from "./fanout.js";
+import { RateLimiter } from "./limiter.js";
+/**
+ * What OpenAI documents for `text-embedding-*`: 2048 inputs, and 300k tokens
+ * across them. The token figure is the one that fires in practice, and it is
+ * held under the real cap because the estimate feeding it counts characters.
+ */
+const MAX_BATCH = 2048;
+const MAX_BATCH_TOKENS = 250_000;
+/**
+ * A gateway is not OpenAI. `openai-compatible` reaches vLLM, Ollama and
+ * whatever else speaks the shape, none of which promise those numbers, so an
+ * unrecognised id gets something any implementation can be expected to hold.
+ */
+const GATEWAY_BATCH = 96;
+const GATEWAY_BATCH_TOKENS = 100_000;
+const OPENAI_MODEL = /^text-embedding-/;
 export class OpenAIEmbedder {
     id;
     #client;
     #dimensions;
+    #limiter;
+    #maxBatch;
+    #maxBatchTokens;
+    #site;
     constructor(id, client, options = {}) {
         this.id = id;
         this.#client = client;
         this.#dimensions = options.dimensions;
+        this.#limiter = options.limiter ?? new RateLimiter();
+        this.#site = callSites(id, options.provider);
+        const known = OPENAI_MODEL.test(id);
+        this.#maxBatch = options.maxBatch ?? (known ? MAX_BATCH : GATEWAY_BATCH);
+        this.#maxBatchTokens =
+            options.maxBatchTokens ?? (known ? MAX_BATCH_TOKENS : GATEWAY_BATCH_TOKENS);
     }
     async embed(req) {
-        const res = await this.#client.embeddings.create({
-            model: this.id,
-            input: req.input,
-            dimensions: req.dimensions ?? this.#dimensions,
-        }, { signal: req.signal });
-        // The API documents the order but does not promise it, and `index` is
-        // the only thing that ties a vector back to its text.
-        const vectors = [...res.data].sort((a, b) => a.index - b.index).map((d) => d.embedding);
-        return embeddingResponse(vectors, req, {
-            ...zeroUsage(),
-            inputTokens: res.usage.prompt_tokens,
+        return await fanout(req, {
+            id: this.id,
+            limiter: this.#limiter,
+            maxBatch: this.#maxBatch,
+            maxBatchTokens: this.#maxBatchTokens,
+            send: async (input) => {
+                const res = await called(this.#site('embedding', 'embeddings.create'), () => this.#client.embeddings.create({
+                    model: this.id,
+                    input,
+                    dimensions: req.dimensions ?? this.#dimensions,
+                }, 
+                // The client's own retrying is off because a retry it
+                // serves is one the limiter never sees: the call
+                // succeeds, nothing learns the provider is refusing, and
+                // the run quietly takes as long as the backoff it did
+                // not report.
+                { signal: req.signal, maxRetries: 0 }));
+                // The API documents the order but does not promise it, and
+                // `index` is the only thing tying a vector back to its text.
+                return {
+                    vectors: [...res.data]
+                        .sort((a, b) => a.index - b.index)
+                        .map((d) => d.embedding),
+                    inputTokens: res.usage.prompt_tokens,
+                };
+            },
         });
     }
 }

package/dist/embeddings/openrouter.d.ts

@@ -1,8 +1,9 @@
 import type { OpenRouter } from '@openrouter/sdk';
 import type { ProviderPreferences } from '@openrouter/sdk/models';
 import type { Embedder, EmbeddingRequest, EmbeddingResponse } from '../embedding.ts';
+import { type BatchOptions } from './fanout.ts';
 /** Provider-specific knobs. */
-export interface OpenRouterEmbedderOptions {
+export interface OpenRouterEmbedderOptions extends BatchOptions {
     /** default width; a request may override it */
     dimensions?: number;
     /**

package/dist/embeddings/openrouter.js

@@ -1,6 +1,6 @@
-import { embeddingResponse } from "../embedding.js";
-import { RETRY_STATUS_CODES } from "../models/openrouter.js";
-import { zeroUsage } from "../types.js";
+import { callSites, called } from "../failure.js";
+import { fanout } from "./fanout.js";
+import { RateLimiter } from "./limiter.js";
 // ---------------------------------------------------------------------------
 // OpenRouter embeddings adapter
 //
@@ -10,36 +10,65 @@ import { zeroUsage } from "../types.js";
 // ---------------------------------------------------------------------------
 /** OpenRouter's word for the retrieval side, which it takes as a free string. */
 const INPUT_TYPES = { query: 'search_query', document: 'search_document' };
+/**
+ * Conservative on purpose. What a batch may hold is whatever the upstream
+ * provider routing picked accepts, which is not knowable from here and can
+ * differ between two requests for the same model.
+ */
+const MAX_BATCH = 96;
+const MAX_BATCH_TOKENS = 100_000;
 export class OpenRouterEmbedder {
     id;
     #client;
     #options;
+    #limiter;
+    #site;
     constructor(id, client, options = {}) {
         this.id = id;
         this.#client = client;
         this.#options = options;
+        this.#limiter = options.limiter ?? new RateLimiter();
+        this.#site = callSites(id, options.provider);
     }
     async embed(req) {
-        const requestBody = {
-            model: this.id,
-            input: req.input,
-            dimensions: req.dimensions ?? this.#options.dimensions,
-            inputType: req.taskType ? INPUT_TYPES[req.taskType] : undefined,
-            provider: this.#options.routing,
-            // `encodingFormat` is deliberately not sent. Its type is an open
-            // enum, whose values are branded and so unreachable without a value
-            // import of the SDK — and float is the default anyway. The string
-            // branch of `embedding` below is what catches a gateway that
-            // decides otherwise.
-        };
-        const res = await this.#client.embeddings.generate({ requestBody }, { fetchOptions: { signal: req.signal }, retryCodes: RETRY_STATUS_CODES });
-        // The response is declared as the body *or* a bare string, so the
-        // parsed shape has to be established before anything is read off it.
-        if (typeof res === 'string') {
-            throw new Error(`embedder "${this.id}": expected an embeddings response, got text`);
-        }
-        const vectors = [...res.data].sort(byIndex).map((d) => vectorOf(d, this.id));
-        return embeddingResponse(vectors, req, res.usage ? { ...zeroUsage(), inputTokens: res.usage.promptTokens } : undefined);
+        return await fanout(req, {
+            id: this.id,
+            limiter: this.#limiter,
+            maxBatch: this.#options.maxBatch ?? MAX_BATCH,
+            maxBatchTokens: this.#options.maxBatchTokens ?? MAX_BATCH_TOKENS,
+            send: async (input) => {
+                const requestBody = {
+                    model: this.id,
+                    input,
+                    dimensions: req.dimensions ?? this.#options.dimensions,
+                    inputType: req.taskType ? INPUT_TYPES[req.taskType] : undefined,
+                    provider: this.#options.routing,
+                    // `encodingFormat` is deliberately not sent. Its type is an
+                    // open enum, whose values are branded and so unreachable
+                    // without a value import of the SDK — and float is the
+                    // default anyway. The string branch of `embedding` below is
+                    // what catches a gateway that decides otherwise.
+                };
+                const res = await called(this.#site('embedding', 'embeddings.generate'), () => this.#client.embeddings.generate({ requestBody }, {
+                    fetchOptions: { signal: req.signal },
+                    // No retrying here: a 429 the SDK swallows is a 429
+                    // the limiter never hears about, and it is the only
+                    // thing that can slow the whole run down rather than
+                    // this one request.
+                    retries: { strategy: 'none' },
+                }));
+                // The response is declared as the body *or* a bare string, so
+                // the parsed shape has to be established before anything is
+                // read off it.
+                if (typeof res === 'string') {
+                    throw new Error(`embedder "${this.id}": expected an embeddings response, got text`);
+                }
+                return {
+                    vectors: [...res.data].sort(byIndex).map((d) => vectorOf(d, this.id)),
+                    inputTokens: res.usage?.promptTokens,
+                };
+            },
+        });
     }
 }
 /** `index` is optional here, so its absence has to mean "keep the order given". */

package/dist/embeddings/rate-limit.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Three answers, because a limiter can only do three things: send it again
+ * later and slow down, send it again later, or give up.
+ */
+export type Failure = 'rate-limit' | 'transient' | 'fatal';
+/** The HTTP status, wherever this vendor's SDK decided to keep it. */
+export declare function statusOf(err: unknown): number | undefined;
+export declare function classify(err: unknown): Failure;
+export declare function isAbort(err: unknown): boolean;
+/**
+ * How long the provider asked to be left alone. Absent far more often than
+ * not — only some vendors send it, and only on some of their limits — so the
+ * limiter has to have a schedule of its own regardless.
+ */
+export declare function retryAfterMs(err: unknown): number | undefined;
+//# sourceMappingURL=rate-limit.d.ts.map

package/dist/embeddings/rate-limit.js

@@ -0,0 +1,131 @@
+// ---------------------------------------------------------------------------
+// What a failed request means
+//
+// Four SDKs, four error shapes, one question: may this be sent again, and does
+// the provider want to be called less often. Everything below is structural —
+// no SDK type is imported, because three of the four are optional peers and an
+// error that arrives from a gateway is not an instance of anything anyway.
+//
+// The one difference worth naming out loud is the status field. OpenAI,
+// Anthropic and @google/genai put it on `status`; the OpenRouter client, being
+// generated, calls it `statusCode`. Reading only one of them reports a revoked
+// key or an exhausted quota as an unknown failure, which is the same trap the
+// CLI's liveness probe already documents.
+// ---------------------------------------------------------------------------
+/** The provider is asking to be called less often, not saying the call was wrong. */
+const RATE_LIMITED = new Set([429, 503]);
+/** Worth another attempt, but says nothing about the rate. */
+const TRANSIENT = new Set([408, 409, 500, 502, 504]);
+/** Undici's words for a connection that never produced a response. */
+const NETWORK = new Set([
+    'ECONNRESET',
+    'ECONNREFUSED',
+    'ETIMEDOUT',
+    'EPIPE',
+    'EAI_AGAIN',
+    'ENOTFOUND',
+    'UND_ERR_SOCKET',
+    'UND_ERR_CONNECT_TIMEOUT',
+    'UND_ERR_HEADERS_TIMEOUT',
+    'UND_ERR_BODY_TIMEOUT',
+    'ERR_STREAM_PREMATURE_CLOSE',
+]);
+/** The error and everything it blames, guarded against a self-referential cause. */
+function* chain(err) {
+    const seen = new Set();
+    for (let at = err; at && typeof at === 'object' && !seen.has(at); at = at.cause) {
+        seen.add(at);
+        yield at;
+    }
+}
+/** The HTTP status, wherever this vendor's SDK decided to keep it. */
+export function statusOf(err) {
+    for (const link of chain(err)) {
+        const raw = link.status ?? link.statusCode;
+        const status = typeof raw === 'string' ? Number(raw) : raw;
+        if (typeof status === 'number' && Number.isFinite(status)) {
+            return status;
+        }
+    }
+    return undefined;
+}
+export function classify(err) {
+    // An abort is the caller changing its mind, so it is never retried.
+    if (isAbort(err)) {
+        return 'fatal';
+    }
+    const status = statusOf(err);
+    if (status !== undefined) {
+        if (RATE_LIMITED.has(status)) {
+            return 'rate-limit';
+        }
+        return TRANSIENT.has(status) || status >= 500 ? 'transient' : 'fatal';
+    }
+    for (const link of chain(err)) {
+        if (typeof link.code === 'string' && NETWORK.has(link.code)) {
+            return 'transient';
+        }
+    }
+    return 'fatal';
+}
+export function isAbort(err) {
+    for (const link of chain(err)) {
+        if (link.name === 'AbortError' || link.code === 'ABORT_ERR') {
+            return true;
+        }
+    }
+    return false;
+}
+/**
+ * How long the provider asked to be left alone. Absent far more often than
+ * not — only some vendors send it, and only on some of their limits — so the
+ * limiter has to have a schedule of its own regardless.
+ */
+export function retryAfterMs(err) {
+    for (const link of chain(err)) {
+        const headers = link.headers;
+        if (!headers) {
+            continue;
+        }
+        // OpenAI's own, and the more precise of the two when both are sent.
+        const ms = header(headers, 'retry-after-ms');
+        if (ms !== undefined) {
+            const value = Number(ms);
+            if (Number.isFinite(value) && value >= 0) {
+                return value;
+            }
+        }
+        const after = header(headers, 'retry-after');
+        if (after === undefined) {
+            continue;
+        }
+        // Seconds, or an HTTP date. The date form is rare but legal.
+        const seconds = Number(after);
+        if (Number.isFinite(seconds) && seconds >= 0) {
+            return seconds * 1000;
+        }
+        const at = Date.parse(after);
+        if (Number.isFinite(at)) {
+            return Math.max(0, at - Date.now());
+        }
+    }
+    return undefined;
+}
+/** Headers arrive as a `Headers` instance from one SDK and a plain object from another. */
+function header(headers, name) {
+    if (typeof headers !== 'object' || headers === null) {
+        return undefined;
+    }
+    const get = headers.get;
+    if (typeof get === 'function') {
+        const value = get.call(headers, name);
+        return typeof value === 'string' ? value : undefined;
+    }
+    for (const [key, value] of Object.entries(headers)) {
+        if (key.toLowerCase() === name && typeof value === 'string') {
+            return value;
+        }
+    }
+    return undefined;
+}
+//# sourceMappingURL=rate-limit.js.map

package/dist/failure.d.ts

@@ -0,0 +1,82 @@
+/** What was being asked of a provider, in the runtime's own words. */
+export type Doing = 'llm generation' | 'llm streaming' | 'embedding';
+/** Which call, on whose connection, for which model. */
+export interface CallSite {
+    doing: Doing;
+    /** the SDK call actually made, spelled the way the vendor spells it */
+    api: string;
+    /** the model or embedding id it was spent on */
+    model: string;
+    /** the declared provider name, when a registry built the adapter */
+    provider?: string;
+}
+/**
+ * The half of a call site an adapter knows before it is called, bound once so
+ * that each call names only itself.
+ */
+export type CallSites = (doing: Doing, api: string) => CallSite;
+export declare const callSites: (model: string, provider?: string) => CallSites;
+/**
+ * Carried by every adapter's options so a failure can name the connection.
+ * Reporting only: the client is already built by the time an adapter sees it,
+ * so the name is here to be printed, not to be resolved.
+ */
+export interface ProviderNamed {
+    /** the registered provider name this adapter speaks through */
+    provider?: string;
+}
+/** How a call went before it failed, which is most of what makes a failure readable. */
+export interface Attempted {
+    /**
+     * How many times THIS runtime opened the call. Usually one: every SDK here
+     * is configured to retry the request itself, and all of that happens inside
+     * a single attempt — which is why the elapsed time matters more than the
+     * count. Above one only where a broken stream was reopened.
+     */
+    attempts: number;
+    elapsedMs: number;
+    /**
+     * Chunks delivered before the stream broke. Any at all means the answer had
+     * already started, and a stream that has emitted cannot be replayed: the
+     * deltas are on screen.
+     */
+    chunks?: number;
+}
+/** A provider call that failed, with the call it was. */
+export declare class ProviderError extends Error {
+    readonly site: CallSite;
+    /** the vendor's own sentence, unwrapped out of whatever it arrived in */
+    readonly detail: string;
+    /** the HTTP status, wherever the SDK or the body left it */
+    readonly status?: number;
+    readonly attempts: number;
+    readonly elapsedMs: number;
+    readonly chunks?: number;
+    constructor(site: CallSite, detail: string, status: number | undefined, cause: unknown, tried?: Attempted);
+}
+/** Runs one provider call, and makes sure a refusal says what it refused. */
+export declare function called<T>(site: CallSite, run: () => Promise<T>): Promise<T>;
+/**
+ * The same for a response body: an HTTP error, and every mid-stream failure,
+ * arrives while the stream is being read — long after the call that opened it
+ * returned successfully. None of the SDKs retry that: their backoff wraps the
+ * request, and by the time a body is being consumed the request has succeeded.
+ *
+ * So this does, but only while nothing has come out of it. A connection reset
+ * before the first chunk is indistinguishable from one that never opened, and
+ * reopening is invisible to the caller. One chunk later it is not: the deltas
+ * have been delivered, a second stream would repeat them, and the failure has
+ * to be reported instead.
+ */
+export declare function reading<T>(site: CallSite, source: AsyncIterable<T>, reopen?: () => Promise<AsyncIterable<T>>): AsyncIterable<T>;
+/** The error to throw in place of `err`, which may be `err` itself. */
+export declare function failed(site: CallSite, err: unknown, tried?: Attempted): unknown;
+/**
+ * The sentence a vendor buried, and the status it came with. Exported because
+ * it is useful anywhere a raw provider error has to be shown to someone.
+ */
+export declare function explain(err: unknown): {
+    detail: string;
+    status?: number;
+};
+//# sourceMappingURL=failure.d.ts.map