@lloyal-labs/sdk 2.0.0 → 3.0.0

package/dist/Rerank.d.ts

@@ -1,50 +1,184 @@
 import type { SessionContext, RerankProgress } from './types';
+/**
+ * Thrown by {@link Rerank.create} when calibration gates fail (single-token
+ * yes/no, BPE-boundary invariance, boot canary signs). Each instance includes
+ * the specific gate and the empirical evidence so callers can fix at the
+ * right layer (model swap, sentinel change, template drift).
+ */
+export declare class RerankCalibrationError extends Error {
+    readonly name = "RerankCalibrationError";
+}
+/**
+ * Thrown when Rerank's internal invariants are violated (lease exhaustion
+ * mid-query, fork returning a disposed handle, etc.). These represent state
+ * the consumer cannot fix; the diagnostic exists for framework-side triage.
+ */
+export declare class RerankInternalError extends Error {
+    readonly name = "RerankInternalError";
+}
+/**
+ * Truncation event emitted via {@link RerankOpts.onTruncate} when a document's
+ * token count exceeds the per-leaf budget and gets sliced from the head.
+ */
+export interface RerankTruncation {
+    /** Position of the truncated document in the score() / scoreBatch() input array. */
+    docIndex: number;
+    /** Original token count of the document. */
+    origLen: number;
+    /** Tokens kept (slice from start to maxLen). */
+    maxLen: number;
+}
+/**
+ * Construction options for {@link Rerank.create}.
+ */
+export interface RerankOpts {
+    /**
+     * Maximum parallel scoring sequences in the underlying BranchStore.
+     * Effective per-group leaf budget is `nSeqMax - 2` because the warm trunk
+     * and per-query branch each hold one lease. Default 10 (= 8 leaves), which
+     * preserves the prior default leaf width while introducing the warm-trunk
+     * lease.
+     */
+    nSeqMax?: number;
+    /**
+     * Per-context KV budget. Defaults to the value reported by the underlying
+     * SessionContext. Per-sequence token budget is `floor(nCtx / nSeqMax)`.
+     */
+    nCtx?: number;
+    /**
+     * Optional callback invoked once per document whose tokens exceeded the
+     * per-leaf budget. Consumers can forward to a trace surface
+     * (`rerank:truncate`) or to a metric. Silent in the SDK by default.
+     */
+    onTruncate?: (event: RerankTruncation) => void;
+}
+/**
+ * Cross-encoder reranker composed over the SDK's Branch / BranchStore primitives.
+ *
+ * # Lifetime + concurrency contract
+ *
+ * Rerank takes **exclusive ownership** of its SessionContext. Routing any other
+ * decode through the same context concurrently is undefined behavior — the
+ * kernel's `llama_context::decode` carries no internal mutex (verified at
+ * llama.cpp b9581). Enforcement is at construction: `Rerank.create()` marks
+ * the context with a `__decodeOwner` flag and refuses a second instance.
+ * The flag is cleared by `dispose()`, so test/REPL re-creation works.
+ *
+ * Concurrent `score()` / `scoreBatch()` calls **on the same Rerank instance**
+ * are serialized by a per-instance Promise chain (~10 LOC). The kernel sees
+ * them in arrival order; consumers still get a concurrent-looking API.
+ *
+ * # Architecture
+ *
+ *   [SYSTEM][USER_PREFIX][QUERY][MID][DOC_i][SUFFIX][GEN_PROMPT]
+ *   └── permanent trunk ─┘ └── per-query branch ──┘ └─── per-chunk leaves ─┘
+ *
+ * - **trunk**: prefilled with the static [SYSTEM][USER_PREFIX] segment ONCE
+ *   at `Rerank.create()`; lives for the instance lifetime. Warm KV is
+ *   amortized across every score() via multi-tag KV survival.
+ * - **queryBranch**: forked from trunk per score() call, prefilled with
+ *   `[query, ...midTokens]`. Forked with `cloneLogits: false` because we
+ *   immediately overwrite the logits with the prefill.
+ * - **leaves**: forked from queryBranch in groups of `BranchStore.available`,
+ *   scatter-prefilled with `[doc_i, ...suffixTokens]` via `BranchStore.prefill`
+ *   (one `llama_decode` per group), scored via `_branchLogitsAt` reading
+ *   exactly two floats per leaf, pruned.
+ *
+ * # Calibration gates (fail-loud at create() time)
+ *
+ *   1. `yes` and `no` must tokenize as single tokens (the score formula
+ *      `logit("yes") − logit("no")` assumes this; broader support requires
+ *      log-sum-exp over label sequences, a 3.x ticket).
+ *   2. BPE-boundary invariance — tokenizing a canary full prompt must equal
+ *      the concat of (prefix, query, mid, doc, suffix) tokenized separately,
+ *      so segment seams don't silently shift the leaf prompts.
+ *   3. Boot canary — score a known relevant + irrelevant pair; relevant
+ *      must outscore irrelevant by > 1.0 logit unit. Asserts the *gap*,
+ *      NOT absolute signs — quantization shifts calibration enough that
+ *      sign assertions are brittle, while the ordering gap still catches
+ *      yes/no token swap, model swap, and template drift.
+ *
+ * # Score formula
+ *
+ *   score = `logit("yes") − logit("no")` (unbounded).
+ *
+ *   **This is the log-odds of an absolute yes/no relevance judgment.** The
+ *   model is a pointwise binary cross-encoder; the official Qwen3-Reranker
+ *   score is the two-token softmax over {yes,no} — i.e. `sigmoid(score)` =
+ *   P(yes) ∈ [0,1] — and our log-odds is its monotone equivalent (identical
+ *   rankings, full dynamic range). Scores ARE thresholdable (0 ≡ P 0.5) and
+ *   comparable across queries to the extent of the model's calibration;
+ *   quantization adds noise at the extremes. Top-1 routinely goes negative
+ *   on real corpora when no document is strongly relevant — an honest
+ *   "probably not", with the ranking still useful. Production traces show
+ *   top-1 ranging from +10 (P≈.9999) to -3 (P≈.05, weak best match).
+ *
+ *   The previous softmax form compressed small logit gaps into extreme
+ *   probabilities (gap of 5 → 0.993; gap of 10 → 0.99995), saturating top-K
+ *   ordering. Logit-diff preserves the full dynamic range. See
+ *   `reasoning.run/scripts/inspect-rerank.mjs` for empirical evidence.
+ *
+ *   Consumers that want a confidence threshold should calibrate against their
+ *   own corpus rather than assuming `> 0` means "relevant" — see SearchTool's
+ *   threshold envelope.
+ */
 export declare class Rerank {
     private _ctx;
+    private _store;
+    private _trunk;
     private _nSeqMax;
     private _nCtx;
     private _yesId;
     private _noId;
-    private _prefixTokens;
     private _midTokens;
     private _suffixTokens;
-    private _pending;
-    private _draining;
+    private _staticPrefix;
+    private _onTruncate?;
+    private _inflight;
     private _disposed;
     private constructor();
     /**
-     * Create a Rerank instance from a pre-created SessionContext
+     * Create a Rerank instance bound to a pre-created SessionContext.
      *
-     * The caller is responsible for creating the context with appropriate
-     * settings (nSeqMax, nCtx, typeK, typeV). Rerank takes ownership of
-     * the context and will dispose it on `dispose()`.
+     * Rerank takes exclusive ownership of `ctx` (see class docstring). The
+     * caller must construct `ctx` with `nSeqMax` ≥ 3 (one slot each for trunk
+     * + queryBranch + at least one leaf).
      *
-     * @param ctx - SessionContext configured for reranking
-     * @param opts - Capacity hints (nSeqMax, nCtx) — must match context creation params
+     * Fires three calibration gates at boot. If any gate fails, throws
+     * {@link RerankCalibrationError} with a diagnostic naming the failure and
+     * cleans up partial state (no ctx leak).
      */
-    static create(ctx: SessionContext, opts?: {
-        nSeqMax?: number;
-        nCtx?: number;
-    }): Promise<Rerank>;
-    score(query: string, documents: number[][], topK?: number): AsyncIterable<RerankProgress>;
+    static create(ctx: SessionContext, opts?: RerankOpts): Promise<Rerank>;
     /**
-     * Score raw text strings against a query in one batch.
+     * Stream progressive ranking results for `documents` against `query`.
      *
-     * Tokenizes texts synchronously, builds reranker prompt arrays, and
-     * dispatches via `_scoreGroup` for parallel cross-encoder scoring.
-     * Up to `nSeqMax` texts are scored per batch call.
+     * Pre-tokenized documents must come from {@link tokenize} or a reranker-
+     * compatible tokenizer; mismatched tokenizers silently produce wrong scores.
      *
-     * @param query - Reference query to score against
-     * @param texts - Raw text strings to score
-     * @returns Scores (0–1) in input order
+     * Consumers may cancel by calling `iterator.return()` directly or by
+     * `for-await break`. Cancellation bounds the post-cancel cost at the one
+     * leaf group already in flight; subsequent groups are skipped.
+     */
+    score(query: string, documents: number[][], topK?: number): AsyncIterable<RerankProgress>;
+    /**
+     * Batch-score raw text strings against a query. Returns logit-diff scores
+     * (unbounded; positive = "yes", negative = "no", magnitude = confidence) in
+     * input order.
      */
     scoreBatch(query: string, texts: string[]): Promise<number[]>;
+    /** Tokenize text using the reranker's underlying tokenizer. */
     tokenize(text: string): Promise<number[]>;
+    /** Release Rerank state, clear ctx ownership, dispose ctx. Idempotent. */
     dispose(): void;
-    private _sortResults;
-    private _enqueue;
-    private _fillGroup;
-    private _drain;
-    private _rerankScore;
+    /**
+     * The shared scoring driver. Both `score()` (async-iterable) and
+     * `scoreBatch()` (Promise) call into this once the serializer is held.
+     */
+    private _scoreInternal;
+    /**
+     * Sort scores descending, raw (unrounded). Consumers that want display
+     * rounding apply `Math.round(score * 1000) / 1000` themselves.
+     */
+    private _sortRaw;
 }
 //# sourceMappingURL=Rerank.d.ts.map

package/dist/Rerank.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Rerank.d.ts","sourceRoot":"","sources":["../src/Rerank.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAgB,cAAc,EAAE,MAAM,SAAS,CAAC;AAmE5E,qBAAa,MAAM;IACjB,OAAO,CAAC,IAAI,CAAiB;IAC7B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,aAAa,CAAW;IAChC,OAAO,CAAC,UAAU,CAAW;IAC7B,OAAO,CAAC,aAAa,CAAW;IAChC,OAAO,CAAC,QAAQ,CAAwB;IACxC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,SAAS,CAAS;IAE1B,OAAO;IAoBP;;;;;;;;;OASG;WACU,MAAM,CAAC,GAAG,EAAE,cAAc,EAAE,IAAI,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAwBrG,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,cAAc,CAAC;IA0BzF;;;;;;;;;;OAUG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAqB7D,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAI/C,OAAO,IAAI,IAAI;IAUf,OAAO,CAAC,YAAY;IAOpB,OAAO,CAAC,QAAQ;IAkBhB,OAAO,CAAC,UAAU;YAiBJ,MAAM;IA+CpB,OAAO,CAAC,YAAY;CAMrB"}
+{"version":3,"file":"Rerank.d.ts","sourceRoot":"","sources":["../src/Rerank.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAgB,cAAc,EAAE,MAAM,SAAS,CAAC;AA2B5E;;;;;GAKG;AACH,qBAAa,sBAAuB,SAAQ,KAAK;IAC/C,QAAQ,CAAC,IAAI,4BAA4B;CAC1C;AAED;;;;GAIG;AACH,qBAAa,mBAAoB,SAAQ,KAAK;IAC5C,QAAQ,CAAC,IAAI,yBAAyB;CACvC;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,oFAAoF;IACpF,QAAQ,EAAE,MAAM,CAAC;IACjB,4CAA4C;IAC5C,OAAO,EAAE,MAAM,CAAC;IAChB,gDAAgD;IAChD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;CAChD;AAyED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqEG;AACH,qBAAa,MAAM;IACjB,OAAO,CAAC,IAAI,CAAiB;IAC7B,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,UAAU,CAAW;IAC7B,OAAO,CAAC,aAAa,CAAW;IAChC,OAAO,CAAC,aAAa,CAAW;IAChC,OAAO,CAAC,WAAW,CAAC,CAAoC;IACxD,OAAO,CAAC,SAAS,CAAoC;IACrD,OAAO,CAAC,SAAS,CAAS;IAE1B,OAAO;IA0BP;;;;;;;;;;OAUG;WACU,MAAM,CAAC,GAAG,EAAE,cAAc,EAAE,IAAI,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;IA2K5E;;;;;;;;;OASG;IACH,KAAK,CACH,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EAAE,EAAE,EACrB,IAAI,CAAC,EAAE,MAAM,GACZ,aAAa,CAAC,cAAc,CAAC;IAgChC;;;;OAIG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IA0CnE,+DAA+D;IACzD,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAI/C,0EAA0E;IAC1E,OAAO,IAAI,IAAI;IAcf;;;OAGG;YACW,cAAc;IAoJ5B;;;OAGG;IACH,OAAO,CAAC,QAAQ;CAMjB"}