@hicaru/pi-rlm 0.3.16 → 0.3.18

package/src/util/bm25.ts

@@ -0,0 +1,97 @@
+/**
+ * Host-side Okapi BM25 (Workstreams C/D/E ranking).
+ *
+ * BM25 DUALITY, documented not accidental: this file and `sandbox/py/retrieval.py`
+ * (`_Bm25Index`) implement the SAME scoring for two runtimes — identical constants, identical
+ * tokenizer, identical idf/norm formulas — so a note ranked here lands in the same order the
+ * sandbox-side `search` would put it. Keep the two files in lockstep (AGENTS.md convention).
+ */
+
+const BM25_K1 = 1.2; // mirrors retrieval.py:_BM25_K1
+const BM25_B = 0.75; // mirrors retrieval.py:_BM25_B
+// Mirrors retrieval.py:_TOKEN_SPLIT / _CAMEL_SPLIT: lowercased alphanumeric runs plus
+// camelCase parts, so `resolveModelId`-style identifiers match their prose spellings.
+const TOKEN_SPLIT = /[^0-9A-Za-z]+/;
+const CAMEL_SPLIT = /(?<=[a-z0-9])(?=[A-Z])/;
+
+export function bm25Tokenize(text: string): readonly string[] {
+  const out: string[] = [];
+  for (const raw of text.split(TOKEN_SPLIT)) {
+    if (raw === "") continue;
+    const lowered = raw.toLowerCase();
+    out.push(lowered);
+    if (raw.length > 3) {
+      const parts = raw.split(CAMEL_SPLIT);
+      if (parts.length > 1) {
+        for (const part of parts) {
+          const piece = part.toLowerCase();
+          if (piece !== "" && piece !== lowered) out.push(piece);
+        }
+      }
+    }
+  }
+  return out;
+}
+
+export interface Bm25Entry<T> {
+  readonly item: T;
+  readonly text: string;
+}
+
+export interface Bm25Hit<T> {
+  readonly item: T;
+  readonly score: number;
+}
+
+/**
+ * Rank entries against a query, best first, top-k, score > 0 only.
+ * Pre-allocated score/index arrays; no growth in the scoring loops.
+ */
+export function bm25Rank<T>(
+  query: string,
+  entries: readonly Bm25Entry<T>[],
+  k: number,
+): readonly Bm25Hit<T>[] {
+  const n = entries.length;
+  if (n === 0 || k <= 0) return [];
+  const docLens = new Array<number>(n);
+  const postings = new Map<string, Array<readonly [number, number]>>();
+  for (let i = 0; i < n; i++) {
+    const terms = bm25Tokenize(entries[i].text);
+    docLens[i] = terms.length;
+    const freq = new Map<string, number>();
+    for (const term of terms) freq.set(term, (freq.get(term) ?? 0) + 1);
+    for (const [term, tf] of freq) {
+      const list = postings.get(term);
+      if (list !== undefined) list.push([i, tf]);
+      else postings.set(term, [[i, tf]]);
+    }
+  }
+  let totalLen = 0;
+  for (let i = 0; i < n; i++) totalLen += docLens[i];
+  const avgLen = totalLen > 0 ? totalLen / n : 1.0;
+
+  const scores = new Array<number>(n).fill(0);
+  const seen = new Set<string>();
+  for (const term of bm25Tokenize(query)) {
+    if (seen.has(term)) continue; // Python twin scores set(terms)
+    seen.add(term);
+    const posting = postings.get(term);
+    if (posting === undefined) continue;
+    const df = posting.length;
+    const idf = Math.log(1.0 + (n - df + 0.5) / (df + 0.5));
+    for (const [idx, tf] of posting) {
+      const norm = BM25_K1 * (1.0 - BM25_B + BM25_B * (docLens[idx] / avgLen));
+      scores[idx] += (idf * (tf * (BM25_K1 + 1.0))) / (tf + norm);
+    }
+  }
+
+  const order = Array.from({ length: n }, (_, i) => i);
+  order.sort((a, b) => scores[b] - scores[a] || a - b); // deterministic tie-break
+  const out: Bm25Hit<T>[] = [];
+  for (let i = 0; i < n && out.length < k; i++) {
+    const idx = order[i];
+    if (scores[idx] > 0) out.push({ item: entries[idx].item, score: scores[idx] });
+  }
+  return out;
+}

package/src/util/concurrency.ts

@@ -93,7 +93,7 @@ export function createSubcallGates(leafLimit: number, childLimit: number = leafL
 }
 
 /** Provider-capped config slice (RlmConfig satisfies this structurally). */
-export interface ProviderCapConfig {
+interface ProviderCapConfig {
   readonly maxConcurrentSubcalls: number;
   readonly maxConcurrentChildren: number;
   /** v5: per-provider concurrent-request caps, e.g. { zai: 4 }. Caps only ever LOWER a limit. */

package/src/util/errors.ts

@@ -12,7 +12,7 @@ export function err<T = never, E = string>(error: E): Result<T, E> {
   return { ok: false, error };
 }
 
-export const ERROR_PREFIX = "Error:";
+const ERROR_PREFIX = "Error:";
 
 export function formatError(message: string): string {
   return `${ERROR_PREFIX} ${message}`;

package/src/util/retry.ts

@@ -95,8 +95,10 @@ export interface RetryPolicy {
 }
 
 export const DEFAULT_RETRY_POLICY: Readonly<RetryPolicy> = Object.freeze({
-  maxAttempts: 3,
-  rateLimitMaxAttempts: 8,
+  // DOCTRINE: 15 attempts on the SAME model, then the call fails loudly. No cross-model
+  // or cross-provider fallback exists anywhere in the runtime — by design.
+  maxAttempts: 15,
+  rateLimitMaxAttempts: 15,
   baseDelayMs: 500,
   maxDelayMs: 15_000,
   throttleBaseMs: 2_000,
@@ -104,7 +106,7 @@ export const DEFAULT_RETRY_POLICY: Readonly<RetryPolicy> = Object.freeze({
 });
 
 /** Shape of the optional retry knobs on RlmConfig — kept structural to avoid a cycle. */
-export interface RetryConfigNumbers {
+interface RetryConfigNumbers {
   readonly retryMaxAttempts?: number;
   readonly rateLimitMaxAttempts?: number;
   readonly retryBaseDelayMs?: number;
@@ -142,7 +144,16 @@ export async function completeWithRetry<T>(
     readonly onRelease?: () => void;
   },
 ): Promise<T> {
-  const { policy, provider, signal, onPark, onRelease } = opts;
+  const { policy, provider, signal } = opts;
+  // Bench rec #5: a silent park is indistinguishable from a hang (the campaign watched 15
+  // attempts × 15 cooldown windows with zero output). Callers may own observability via
+  // opts.onPark; otherwise one [rlm]-prefixed warn fires per park — fail-soft, never throws.
+  const onPark =
+    opts.onPark ??
+    ((ms: number): void => {
+      console.warn(`[rlm] ${provider} parked ${Math.round(ms)}ms on provider cooldown`);
+    });
+  const { onRelease } = opts;
   const cooldown = policy.cooldown ?? sharedCooldown;
   let status: number | undefined;
   let headers: Record<string, string> | undefined;
@@ -174,10 +185,14 @@ export async function completeWithRetry<T>(
       }
       if (tries + 1 >= policy.maxAttempts) throw err;
       if (!retryableError(status, msg)) throw err;
-      await sleepMs(
-        Math.min(retryAfterMs(headers) ?? backoffMs(tries, policy.baseDelayMs, policy.maxDelayMs), policy.maxDelayMs),
-        signal,
+      const delay = Math.min(
+        retryAfterMs(headers) ?? backoffMs(tries, policy.baseDelayMs, policy.maxDelayMs),
+        policy.maxDelayMs,
       );
+      console.warn(
+        `[rlm] ${provider} attempt ${tries + 1}/${policy.maxAttempts} failed (${msg.slice(0, 140)}) — retrying in ${delay}ms`,
+      );
+      await sleepMs(delay, signal);
     }
   }
 }

package/src/util/state-merge.ts

@@ -0,0 +1,34 @@
+/**
+ * THE deep-merge with null-deletion (SKILL.state paper §3.2, the `⊕` operator) — one
+ * implementation shared by the RunState patch path (core/run-state.ts, Workstream A) and the
+ * SkillState note merge (config/skillstate.ts, Workstream B). Never inline a second merge.
+ *
+ *   x ⊕ {k: null} = x without k            (delete — must be explicit, never silent)
+ *   x ⊕ {k: v}    = x[k] ⊕ v if both are plain objects (deep merge)
+ *                   else v                  (replace)
+ *
+ * Arrays are replaced wholesale: array semantics (append, index-set, dedup, eviction) are
+ * domain policy and live with the callers that understand them (see core/run-state.ts).
+ * A non-object patch replaces the base as-is.
+ */
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+export function deepMergeWithNullDeletion(base: unknown, patch: unknown): unknown {
+  if (!isPlainObject(base) || !isPlainObject(patch)) return patch;
+  const out: Record<string, unknown> = { ...base };
+  for (const [key, value] of Object.entries(patch)) {
+    if (value === null) {
+      // eslint-disable-next-line @typescript-eslint/no-dynamic-delete -- ⊕ explicit null = delete (paper §3.2 +)
+      delete out[key]; // explicit null = delete (paper §3.2)
+      continue;
+    }
+    const current = out[key];
+    out[key] = isPlainObject(current) && isPlainObject(value)
+      ? deepMergeWithNullDeletion(current, value)
+      : value;
+  }
+  return out;
+}

package/src/util/throttle.ts

@@ -81,7 +81,7 @@ export class ProviderCooldown {
 }
 
 /** Mirrored into DEFAULT_RETRY_POLICY (util/retry.ts imports these — keep one-way). */
-export const THROTTLE_DEFAULTS = Object.freeze({ baseMs: 2_000, maxMs: 60_000 } as const);
+const THROTTLE_DEFAULTS = Object.freeze({ baseMs: 2_000, maxMs: 60_000 } as const);
 
 /** Process-wide instance: every completion in this pi session shares it. */
 export const sharedCooldown: ProviderCooldown = new ProviderCooldown(

package/src/util/type-guards.ts

@@ -0,0 +1,6 @@
+/** Shared runtime type guards — one implementation, imported everywhere (DRY). */
+
+/** True for any non-null object value; the standard probe before field checks. */
+export function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null;
+}