auto-model-router 0.2.2 → 0.2.8

package/src/config/types.ts

@@ -139,14 +139,21 @@ export interface FilterConfig {
 	 */
 	contextHeadroom: number;
 	/**
-	 * How hard to penalise slow models in candidate scoring. A model's mean
-	 * time-to-first-token above `latencyReferenceMs` inflates its effective
-	 * cost — the same lever trust uses for flakiness — so a faster model of
-	 * equal quality and price wins. 0 disables latency scoring entirely.
+	 * How hard to penalise slow models in candidate scoring. A model's expected
+	 * total wait — TTFT plus streaming the expected completion at its measured
+	 * throughput — above the reference inflates its effective cost, the same lever
+	 * trust uses for flakiness, so a faster model of equal quality and price wins.
+	 * 0 disables latency scoring entirely.
 	 */
 	latencyWeight: number;
-	/** TTFT (ms) below which a model is considered fully responsive (no penalty). */
+	/** Reference TTFT (ms): the start-latency component that accrues no penalty. */
 	latencyReferenceMs: number;
+	/**
+	 * Reference throughput (tokens/second): the streaming speed that accrues no
+	 * penalty. Below it, a slow-streaming model's expected wait exceeds the
+	 * reference and its effective cost is inflated.
+	 */
+	latencyReferenceTokensPerSec: number;
 	/** Streamed samples required before latency is scored against a model. */
 	latencyMinSamples: number;
 }
@@ -326,6 +333,71 @@ export interface LedgerConfig {
 	conversationTtlMs: number;
 }
 
+/**
+ * agentdox bridge: one shared project context that follows a conversation
+ * across model switches, plus a model-attributed transcript written back.
+ *
+ * Off by default. Enabling it injects a project-context block into the system
+ * prefix; the block is PINNED per conversation and refreshed only when the
+ * prompt cache is already cold (see `src/context/bridge.ts`), so sharing
+ * context does not cost a cache miss every turn.
+ */
+export interface ContextConfig {
+	enabled: boolean;
+	/** agentdox REST base URL, e.g. `http://localhost:3003`. */
+	baseUrl: string;
+	/** Bearer token with read+write on the project scope. From `AGENTDOX_TOKEN`. */
+	token: string;
+	/**
+	 * Fallback project scope when a request carries no `X-Agentdox-Scope`
+	 * header. Empty ⇒ the bridge is inert for unlabelled requests rather than
+	 * guessing, so one harness cannot leak context into another's project.
+	 */
+	defaultScope: string;
+	/** Per-request timeout against agentdox, ms. */
+	timeoutMs: number;
+	/**
+	 * Upper bound on how stale a pinned context block may get, ms. Reaching it
+	 * forces a refresh on the next turn even if the model did not change —
+	 * the one case where the bridge knowingly spends a cache miss. 0 disables
+	 * the TTL, refreshing only on turns whose cache is already forfeit.
+	 */
+	maxStalenessMs: number;
+	/** Hard cap on injected block size, characters. */
+	maxBlockChars: number;
+	/** Write settled turns back to agentdox sessions, tagged with the served model. */
+	recordTurns: boolean;
+	/** Bound on queued write-backs; excess turns are dropped, never buffered unbounded. */
+	maxQueue: number;
+}
+
+/**
+ * Context optimization (compaction): before dispatch, shrink stale, low-value
+ * bulk — chiefly old tool output — so long agentic conversations cost less and
+ * keep fitting narrower-window models. Deterministic and reversible-by-reference:
+ * every elision leaves an in-band breadcrumb so the model can re-run the tool.
+ * Off by default. See docs/context-optimization.md.
+ */
+export interface CompactionConfig {
+	enabled: boolean;
+	/** Compact when the estimated prompt exceeds this many tokens. */
+	budgetTokens: number;
+	/** Also compact when the prompt would overflow the profile's context window. */
+	fitToWindow: boolean;
+	/** Never touch the last N user/assistant turns or the volatile tail. */
+	protectRecentTurns: number;
+	/** Tool results larger than this (outside the protected window) are truncated. */
+	maxToolResultBytes: number;
+	/** Bytes of a truncated tool result's head to keep. */
+	keepHeadBytes: number;
+	/** Bytes of a truncated tool result's tail to keep. */
+	keepTailBytes: number;
+	/** Elide an older tool result when a newer call to the same resource supersedes it. */
+	elideSupersededReads: boolean;
+	/** Collapse byte-identical repeated tool results to a single copy. */
+	collapseDuplicateResults: boolean;
+}
+
 export interface RouterConfig {
 	server: ServerConfig;
 	openrouter: OpenRouterConfig;
@@ -338,6 +410,8 @@ export interface RouterConfig {
 	hysteresis: HysteresisConfig;
 	exploration: ExplorationConfig;
 	cache: CacheConfig;
+	context: ContextConfig;
+	compaction: CompactionConfig;
 	budget: BudgetConfig;
 	profiles: ProfileConfig[];
 	ledger: LedgerConfig;

package/src/context/agentdox.ts

@@ -0,0 +1,113 @@
+/**
+ * agentdox REST client.
+ *
+ * Deliberately tiny and total: every method resolves to a value or null and
+ * never throws. agentdox is an ENRICHMENT, not a dependency — if it is down,
+ * slow, or unauthorized, the turn must still route and dispatch normally.
+ */
+
+import type { Logger } from "../util/log.ts";
+
+export interface AgentDoxClientOptions {
+	baseUrl: string;
+	token: string;
+	timeoutMs: number;
+	log: Logger;
+}
+
+export interface AgentDoxClient {
+	/**
+	 * Assembles a context slice for `scope`, biased by `query`. Falls back to
+	 * the server's pre-assembled baseline when assembly is unavailable (older
+	 * server, or no query-relevant content).
+	 */
+	assemble(scope: string, query: string): Promise<string | null>;
+	createSession(scope: string, title: string): Promise<string | null>;
+	append(sessionId: string, role: "user" | "assistant", content: string, refs: string[]): Promise<boolean>;
+}
+
+export function createAgentDoxClient(opts: AgentDoxClientOptions): AgentDoxClient {
+	const { baseUrl, token, timeoutMs, log } = opts;
+	const root = baseUrl.replace(/\/+$/, "");
+
+	const request = async (
+		method: string,
+		path: string,
+		body?: unknown,
+	): Promise<{ status: number; json: unknown } | null> => {
+		const ctl = new AbortController();
+		const timer = setTimeout(() => ctl.abort(), timeoutMs);
+		try {
+			const headers: Record<string, string> = { authorization: `Bearer ${token}` };
+			if (body !== undefined) headers["content-type"] = "application/json";
+			const res = await fetch(`${root}${path}`, {
+				method,
+				headers,
+				signal: ctl.signal,
+				...(body === undefined ? {} : { body: JSON.stringify(body) }),
+			});
+			// A 404 is meaningful data (no snapshot/brief yet), not a failure.
+			const text = await res.text();
+			let parsed: unknown = null;
+			try {
+				parsed = text === "" ? null : JSON.parse(text);
+			} catch {
+				parsed = null;
+			}
+			return { status: res.status, json: parsed };
+		} catch (err) {
+			// Timeouts, connection refused, DNS — all the same to the caller.
+			log.debug("agentdox request failed", {
+				method,
+				path,
+				error: err instanceof Error ? err.message : String(err),
+			});
+			return null;
+		} finally {
+			clearTimeout(timer);
+		}
+	};
+
+	const promptOf = (json: unknown): string | null => {
+		if (typeof json !== "object" || json === null) return null;
+		const p = (json as { prompt?: unknown }).prompt;
+		return typeof p === "string" && p.trim() !== "" ? p : null;
+	};
+
+	return {
+		async assemble(scope, query) {
+			const res = await request("POST", "/context/assemble", { scope, query });
+			if (res !== null && res.status === 200) {
+				const prompt = promptOf(res.json);
+				if (prompt !== null) return prompt;
+			}
+			if (res !== null && (res.status === 401 || res.status === 403)) {
+				log.warn("agentdox rejected the router token; context injection is off for this scope", {
+					scope,
+					status: res.status,
+				});
+				return null;
+			}
+			// Baseline fallback: the server-side auto-context job keeps this fresh.
+			const snap = await request("GET", `/context/snapshot?scope=${encodeURIComponent(scope)}`);
+			if (snap === null || snap.status !== 200) return null;
+			return promptOf(snap.json);
+		},
+
+		async createSession(scope, title) {
+			const res = await request("POST", "/sessions", { scope, title });
+			if (res === null || (res.status !== 200 && res.status !== 201)) return null;
+			const id = (res.json as { id?: unknown } | null)?.id;
+			return typeof id === "string" ? id : null;
+		},
+
+		async append(sessionId, role, content, refs) {
+			const res = await request("POST", `/sessions/${encodeURIComponent(sessionId)}/messages`, {
+				role,
+				content,
+				refs,
+			});
+			return res !== null && (res.status === 200 || res.status === 201);
+		},
+	};
+}

package/src/context/bridge.ts

@@ -0,0 +1,166 @@
+/**
+ * The refresh policy, which is the whole design.
+ *
+ * Injecting fresh context every turn would defeat the router's single largest
+ * cost lever: the prompt cache. The block sits at the front of the prefix, so
+ * changing it invalidates everything after it. The rule is therefore to
+ * re-fetch ONLY when the prefix is already being paid for:
+ *
+ *   - no pin yet          first turn; nothing is warm
+ *   - model switching     the router already decided to forfeit the cache
+ *   - retrying            an escalation/failover dispatch is cold by definition
+ *   - staleness TTL       a bounded upper limit on how old context may get
+ *
+ * Between those moments the same bytes are re-injected verbatim and the cache
+ * survives. This is what makes "context refreshes as I switch models" cheap
+ * rather than ruinous: the refresh rides on a cache miss that was happening
+ * anyway.
+ */
+
+import { sha256Hex } from "../util/hash.ts";
+import type { Logger } from "../util/log.ts";
+import type { AgentDoxClient } from "./agentdox.ts";
+import type { ContextBlockStore, ContextBridge, ContextPin, ContextResolveInput, TurnRecord } from "./types.ts";
+
+export interface BridgeOptions {
+	client: AgentDoxClient;
+	store: ContextBlockStore;
+	log: Logger;
+	/** Upper bound on pinned-context age, ms. 0 ⇒ refresh only on cache-cold turns. */
+	maxStalenessMs: number;
+	/** Hard cap on injected block size; a runaway context must not dominate the prompt. */
+	maxBlockChars: number;
+	/** Record settled turns back into agentdox sessions. */
+	recordTurns: boolean;
+	/** Bound on queued write-backs; excess is dropped rather than grown unbounded. */
+	maxQueue: number;
+}
+
+/** Wraps the raw agentdox slice in a delimiter the model can reason about. */
+function renderBlock(raw: string, maxChars: number): string {
+	const body = raw.length > maxChars ? `${raw.slice(0, maxChars)}\n[...truncated]` : raw;
+	return [
+		"<project-context source=\"agentdox\">",
+		"Durable project memory, documentation, and history shared across every model that",
+		"serves this conversation. Treat it as established fact; prefer it over re-deriving.",
+		"",
+		body,
+		"</project-context>",
+	].join("\n");
+}
+
+export function createContextBridge(opts: BridgeOptions): ContextBridge {
+	const { client, store, log, maxStalenessMs, maxBlockChars, recordTurns, maxQueue } = opts;
+
+	// Serialized write-back queue. Session appends for one conversation must
+	// stay ordered, and agentdox is a local service — one worker is plenty.
+	let queue: Promise<void> = Promise.resolve();
+	let queued = 0;
+	let closed = false;
+
+	const shouldRefresh = (input: ContextResolveInput, pin: ContextPin | null): boolean => {
+		if (pin === null) return true;
+		if (input.modelSwitching || input.retrying) return true;
+		// Staleness is measured from when THIS conversation last refreshed, not
+		// from the shared block row: another conversation confirming the same
+		// content must not silently extend our TTL. The re-check is cheap
+		// anyway — identical content hashes to the same version, so a
+		// confirming refresh re-sends identical bytes and the cache survives.
+		if (maxStalenessMs > 0 && Date.now() - input.pinnedFetchedAtMs > maxStalenessMs) return true;
+		return false;
+	};
+
+	return {
+		enabled: true,
+
+		async resolve(input) {
+			if (input.scope === "") return null;
+
+			const pinned = input.pinnedVersion === null ? null : store.get(input.pinnedVersion);
+			if (!shouldRefresh(input, pinned) && pinned !== null) {
+				// Carry the conversation's own pin time forward, so the TTL keeps
+				// counting from its last real refresh rather than resetting to
+				// whenever some other conversation last touched this block.
+				return { ...pinned, fetchedAtMs: input.pinnedFetchedAtMs };
+			}
+
+			const raw = await client.assemble(input.scope, input.query);
+			if (raw === null) {
+				// agentdox unreachable or empty. Keep serving the pinned block if we
+				// have one: stale shared context beats none, and re-using it also
+				// keeps the prefix stable.
+				return pinned;
+			}
+
+			const block = renderBlock(raw, maxBlockChars);
+			// Content hash, not a timestamp: agentdox re-assembles on a timer, and
+			// an unchanged assembly MUST keep its version so the cache survives.
+			const version = sha256Hex(block).slice(0, 32);
+			const pin: ContextPin = { version, block, fetchedAtMs: Date.now() };
+			try {
+				store.put(input.scope, pin);
+			} catch (err) {
+				log.debug("context block persist failed", { error: err instanceof Error ? err.message : String(err) });
+			}
+			if (pinned !== null && pinned.version !== version) {
+				log.debug("context refreshed", {
+					scope: input.scope,
+					from: pinned.version.slice(0, 8),
+					to: version.slice(0, 8),
+					reason: input.retrying ? "retry" : input.modelSwitching ? "model-switch" : "stale",
+				});
+			}
+			return pin;
+		},
+
+		recordTurn(rec: TurnRecord) {
+			if (!recordTurns || closed || rec.scope === "") return;
+			if (rec.userText === "" && rec.assistantText === "") return;
+			if (queued >= maxQueue) {
+				log.debug("agentdox write-back queue full; dropping turn record", { queued });
+				return;
+			}
+			queued++;
+			queue = queue
+				.then(async () => {
+					let sessionId = store.sessionFor(rec.conversationKey);
+					if (sessionId === null) {
+						sessionId = await client.createSession(rec.scope, rec.title);
+						if (sessionId === null) return;
+						store.bindSession(rec.conversationKey, rec.scope, sessionId);
+					}
+					// Model attribution rides on refs, which agentdox already carries
+					// per message. This is what makes the transcript newly useful:
+					// every turn shows WHICH model produced it.
+					const refs = [`model:${rec.slug}`, `tier:${rec.tier}`];
+					if (rec.userText !== "") await client.append(sessionId, "user", rec.userText, []);
+					if (rec.assistantText !== "") await client.append(sessionId, "assistant", rec.assistantText, refs);
+				})
+				.catch((err: unknown) => {
+					log.debug("agentdox write-back failed", { error: err instanceof Error ? err.message : String(err) });
+				})
+				.finally(() => {
+					queued--;
+				});
+		},
+
+		async flush() {
+			await queue;
+		},
+
+		close() {
+			closed = true;
+		},
+	};
+}
+
+/** The inert bridge used when agentdox is not configured. Every call is free. */
+export function createDisabledBridge(): ContextBridge {
+	return {
+		enabled: false,
+		resolve: async () => null,
+		recordTurn: () => {},
+		flush: async () => {},
+		close: () => {},
+	};
+}

package/src/context/index.ts

@@ -0,0 +1,33 @@
+/**
+ * Bridge factory. Returns the inert bridge unless agentdox is fully
+ * configured, so every call site can stay unconditional.
+ */
+
+import type { Database } from "bun:sqlite";
+
+import type { RouterConfig } from "../config/types.ts";
+import { createLogger } from "../util/log.ts";
+import { createAgentDoxClient } from "./agentdox.ts";
+import { createContextBridge, createDisabledBridge } from "./bridge.ts";
+import { createContextStore } from "./store.ts";
+import type { ContextBridge } from "./types.ts";
+
+export type { ContextBridge, ContextPin, ContextResolveInput, TurnRecord } from "./types.ts";
+export { createContextBridge, createDisabledBridge } from "./bridge.ts";
+export { createContextStore } from "./store.ts";
+export { createAgentDoxClient } from "./agentdox.ts";
+
+export function createBridgeFromConfig(cfg: RouterConfig, db: Database): ContextBridge {
+	const c = cfg.context;
+	if (!c.enabled || c.baseUrl === "" || c.token === "") return createDisabledBridge();
+	const log = createLogger(cfg.logLevel);
+	return createContextBridge({
+		client: createAgentDoxClient({ baseUrl: c.baseUrl, token: c.token, timeoutMs: c.timeoutMs, log }),
+		store: createContextStore(db),
+		log,
+		maxStalenessMs: c.maxStalenessMs,
+		maxBlockChars: c.maxBlockChars,
+		recordTurns: c.recordTurns,
+		maxQueue: c.maxQueue,
+	});
+}

package/src/context/store.ts

@@ -0,0 +1,82 @@
+/**
+ * Persistence for the agentdox bridge.
+ *
+ * Blocks are content-addressed by version and shared across conversations, so
+ * ten conversations on one project store one copy. Persisting them (rather
+ * than caching in memory) means a router restart can re-inject the SAME bytes
+ * a conversation was already using — OpenRouter's prompt cache outlives our
+ * process, and re-fetching would needlessly change the prefix.
+ *
+ * Tables are created by `util/sqlite.ts`, the single migration path.
+ */
+
+import type { Database, Statement } from "bun:sqlite";
+
+import type { ContextBlockStore, ContextPin } from "./types.ts";
+
+interface BlockRow {
+	version: string;
+	block: string;
+	fetched_at_ms: number;
+}
+
+interface SessionRow {
+	session_id: string;
+}
+
+export function createContextStore(db: Database): ContextBlockStore {
+	// Hoisted: these run on the turn hot path.
+	const selectBlock: Statement<BlockRow, [string]> = db.query(
+		"SELECT version, block, fetched_at_ms FROM context_blocks WHERE version = ?",
+	);
+	const insertBlock = db.query(`
+		INSERT INTO context_blocks (version, scope, block, fetched_at_ms)
+		VALUES ($version, $scope, $block, $fetchedAtMs)
+		ON CONFLICT(version) DO UPDATE SET fetched_at_ms = excluded.fetched_at_ms
+	`);
+	const selectSession: Statement<SessionRow, [string]> = db.query(
+		"SELECT session_id FROM agentdox_sessions WHERE conversation_key = ?",
+	);
+	const insertSession = db.query(`
+		INSERT INTO agentdox_sessions (conversation_key, scope, session_id, created_at_ms)
+		VALUES ($key, $scope, $sessionId, $createdAtMs)
+		ON CONFLICT(conversation_key) DO UPDATE SET session_id = excluded.session_id
+	`);
+	const deleteStale: Statement<unknown, [number]> = db.query("DELETE FROM context_blocks WHERE fetched_at_ms < ?");
+
+	return {
+		get(version) {
+			const row = selectBlock.get(version);
+			if (row === null) return null;
+			return { version: row.version, block: row.block, fetchedAtMs: row.fetched_at_ms };
+		},
+
+		put(scope, pin: ContextPin) {
+			insertBlock.run({
+				$version: pin.version,
+				$scope: scope,
+				$block: pin.block,
+				$fetchedAtMs: pin.fetchedAtMs,
+			});
+		},
+
+		sessionFor(conversationKey) {
+			const row = selectSession.get(conversationKey);
+			return row === null ? null : row.session_id;
+		},
+
+		bindSession(conversationKey, scope, sessionId) {
+			insertSession.run({
+				$key: conversationKey,
+				$scope: scope,
+				$sessionId: sessionId,
+				$createdAtMs: Date.now(),
+			});
+		},
+
+		prune(maxAgeMs) {
+			const res = deleteStale.run(Date.now() - maxAgeMs) as unknown as { changes?: number };
+			return res.changes ?? 0;
+		},
+	};
+}

package/src/context/types.ts

@@ -0,0 +1,78 @@
+/**
+ * agentdox bridge — one shared project context that survives model switches.
+ *
+ * The router is the only choke point that sees every model in every harness,
+ * so injecting project memory/docs/brief HERE gives every candidate model the
+ * same context without per-harness MCP wiring.
+ *
+ * The load-bearing constraint is the prompt cache. A context block that
+ * changes per turn sits at the front of the prefix and turns every turn into a
+ * full cache miss — which would cost far more than routing saves. So a block
+ * is PINNED per conversation and only re-fetched at moments the prefix is
+ * already forfeit (a model switch, an escalation retry) or after a staleness
+ * TTL. See `bridge.ts` for the policy.
+ */
+
+/** A pinned, prompt-ready context block. */
+export interface ContextPin {
+	/**
+	 * Content hash of `block`. Deliberately NOT agentdox's `assembledAt`: the
+	 * server re-assembles on a timer, so a timestamp changes even when nothing
+	 * about the content did, and that would break a warm cache for no reason.
+	 */
+	version: string;
+	/** Text appended to the system prefix by the wire. */
+	block: string;
+	fetchedAtMs: number;
+}
+
+export interface ContextResolveInput {
+	/** agentdox project slug. Empty ⇒ the bridge is inert for this turn. */
+	scope: string;
+	conversationKey: string;
+	/** Version pinned by the previous turn; null on a fresh conversation. */
+	pinnedVersion: string | null;
+	/** When that pin was fetched, for the staleness TTL. */
+	pinnedFetchedAtMs: number;
+	/** This turn dispatches to a different slug than the last committed turn. */
+	modelSwitching: boolean;
+	/** An escalation or failover retry — the prefix is cold either way. */
+	retrying: boolean;
+	/** Latest user text, used to bias agentdox relevance ranking. */
+	query: string;
+}
+
+/** One settled turn, recorded to agentdox with the model that served it. */
+export interface TurnRecord {
+	scope: string;
+	conversationKey: string;
+	/** Title used if this is the first turn and a session must be created. */
+	title: string;
+	userText: string;
+	assistantText: string;
+	/** The slug that actually served the turn — the model attribution. */
+	slug: string;
+	tier: string;
+}
+
+export interface ContextBridge {
+	/** False when no agentdox URL/token is configured; every call is then a no-op. */
+	readonly enabled: boolean;
+	/** Resolves the block to inject. Never throws: agentdox being down degrades to null. */
+	resolve(input: ContextResolveInput): Promise<ContextPin | null>;
+	/** Queues a turn for write-back. Returns immediately; never blocks the turn. */
+	recordTurn(rec: TurnRecord): void;
+	/** Drains the write queue. For tests and shutdown. */
+	flush(): Promise<void>;
+	close(): void;
+}
+
+/** Content-addressed store of fetched blocks, so a restart keeps a warm prefix. */
+export interface ContextBlockStore {
+	get(version: string): ContextPin | null;
+	put(scope: string, pin: ContextPin): void;
+	/** agentdox session id previously opened for a conversation. */
+	sessionFor(conversationKey: string): string | null;
+	bindSession(conversationKey: string, scope: string, sessionId: string): void;
+	prune(maxAgeMs: number): number;
+}