@namzu/sdk 34.2.0 → 35.0.0

package/src/compaction/plan.ts

@@ -1,7 +1,10 @@
 import type { CompactionConfig } from '../config/runtime.js'
-import { CHARS_PER_TOKEN } from '../constants/limits.js'
 import type { Message } from '../types/message/index.js'
 import { findSafeTrimIndex } from './dangling.js'
+import { buildGoal } from './salience/goal.js'
+import { scoreMessages } from './salience/score.js'
+import { type WorkingSetPlan, planWorkingSet } from './salience/working-set.js'
+import { estimateMessageTokens, estimateMessagesTokens } from './token-estimate.js'
 import { clearStaleToolResults } from './tool-result-editing.js'
 
 /**
@@ -43,6 +46,8 @@ export type CompactionPlan =
 			readonly kind: 'cleared'
 			readonly messages: readonly Message[]
 			readonly clearedCount: number
+			/** Assistant narrations cut to their first sentence by the salience pass. */
+			readonly stubbedCount?: number
 			readonly charsReclaimed: number
 			readonly reclaimedTokens: number
 			readonly reliefWasEnough: boolean
@@ -86,24 +91,6 @@ export interface CompactionPlanInput {
 	readonly skipToolResultClear?: boolean
 }
 
-/**
- * How many characters a message's content is worth.
- *
- * An image costs far fewer tokens than its base64 length divided by four —
- * but under-counting it to zero is the worse error: it let a run full of
- * screenshots read as an empty context.
- */
-function measureContentChars(content: unknown): number {
-	if (typeof content === 'string') return content.length
-	if (!Array.isArray(content)) return 0
-	let total = 0
-	for (const block of content as readonly Record<string, unknown>[]) {
-		if (block.type === 'text' && typeof block.text === 'string') total += block.text.length
-		else if (block.type === 'image' && typeof block.data === 'string') total += block.data.length
-	}
-	return total
-}
-
 /**
  * The last index whose tail fits in `budgetTokens`, walking backwards.
  *
@@ -125,7 +112,7 @@ export function naiveKeepStartByTokens(messages: readonly Message[], budgetToken
 	let tokens = 0
 	let start = messages.length
 	for (let index = messages.length - 1; index >= 0; index--) {
-		const cost = Math.ceil(measureContentChars(messages[index]?.content) / CHARS_PER_TOKEN)
+		const cost = estimateMessageTokens(messages[index] as Message)
 		// Checked BEFORE adding, so the boundary never includes a message
 		// that pushes the tail over. Adding first and trimming after would
 		// admit one oversized message on every run.
@@ -136,6 +123,49 @@ export function naiveKeepStartByTokens(messages: readonly Message[], budgetToken
 	return start
 }
 
+/**
+ * The salience pass: score every message and evict the least salient
+ * until the context is under `softTarget`. Returned as a `cleared` plan
+ * so the phase commits it the way it commits the stale-result pass; the
+ * summary path runs after it only when the trigger threshold is still
+ * exceeded.
+ */
+/** Where the salience pass holds the context, as a fraction of the window. */
+export const DEFAULT_SOFT_TARGET = 0.5
+
+export function planSalienceWorkingSet(input: {
+	readonly messages: readonly Message[]
+	readonly config: CompactionConfig
+	readonly contextWindowTokens: number
+	readonly estimatedTokens: number
+	readonly openTasks?: readonly string[]
+}): Extract<CompactionPlan, { kind: 'cleared' }> & { readonly working: WorkingSetPlan } {
+	const { config, estimatedTokens, contextWindowTokens: budget } = input
+	const goal = buildGoal(input.messages, {
+		...(input.openTasks ? { openTasks: input.openTasks } : {}),
+	})
+	const scored = scoreMessages(input.messages, {
+		goal,
+		keepRecentMessages: config.keepRecentMessages,
+	})
+	const working = planWorkingSet(input.messages, scored, {
+		estimatedTokens,
+		targetTokens: Math.floor(budget * (config.softTarget ?? DEFAULT_SOFT_TARGET)),
+		minToolResultChars: config.minToolResultCharsToClear,
+		...(config.preserveToolResultsFrom ? { preserveTools: config.preserveToolResultsFrom } : {}),
+	})
+	return {
+		kind: 'cleared',
+		messages: working.messages,
+		clearedCount: working.clearedCount,
+		stubbedCount: working.stubbedCount,
+		charsReclaimed: working.charsReclaimed,
+		reclaimedTokens: working.reclaimedTokens,
+		reliefWasEnough: (estimatedTokens - working.reclaimedTokens) / budget < config.triggerThreshold,
+		working,
+	}
+}
+
 export function planCompaction(input: CompactionPlanInput): CompactionPlan {
 	const { config, estimatedTokens, contextWindowTokens: budget } = input
 
@@ -151,7 +181,8 @@ export function planCompaction(input: CompactionPlanInput): CompactionPlan {
 		})
 
 		if (edit.clearedCount > 0) {
-			const reclaimedTokens = Math.ceil(edit.charsReclaimed / CHARS_PER_TOKEN)
+			const reclaimedTokens =
+				estimateMessagesTokens(input.messages) - estimateMessagesTokens(edit.messages)
 			// NOT on a forced pass. A forced pass runs because the provider
 			// REJECTED the prompt as too long, which is a measurement — and
 			// answering it with the same estimate the provider just refuted

package/src/compaction/salience/bm25.ts

@@ -0,0 +1,72 @@
+/**
+ * BM25 over the run's own messages.
+ *
+ * The corpus is small — a few hundred documents at most — so the index
+ * is rebuilt from term frequencies each pass rather than maintained
+ * incrementally; the arithmetic is the cost of a few string splits. The
+ * constants are the textbook ones (k1 = 1.2, b = 0.75). What matters
+ * here is not ranking quality against a web corpus but that a message
+ * mentioning the goal's identifiers outscores one that does not, and
+ * that a term every message contains (the run's own file name, say)
+ * stops counting.
+ */
+
+import { termFrequencies, tokenize } from './tokenize.js'
+
+export interface Bm25Document {
+	readonly tf: ReadonlyMap<string, number>
+	readonly length: number
+}
+
+export interface Bm25Index {
+	readonly documents: readonly Bm25Document[]
+	readonly documentFrequency: ReadonlyMap<string, number>
+	readonly averageLength: number
+}
+
+const K1 = 1.2
+const B = 0.75
+
+export function indexDocument(text: string): Bm25Document {
+	const tokens = tokenize(text)
+	return { tf: termFrequencies(tokens), length: tokens.length }
+}
+
+export function buildIndex(documents: readonly Bm25Document[]): Bm25Index {
+	const documentFrequency = new Map<string, number>()
+	let total = 0
+	for (const doc of documents) {
+		total += doc.length
+		for (const term of doc.tf.keys()) {
+			documentFrequency.set(term, (documentFrequency.get(term) ?? 0) + 1)
+		}
+	}
+	return {
+		documents,
+		documentFrequency,
+		averageLength: documents.length === 0 ? 0 : total / documents.length,
+	}
+}
+
+/** BM25 score of one document for a query given as tokens. */
+export function bm25Score(
+	index: Bm25Index,
+	doc: Bm25Document,
+	queryTokens: readonly string[],
+): number {
+	if (doc.length === 0 || index.documents.length === 0) return 0
+	const n = index.documents.length
+	let score = 0
+	const seen = new Set<string>()
+	for (const term of queryTokens) {
+		if (seen.has(term)) continue
+		seen.add(term)
+		const tf = doc.tf.get(term)
+		if (!tf) continue
+		const df = index.documentFrequency.get(term) ?? 0
+		const idf = Math.log(1 + (n - df + 0.5) / (df + 0.5))
+		const norm = tf + K1 * (1 - B + (B * doc.length) / Math.max(1, index.averageLength))
+		score += idf * ((tf * (K1 + 1)) / norm)
+	}
+	return score
+}

package/src/compaction/salience/goal.ts

@@ -0,0 +1,44 @@
+/**
+ * The goal vector: what "relevant" is measured against.
+ *
+ * Not the whole history — that would make every message relevant to
+ * itself — but the four things that say what the run is for right now:
+ * the first user message (the task), the most recent user turns (the
+ * requirements as they were refined), the open items of the task list
+ * when the run keeps one, and the latest assistant text (its stated
+ * intent). Weighted by repetition rather than a coefficient: the task
+ * statement is included twice so a run that has drifted into detail is
+ * still scored against what it was asked to do.
+ */
+
+import type { Message } from '../../types/message/index.js'
+
+export interface GoalSources {
+	/** Titles of task-list items not yet completed, when the host keeps a task store. */
+	readonly openTasks?: readonly string[]
+	/** How many of the latest user turns count as live requirements. */
+	readonly recentUserTurns?: number
+}
+
+export function buildGoal(messages: readonly Message[], sources: GoalSources = {}): string {
+	const users = messages.filter(
+		(m): m is Extract<Message, { role: 'user' }> =>
+			m.role === 'user' && typeof m.content === 'string',
+	)
+	const first = users[0]?.content ?? ''
+	const recent = users
+		.slice(-(sources.recentUserTurns ?? 3))
+		.map((m) => m.content)
+		.filter((text) => text !== first)
+	let latestIntent = ''
+	for (let i = messages.length - 1; i >= 0; i -= 1) {
+		const m = messages[i] as Message
+		if (m.role === 'assistant' && typeof m.content === 'string' && m.content.trim().length > 0) {
+			latestIntent = m.content
+			break
+		}
+	}
+	return [first, first, ...recent, ...(sources.openTasks ?? []), latestIntent]
+		.filter((part) => part.length > 0)
+		.join('\n')
+}

package/src/compaction/salience/index.ts

@@ -0,0 +1,19 @@
+export { buildGoal, type GoalSources } from './goal.js'
+export {
+	DEFAULT_SALIENCE_CONFIG,
+	DEFAULT_SALIENCE_WEIGHTS,
+	type ProtectedReason,
+	type SalienceConfig,
+	type SalienceWeights,
+	type ScoreOptions,
+	type ScoredMessage,
+	messageText,
+	scoreMessages,
+} from './score.js'
+export {
+	type WorkingSetAction,
+	type WorkingSetOptions,
+	type WorkingSetPlan,
+	isStubbedNarration,
+	planWorkingSet,
+} from './working-set.js'

package/src/compaction/salience/minhash.ts

@@ -0,0 +1,53 @@
+/**
+ * Near-duplicate detection with MinHash over word shingles.
+ *
+ * Two tool results are rarely byte-identical — a timestamp, a duration,
+ * a line number moves — and still the same output. Three-word shingles
+ * hashed 64 ways give a Jaccard estimate that is stable against those
+ * edits and cheap enough to compare every message against every other
+ * once per pass at the sizes a run reaches. Hashing is FNV-1a with a
+ * per-slot seed; nothing here needs to be cryptographic, only consistent
+ * within a process.
+ */
+
+const HASHES = 64
+const SHINGLE = 3
+
+function fnv1a(text: string, seed: number): number {
+	let hash = (0x811c9dc5 ^ seed) >>> 0
+	for (let i = 0; i < text.length; i += 1) {
+		hash ^= text.charCodeAt(i)
+		hash = Math.imul(hash, 0x01000193) >>> 0
+	}
+	return hash
+}
+
+/** The signature of `tokens`; an empty signature for fewer than one shingle. */
+export function minhash(tokens: readonly string[]): Uint32Array {
+	const signature = new Uint32Array(HASHES).fill(0xffffffff)
+	if (tokens.length < SHINGLE) {
+		if (tokens.length === 0) return signature
+		const single = tokens.join(' ')
+		for (let slot = 0; slot < HASHES; slot += 1) signature[slot] = fnv1a(single, slot)
+		return signature
+	}
+	for (let i = 0; i + SHINGLE <= tokens.length; i += 1) {
+		const shingle = `${tokens[i]} ${tokens[i + 1]} ${tokens[i + 2]}`
+		for (let slot = 0; slot < HASHES; slot += 1) {
+			const h = fnv1a(shingle, slot)
+			if (h < (signature[slot] as number)) signature[slot] = h
+		}
+	}
+	return signature
+}
+
+/** Estimated Jaccard similarity of two signatures, 0..1. */
+export function similarity(a: Uint32Array, b: Uint32Array): number {
+	let same = 0
+	for (let slot = 0; slot < HASHES; slot += 1) if (a[slot] === b[slot]) same += 1
+	return same / HASHES
+}
+
+export function isEmptySignature(signature: Uint32Array): boolean {
+	return signature.every((h) => h === 0xffffffff)
+}

package/src/compaction/salience/score.ts

@@ -0,0 +1,305 @@
+/**
+ * Salience: how much the next step depends on each message.
+ *
+ * The compaction pass chose what to keep by position — the floor, the
+ * last few turns, the last few tool results — and a fact stated in the
+ * middle of a long run aged out at the same rate as chatter. This gives
+ * every message a number instead, from four things the kernel can
+ * observe without a model:
+ *
+ *   recency     how many messages ago, decayed with a half-life;
+ *   relevance   BM25 against the goal — the task, the requirements, the
+ *               open plan items, the latest intent;
+ *   utility     whether it was USED: a tool result whose paths and
+ *               identifiers reappear in a later tool input or assistant
+ *               turn; an instruction from the user;
+ *   redundancy  whether a later message says the same thing — a repeated
+ *               read, the same error twice — so the older copy yields.
+ *
+ * What no score may cross is stated as `protected`: the leading system
+ * run, `retain` markers with the turn they pull in, the most recent
+ * turns, and the other half of a tool-call pair. Pure, like `plan.ts`:
+ * no run, no logger, no provider, so it can be asked without one.
+ */
+
+import type { AssistantMessage, Message, ToolMessage } from '../../types/message/index.js'
+import { findRetainedIndices } from '../retention.js'
+import { estimateMessageTokens } from '../token-estimate.js'
+import { type Bm25Document, bm25Score, buildIndex, indexDocument } from './bm25.js'
+import { isEmptySignature, minhash, similarity } from './minhash.js'
+import { tokenize } from './tokenize.js'
+
+export interface SalienceWeights {
+	readonly recency: number
+	readonly relevance: number
+	readonly utility: number
+	readonly redundancy: number
+}
+
+export const DEFAULT_SALIENCE_WEIGHTS: SalienceWeights = {
+	recency: 1,
+	relevance: 1,
+	utility: 1,
+	redundancy: 1,
+}
+
+export interface SalienceConfig {
+	/** Messages after which recency has halved. */
+	readonly halfLifeMessages: number
+	/** Estimated Jaccard at or above which two messages are the same thing. */
+	readonly duplicateThreshold: number
+	readonly weights: SalienceWeights
+}
+
+export const DEFAULT_SALIENCE_CONFIG: SalienceConfig = {
+	halfLifeMessages: 12,
+	duplicateThreshold: 0.8,
+	weights: DEFAULT_SALIENCE_WEIGHTS,
+}
+
+export type ProtectedReason = 'system-floor' | 'retain' | 'recent' | 'pair'
+
+export interface ScoredMessage {
+	readonly index: number
+	readonly role: Message['role']
+	/** Estimated tokens of the message, including rich content and tool inputs. */
+	readonly tokens: number
+	readonly recency: number
+	readonly relevance: number
+	readonly utility: number
+	readonly redundancy: number
+	/** The weighted sum; meaningful only relative to the other messages of the same pass. */
+	readonly salience: number
+	readonly protected: ProtectedReason | null
+}
+
+export interface ScoreOptions {
+	/** The goal vector as text: task statement, requirements, open plan items, latest intent. */
+	readonly goal: string
+	/** How many trailing messages are protected as recent. */
+	readonly keepRecentMessages: number
+	readonly config?: Partial<SalienceConfig>
+}
+
+/** The model-visible text of a message, with an assistant's tool inputs. */
+export function messageText(message: Message): string {
+	if (message.role === 'tool') {
+		return typeof message.content === 'string'
+			? message.content
+			: message.content.flatMap((block) => (block.type === 'text' ? [block.text] : [])).join('\n')
+	}
+	const parts: string[] = []
+	if (typeof message.content === 'string') parts.push(message.content)
+	if (message.role === 'assistant') {
+		for (const call of (message as AssistantMessage).toolCalls ?? []) {
+			parts.push(call.function.name, call.function.arguments)
+		}
+	}
+	return parts.join('\n')
+}
+
+function hasRichContent(message: Message): boolean {
+	return (
+		(message.role === 'user' && (message.attachments?.length ?? 0) > 0) ||
+		(message.role === 'tool' &&
+			typeof message.content !== 'string' &&
+			message.content.some((block) => block.type !== 'text'))
+	)
+}
+
+const isAssistantWithCalls = (m: Message): m is AssistantMessage =>
+	m.role === 'assistant' && Array.isArray((m as AssistantMessage).toolCalls)
+
+/** Tokens that look like something a later turn could cite: paths, identifiers, dotted keys. */
+function citable(tokens: readonly string[]): Set<string> {
+	const out = new Set<string>()
+	for (const token of tokens) {
+		if (token.length >= 5 || token.includes('/') || token.includes('.')) out.add(token)
+	}
+	return out
+}
+
+export function scoreMessages(
+	messages: readonly Message[],
+	options: ScoreOptions,
+): ScoredMessage[] {
+	const config: SalienceConfig = {
+		...DEFAULT_SALIENCE_CONFIG,
+		...options.config,
+		weights: { ...DEFAULT_SALIENCE_WEIGHTS, ...options.config?.weights },
+	}
+	const n = messages.length
+	const texts = messages.map(messageText)
+	const richContent = messages.map(hasRichContent)
+	const tokenLists = texts.map(tokenize)
+	const documents: Bm25Document[] = texts.map(indexDocument)
+	const index = buildIndex(documents)
+	const goalTokens = tokenize(options.goal)
+
+	// Relevance, normalised to the best message of the pass so weights mean
+	// the same thing in a short run and a long one.
+	// Normalised to the best message the pass could evict, not to the user
+	// turn that stated the goal: that turn matches the goal by definition,
+	// and measured against it every result reads as barely relevant.
+	const rawRelevance = documents.map((doc) => bm25Score(index, doc, goalTokens))
+	// Nor to an assistant turn that only issued a call: its arguments echo
+	// the goal's path in a dozen tokens, which BM25 scores above the
+	// four-thousand-token result that actually holds the file.
+	const maxRelevance = Math.max(
+		0,
+		...rawRelevance.filter((_, i) => {
+			const m = messages[i] as Message
+			return m.role === 'tool' || (m.role === 'assistant' && !isAssistantWithCalls(m))
+		}),
+	)
+
+	// Citation index: everything a LATER assistant turn or tool input names.
+	// Built once, suffix-wise, so each message asks "does anything after me
+	// cite my identifiers" in one lookup.
+	const citedAfter: Set<string>[] = new Array(n)
+	let running = new Set<string>()
+	for (let i = n - 1; i >= 0; i -= 1) {
+		citedAfter[i] = running
+		const message = messages[i] as Message
+		if (message.role === 'assistant' || message.role === 'user') {
+			running = new Set([...running, ...citable(tokenLists[i] as string[])])
+		}
+	}
+
+	// Redundancy: the highest similarity to any later message of the same
+	// role, and a full demotion when a later assistant turn repeats this
+	// message's tool call exactly.
+	const signatures = tokenLists.map(minhash)
+	const callKeys = messages.map((m) =>
+		isAssistantWithCalls(m)
+			? (m.toolCalls ?? []).map((c) => `${c.function.name}:${c.function.arguments}`)
+			: [],
+	)
+	const laterCallKeys: Set<string>[] = new Array(n)
+	let calls = new Set<string>()
+	for (let i = n - 1; i >= 0; i -= 1) {
+		laterCallKeys[i] = calls
+		if ((callKeys[i] as string[]).length > 0)
+			calls = new Set([...calls, ...(callKeys[i] as string[])])
+	}
+
+	// Protection.
+	const protectedBy: (ProtectedReason | null)[] = new Array(n).fill(null)
+	let floorEnd = 0
+	while (floorEnd < n && messages[floorEnd]?.role === 'system') {
+		protectedBy[floorEnd] = 'system-floor'
+		floorEnd += 1
+	}
+	for (const i of findRetainedIndices(messages))
+		if (protectedBy[i] === null) protectedBy[i] = 'retain'
+	for (let i = Math.max(floorEnd, n - options.keepRecentMessages); i < n; i += 1) {
+		if (protectedBy[i] === null) protectedBy[i] = 'recent'
+	}
+	// A protected half of a pair protects the other half.
+	for (let i = 0; i < n; i += 1) {
+		const message = messages[i] as Message
+		if (!isAssistantWithCalls(message) || protectedBy[i] === null) continue
+		const ids = new Set((message.toolCalls ?? []).map((c) => c.id))
+		for (let j = i + 1; j < n && ids.size > 0; j += 1) {
+			const candidate = messages[j] as Message
+			if (candidate.role !== 'tool') {
+				if (candidate.role === 'assistant') break
+				continue
+			}
+			if (ids.has((candidate as ToolMessage).toolCallId)) {
+				ids.delete((candidate as ToolMessage).toolCallId)
+				if (protectedBy[j] === null) protectedBy[j] = 'pair'
+			}
+		}
+	}
+	for (let j = n - 1; j >= 0; j -= 1) {
+		const message = messages[j] as Message
+		if (message.role !== 'tool' || protectedBy[j] === null) continue
+		const id = (message as ToolMessage).toolCallId
+		for (let i = j - 1; i >= 0; i -= 1) {
+			const candidate = messages[i] as Message
+			if (isAssistantWithCalls(candidate) && (candidate.toolCalls ?? []).some((c) => c.id === id)) {
+				if (protectedBy[i] === null) protectedBy[i] = 'pair'
+				break
+			}
+		}
+	}
+
+	const scored: ScoredMessage[] = []
+	for (let i = 0; i < n; i += 1) {
+		const message = messages[i] as Message
+		const distance = n - 1 - i
+		const recency = 2 ** (-distance / Math.max(1, config.halfLifeMessages))
+		const relevance = maxRelevance > 0 ? (rawRelevance[i] as number) / maxRelevance : 0
+
+		let utility = 0
+		if (message.role === 'user') {
+			// An instruction is used by everything after it; an acknowledgment
+			// ("ok, go on") is not an instruction. The difference is whether it
+			// names anything or says more than a few words.
+			const tokens = tokenLists[i] as string[]
+			utility = citable(tokens).size > 0 || tokens.length >= 6 ? 1 : 0.2
+		} else if (message.role === 'tool') {
+			// Used is used: one identifier from this result named by a later
+			// turn is the evidence, and a count would let the recency of
+			// everything after it outvote the one result the run came back
+			// for. The eval that placed the account id deep in an early dump
+			// and cited it later watched the salience pass clear it under a
+			// thirds rule.
+			const mine = citable(tokenLists[i] as string[])
+			let hits = 0
+			for (const token of mine) if ((citedAfter[i] as Set<string>).has(token)) hits += 1
+			utility = hits > 0 ? 1 : 0
+		} else if (isAssistantWithCalls(message)) utility = 0.5
+
+		let redundancy = 0
+		const signature = signatures[i] as Uint32Array
+		// A caption or a repeated call cannot prove that two screenshots or
+		// documents contain the same evidence. The lexical scorer has no
+		// access to their meaning, so it must not call those messages duplicates.
+		if (!richContent[i] && !isEmptySignature(signature)) {
+			for (let j = i + 1; j < n; j += 1) {
+				if (richContent[j] || (messages[j] as Message).role !== message.role) continue
+				const sim = similarity(signature, signatures[j] as Uint32Array)
+				if (sim > redundancy) redundancy = sim
+			}
+		}
+		if (redundancy < config.duplicateThreshold) redundancy = redundancy >= 0.5 ? redundancy : 0
+		else redundancy = 1
+		if (message.role === 'tool' && !richContent[i]) {
+			// The result of a call a later turn repeats exactly: the newer
+			// result is the one that counts.
+			for (let k = i - 1; k >= 0; k -= 1) {
+				const producer = messages[k] as Message
+				if (!isAssistantWithCalls(producer)) continue
+				const call = (producer.toolCalls ?? []).find(
+					(c) => c.id === (message as ToolMessage).toolCallId,
+				)
+				if (call) {
+					const key = `${call.function.name}:${call.function.arguments}`
+					if ((laterCallKeys[i] as Set<string>).has(key)) redundancy = 1
+					break
+				}
+			}
+		}
+
+		const w = config.weights
+		const salience =
+			w.recency * recency +
+			w.relevance * relevance +
+			w.utility * utility -
+			w.redundancy * redundancy
+		scored.push({
+			index: i,
+			role: message.role,
+			tokens: estimateMessageTokens(message),
+			recency,
+			relevance,
+			utility,
+			redundancy,
+			salience,
+			protected: protectedBy[i] as ProtectedReason | null,
+		})
+	}
+	return scored
+}