@namzu/sdk 34.2.0 → 36.0.0

package/src/compaction/salience/bm25.ts

@@ -0,0 +1,72 @@
+/**
+ * BM25 over the run's own messages.
+ *
+ * The corpus is small — a few hundred documents at most — so the index
+ * is rebuilt from term frequencies each pass rather than maintained
+ * incrementally; the arithmetic is the cost of a few string splits. The
+ * constants are the textbook ones (k1 = 1.2, b = 0.75). What matters
+ * here is not ranking quality against a web corpus but that a message
+ * mentioning the goal's identifiers outscores one that does not, and
+ * that a term every message contains (the run's own file name, say)
+ * stops counting.
+ */
+
+import { termFrequencies, tokenize } from './tokenize.js'
+
+export interface Bm25Document {
+	readonly tf: ReadonlyMap<string, number>
+	readonly length: number
+}
+
+export interface Bm25Index {
+	readonly documents: readonly Bm25Document[]
+	readonly documentFrequency: ReadonlyMap<string, number>
+	readonly averageLength: number
+}
+
+const K1 = 1.2
+const B = 0.75
+
+export function indexDocument(text: string): Bm25Document {
+	const tokens = tokenize(text)
+	return { tf: termFrequencies(tokens), length: tokens.length }
+}
+
+export function buildIndex(documents: readonly Bm25Document[]): Bm25Index {
+	const documentFrequency = new Map<string, number>()
+	let total = 0
+	for (const doc of documents) {
+		total += doc.length
+		for (const term of doc.tf.keys()) {
+			documentFrequency.set(term, (documentFrequency.get(term) ?? 0) + 1)
+		}
+	}
+	return {
+		documents,
+		documentFrequency,
+		averageLength: documents.length === 0 ? 0 : total / documents.length,
+	}
+}
+
+/** BM25 score of one document for a query given as tokens. */
+export function bm25Score(
+	index: Bm25Index,
+	doc: Bm25Document,
+	queryTokens: readonly string[],
+): number {
+	if (doc.length === 0 || index.documents.length === 0) return 0
+	const n = index.documents.length
+	let score = 0
+	const seen = new Set<string>()
+	for (const term of queryTokens) {
+		if (seen.has(term)) continue
+		seen.add(term)
+		const tf = doc.tf.get(term)
+		if (!tf) continue
+		const df = index.documentFrequency.get(term) ?? 0
+		const idf = Math.log(1 + (n - df + 0.5) / (df + 0.5))
+		const norm = tf + K1 * (1 - B + (B * doc.length) / Math.max(1, index.averageLength))
+		score += idf * ((tf * (K1 + 1)) / norm)
+	}
+	return score
+}

package/src/compaction/salience/goal.ts

@@ -0,0 +1,44 @@
+/**
+ * The goal vector: what "relevant" is measured against.
+ *
+ * Not the whole history — that would make every message relevant to
+ * itself — but the four things that say what the run is for right now:
+ * the first user message (the task), the most recent user turns (the
+ * requirements as they were refined), the open items of the task list
+ * when the run keeps one, and the latest assistant text (its stated
+ * intent). Weighted by repetition rather than a coefficient: the task
+ * statement is included twice so a run that has drifted into detail is
+ * still scored against what it was asked to do.
+ */
+
+import type { Message } from '../../types/message/index.js'
+
+export interface GoalSources {
+	/** Titles of task-list items not yet completed, when the host keeps a task store. */
+	readonly openTasks?: readonly string[]
+	/** How many of the latest user turns count as live requirements. */
+	readonly recentUserTurns?: number
+}
+
+export function buildGoal(messages: readonly Message[], sources: GoalSources = {}): string {
+	const users = messages.filter(
+		(m): m is Extract<Message, { role: 'user' }> =>
+			m.role === 'user' && typeof m.content === 'string',
+	)
+	const first = users[0]?.content ?? ''
+	const recent = users
+		.slice(-(sources.recentUserTurns ?? 3))
+		.map((m) => m.content)
+		.filter((text) => text !== first)
+	let latestIntent = ''
+	for (let i = messages.length - 1; i >= 0; i -= 1) {
+		const m = messages[i] as Message
+		if (m.role === 'assistant' && typeof m.content === 'string' && m.content.trim().length > 0) {
+			latestIntent = m.content
+			break
+		}
+	}
+	return [first, first, ...recent, ...(sources.openTasks ?? []), latestIntent]
+		.filter((part) => part.length > 0)
+		.join('\n')
+}

package/src/compaction/salience/index.ts

@@ -0,0 +1,19 @@
+export { buildGoal, type GoalSources } from './goal.js'
+export {
+	DEFAULT_SALIENCE_CONFIG,
+	DEFAULT_SALIENCE_WEIGHTS,
+	type ProtectedReason,
+	type SalienceConfig,
+	type SalienceWeights,
+	type ScoreOptions,
+	type ScoredMessage,
+	messageText,
+	scoreMessages,
+} from './score.js'
+export {
+	type WorkingSetAction,
+	type WorkingSetOptions,
+	type WorkingSetPlan,
+	isStubbedNarration,
+	planWorkingSet,
+} from './working-set.js'

package/src/compaction/salience/minhash.ts

@@ -0,0 +1,53 @@
+/**
+ * Near-duplicate detection with MinHash over word shingles.
+ *
+ * Two tool results are rarely byte-identical — a timestamp, a duration,
+ * a line number moves — and still the same output. Three-word shingles
+ * hashed 64 ways give a Jaccard estimate that is stable against those
+ * edits and cheap enough to compare every message against every other
+ * once per pass at the sizes a run reaches. Hashing is FNV-1a with a
+ * per-slot seed; nothing here needs to be cryptographic, only consistent
+ * within a process.
+ */
+
+const HASHES = 64
+const SHINGLE = 3
+
+function fnv1a(text: string, seed: number): number {
+	let hash = (0x811c9dc5 ^ seed) >>> 0
+	for (let i = 0; i < text.length; i += 1) {
+		hash ^= text.charCodeAt(i)
+		hash = Math.imul(hash, 0x01000193) >>> 0
+	}
+	return hash
+}
+
+/** The signature of `tokens`; an empty signature for fewer than one shingle. */
+export function minhash(tokens: readonly string[]): Uint32Array {
+	const signature = new Uint32Array(HASHES).fill(0xffffffff)
+	if (tokens.length < SHINGLE) {
+		if (tokens.length === 0) return signature
+		const single = tokens.join(' ')
+		for (let slot = 0; slot < HASHES; slot += 1) signature[slot] = fnv1a(single, slot)
+		return signature
+	}
+	for (let i = 0; i + SHINGLE <= tokens.length; i += 1) {
+		const shingle = `${tokens[i]} ${tokens[i + 1]} ${tokens[i + 2]}`
+		for (let slot = 0; slot < HASHES; slot += 1) {
+			const h = fnv1a(shingle, slot)
+			if (h < (signature[slot] as number)) signature[slot] = h
+		}
+	}
+	return signature
+}
+
+/** Estimated Jaccard similarity of two signatures, 0..1. */
+export function similarity(a: Uint32Array, b: Uint32Array): number {
+	let same = 0
+	for (let slot = 0; slot < HASHES; slot += 1) if (a[slot] === b[slot]) same += 1
+	return same / HASHES
+}
+
+export function isEmptySignature(signature: Uint32Array): boolean {
+	return signature.every((h) => h === 0xffffffff)
+}

package/src/compaction/salience/score.ts

@@ -0,0 +1,305 @@
+/**
+ * Salience: how much the next step depends on each message.
+ *
+ * The compaction pass chose what to keep by position — the floor, the
+ * last few turns, the last few tool results — and a fact stated in the
+ * middle of a long run aged out at the same rate as chatter. This gives
+ * every message a number instead, from four things the kernel can
+ * observe without a model:
+ *
+ *   recency     how many messages ago, decayed with a half-life;
+ *   relevance   BM25 against the goal — the task, the requirements, the
+ *               open plan items, the latest intent;
+ *   utility     whether it was USED: a tool result whose paths and
+ *               identifiers reappear in a later tool input or assistant
+ *               turn; an instruction from the user;
+ *   redundancy  whether a later message says the same thing — a repeated
+ *               read, the same error twice — so the older copy yields.
+ *
+ * What no score may cross is stated as `protected`: the leading system
+ * run, `retain` markers with the turn they pull in, the most recent
+ * turns, and the other half of a tool-call pair. Pure, like `plan.ts`:
+ * no run, no logger, no provider, so it can be asked without one.
+ */
+
+import type { AssistantMessage, Message, ToolMessage } from '../../types/message/index.js'
+import { findRetainedIndices } from '../retention.js'
+import { estimateMessageTokens } from '../token-estimate.js'
+import { type Bm25Document, bm25Score, buildIndex, indexDocument } from './bm25.js'
+import { isEmptySignature, minhash, similarity } from './minhash.js'
+import { tokenize } from './tokenize.js'
+
+export interface SalienceWeights {
+	readonly recency: number
+	readonly relevance: number
+	readonly utility: number
+	readonly redundancy: number
+}
+
+export const DEFAULT_SALIENCE_WEIGHTS: SalienceWeights = {
+	recency: 1,
+	relevance: 1,
+	utility: 1,
+	redundancy: 1,
+}
+
+export interface SalienceConfig {
+	/** Messages after which recency has halved. */
+	readonly halfLifeMessages: number
+	/** Estimated Jaccard at or above which two messages are the same thing. */
+	readonly duplicateThreshold: number
+	readonly weights: SalienceWeights
+}
+
+export const DEFAULT_SALIENCE_CONFIG: SalienceConfig = {
+	halfLifeMessages: 12,
+	duplicateThreshold: 0.8,
+	weights: DEFAULT_SALIENCE_WEIGHTS,
+}
+
+export type ProtectedReason = 'system-floor' | 'retain' | 'recent' | 'pair'
+
+export interface ScoredMessage {
+	readonly index: number
+	readonly role: Message['role']
+	/** Estimated tokens of the message, including rich content and tool inputs. */
+	readonly tokens: number
+	readonly recency: number
+	readonly relevance: number
+	readonly utility: number
+	readonly redundancy: number
+	/** The weighted sum; meaningful only relative to the other messages of the same pass. */
+	readonly salience: number
+	readonly protected: ProtectedReason | null
+}
+
+export interface ScoreOptions {
+	/** The goal vector as text: task statement, requirements, open plan items, latest intent. */
+	readonly goal: string
+	/** How many trailing messages are protected as recent. */
+	readonly keepRecentMessages: number
+	readonly config?: Partial<SalienceConfig>
+}
+
+/** The model-visible text of a message, with an assistant's tool inputs. */
+export function messageText(message: Message): string {
+	if (message.role === 'tool') {
+		return typeof message.content === 'string'
+			? message.content
+			: message.content.flatMap((block) => (block.type === 'text' ? [block.text] : [])).join('\n')
+	}
+	const parts: string[] = []
+	if (typeof message.content === 'string') parts.push(message.content)
+	if (message.role === 'assistant') {
+		for (const call of (message as AssistantMessage).toolCalls ?? []) {
+			parts.push(call.function.name, call.function.arguments)
+		}
+	}
+	return parts.join('\n')
+}
+
+function hasRichContent(message: Message): boolean {
+	return (
+		(message.role === 'user' && (message.attachments?.length ?? 0) > 0) ||
+		(message.role === 'tool' &&
+			typeof message.content !== 'string' &&
+			message.content.some((block) => block.type !== 'text'))
+	)
+}
+
+const isAssistantWithCalls = (m: Message): m is AssistantMessage =>
+	m.role === 'assistant' && Array.isArray((m as AssistantMessage).toolCalls)
+
+/** Tokens that look like something a later turn could cite: paths, identifiers, dotted keys. */
+function citable(tokens: readonly string[]): Set<string> {
+	const out = new Set<string>()
+	for (const token of tokens) {
+		if (token.length >= 5 || token.includes('/') || token.includes('.')) out.add(token)
+	}
+	return out
+}
+
+export function scoreMessages(
+	messages: readonly Message[],
+	options: ScoreOptions,
+): ScoredMessage[] {
+	const config: SalienceConfig = {
+		...DEFAULT_SALIENCE_CONFIG,
+		...options.config,
+		weights: { ...DEFAULT_SALIENCE_WEIGHTS, ...options.config?.weights },
+	}
+	const n = messages.length
+	const texts = messages.map(messageText)
+	const richContent = messages.map(hasRichContent)
+	const tokenLists = texts.map(tokenize)
+	const documents: Bm25Document[] = texts.map(indexDocument)
+	const index = buildIndex(documents)
+	const goalTokens = tokenize(options.goal)
+
+	// Relevance, normalised to the best message of the pass so weights mean
+	// the same thing in a short run and a long one.
+	// Normalised to the best message the pass could evict, not to the user
+	// turn that stated the goal: that turn matches the goal by definition,
+	// and measured against it every result reads as barely relevant.
+	const rawRelevance = documents.map((doc) => bm25Score(index, doc, goalTokens))
+	// Nor to an assistant turn that only issued a call: its arguments echo
+	// the goal's path in a dozen tokens, which BM25 scores above the
+	// four-thousand-token result that actually holds the file.
+	const maxRelevance = Math.max(
+		0,
+		...rawRelevance.filter((_, i) => {
+			const m = messages[i] as Message
+			return m.role === 'tool' || (m.role === 'assistant' && !isAssistantWithCalls(m))
+		}),
+	)
+
+	// Citation index: everything a LATER assistant turn or tool input names.
+	// Built once, suffix-wise, so each message asks "does anything after me
+	// cite my identifiers" in one lookup.
+	const citedAfter: Set<string>[] = new Array(n)
+	let running = new Set<string>()
+	for (let i = n - 1; i >= 0; i -= 1) {
+		citedAfter[i] = running
+		const message = messages[i] as Message
+		if (message.role === 'assistant' || message.role === 'user') {
+			running = new Set([...running, ...citable(tokenLists[i] as string[])])
+		}
+	}
+
+	// Redundancy: the highest similarity to any later message of the same
+	// role, and a full demotion when a later assistant turn repeats this
+	// message's tool call exactly.
+	const signatures = tokenLists.map(minhash)
+	const callKeys = messages.map((m) =>
+		isAssistantWithCalls(m)
+			? (m.toolCalls ?? []).map((c) => `${c.function.name}:${c.function.arguments}`)
+			: [],
+	)
+	const laterCallKeys: Set<string>[] = new Array(n)
+	let calls = new Set<string>()
+	for (let i = n - 1; i >= 0; i -= 1) {
+		laterCallKeys[i] = calls
+		if ((callKeys[i] as string[]).length > 0)
+			calls = new Set([...calls, ...(callKeys[i] as string[])])
+	}
+
+	// Protection.
+	const protectedBy: (ProtectedReason | null)[] = new Array(n).fill(null)
+	let floorEnd = 0
+	while (floorEnd < n && messages[floorEnd]?.role === 'system') {
+		protectedBy[floorEnd] = 'system-floor'
+		floorEnd += 1
+	}
+	for (const i of findRetainedIndices(messages))
+		if (protectedBy[i] === null) protectedBy[i] = 'retain'
+	for (let i = Math.max(floorEnd, n - options.keepRecentMessages); i < n; i += 1) {
+		if (protectedBy[i] === null) protectedBy[i] = 'recent'
+	}
+	// A protected half of a pair protects the other half.
+	for (let i = 0; i < n; i += 1) {
+		const message = messages[i] as Message
+		if (!isAssistantWithCalls(message) || protectedBy[i] === null) continue
+		const ids = new Set((message.toolCalls ?? []).map((c) => c.id))
+		for (let j = i + 1; j < n && ids.size > 0; j += 1) {
+			const candidate = messages[j] as Message
+			if (candidate.role !== 'tool') {
+				if (candidate.role === 'assistant') break
+				continue
+			}
+			if (ids.has((candidate as ToolMessage).toolCallId)) {
+				ids.delete((candidate as ToolMessage).toolCallId)
+				if (protectedBy[j] === null) protectedBy[j] = 'pair'
+			}
+		}
+	}
+	for (let j = n - 1; j >= 0; j -= 1) {
+		const message = messages[j] as Message
+		if (message.role !== 'tool' || protectedBy[j] === null) continue
+		const id = (message as ToolMessage).toolCallId
+		for (let i = j - 1; i >= 0; i -= 1) {
+			const candidate = messages[i] as Message
+			if (isAssistantWithCalls(candidate) && (candidate.toolCalls ?? []).some((c) => c.id === id)) {
+				if (protectedBy[i] === null) protectedBy[i] = 'pair'
+				break
+			}
+		}
+	}
+
+	const scored: ScoredMessage[] = []
+	for (let i = 0; i < n; i += 1) {
+		const message = messages[i] as Message
+		const distance = n - 1 - i
+		const recency = 2 ** (-distance / Math.max(1, config.halfLifeMessages))
+		const relevance = maxRelevance > 0 ? (rawRelevance[i] as number) / maxRelevance : 0
+
+		let utility = 0
+		if (message.role === 'user') {
+			// An instruction is used by everything after it; an acknowledgment
+			// ("ok, go on") is not an instruction. The difference is whether it
+			// names anything or says more than a few words.
+			const tokens = tokenLists[i] as string[]
+			utility = citable(tokens).size > 0 || tokens.length >= 6 ? 1 : 0.2
+		} else if (message.role === 'tool') {
+			// Used is used: one identifier from this result named by a later
+			// turn is the evidence, and a count would let the recency of
+			// everything after it outvote the one result the run came back
+			// for. The eval that placed the account id deep in an early dump
+			// and cited it later watched the salience pass clear it under a
+			// thirds rule.
+			const mine = citable(tokenLists[i] as string[])
+			let hits = 0
+			for (const token of mine) if ((citedAfter[i] as Set<string>).has(token)) hits += 1
+			utility = hits > 0 ? 1 : 0
+		} else if (isAssistantWithCalls(message)) utility = 0.5
+
+		let redundancy = 0
+		const signature = signatures[i] as Uint32Array
+		// A caption or a repeated call cannot prove that two screenshots or
+		// documents contain the same evidence. The lexical scorer has no
+		// access to their meaning, so it must not call those messages duplicates.
+		if (!richContent[i] && !isEmptySignature(signature)) {
+			for (let j = i + 1; j < n; j += 1) {
+				if (richContent[j] || (messages[j] as Message).role !== message.role) continue
+				const sim = similarity(signature, signatures[j] as Uint32Array)
+				if (sim > redundancy) redundancy = sim
+			}
+		}
+		if (redundancy < config.duplicateThreshold) redundancy = redundancy >= 0.5 ? redundancy : 0
+		else redundancy = 1
+		if (message.role === 'tool' && !richContent[i]) {
+			// The result of a call a later turn repeats exactly: the newer
+			// result is the one that counts.
+			for (let k = i - 1; k >= 0; k -= 1) {
+				const producer = messages[k] as Message
+				if (!isAssistantWithCalls(producer)) continue
+				const call = (producer.toolCalls ?? []).find(
+					(c) => c.id === (message as ToolMessage).toolCallId,
+				)
+				if (call) {
+					const key = `${call.function.name}:${call.function.arguments}`
+					if ((laterCallKeys[i] as Set<string>).has(key)) redundancy = 1
+					break
+				}
+			}
+		}
+
+		const w = config.weights
+		const salience =
+			w.recency * recency +
+			w.relevance * relevance +
+			w.utility * utility -
+			w.redundancy * redundancy
+		scored.push({
+			index: i,
+			role: message.role,
+			tokens: estimateMessageTokens(message),
+			recency,
+			relevance,
+			utility,
+			redundancy,
+			salience,
+			protected: protectedBy[i] as ProtectedReason | null,
+		})
+	}
+	return scored
+}

package/src/compaction/salience/tokenize.ts

@@ -0,0 +1,89 @@
+/**
+ * An identifier-aware tokenizer for scoring, not for search.
+ *
+ * A coding conversation's vocabulary is paths, identifiers and error
+ * names. A word tokenizer sees `src/store.mjs` as one token that matches
+ * nothing and `removeTodo` as another; the model, asked about `store`,
+ * is looking at both. So a path splits on its separators, a `camelCase`
+ * or `snake_case` identifier splits into its words AND keeps its whole
+ * form, and a dotted attribute key (`namzu.run.id`) yields both the key
+ * and its parts. Everything is lower-cased; nothing is stemmed, because
+ * a stemmer that turns `tests` into `test` also turns `testing` into it,
+ * and the false matches cost more than the missed ones in a corpus this
+ * small. Stop words are the handful that carry no relevance in any
+ * sentence and would otherwise dominate term frequency.
+ */
+
+const STOP_WORDS: ReadonlySet<string> = new Set([
+	'a',
+	'an',
+	'and',
+	'are',
+	'as',
+	'at',
+	'be',
+	'by',
+	'for',
+	'from',
+	'i',
+	'if',
+	'in',
+	'is',
+	'it',
+	'of',
+	'on',
+	'or',
+	'that',
+	'the',
+	'then',
+	'this',
+	'to',
+	'was',
+	'will',
+	'with',
+	'you',
+])
+
+const WORD = /[\p{L}\p{N}_$][\p{L}\p{N}_$.\-/\\:]*/gu
+
+function splitIdentifier(raw: string): string[] {
+	const parts = raw
+		.replace(/([a-z0-9])([A-Z])/g, '$1 $2')
+		.replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2')
+		.split(/[_\s]+/)
+		.filter((p) => p.length > 0)
+	return parts.length > 1 ? parts : []
+}
+
+/** Tokens of `text`, lower-cased, with identifiers and paths opened up. */
+export function tokenize(text: string): string[] {
+	const out: string[] = []
+	for (const match of text.matchAll(WORD)) {
+		const raw = match[0].replace(/^[.\-/\\:]+|[.\-/\\:]+$/g, '')
+		if (raw.length === 0) continue
+		const whole = raw.toLowerCase()
+		if (!STOP_WORDS.has(whole)) out.push(whole)
+		// Separators first: a path or dotted key contributes each segment.
+		const segments = raw.split(/[/\\.:\-]+/).filter((s) => s.length > 0)
+		if (segments.length > 1) {
+			for (const segment of segments) {
+				const lower = segment.toLowerCase()
+				if (!STOP_WORDS.has(lower) && lower !== whole) out.push(lower)
+				for (const part of splitIdentifier(segment)) out.push(part.toLowerCase())
+			}
+		} else {
+			for (const part of splitIdentifier(raw)) {
+				const lower = part.toLowerCase()
+				if (!STOP_WORDS.has(lower)) out.push(lower)
+			}
+		}
+	}
+	return out
+}
+
+/** Term frequencies of `tokens`. */
+export function termFrequencies(tokens: readonly string[]): Map<string, number> {
+	const tf = new Map<string, number>()
+	for (const token of tokens) tf.set(token, (tf.get(token) ?? 0) + 1)
+	return tf
+}