@namzu/sdk 21.0.0 → 21.1.0

package/src/run/index.ts

@@ -7,6 +7,23 @@ export type { RunReporter } from './reporter.js'
 
 export { DEFAULT_DRAIN_PAGE_SIZE, drainRuns } from './drain.js'
 export type { DrainFailure, DrainRun, DrainRunsParams, DrainRunsResult } from './drain.js'
+export {
+	DEFAULT_GATE_MAX_RETRIES,
+	DEFAULT_GATE_OUTPUT_CHARS,
+	DEFAULT_GATE_TIMEOUT_MS,
+	clipOutput,
+	createCommandGate,
+} from './command-gate.js'
+export type { CommandGateOptions, GateExec } from './command-gate.js'
+export {
+	FINGERPRINT_MAX_BYTES,
+	FINGERPRINT_TIMEOUT_MS,
+	fingerprintWorkspace,
+} from './workspace-fingerprint.js'
+export type { FingerprintExec, WorkspaceFingerprintOptions } from './workspace-fingerprint.js'
 
 export { checkLimitsDetailed, buildLimitConfig } from './LimitChecker.js'
 export type { LimitCheckerState, LimitCheckResult } from './LimitChecker.js'
+
+export { RUN_MEMORY_TAG, createMemoryPromoter } from './memory-promoter.js'
+export type { MemoryPromoterOptions } from './memory-promoter.js'

package/src/run/memory-promoter.ts

@@ -0,0 +1,155 @@
+/**
+ * The default {@link PromoteMemory}: write what a run learned into a
+ * {@link MemoryStore}, or write nothing at all.
+ *
+ * `promoteMemory` is called once at settle with the compaction extractor's
+ * already-structured output — decisions, discoveries, user requirements,
+ * failures, environment facts — and **nothing shipped supplied the hook**.
+ * So the structure the compaction pass had spent tokens producing was
+ * serialized into one system message and dropped on the floor when the run
+ * ended, exactly as its own module comment says. This is the supplier, and
+ * it is mostly a filter: the hard part — extracting facts from a transcript
+ * — already happened.
+ *
+ * ## The filter, which is the only decision here
+ *
+ * **A run that learned nothing must leave nothing.** Not an empty record,
+ * not a record whose body says "no decisions" — nothing. A promoter that
+ * wrote a row per run would fill the store with the runs least worth
+ * remembering, and `search_memory` would then return them: the model reads
+ * that store on later runs, so noise here is not merely wasted disk, it is
+ * context spent on a run that did nothing.
+ *
+ * What counts as having learned something is the five KNOWLEDGE categories —
+ * decisions, discoveries, user requirements, failures, environment. Not
+ * `task`, which every run has because it is the prompt restated. Not
+ * `files`, which every run that opened anything has, and which says what was
+ * touched rather than what was learned. A run whose only trace is "it read
+ * six files" is the exact record this filter exists to refuse.
+ *
+ * ## What it does NOT do
+ *
+ * Deduplicate against what is already stored, merge with a previous run's
+ * record, or expire anything. Each is a policy with real trade-offs and a
+ * host that wants one owns it — `promoteMemory` is a callback precisely so
+ * that the runtime does not decide this. This is the obvious default, not
+ * the only possible one.
+ */
+
+import type { MemoryStore } from '../types/memory/index.js'
+import type { PromoteMemory, RunMemoryCandidate } from '../types/run/memory-promotion.js'
+
+/**
+ * The categories that make a run worth remembering.
+ *
+ * Ordered as they are rendered. `userRequirements` first because it is the
+ * most durable of the five — a constraint the user stated outlives the run
+ * that heard it, whereas a discovery about a codebase expires when the
+ * codebase moves.
+ */
+const KNOWLEDGE = [
+	['userRequirements', 'What the user requires'],
+	['decisions', 'Decisions'],
+	['discoveries', 'Discoveries'],
+	['failures', 'What did not work'],
+	['environment', 'Environment'],
+] as const satisfies readonly (readonly [keyof RunMemoryCandidate, string])[]
+
+/** Tag every record this promoter writes, so a host can find or prune them. */
+export const RUN_MEMORY_TAG = 'run-memory'
+
+export interface MemoryPromoterOptions {
+	/** Where records go. The same store `save_memory` writes through. */
+	readonly store: MemoryStore
+	/**
+	 * Extra tags on every record, beyond {@link RUN_MEMORY_TAG}.
+	 *
+	 * A host running several agents against one store uses this to tell whose
+	 * memory is whose; without it a later search cannot.
+	 */
+	readonly tags?: readonly string[]
+	/**
+	 * Cap on entries rendered per category. Defaults to 20.
+	 *
+	 * The extractor already caps its lists, and this is the second cap for
+	 * the same reason the first exists: a record nobody will read is a record
+	 * that costs context every time it is retrieved.
+	 */
+	readonly maxPerCategory?: number
+}
+
+/** Everything the candidate knows, as `[heading, items]`, empties dropped. */
+function knowledge(
+	candidate: RunMemoryCandidate,
+	cap: number,
+): readonly (readonly [string, readonly string[]])[] {
+	const out: (readonly [string, readonly string[]])[] = []
+	for (const [key, heading] of KNOWLEDGE) {
+		const items = candidate[key] as readonly string[]
+		if (items.length > 0) out.push([heading, items.slice(0, cap)])
+	}
+	return out
+}
+
+/** A one-line summary naming what kind of knowledge the record holds. */
+function summarize(sections: readonly (readonly [string, readonly string[]])[]): string {
+	return sections.map(([heading, items]) => `${heading.toLowerCase()} (${items.length})`).join(', ')
+}
+
+function render(
+	candidate: RunMemoryCandidate,
+	sections: readonly (readonly [string, readonly string[]])[],
+): string {
+	const body = sections.map(
+		([heading, items]) => `## ${heading}\n\n${items.map((i) => `- ${i}`).join('\n')}`,
+	)
+	// The eviction counts, when there are any. Carried rather than hidden for
+	// the reason the candidate carries them: somebody reading this record
+	// should know they are reading a truncated account of the run, not a
+	// complete one.
+	const evicted = Object.entries(candidate.evicted).filter(([, n]) => n > 0)
+	if (evicted.length > 0) {
+		body.push(
+			`## Dropped during the run\n\n${evicted
+				.map(([category, n]) => `- ${category}: ${n} entr${n === 1 ? 'y' : 'ies'} evicted`)
+				.join('\n')}`,
+		)
+	}
+	if (candidate.files.length > 0) {
+		body.push(`## Files touched\n\n${candidate.files.map((f) => `- ${f}`).join('\n')}`)
+	}
+	return `# ${candidate.task}\n\n${body.join('\n\n')}\n`
+}
+
+/**
+ * Build a promoter that writes one record per run that learned something.
+ *
+ * Never throws out to the runtime — but it does not swallow either: the
+ * runtime already catches and logs a promoter's failure at settle, and
+ * catching here as well would hide a broken store from the one place that
+ * reports it.
+ */
+export function createMemoryPromoter(options: MemoryPromoterOptions): PromoteMemory {
+	const cap = options.maxPerCategory ?? 20
+	const tags = [RUN_MEMORY_TAG, ...(options.tags ?? [])]
+
+	return async (candidate: RunMemoryCandidate): Promise<void> => {
+		const sections = knowledge(candidate, cap)
+		// Nothing learned, nothing written. Not an empty record: a store full
+		// of rows describing runs that discovered nothing is a store whose
+		// search results are mostly noise, and the model reads that store.
+		if (sections.length === 0) return
+
+		await options.store.create({
+			title: candidate.task.trim() || `Run ${candidate.runId}`,
+			summary: summarize(sections),
+			content: render(candidate, sections),
+			tags,
+			format: 'markdown',
+			// The run id, so a record can be traced back to the run that formed
+			// it. Evidence rather than decoration: without it a surprising
+			// memory cannot be checked against what actually happened.
+			metadata: { runId: candidate.runId, source: RUN_MEMORY_TAG },
+		})
+	}
+}

package/src/run/workspace-fingerprint.ts

@@ -0,0 +1,193 @@
+/**
+ * A hash of everything a run could have changed in its working tree.
+ *
+ * It exists to answer one question, asked between two attempts at the same
+ * verification: **did anything happen since it last failed?** A verify-then-fix
+ * loop that re-runs the build after a turn which edited nothing spends a full
+ * command execution to learn what a comparison already knew, and does it once
+ * per remaining attempt — so a model that has stopped making progress burns
+ * the entire budget confirming the same failure.
+ *
+ * ## What is hashed, and why each part
+ *
+ * Three sources, because no one of them is complete:
+ *
+ *  1. **`git status --porcelain`** — which paths differ from the index at all.
+ *     Cheap, and it catches additions, deletions and mode changes. On its own
+ *     it is not enough: editing a tracked file that was ALREADY modified
+ *     leaves the status output byte-identical.
+ *  2. **`git diff --binary HEAD`** — the content of every tracked change.
+ *     `--binary` so an edit to a file git treats as binary is a real diff
+ *     rather than the constant line `Binary files … differ`, which would make
+ *     every edit to such a file invisible.
+ *  3. **Untracked file contents**, which no `git diff` covers. A new file is
+ *     named by `status` but its CONTENT is not, so successive edits to a
+ *     brand-new file would otherwise look like no change at all.
+ *
+ * ### Symlinks are recorded as their target, not read through
+ *
+ * Reading a link follows it, so a link repointed from one file to another
+ * with identical contents hashes the same — while the thing the workspace
+ * actually resolves has changed. The link's target path is the fact that
+ * moved, so that is what goes in.
+ *
+ * ## Failing open, on the cheap side
+ *
+ * Every uncertainty returns `null`, meaning *no fingerprint*, and a caller
+ * that cannot fingerprint re-runs its command. That is the correct direction:
+ * the cost of a wrong `null` is one command execution, and the cost of a
+ * wrong MATCH is a verification silently skipped — the loop would report
+ * "nothing changed" about a workspace that did change, and the model would be
+ * told to edit something it had already edited.
+ *
+ * So: a non-zero exit from any git invocation, a repository with no commits,
+ * a timeout, or output past the size cap all produce `null` rather than a
+ * partial hash. A truncated diff that hashed successfully would be the worst
+ * outcome available here, because two different workspaces truncated at the
+ * same point collide.
+ */
+
+import { createHash } from 'node:crypto'
+import { lstat, readFile, readlink } from 'node:fs/promises'
+import { join } from 'node:path'
+
+import type { CommandOptions, CommandResult } from '../types/execution/index.js'
+
+/** How a fingerprint runs git. Injected so a test needs no repository. */
+export type FingerprintExec = (
+	command: string,
+	args: string[],
+	options?: CommandOptions,
+) => Promise<CommandResult>
+
+/**
+ * The three filesystem reads an untracked entry needs.
+ *
+ * Injectable for one specific reason, written down because a seam that
+ * exists only for tests is usually a smell: **creating a symlink requires a
+ * privilege that is not granted by default on Windows**, so the symlink rule
+ * below — the one that says a repointed link changes the fingerprint even
+ * when the bytes behind it do not — cannot be exercised on a developer
+ * machine without it. A rule that can only be checked on some machines is a
+ * rule nobody checks.
+ *
+ * The default is `node:fs/promises` and every other test uses it against a
+ * real repository, so this is not a fixture standing in for production; it is
+ * one branch of one function reached without a privilege.
+ */
+export interface FingerprintFs {
+	lstat(path: string): Promise<{ isSymbolicLink(): boolean; isFile(): boolean }>
+	readlink(path: string): Promise<string>
+	readFile(path: string): Promise<Buffer>
+}
+
+const NODE_FS: FingerprintFs = { lstat, readlink, readFile }
+
+/**
+ * Cap on the bytes any single git invocation may produce.
+ *
+ * Past it the fingerprint is abandoned rather than hashed. A diff big enough
+ * to hit this is a diff nobody is going to iterate on anyway, and hashing a
+ * clipped one would let two different trees agree.
+ */
+export const FINGERPRINT_MAX_BYTES = 4 * 1024 * 1024
+
+/** Default deadline per git invocation. */
+export const FINGERPRINT_TIMEOUT_MS = 20_000
+
+export interface WorkspaceFingerprintOptions {
+	/** Repository root, or any directory inside it. */
+	readonly cwd: string
+	/** How to run git. */
+	readonly exec: FingerprintExec
+	/** Per-invocation deadline. See {@link FINGERPRINT_TIMEOUT_MS}. */
+	readonly timeoutMs?: number
+	/** See {@link FINGERPRINT_MAX_BYTES}. */
+	readonly maxBytes?: number
+	/** Filesystem reads. See {@link FingerprintFs}. */
+	readonly fs?: FingerprintFs
+}
+
+/** One untracked path's contribution, or `null` when it could not be read. */
+async function untrackedEntry(cwd: string, rel: string, fs: FingerprintFs): Promise<string | null> {
+	const abs = join(cwd, rel)
+	try {
+		const stats = await fs.lstat(abs)
+		if (stats.isSymbolicLink()) {
+			// The TARGET, not what is behind it. Following the link would hash a
+			// repointed link to the same value whenever the new target happens
+			// to hold the same bytes, and a repoint is a change to the workspace
+			// by any reading that matters.
+			return `L ${rel}\0${await fs.readlink(abs)}`
+		}
+		if (!stats.isFile()) return `? ${rel}`
+		const body = await fs.readFile(abs)
+		return `F ${rel}\0${createHash('sha256').update(body).digest('hex')}`
+	} catch {
+		// Vanished between the listing and the read, or unreadable. Neither is
+		// a fingerprint this function may guess at.
+		return null
+	}
+}
+
+/**
+ * A hash of the working tree's uncommitted state, or `null` when it cannot be
+ * established.
+ *
+ * **`null` is never "unchanged".** It means "I cannot tell", and the caller
+ * must treat it as a reason to do the work rather than to skip it.
+ */
+export async function fingerprintWorkspace(
+	options: WorkspaceFingerprintOptions,
+): Promise<string | null> {
+	const { cwd, exec } = options
+	const timeoutMs = options.timeoutMs ?? FINGERPRINT_TIMEOUT_MS
+	const maxBytes = options.maxBytes ?? FINGERPRINT_MAX_BYTES
+	const fs = options.fs ?? NODE_FS
+
+	const git = async (args: string[]): Promise<string | null> => {
+		let result: CommandResult
+		try {
+			result = await exec('git', args, { cwd, timeoutMs })
+		} catch {
+			return null
+		}
+		// A timeout surfaces here as a non-zero exit, and so does "not a
+		// repository" and "no commits yet". All three mean the same thing to
+		// this function: it has no basis for a comparison.
+		if (result.exitCode !== 0) return null
+		if (Buffer.byteLength(result.stdout, 'utf8') > maxBytes) return null
+		return result.stdout
+	}
+
+	const status = await git(['status', '--porcelain'])
+	if (status === null) return null
+
+	const diff = await git(['diff', '--binary', 'HEAD'])
+	if (diff === null) return null
+
+	const untracked = await git(['ls-files', '--others', '--exclude-standard', '-z'])
+	if (untracked === null) return null
+
+	const parts = [`status ${status}`, `diff ${diff}`]
+	// Split on NUL, which is what `-z` is for: a path may contain a newline,
+	// and splitting on one would turn a single strange filename into two
+	// ordinary-looking ones.
+	//
+	// Sorted, because `ls-files` order is not part of any contract and a
+	// fingerprint that moved when the listing order did would report a change
+	// nobody made.
+	for (const rel of untracked.split('\0').filter(Boolean).sort()) {
+		const entry = await untrackedEntry(cwd, rel, fs)
+		if (entry === null) return null
+		parts.push(entry)
+	}
+
+	// Length-prefixed rather than delimiter-joined. A diff can contain any
+	// byte, so any separator is a separator the content can forge — and two
+	// different trees that agreed after forgery would be reported as
+	// unchanged, which is the one wrong answer this file is arranged to avoid.
+	const hash = createHash('sha256')
+	for (const part of parts) hash.update(`${Buffer.byteLength(part, 'utf8')}:${part}`)
+	return hash.digest('hex')
+}