@namzu/sdk 43.0.0 → 44.1.0

package/src/store/evidence/compaction-archive.ts

@@ -1,139 +0,0 @@
-import { randomUUID } from 'node:crypto'
-import { lstat, mkdir, writeFile } from 'node:fs/promises'
-import { join } from 'node:path'
-import { z } from 'zod'
-import type { RunEvent } from '../../types/run/events.js'
-import type { CompactedToolMetadata } from './compaction-provenance.js'
-import { compactedTexts } from './compaction-text.js'
-import { digest, spillManifest } from './format.js'
-import { RECORD_BYTES, decode, noLinks, openEvidence, stamp } from './io.js'
-
-const integer = z.number().int().nonnegative().safe()
-const hash = z.string().regex(/^[a-f0-9]{64}$/)
-const role = z.enum(['system', 'user', 'assistant', 'tool'])
-export const compactionArchiveSchema = z.object({
-	type: z.literal('compaction_archive'),
-	runId: z.string().uuid(),
-	seq: integer.positive().optional(),
-	iteration: integer,
-	reason: z.enum(['threshold', 'overflow', 'manual']),
-	archive: z.object({
-		version: z.literal(1),
-		id: z.string().uuid(),
-		bytes: integer,
-		sha256: hash,
-		messageCount: integer,
-		parts: z
-			.array(
-				z.object({
-					role,
-					summary: z.literal(true).optional(),
-					manifest: hash,
-					toolName: z.string().min(1).max(1024).optional(),
-					isError: z.boolean().optional(),
-				}),
-			)
-			.max(8192),
-	}),
-})
-
-/** Internal storage encoding only; RunStore readers restore compaction_shed. */
-export function compactionArchiveDirectory(runDir: string, id: string): string {
-	return join(runDir, 'compaction-output', z.string().uuid().parse(id))
-}
-
-export function compactionPartPath(runDir: string, id: string, part: number): string {
-	return join(compactionArchiveDirectory(runDir, id), `${integer.parse(part)}.txt`)
-}
-
-/** Keep JSONL index records bounded while preserving whole original messages. */
-export async function retainCompactionRecord(
-	event: RunEvent,
-	runDir: string,
-): Promise<RunEvent | z.infer<typeof compactionArchiveSchema>> {
-	if (event.type !== 'compaction_shed') return event
-	const { messages: removed, type: _type, ...envelope } = event
-	const messageCount = removed.length
-	const original = Buffer.from(JSON.stringify(event.messages), 'utf8')
-	if (original.length <= 3 * 1024 * 1024) return event
-	const texts = compactedTexts(event.messages)
-	if (texts.length > 8192) throw new Error('Compaction archive exceeds 8192 textual parts.')
-	// The same 4096-chunk ceiling the reader enforces for retained tool text.
-	if (texts.some((part) => Buffer.byteLength(part.text, 'utf8') > 256 * 1024 * 1024))
-		throw new Error('Compaction archive text exceeds 256 MiB per part.')
-	const id = randomUUID()
-	const dir = compactionArchiveDirectory(runDir, id)
-	await noLinks(runDir)
-	await mkdir(join(runDir, 'compaction-output'), { recursive: true, mode: 0o700 })
-	await noLinks(join(runDir, 'compaction-output'))
-	await mkdir(dir, { mode: 0o700 })
-	await writeFile(join(dir, 'messages.json'), original, { flag: 'wx', mode: 0o600 })
-	const parts: ({
-		role: z.infer<typeof role>
-		manifest: string
-		summary?: true
-	} & CompactedToolMetadata)[] = []
-	for (const [part, value] of texts.entries()) {
-		const bytes = Buffer.from(value.text, 'utf8')
-		const manifest = spillManifest(bytes)
-		if (Buffer.byteLength(manifest, 'utf8') > RECORD_BYTES)
-			throw new Error('Compaction text manifest exceeds its 4 MiB read limit.')
-		const path = compactionPartPath(runDir, id, part)
-		await writeFile(path, bytes, { flag: 'wx', mode: 0o600 })
-		await writeFile(`${path}.manifest.json`, manifest, { flag: 'wx', mode: 0o600 })
-		parts.push({
-			role: value.role,
-			summary: value.summary,
-			manifest: digest(manifest),
-			toolName: value.toolName,
-			isError: value.isError,
-		})
-	}
-	return {
-		...envelope,
-		...compactionArchiveSchema.parse({
-			type: 'compaction_archive',
-			runId: event.runId,
-			seq: event.seq,
-			iteration: event.iteration,
-			reason: event.reason,
-			archive: {
-				version: 1,
-				id,
-				bytes: original.length,
-				sha256: digest(original),
-				messageCount,
-				parts,
-			},
-		}),
-	}
-}
-
-/** Full event readback is explicit and may load large bodies; search never calls this. */
-export async function restoreCompactionRecord(
-	record: Record<string, unknown>,
-	runDir: string,
-): Promise<Record<string, unknown>> {
-	if (record.type !== 'compaction_archive') return record
-	const saved = compactionArchiveSchema.parse(record)
-	const path = join(compactionArchiveDirectory(runDir, saved.archive.id), 'messages.json')
-	const file = await openEvidence(path)
-	try {
-		const before = await file.stat()
-		if (before.size !== saved.archive.bytes) throw new Error('Compaction archive size changed.')
-		const bytes = await file.readFile()
-		if (
-			stamp(before) !== stamp(await file.stat()) ||
-			stamp(before) !== stamp(await lstat(path)) ||
-			digest(bytes) !== saved.archive.sha256
-		)
-			throw new Error('Compaction archive bytes changed.')
-		const messages: unknown = JSON.parse(decode(bytes))
-		if (!Array.isArray(messages) || messages.length !== saved.archive.messageCount)
-			throw new Error('Compaction archive message count changed.')
-		const { archive: _archive, ...rest } = record
-		return { ...rest, type: 'compaction_shed', messages }
-	} finally {
-		await file.close()
-	}
-}

package/src/store/evidence/linked.ts

@@ -1,365 +0,0 @@
-import { lstat } from 'node:fs/promises'
-import type { FileHandle } from 'node:fs/promises'
-import { join } from 'node:path'
-import { z } from 'zod'
-import { EVIDENCE_CHUNK_BYTES, digest } from './format.js'
-import { type EvidenceSeal, eventTexts, evidenceSeal } from './index-page.js'
-import {
-	type EvidenceBudget,
-	EvidencePageLimit,
-	PAGE_BYTES,
-	decode,
-	openEvidence,
-	readBytes,
-	readSmall,
-} from './io.js'
-import { passageMatcher, passagesInWindow } from './passages.js'
-import { type RecordPointer, recordPointerSchema, recordPredecessors } from './record-chain.js'
-import { evidenceSearchInput, evidenceTermsSchema } from './search-input.js'
-import {
-	evidenceExclusionsKey,
-	evidenceExclusionsSchema,
-	excludesSuccessfulTool,
-} from './selection.js'
-import { createTextSourceReader, readTextPage, textPointerSchema } from './source-text.js'
-import type {
-	DiskRunEvidenceOptions,
-	RunEvidenceReadOptions,
-	RunTextEvidenceMatch,
-	RunTextEvidenceSearchOptions,
-	RunTextEvidenceSource,
-} from './types.js'
-
-const integer = z.number().int().nonnegative().safe()
-const scopeSchema = z
-	.object({
-		tenantId: z.string().uuid(),
-		projectId: z.string().uuid(),
-		sessionId: z.string().uuid(),
-		runId: z.string().uuid(),
-	})
-	.strict()
-const cursorSchema = z.object({
-	kind: z.enum(['linked-search', 'linked-search-terms', 'linked-search-tokens']),
-	query: z.string().max(256),
-	termsKey: z
-		.string()
-		.regex(/^[a-f0-9]{64}$/)
-		.optional(),
-	caseSensitive: z.boolean().default(true),
-	matchMode: z.enum(['literal', 'token']).default('literal'),
-	exclusionsKey: z
-		.string()
-		.regex(/^[a-f0-9]{64}$/)
-		.optional(),
-	seq: integer.optional(),
-	part: integer.optional(),
-	next: recordPointerSchema.nullable(),
-	textIndex: integer,
-	chunk: integer,
-	within: integer.default(0),
-})
-const addressSchema = z.object({ kind: z.literal('linked-text'), entry: textPointerSchema })
-
-/** Internal: only the durable writer can supply this anchor. Never accept it from a tool input. */
-export function createLinkedRunTextEvidenceSource(
-	options: DiskRunEvidenceOptions,
-	anchor: { tip: RecordPointer; identity: string; epoch: string },
-): RunTextEvidenceSource {
-	const scope = Object.freeze(scopeSchema.parse(options.scope))
-	const runDir = options.runDir
-	const tip = recordPointerSchema.parse(anchor.tip)
-	const ceiling = tip.offset + tip.length
-	const scopeKey = digest(JSON.stringify(scope))
-	const sourceKey = digest(`linked-v1:${scopeKey}:${anchor.identity}:${anchor.epoch}`)
-	const maxReadBytes = integer
-		.min(1024 * 1024)
-		.max(PAGE_BYTES)
-		.parse(options.maxReadBytes ?? PAGE_BYTES)
-	const inBoundary = (pointer: RecordPointer) => {
-		if (pointer.seq > tip.seq || pointer.offset + pointer.length > ceiling)
-			throw new Error('Evidence address exceeds the captured boundary.')
-	}
-	async function access<T extends { scannedBytes: number }>(
-		signal: AbortSignal | undefined,
-		requestedBytes: number | undefined,
-		action: (handle: FileHandle, seal: EvidenceSeal, budget: EvidenceBudget) => Promise<T>,
-	): Promise<T> {
-		const limit = Math.min(maxReadBytes, requestedBytes ?? maxReadBytes)
-		const budget: EvidenceBudget = { bytes: 0, limit, signal }
-		signal?.throwIfAborted()
-		const verifyOwner = async () => {
-			const metadata = JSON.parse(
-				decode(await readSmall(join(runDir, 'run.json'), budget, 512 * 1024)),
-			)
-			if (
-				metadata.id !== scope.runId ||
-				JSON.stringify(scopeSchema.parse(metadata.metadata?.scope)) !== JSON.stringify(scope)
-			)
-				throw new Error('Evidence run does not belong to the authorized scope.')
-		}
-		await verifyOwner()
-		// Reserve a final ownership check; counters/status may change while this run works.
-		budget.limit = limit - 512 * 1024
-
-		const path = join(options.runDir, 'transcript.jsonl')
-		const handle = await openEvidence(path)
-		const check = (stat: { dev: number; ino: number; size: number }) => {
-			if (`${stat.dev}:${stat.ino}` !== anchor.identity || stat.size < ceiling)
-				throw new Error('Captured transcript was replaced or shortened.')
-		}
-		try {
-			check(await handle.stat())
-			const seal = await evidenceSeal(options.indexDir, scopeKey, sourceKey, budget)
-			const result = await action(handle, seal, budget)
-			signal?.throwIfAborted()
-			check(await handle.stat())
-			check(await lstat(path))
-			budget.limit = limit
-			await verifyOwner()
-			return { ...result, scannedBytes: budget.bytes }
-		} finally {
-			await handle.close()
-		}
-	}
-	return Object.freeze({
-		scope,
-		async search(options: RunTextEvidenceSearchOptions = {}, signal?: AbortSignal) {
-			const input = z
-				.object({
-					maxReadBytes: integer
-						.min(1024 * 1024)
-						.max(PAGE_BYTES)
-						.optional(),
-					query: z.string().max(256).optional(),
-					terms: evidenceTermsSchema,
-					caseSensitive: z.boolean().default(true),
-					matchMode: z.enum(['literal', 'token']).default('literal'),
-					excludeSuccessfulTools: evidenceExclusionsSchema,
-					excludeDerivedSummaries: z.boolean().default(false),
-					cursor: z.string().max(4096).optional(),
-					seq: integer.positive().optional(),
-					part: integer.optional(),
-					limit: integer.min(1).max(4).default(4),
-				})
-				.strict()
-				.parse(options)
-			if (input.part !== undefined && input.seq === undefined)
-				throw new Error('Part requires an event sequence.')
-			const { query, terms, termsKey, browse } = evidenceSearchInput(input)
-			const exclusionsKey = evidenceExclusionsKey(
-				input.excludeSuccessfulTools,
-				input.excludeDerivedSummaries,
-			)
-			const kind =
-				input.matchMode === 'token'
-					? ('linked-search-tokens' as const)
-					: terms
-						? ('linked-search-terms' as const)
-						: ('linked-search' as const)
-			return access(signal, input.maxReadBytes, async (handle, seal, budget) => {
-				const readSource = createTextSourceReader(handle, runDir, scope.runId, budget)
-				const cursor = input.cursor
-					? cursorSchema.parse(seal.unpack(input.cursor))
-					: {
-							kind,
-							query,
-							termsKey,
-							exclusionsKey,
-							seq: input.seq,
-							part: input.part,
-							next: tip as RecordPointer | null,
-							textIndex: 0,
-							chunk: 0,
-							within: 0,
-							caseSensitive: input.caseSensitive,
-							matchMode: input.matchMode,
-						}
-				if (
-					cursor.kind !== kind ||
-					cursor.query !== query ||
-					cursor.termsKey !== termsKey ||
-					cursor.exclusionsKey !== exclusionsKey ||
-					cursor.caseSensitive !== input.caseSensitive ||
-					cursor.matchMode !== input.matchMode ||
-					cursor.seq !== input.seq ||
-					cursor.part !== input.part
-				)
-					throw new Error('Search cursor query changed.')
-				const matchPassage = passageMatcher(terms ?? query, input.caseSensitive, input.matchMode)
-				const matches: RunTextEvidenceMatch[] = []
-				const unavailable: string[] = []
-				let records = 0
-				let parts = 0
-				let chunks = 0
-				let incomplete = false
-				let excludedToolResults = 0
-				let excludedSummaries = 0
-				while (
-					cursor.next &&
-					records < 64 &&
-					parts < 64 &&
-					chunks < 64 &&
-					matches.length < input.limit
-				) {
-					const pointer = cursor.next
-					inBoundary(pointer)
-					if (input.seq !== undefined && pointer.seq < input.seq) {
-						cursor.next = null
-						break
-					}
-					let raw: Buffer
-					try {
-						raw = await readBytes(handle, pointer.offset, pointer.length, budget)
-					} catch (error) {
-						if (error instanceof EvidencePageLimit) {
-							if (records === 0) throw new Error('Recorded text exceeds the available page budget.')
-							break
-						}
-						throw error
-					}
-					if (digest(raw) !== pointer.sha256)
-						throw new Error('Recorded text integrity chain changed.')
-					const event = JSON.parse(decode(raw))
-					if (event.runId !== scope.runId || event.seq !== pointer.seq)
-						throw new Error('Recorded text identity changed.')
-					records++
-					const links = recordPredecessors(event, pointer)
-					incomplete ||= links.incomplete
-					const texts =
-						input.seq === undefined || input.seq === pointer.seq ? eventTexts(event) : []
-					while (
-						cursor.textIndex < texts.length &&
-						parts < 64 &&
-						chunks < 64 &&
-						matches.length < input.limit
-					) {
-						const part = cursor.textIndex
-						if (input.part !== undefined && part !== input.part) {
-							cursor.textIndex++
-							continue
-						}
-						parts++
-						try {
-							const derived =
-								input.excludeDerivedSummaries && texts[part]?.source === 'compaction_shed:summary'
-							if (
-								derived ||
-								excludesSuccessfulTool(
-									event.type === 'tool_completed' && typeof event.toolUseId === 'string'
-										? event
-										: (texts[part] ?? {}),
-									input.excludeSuccessfulTools,
-								)
-							) {
-								if (derived) excludedSummaries++
-								else excludedToolResults++
-								cursor.textIndex++
-								cursor.chunk = 0
-								cursor.within = 0
-								continue
-							}
-							const source = await readSource({ ...pointer, part }, raw)
-							if (source.retained === 'preview') incomplete = true
-							while (cursor.chunk < source.chunks && chunks < 64 && matches.length < input.limit) {
-								signal?.throwIfAborted()
-								chunks++
-								if (
-									(terms ?? [query]).some((term) =>
-										source.mayMatch(cursor.chunk, term, input.matchMode, input.caseSensitive),
-									)
-								) {
-									const window = await source.window(cursor.chunk, input.matchMode === 'token')
-									const text = decode(window.bytes)
-									const page = passagesInWindow(
-										text,
-										Math.max(cursor.within, window.searchFrom ?? 0),
-										matchPassage,
-										input.limit - matches.length,
-										source.entry.spill
-											? (cursor.chunk + 1) * EVIDENCE_CHUNK_BYTES - window.offset
-											: undefined,
-									)
-									cursor.within = page.next
-									for (const { start, end } of page.passages) {
-										const excerpt = text.slice(start, end)
-										const byteOffset = window.offset + Buffer.byteLength(text.slice(0, start))
-										matches.push({
-											address: seal.pack({ kind: 'linked-text', entry: { ...pointer, part } }),
-											seq: pointer.seq,
-											recordedAt: source.recordedAt,
-											source: source.entry.source,
-											part,
-											toolName: source.entry.toolName,
-											isError: source.entry.isError,
-											retained: source.retained,
-											excerpt,
-											excerptComplete:
-												source.retained === 'full' &&
-												byteOffset === 0 &&
-												Buffer.byteLength(excerpt) === source.bytes,
-											byteOffset,
-											characterOffset:
-												window.characterOffset === undefined
-													? undefined
-													: window.characterOffset + start,
-										})
-									}
-									if (cursor.within < text.length) break
-								}
-								cursor.within = 0
-								cursor.chunk++
-								if (browse) cursor.chunk = source.chunks
-							}
-							if (cursor.chunk < source.chunks) break
-						} catch (error) {
-							signal?.throwIfAborted()
-							if (error instanceof EvidencePageLimit && (records > 1 || parts > 1 || chunks > 1))
-								break
-							unavailable.push(`Text record ${pointer.seq}/${part} is unavailable or changed.`)
-						}
-						cursor.textIndex++
-						cursor.chunk = 0
-						cursor.within = 0
-					}
-					if (cursor.textIndex < texts.length) break
-					cursor.next = links.next
-					cursor.textIndex = 0
-					cursor.chunk = 0
-					cursor.within = 0
-				}
-				return {
-					scope,
-					matches,
-					nextCursor: cursor.next ? seal.pack(cursor) : null,
-					scannedBytes: budget.bytes,
-					indexedRecords: records,
-					cacheHit: false,
-					incomplete: incomplete || unavailable.length > 0,
-					unavailable,
-					...(excludedToolResults ? { excludedToolResults } : {}),
-					...(excludedSummaries ? { excludedSummaries } : {}),
-				}
-			})
-		},
-		async read(options: RunEvidenceReadOptions, signal?: AbortSignal) {
-			const input = z
-				.object({
-					address: z.string().max(8192),
-					byteOffset: integer.default(0),
-					maxReadBytes: integer
-						.min(1024 * 1024)
-						.max(PAGE_BYTES)
-						.optional(),
-				})
-				.strict()
-				.parse(options)
-			return access(signal, input.maxReadBytes, async (handle, seal, budget) => {
-				const pointer = addressSchema.parse(seal.unpack(input.address)).entry
-				inBoundary(pointer)
-				const source = await createTextSourceReader(handle, runDir, scope.runId, budget)(pointer)
-				return readTextPage(source, scope, input.byteOffset, budget)
-			})
-		},
-	})
-}

package/src/store/evidence/record-chain.ts

@@ -1,108 +0,0 @@
-import type { FileHandle } from 'node:fs/promises'
-import { z } from 'zod'
-import { digest } from './format.js'
-import { eventTexts } from './index-page.js'
-import { RECORD_BYTES, decode, openEvidence, readBytes } from './io.js'
-
-const integer = z.number().int().nonnegative().safe()
-export const recordPointerSchema = z.object({
-	offset: integer,
-	length: integer.positive().max(RECORD_BYTES),
-	sha256: z.string().regex(/^[a-f0-9]{64}$/),
-	seq: integer.positive(),
-})
-export type RecordPointer = z.infer<typeof recordPointerSchema>
-
-/** Malformed content must still be visited and rejected by the reader. */
-export function hasEvidenceText(event: Record<string, unknown>): boolean {
-	try {
-		return eventTexts(event).length > 0
-	} catch {
-		return true
-	}
-}
-
-/** Validate links even when a text-only traversal skips intervening operational records. */
-export function recordPredecessors(event: Record<string, unknown>, pointer: RecordPointer) {
-	let previous: RecordPointer | null = null
-	let incomplete = false
-	if (event.previousRecord != null) {
-		previous = recordPointerSchema.parse(event.previousRecord)
-		if (previous.offset + previous.length !== pointer.offset || previous.seq + 1 !== pointer.seq)
-			throw new Error('Invalid text integrity chain.')
-	} else if (pointer.seq === 1) {
-		if (pointer.offset !== 0 || event.type !== 'run_started')
-			throw new Error('Invalid transcript start.')
-	} else {
-		incomplete = true
-	}
-	let next = previous
-	if (Object.hasOwn(event, 'previousTextRecord')) {
-		next = recordPointerSchema.nullable().parse(event.previousTextRecord)
-		if (
-			(next === null && previous !== null) ||
-			(next !== null &&
-				(previous === null ||
-					next.seq >= pointer.seq ||
-					next.offset + next.length > pointer.offset ||
-					(next.seq === previous.seq &&
-						(next.offset !== previous.offset ||
-							next.length !== previous.length ||
-							next.sha256 !== previous.sha256))))
-		)
-			throw new Error('Invalid text predecessor link.')
-	}
-	return { previous, next, incomplete }
-}
-
-/** Bootstrap only the last complete record, never load a growing log to capture a boundary. */
-export async function transcriptTail(
-	path: string,
-	runId: string,
-): Promise<{ tip: RecordPointer; textTip?: RecordPointer | null } | undefined> {
-	let handle: FileHandle
-	try {
-		handle = await openEvidence(path)
-	} catch (error) {
-		if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined
-		throw error
-	}
-	try {
-		const stat = await handle.stat()
-		if (!stat.size) return undefined
-		const offset = Math.max(0, stat.size - RECORD_BYTES)
-		const tail = await readBytes(handle, offset, stat.size - offset, { bytes: 0 })
-		if (tail.at(-1) !== 10) return undefined
-		const start = tail.lastIndexOf(10, tail.length - 2) + 1
-		if (start === 0 && offset !== 0) return undefined
-		const raw = tail.subarray(start)
-		let event: Record<string, unknown> | null
-		try {
-			event = JSON.parse(decode(raw))
-		} catch {
-			return undefined
-		}
-		if (event?.runId !== runId) return undefined
-		const pointer = recordPointerSchema.safeParse({
-			offset: offset + start,
-			length: raw.length,
-			sha256: digest(raw),
-			seq: event.seq,
-		})
-		if (!pointer.success) return undefined
-		const tip = pointer.data
-		// Starts, gaps and malformed records remain traversal barriers. They may
-		// never disappear behind a later text link after reopening the writer.
-		let textTip: RecordPointer | null | undefined = tip
-		try {
-			const links = recordPredecessors(event, tip)
-			if (!hasEvidenceText(event) && links.previous !== null)
-				textTip = Object.hasOwn(event, 'previousTextRecord') ? links.next : undefined
-		} catch {
-			// Keep the tail itself; retrieval will reject its malformed links.
-		}
-		return { tip, textTip }
-	} finally {
-		await handle.close()
-	}
-}