@namzu/sdk 62.7.0 → 62.9.0

package/src/store/session-log/core.ts

@@ -52,6 +52,12 @@ import {
 } from './lease.js'
 import type { SpillReadOptions, SpillRef, SpillStore } from './spill.js'
 import { TOOL_INPUT_RECORD_TYPES, spilledInput, toolInputSlots } from './spilled-input.js'
+import {
+	type VerifiedPrefix,
+	rehashPrefix,
+	startVerifiedPrefix,
+	verifiedPrefixOf,
+} from './verified-prefix.js'
 
 // ─── public types ─────────────────────────────────────────────────────────
 
@@ -91,6 +97,17 @@ export interface ReadSessionLogOptions {
 	readonly mode?: 'strict' | 'tolerant'
 	/** Resume after this record: its bytes are checked, then the walk continues from it. */
 	readonly after?: RecordPointer
+	/**
+	 * With `after`, and in `strict` mode only: also confirm that every byte
+	 * before the cursor is what this process verified. The prefix is read and
+	 * hashed line by line against what was remembered (no parsing, nothing
+	 * kept); when this process remembers nothing of the log, or any byte
+	 * differs, it is verified in full from byte 0, which refuses a broken chain
+	 * as a read from byte 0 does. Without it a resumed read trusts the records
+	 * before the cursor, as a reduction that is advanced rather than recomputed
+	 * does. Ignored when the medium has no `identity`.
+	 */
+	readonly reverify?: boolean
 	/**
 	 * The record the log must hold at `expectHead.seq`, byte for byte. Anchors
 	 * the tail, which no later record vouches for.
@@ -288,6 +305,25 @@ export interface LogMedium extends LogBytes {
 	truncate(size: number, expectedSize: number): Promise<void>
 	/** The bytes from `offset` to the end, in chunks. */
 	stream(offset: number): AsyncIterable<Uint8Array>
+	/**
+	 * A string that names this log's bytes for as long as the same file stands
+	 * (the disk medium: device, inode and path), or `undefined` when the log
+	 * has none yet or the medium cannot say. A log that has one lets a later
+	 * claim in the same process re-check the bytes it already verified without
+	 * parsing them again; the content is still compared, so a wrong or reused
+	 * identity can only cost a full verification, never skip one.
+	 */
+	identity?(): Promise<string | undefined>
+	/**
+	 * Hand the bytes from `offset` to `visit`, chunk by chunk, until `visit`
+	 * answers `true` or the log ends. The chunk is only valid for the call: the
+	 * medium may fill the same buffer again for the next one, so a visitor keeps
+	 * no slice of it. It lets a re-check of a long log read through one buffer
+	 * instead of allocating a fresh native buffer for every megabyte, which
+	 * otherwise makes the collector run on every few walks. Optional; `stream`
+	 * is used when a medium has no `scan`.
+	 */
+	scan?(offset: number, visit: (chunk: Uint8Array) => boolean): Promise<void>
 }
 
 export interface SessionLogCoreOptions {
@@ -397,6 +433,8 @@ export class SessionLogCore implements SessionLog {
 	readonly #holding: HeldSessionLease = { release: () => this.#releaseHeld() }
 	/** Bytes of the log this instance has verified and applied. */
 	#synced = 0
+	/** What this process remembers of the log's verified prefix (see `verified-prefix.ts`). */
+	#ledgerKey: string | undefined
 	#torn = 0
 	readonly #degradedBufferBytes: number
 	/** Records accepted into the chain that the medium has not taken yet, oldest first. */
@@ -462,13 +500,48 @@ export class SessionLogCore implements SessionLog {
 
 	// ── state ──
 
-	/** Catch up with the medium: verify and apply whatever was appended since the last sync. */
+	/**
+	 * Catch up with the medium: verify and apply whatever was appended since the last sync.
+	 *
+	 * An instance that has nothing yet (a fresh one) first asks whether this
+	 * process already verified a prefix of this very log: it then re-reads and
+	 * hashes every byte of that prefix, and takes it as verified only when
+	 * every line's length and SHA-256 equal what was verified (no parsing, no
+	 * schema check). The bytes after it, and every byte when nothing matches,
+	 * are verified in full.
+	 */
 	async #catchUp(): Promise<void> {
 		// Records waiting for the disk are ahead of it; the lease keeps others out.
 		if (this.#pending.length > 0) return
 		const size = await this.#medium.size()
 		if (size === this.#synced + this.#torn && size >= this.#synced) return
 		if (size < this.#synced) this.#reset()
+		let ledger: VerifiedPrefix | undefined
+		const key = await this.#medium.identity?.()
+		this.#ledgerKey = key
+		if (key !== undefined) {
+			ledger = verifiedPrefixOf(key)
+			if (this.#synced === 0 && this.#chain.head === null) {
+				const remembered = ledger
+				if (
+					remembered !== undefined &&
+					remembered.count > 0 &&
+					remembered.sessionId === this.sessionId &&
+					remembered.end <= size &&
+					(await rehashPrefix(this.#medium, remembered, remembered.count))
+				) {
+					this.#chain = new SessionLogChain({
+						head: remembered.head,
+						sessionId: remembered.sessionId,
+						gen: remembered.gen,
+					})
+					this.#turns = remembered.turns()
+					this.#synced = remembered.end
+				} else {
+					ledger = startVerifiedPrefix(key)
+				}
+			}
+		}
 		const splitter = new LineSplitter(this.#synced)
 		for await (const chunk of this.#medium.stream(this.#synced)) {
 			for (const { line, offset } of splitter.push(chunk)) {
@@ -483,6 +556,7 @@ export class SessionLogCore implements SessionLog {
 				}
 				this.#turns.apply(entry.record)
 				this.#synced = offset + line.byteLength
+				ledger?.append(entry)
 			}
 		}
 		this.#torn = splitter.pendingBytes
@@ -761,9 +835,21 @@ export class SessionLogCore implements SessionLog {
 		const entry = this.#chain.accept(bytes, this.#synced)
 		this.#turns.apply(entry.record)
 		this.#synced += bytes.byteLength
+		// On the disk and fully verified: a later claim need not parse it again.
+		if (this.#pending.length === 0) await this.#rememberWritten(entry)
 		return entry
 	}
 
+	/** Let this process remember a record this instance just wrote and verified. */
+	async #rememberWritten(entry: SessionLogEntry): Promise<void> {
+		if (this.#ledgerKey === undefined) this.#ledgerKey = await this.#medium.identity?.()
+		if (this.#ledgerKey === undefined) return
+		const ledger =
+			verifiedPrefixOf(this.#ledgerKey) ??
+			(entry.pointer.seq === 1 ? startVerifiedPrefix(this.#ledgerKey) : undefined)
+		ledger?.append(entry)
+	}
+
 	/**
 	 * Hand a record's bytes to the medium. A volume that refuses writes (full,
 	 * over quota, read-only) does not fail the record: it waits in a bounded
@@ -888,6 +974,7 @@ export class SessionLogCore implements SessionLog {
 		if (
 			type !== 'message' &&
 			type !== 'message_replaced' &&
+			type !== 'message_edited' &&
 			type !== 'compaction' &&
 			type !== 'turn_completed'
 		)
@@ -926,7 +1013,7 @@ export class SessionLogCore implements SessionLog {
 		const key =
 			content.role === 'tool' && typeof content.toolCallId === 'string'
 				? content.toolCallId
-				: type === 'message_replaced'
+				: type === 'message_replaced' || type === 'message_edited'
 					? `record:${String(candidate.id)}`
 					: `message:${String(candidate.messageId)}`
 		const ref = await this.#spillOrLose(key, 'message', JSON.stringify(content))
@@ -1093,7 +1180,7 @@ export async function collect(
  * both backends and by `readSessionLog`.
  */
 export async function* walkSessionLog(
-	medium: Pick<LogMedium, 'size' | 'read' | 'stream'>,
+	medium: Pick<LogMedium, 'size' | 'read' | 'stream' | 'identity' | 'scan'>,
 	options: ReadSessionLogOptions & { readonly sessionId?: SessionId },
 ): AsyncGenerator<SessionLogEntry, SessionLogReadSummary> {
 	const strict = (options.mode ?? 'strict') === 'strict'
@@ -1105,6 +1192,9 @@ export async function* walkSessionLog(
 		brokeAt = error
 	}
 
+	// A resumed read that must also vouch for the bytes before its cursor.
+	let covered = 0
+	let ledger: VerifiedPrefix | undefined
 	if (options.after !== undefined) {
 		const after = options.after
 		if (!(await verifyPointer(medium, after))) {
@@ -1117,15 +1207,42 @@ export async function* walkSessionLog(
 			fail(error)
 			return { intact: false, throughSeq: 0, head: null, tornBytes: 0, break: error }
 		}
-		const parsed = parseSessionLogLine(await medium.read(after.offset, after.length))
-		chain = new SessionLogChain({
-			head: after,
-			sessionId: options.sessionId ?? parsed.record.sessionId,
-			gen: parsed.record.gen,
-		})
-		start = after.offset + after.length
+		const key = options.reverify && strict ? await medium.identity?.() : undefined
+		if (key !== undefined) {
+			const remembered = verifiedPrefixOf(key)
+			if (
+				remembered !== undefined &&
+				remembered.sessionId === options.sessionId &&
+				remembered.holds(after) &&
+				(await rehashPrefix(medium, remembered, after.seq))
+			) {
+				ledger = remembered
+			} else {
+				// Nothing remembered, or it no longer matches: verify from byte 0 and
+				// remember it, handing the caller only what follows the cursor.
+				ledger = startVerifiedPrefix(key)
+				covered = after.seq
+			}
+		}
+		if (covered > 0) {
+			chain = new SessionLogChain({ head: null, sessionId: options.sessionId })
+		} else {
+			const parsed = parseSessionLogLine(await medium.read(after.offset, after.length))
+			chain = new SessionLogChain({
+				head: after,
+				sessionId: options.sessionId ?? parsed.record.sessionId,
+				gen: parsed.record.gen,
+			})
+			start = after.offset + after.length
+		}
 	} else {
 		chain = new SessionLogChain({ head: null, sessionId: options.sessionId })
+		// A whole read that asked to reverify is the first of its kind: what it verifies in
+		// full is remembered, so the resumed reads after it can re-check by hash.
+		if (options.reverify && strict) {
+			const key = await medium.identity?.()
+			if (key !== undefined) ledger = verifiedPrefixOf(key) ?? startVerifiedPrefix(key)
+		}
 	}
 
 	const splitter = new LineSplitter(start)
@@ -1165,6 +1282,8 @@ export async function* walkSessionLog(
 						break outer
 					}
 				}
+				ledger?.append(entry)
+				if (entry.pointer.seq <= covered) continue
 				yield entry
 				if (options.throughSeq !== undefined && entry.pointer.seq >= options.throughSeq) {
 					stopped = true

package/src/store/session-log/disk.ts

@@ -59,6 +59,16 @@ export class DiskLogMedium implements LogMedium {
 		}
 	}
 
+	async identity(): Promise<string | undefined> {
+		try {
+			const stats = await stat(this.file)
+			return `${stats.dev}:${stats.ino}:${this.file}`
+		} catch (error) {
+			if (isErrno(error, 'ENOENT')) return undefined
+			throw error
+		}
+	}
+
 	async read(offset: number, length: number): Promise<Uint8Array> {
 		let handle: FileHandle
 		try {
@@ -76,6 +86,28 @@ export class DiskLogMedium implements LogMedium {
 		}
 	}
 
+	async scan(offset: number, visit: (chunk: Uint8Array) => boolean): Promise<void> {
+		let handle: FileHandle
+		try {
+			handle = await open(this.file, 'r')
+		} catch (error) {
+			if (isErrno(error, 'ENOENT')) return
+			throw error
+		}
+		try {
+			const buffer = Buffer.allocUnsafe(1 << 20)
+			let position = offset
+			for (;;) {
+				const { bytesRead } = await handle.read(buffer, 0, buffer.length, position)
+				if (bytesRead === 0) return
+				position += bytesRead
+				if (visit(buffer.subarray(0, bytesRead))) return
+			}
+		} finally {
+			await handle.close()
+		}
+	}
+
 	async *stream(offset: number): AsyncIterable<Uint8Array> {
 		if ((await this.size()) <= offset) return
 		const stream = createReadStream(this.file, { start: offset, highWaterMark: 1 << 20 })

package/src/store/session-log/fold.ts

@@ -22,7 +22,10 @@ import type { SpillRef } from './spill.js'
  * override or a structured-output answer is what every reader of the fold
  * sees; the model's raw text stays in the log for audit only.
  *
- * Only `message`, `message_replaced` and `compaction` records move the fold.
+ * `message_edited` is applied the same way as `message_replaced`: a compaction
+ * pass's edit of one message, written under the message's own id.
+ *
+ * Only `message`, `message_replaced`, `message_edited` and `compaction` records move the fold.
  * The live compaction events (`compaction_completed` and the rest) are
  * reports, not instructions.
  */
@@ -118,7 +121,8 @@ export class SessionMessageFold {
 			case 'message':
 				this.#message(record as MessageRecord)
 				return
-			case 'message_replaced': {
+			case 'message_replaced':
+			case 'message_edited': {
 				const spill = (record as { spill?: SpillRef }).spill
 				this.#replacements.set(record.targetMessageId, {
 					message: record.content,
@@ -311,6 +315,18 @@ export class SessionTurnState {
 		return this.#active
 	}
 
+	/**
+	 * An independent copy at the records applied so far. `apply` replaces the
+	 * active turn's object rather than editing it, so the copy may share it.
+	 */
+	clone(): SessionTurnState {
+		const copy = new SessionTurnState()
+		copy.#active = this.#active
+		copy.#lastTurnId = this.#lastTurnId
+		copy.#started = this.#started
+		return copy
+	}
+
 	/** The last turn that started, active or not. */
 	get lastTurnId(): TurnId | undefined {
 		return this.#lastTurnId
@@ -412,5 +428,6 @@ const CLOSES_OR_DECIDES: ReadonlySet<string> = new Set([
 	'audit',
 	'log_repaired',
 	'message_replaced',
+	'message_edited',
 	'session_updated',
 ])

package/src/store/session-log/incremental.ts

@@ -0,0 +1,160 @@
+import type { RecordPointer } from '../../types/session/records.js'
+import { type SessionLogEntry, SessionLogIntegrityError } from './chain.js'
+import type { SessionLog } from './core.js'
+
+/**
+ * Reductions over a whole session log, advanced rather than recomputed.
+ *
+ * A session log only grows, and every turn asks it the same questions (which
+ * checkpoints were committed, which turn ids exist, what the folded context
+ * is). Answering each from byte 0 parses and verifies every record the
+ * session ever wrote, so a turn cost more the longer the session. A reduction
+ * held here remembers the last record it folded and, on the next ask, walks
+ * only what was appended after it. `SessionLog.read({ after })` checks that
+ * record against the log byte for byte before continuing, so a log that was
+ * replaced, truncated or rewritten at that record is noticed and the
+ * reduction starts again from byte 0 with the original behaviour (strict reads
+ * refuse a broken chain, tolerant reads stop at the break).
+ *
+ * What a resumed read does not do is re-verify the records before the anchor:
+ * they were verified when they were first folded, by this process. A byte
+ * flipped in them afterwards, with every later record left as it was, is not
+ * noticed by this path; a fresh process (a resume) always starts from byte 0.
+ *
+ * The cache is keyed by session id and the reduction's own name and mode,
+ * holds a few sessions, and is anchored by content, so two logs that share a
+ * session id never share a result.
+ *
+ * Cost that remains: the state of the "every message ever recorded" reduction
+ * holds each message the log ever recorded, superseded and compacted ones
+ * included, and the earlier texts of an edited message (one per edit, until a
+ * replacement clears them), so it grows with the log and not with the live context, and it
+ * lives for the life of the process (for the sessions kept). Reading it
+ * returns a copy of all of them, which is O(history) per turn. Both are small
+ * next to the whole-log walks this file removes, and are left as they are.
+ * More than SESSIONS_KEPT sessions in use at once evict one another, and each
+ * eviction restarts from byte 0, the cost before this file existed.
+ */
+
+const SESSIONS_KEPT = 4
+
+interface Kept {
+	state: unknown
+	/** The last record folded; the next read resumes after it. */
+	anchor: RecordPointer | undefined
+	/** Calls on one reduction take turns: the state is advanced in place. */
+	turn: Promise<unknown>
+}
+
+const sessions = new Map<string, Map<string, Kept>>()
+
+function keptFor(sessionId: string, key: string): Kept {
+	let reductions = sessions.get(sessionId)
+	if (reductions === undefined) {
+		reductions = new Map()
+		if (sessions.size >= SESSIONS_KEPT) {
+			const oldest = sessions.keys().next().value
+			if (oldest !== undefined) sessions.delete(oldest)
+		}
+	} else {
+		sessions.delete(sessionId)
+	}
+	sessions.set(sessionId, reductions)
+	let kept = reductions.get(key)
+	if (kept === undefined) {
+		kept = { state: undefined, anchor: undefined, turn: Promise.resolve() }
+		reductions.set(key, kept)
+	}
+	return kept
+}
+
+export interface IncrementalFold<S, R> {
+	/** The reduction's name; one cache slot per name and read mode. */
+	readonly name: string
+	readonly mode: 'strict' | 'tolerant'
+	/**
+	 * Strict reductions only: before resuming, hash every byte before the
+	 * resume point against what this process verified (`read({ reverify })`),
+	 * so a resumed read refuses a log altered anywhere, as a read from byte 0
+	 * does, without parsing the prefix again. Off by default: the window the
+	 * header describes then applies.
+	 */
+	readonly reverify?: boolean
+	/** A fresh, empty state. */
+	create(): S
+	/** Fold one record into the state, in place. */
+	apply(state: S, entry: SessionLogEntry): void
+	/**
+	 * The answer, from a state that is current for as long as this call runs.
+	 * It must not keep or hand out the state itself, which later calls advance.
+	 */
+	result(state: S): Promise<R> | R
+}
+
+/**
+ * Fold `log` into `fold.create()` through `fold.apply`, resuming from the
+ * record the previous call for this session stopped at when the log still
+ * holds it, and answer with `fold.result`.
+ */
+export async function foldSessionLog<S, R>(
+	log: SessionLog,
+	fold: IncrementalFold<S, R>,
+): Promise<R> {
+	const kept = keptFor(log.sessionId, `${fold.name}:${fold.mode}`)
+	const run = async (): Promise<R> => {
+		let state = kept.state as S | undefined
+		let anchor = kept.anchor
+		for (let attempt = 0; attempt < 2; attempt++) {
+			const resuming = state !== undefined && anchor !== undefined
+			if (!resuming) {
+				state = fold.create()
+				anchor = undefined
+			}
+			let moved = false
+			try {
+				const walk = log.read({
+					mode: fold.mode,
+					...(anchor ? { after: anchor } : {}),
+					...(fold.reverify && fold.mode === 'strict' ? { reverify: true } : {}),
+				})
+				let summary: Awaited<ReturnType<typeof walk.next>>
+				for (;;) {
+					summary = await walk.next()
+					if (summary.done) break
+					moved = true
+					fold.apply(state as S, summary.value)
+				}
+				const ended = summary.value
+				if (resuming && !moved && ended.break?.reason === 'anchor-mismatch') {
+					kept.state = undefined
+					kept.anchor = undefined
+					state = undefined
+					anchor = undefined
+					continue
+				}
+				kept.state = state
+				kept.anchor = ended.head?.pointer ?? anchor
+				return await fold.result(state as S)
+			} catch (error) {
+				// The state may be half-advanced: never reuse it.
+				kept.state = undefined
+				kept.anchor = undefined
+				if (
+					resuming &&
+					!moved &&
+					error instanceof SessionLogIntegrityError &&
+					error.reason === 'anchor-mismatch'
+				) {
+					state = undefined
+					anchor = undefined
+					continue
+				}
+				throw error
+			}
+		}
+		throw new Error('unreachable')
+	}
+	const result = kept.turn.then(run, run)
+	kept.turn = result.catch(() => undefined)
+	return result
+}

package/src/store/session-log/index.ts

@@ -61,6 +61,7 @@ export {
 	withMessageId,
 } from './fold.js'
 export { type TornTailRepair, repairRecordDraft } from './heal.js'
+export { type IncrementalFold, foldSessionLog } from './incremental.js'
 export {
 	foldChildOutcomes,
 	foldTurnOutcomes,