@namzu/sdk 30.2.0 → 31.1.0

package/src/store/memory/disk.ts

@@ -1,5 +1,6 @@
-import { mkdir, unlink } from 'node:fs/promises'
-import { join } from 'node:path'
+import { mkdir, realpath, unlink } from 'node:fs/promises'
+import { dirname, join, resolve } from 'node:path'
+import { NamzuError } from '../../types/errors/index.js'
 import type { MemoryId } from '../../types/ids/index.js'
 import type {
 	CreateMemoryParams,
@@ -9,7 +10,7 @@ import type {
 	MemorySearchResult,
 	MemoryStore,
 } from '../../types/memory/index.js'
-import { generateMemoryId } from '../../utils/id.js'
+import { asMemoryId, generateMemoryId } from '../../utils/id.js'
 import { SCOPE_ATTRIBUTE } from '../../utils/log/types.js'
 import { type Logger, resolveLogger } from '../../utils/logger.js'
 import { DiskRecordStore } from '../kv/record-store.js'
@@ -26,142 +27,315 @@ import { InMemoryMemoryIndex } from './index.js'
  */
 const SCHEMA = defineSchema({ kind: 'memory-store', current: 1, migrations: {} })
 
+const SAFE_STORAGE_ID = /^mem_[A-Za-z0-9_-]+$/
+
+interface StorageLocation {
+	readonly indexPath: string
+	readonly contentDir: string
+}
+
+const operationTails = new Map<string, Promise<void>>()
+
+/** Serialize every operation over one canonical disk projection in this process. */
+async function withMemoryOperationLock<T>(key: string, operation: () => Promise<T>): Promise<T> {
+	const previous = operationTails.get(key) ?? Promise.resolve()
+	let release = () => {}
+	const current = new Promise<void>((resolveCurrent) => {
+		release = resolveCurrent
+	})
+	const tail = previous.then(() => current)
+	operationTails.set(key, tail)
+
+	await previous
+	try {
+		return await operation()
+	} finally {
+		release()
+		if (operationTails.get(key) === tail) operationTails.delete(key)
+	}
+}
+
+function invalidIndex(
+	reason: string,
+	details: { entryIndex?: number; field?: string } = {},
+): never {
+	throw new NamzuError({
+		code: 'storage_error',
+		message: `Memory index is invalid${
+			details.entryIndex === undefined ? '' : ` at entry ${details.entryIndex}`
+		}: ${reason}. Refusing to treat unreadable durable memory as empty.`,
+		details,
+		retryable: false,
+	})
+}
+
+function invalidContent(id: string, reason: string, details: { field?: string } = {}): never {
+	throw new NamzuError({
+		code: 'storage_error',
+		message: `Memory content for ${id} is invalid: ${reason}. Refusing to treat indexed durable memory as absent or valid.`,
+		details: { id, ...details },
+		retryable: false,
+	})
+}
+
+/** A typed ID is not automatically a safe filesystem segment. */
+function assertStorageMemoryId(
+	value: unknown,
+	refuse: (reason: string) => never,
+): asserts value is MemoryId {
+	if (typeof value !== 'string') refuse('id must be a memory ID string')
+	try {
+		asMemoryId(value)
+	} catch {
+		refuse('id must use the recognized mem_ prefix')
+	}
+	if (!SAFE_STORAGE_ID.test(value)) {
+		refuse('id must be one cross-platform safe filename segment')
+	}
+}
+
+/**
+ * Validate the domain shape before anything is installed in the live index.
+ *
+ * `DiskRecordStore` owns the version envelope and JSON parsing; it cannot
+ * know that this particular array contains memory entries. Keeping this
+ * check here avoids turning the generic primitive into a schema registry,
+ * while ensuring valid JSON with a wrong field type cannot be accepted and
+ * later written back beside a new record.
+ */
+function assertMemoryIndexEntries(value: unknown): asserts value is readonly MemoryIndexEntry[] {
+	if (!Array.isArray(value)) invalidIndex('the top-level value must be an array')
+
+	const ids = new Set<string>()
+	for (const [entryIndex, candidate] of value.entries()) {
+		if (candidate === null || typeof candidate !== 'object' || Array.isArray(candidate)) {
+			invalidIndex('each entry must be an object', { entryIndex })
+		}
+		const entry = candidate as Record<string, unknown>
+
+		assertStorageMemoryId(entry.id, (reason) =>
+			invalidIndex(reason, {
+				entryIndex,
+				field: 'id',
+			}),
+		)
+		if (ids.has(entry.id)) {
+			invalidIndex('ids must be unique', { entryIndex, field: 'id' })
+		}
+		ids.add(entry.id)
+
+		for (const field of ['title', 'summary'] as const) {
+			if (typeof entry[field] !== 'string') {
+				invalidIndex(`${field} must be a string`, { entryIndex, field })
+			}
+		}
+		if (!Array.isArray(entry.tags) || !entry.tags.every((tag) => typeof tag === 'string')) {
+			invalidIndex('tags must be an array of strings', { entryIndex, field: 'tags' })
+		}
+		if (entry.status !== 'active' && entry.status !== 'archived') {
+			invalidIndex('status must be active or archived', { entryIndex, field: 'status' })
+		}
+		for (const field of ['createdAt', 'updatedAt'] as const) {
+			if (typeof entry[field] !== 'number' || !Number.isFinite(entry[field])) {
+				invalidIndex(`${field} must be a finite number`, { entryIndex, field })
+			}
+		}
+	}
+}
+
+function assertMemoryContent(value: unknown, expectedId: MemoryId): asserts value is MemoryContent {
+	if (value === null || typeof value !== 'object' || Array.isArray(value)) {
+		invalidContent(expectedId, 'the record must be an object')
+	}
+	const content = value as Record<string, unknown>
+	assertStorageMemoryId(content.id, (reason) => invalidContent(expectedId, reason, { field: 'id' }))
+	if (content.id !== expectedId) {
+		invalidContent(expectedId, 'record id does not match its index owner', { field: 'id' })
+	}
+	if (typeof content.content !== 'string') {
+		invalidContent(expectedId, 'content must be a string', { field: 'content' })
+	}
+	if (content.format !== 'text' && content.format !== 'markdown' && content.format !== 'json') {
+		invalidContent(expectedId, 'format must be text, markdown, or json', { field: 'format' })
+	}
+	if (
+		content.metadata !== undefined &&
+		(content.metadata === null ||
+			typeof content.metadata !== 'object' ||
+			Array.isArray(content.metadata))
+	) {
+		invalidContent(expectedId, 'metadata must be an object when present', { field: 'metadata' })
+	}
+}
+
 export interface DiskMemoryStoreConfig {
 	baseDir: string
 	logger?: Logger
 }
 
 export class DiskMemoryStore implements MemoryStore {
-	private baseDir: string
-	private log: Logger
-	private index = new InMemoryMemoryIndex()
-	private initialized = false
+	private readonly baseDir: string
+	private readonly log: Logger
+	private readonly index = new InMemoryMemoryIndex()
+	private location?: StorageLocation
 	// Two instances of one primitive rather than two copies of its body.
 	// Typed separately because the index file holds an ARRAY and a content
 	// file holds one record, and a single `DiskRecordStore<unknown>` would
 	// have put the cast back at every call site.
-	private readonly records = new DiskRecordStore<MemoryContent>(SCHEMA)
-	private readonly indexRecords = new DiskRecordStore<readonly MemoryIndexEntry[]>(SCHEMA)
+	private readonly records = new DiskRecordStore<unknown>(SCHEMA)
+	private readonly indexRecords = new DiskRecordStore<unknown>(SCHEMA)
 
 	constructor(config: DiskMemoryStoreConfig) {
 		this.baseDir = join(config.baseDir, 'memory')
 		this.log = resolveLogger(config.logger).child({ [SCOPE_ATTRIBUTE]: 'store/memory/disk' })
 	}
 
-	private get indexPath(): string {
-		return join(this.baseDir, 'index.json')
-	}
-
 	private get contentDir(): string {
 		return join(this.baseDir, 'content')
 	}
 
-	private contentPath(id: MemoryId): string {
-		return join(this.contentDir, `${id}.json`)
-	}
-
-	private async ensureInit(): Promise<void> {
-		if (this.initialized) return
+	private async storageLocation(): Promise<StorageLocation> {
+		if (this.location) return this.location
 
 		await mkdir(this.contentDir, { recursive: true })
+		const [canonicalBaseDir, canonicalContentDir] = await Promise.all([
+			realpath(this.baseDir),
+			realpath(this.contentDir),
+		])
+		if (dirname(canonicalContentDir) !== canonicalBaseDir) {
+			invalidIndex('the content directory resolves outside the memory store')
+		}
+		this.location = {
+			indexPath: join(canonicalBaseDir, 'index.json'),
+			contentDir: canonicalContentDir,
+		}
+		return this.location
+	}
+
+	private contentPath(location: StorageLocation, id: MemoryId): string {
+		assertStorageMemoryId(id, (reason) => invalidContent(id, reason, { field: 'id' }))
+		const path = resolve(location.contentDir, `${id}.json`)
+		if (dirname(path) !== location.contentDir) {
+			invalidContent(id, 'resolved path escapes the content directory', { field: 'id' })
+		}
+		return path
+	}
 
+	private async reloadIndex(location: StorageLocation): Promise<void> {
 		try {
-			// `null` for "no index yet", which is an ordinary first-run state
-			// and not something to warn about. The catch below is now only
-			// for a real IO or parse failure — the distinction this method
-			// used to draw by hand with an ENOENT comparison.
-			const entries = await this.indexRecords.read(this.indexPath)
-			if (entries !== null) {
-				this.index.rebuild([...entries])
-				this.log.info('Memory index loaded', { 'namzu.store.count': entries.length })
+			const entries = await this.indexRecords.read(location.indexPath)
+			if (entries === null) {
+				this.index.rebuild([])
+				return
 			}
+			assertMemoryIndexEntries(entries)
+			this.index.rebuild([...entries])
+			this.log.info('Memory index loaded', { 'namzu.store.count': entries.length })
 		} catch (err) {
-			this.log.warn('Failed to read memory index — starting fresh', {
+			this.log.error('Failed to read memory index — refusing to start empty', {
 				'exception.message': String(err),
 			})
+			throw err
 		}
+	}
+
+	private async withAuthoritativeIndex<T>(
+		operation: (location: StorageLocation) => Promise<T>,
+	): Promise<T> {
+		const location = await this.storageLocation()
+		return withMemoryOperationLock(location.indexPath, async () => {
+			await this.reloadIndex(location)
+			return operation(location)
+		})
+	}
 
-		this.initialized = true
+	private async readContent(location: StorageLocation, id: MemoryId): Promise<MemoryContent> {
+		const content = await this.records.read(this.contentPath(location, id))
+		if (content === null) invalidContent(id, 'the indexed content record is missing')
+		assertMemoryContent(content, id)
+		return content
+	}
+
+	private async writeIndex(
+		location: StorageLocation,
+		entries: readonly MemoryIndexEntry[],
+	): Promise<void> {
+		await this.indexRecords.write(location.indexPath, entries)
 	}
 
 	async create(
 		params: CreateMemoryParams,
 	): Promise<{ entry: MemoryIndexEntry; content: MemoryContent }> {
-		await this.ensureInit()
-
-		const id = generateMemoryId()
-		const now = Date.now()
-
-		const entry: MemoryIndexEntry = {
-			id,
-			title: params.title,
-			summary: params.summary,
-			tags: params.tags ? [...params.tags] : [],
-			status: 'active',
-			createdAt: now,
-			updatedAt: now,
-		}
+		return this.withAuthoritativeIndex(async (location) => {
+			const id = generateMemoryId()
+			const now = Date.now()
+
+			const entry: MemoryIndexEntry = {
+				id,
+				title: params.title,
+				summary: params.summary,
+				tags: params.tags ? [...params.tags] : [],
+				status: 'active',
+				createdAt: now,
+				updatedAt: now,
+			}
 
-		const memoryContent: MemoryContent = {
-			id,
-			content: params.content,
-			format: params.format ?? 'text',
-			metadata: params.metadata ? { ...params.metadata } : undefined,
-		}
+			const memoryContent: MemoryContent = {
+				id,
+				content: params.content,
+				format: params.format ?? 'text',
+				metadata: params.metadata ? { ...params.metadata } : undefined,
+			}
+			const contentPath = this.contentPath(location, id)
+			const entries = [...this.index.allEntries(), entry]
 
-		this.index.set(entry)
-		await this.persistIndex()
-		await this.records.write(this.contentPath(id), memoryContent)
+			await this.records.write(contentPath, memoryContent)
+			try {
+				await this.writeIndex(location, entries)
+			} catch (error) {
+				await unlink(contentPath).catch(() => undefined)
+				throw error
+			}
+			this.index.rebuild(entries)
 
-		this.log.info('Memory created', { 'namzu.memory.id': id, 'namzu.store.title': params.title })
+			this.log.info('Memory created', {
+				'namzu.memory.id': id,
+				'namzu.store.title': params.title,
+			})
 
-		return { entry, content: memoryContent }
+			return { entry, content: memoryContent }
+		})
 	}
 
 	async get(id: MemoryId): Promise<MemoryContent | undefined> {
-		await this.ensureInit()
-
-		if (!this.index.getEntry(id)) return undefined
-
-		try {
-			return (await this.records.read(this.contentPath(id))) ?? undefined
-		} catch {
-			this.log.warn('Failed to read memory content', { 'namzu.memory.id': id })
-			return undefined
-		}
+		return this.withAuthoritativeIndex(async (location) => {
+			assertStorageMemoryId(id, (reason) => invalidContent(id, reason, { field: 'id' }))
+			if (!this.index.getEntry(id)) return undefined
+			return this.readContent(location, id)
+		})
 	}
 
 	async update(
 		id: MemoryId,
 		updates: Partial<CreateMemoryParams>,
 	): Promise<MemoryIndexEntry | undefined> {
-		await this.ensureInit()
-
-		const existing = this.index.getEntry(id)
-		if (!existing) return undefined
-
-		const now = Date.now()
-
-		const updated: MemoryIndexEntry = {
-			...existing,
-			title: updates.title ?? existing.title,
-			summary: updates.summary ?? existing.summary,
-			tags: updates.tags ? [...updates.tags] : existing.tags,
-			updatedAt: now,
-		}
-
-		this.index.set(updated)
-		await this.persistIndex()
-
-		if (
-			updates.content !== undefined ||
-			updates.format !== undefined ||
-			updates.metadata !== undefined
-		) {
-			try {
-				const existingContent = await this.records.read(this.contentPath(id))
-				if (existingContent === null) throw new Error('memory content is missing')
-
+		return this.withAuthoritativeIndex(async (location) => {
+			assertStorageMemoryId(id, (reason) => invalidContent(id, reason, { field: 'id' }))
+			const existing = this.index.getEntry(id)
+			if (!existing) return undefined
+			const existingContent = await this.readContent(location, id)
+			const updated: MemoryIndexEntry = {
+				...existing,
+				title: updates.title ?? existing.title,
+				summary: updates.summary ?? existing.summary,
+				tags: updates.tags ? [...updates.tags] : existing.tags,
+				updatedAt: Date.now(),
+			}
+			const updatesContent =
+				updates.content !== undefined ||
+				updates.format !== undefined ||
+				updates.metadata !== undefined
+			if (updatesContent) {
 				const updatedContent: MemoryContent = {
 					...existingContent,
 					content: updates.content ?? existingContent.content,
@@ -169,41 +343,42 @@ export class DiskMemoryStore implements MemoryStore {
 					metadata:
 						updates.metadata !== undefined ? { ...updates.metadata } : existingContent.metadata,
 				}
-
-				await this.records.write(this.contentPath(id), updatedContent)
-			} catch {
-				this.log.warn('Failed to update memory content', { 'namzu.memory.id': id })
+				await this.records.write(this.contentPath(location, id), updatedContent)
 			}
-		}
 
-		this.log.info('Memory updated', { 'namzu.memory.id': id })
-		return updated
+			const entries = this.index.allEntries().map((entry) => (entry.id === id ? updated : entry))
+			await this.writeIndex(location, entries)
+			this.index.rebuild([...entries])
+
+			this.log.info('Memory updated', { 'namzu.memory.id': id })
+			return updated
+		})
 	}
 
 	async delete(id: MemoryId): Promise<boolean> {
-		await this.ensureInit()
+		return this.withAuthoritativeIndex(async (location) => {
+			assertStorageMemoryId(id, (reason) => invalidContent(id, reason, { field: 'id' }))
+			if (!this.index.getEntry(id)) return false
 
-		const existed = this.index.remove(id)
-		if (!existed) return false
-
-		await this.persistIndex()
-		await unlink(this.contentPath(id)).catch(() => undefined)
+			try {
+				await unlink(this.contentPath(location, id))
+			} catch (error) {
+				if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error
+			}
+			const entries = this.index.allEntries().filter((entry) => entry.id !== id)
+			await this.writeIndex(location, entries)
+			this.index.rebuild([...entries])
 
-		this.log.info('Memory deleted', { 'namzu.memory.id': id })
-		return true
+			this.log.info('Memory deleted', { 'namzu.memory.id': id })
+			return true
+		})
 	}
 
 	async list(params?: MemorySearchParams): Promise<MemorySearchResult> {
-		await this.ensureInit()
-		return this.index.search(params ?? {})
+		return this.withAuthoritativeIndex(async () => this.index.search(params ?? {}))
 	}
 
 	getIndex(): InMemoryMemoryIndex {
 		return this.index
 	}
-
-	private async persistIndex(): Promise<void> {
-		const entries = this.index.allEntries()
-		await this.indexRecords.write(this.indexPath, entries)
-	}
 }

package/src/store/run/disk.ts

@@ -1,6 +1,7 @@
 import { appendFile, mkdir, readFile, readdir, unlink } from 'node:fs/promises'
 import { join } from 'node:path'
 import type { CheckpointId, IterationCheckpoint } from '../../types/hitl/index.js'
+import type { Message } from '../../types/message/index.js'
 import type {
 	AuditEvent,
 	PersistedRunEvent,
@@ -8,7 +9,12 @@ import type {
 	RunEvent,
 	RunStoreConfig,
 } from '../../types/run/index.js'
-import type { CompletedToolRecord, ReadRunEventsOptions, RunStore } from '../../types/run/store.js'
+import type {
+	CompletedToolRecord,
+	ReadRunEventsOptions,
+	RunMessageSnapshot,
+	RunStore,
+} from '../../types/run/store.js'
 import { atomicWriteFile } from '../../utils/atomic-write.js'
 import { SCOPE_ATTRIBUTE } from '../../utils/log/types.js'
 import { type Logger, resolveLogger } from '../../utils/logger.js'
@@ -79,6 +85,10 @@ export class RunDiskStore implements RunStore {
 		return readRunEventsIn(this.requireInit(), options)
 	}
 
+	async readMessages(): Promise<RunMessageSnapshot> {
+		return readRunMessagesIn(this.requireInit())
+	}
+
 	async appendAuditEvent(event: AuditEvent): Promise<void> {
 		const dir = this.requireInit()
 		// One JSON object per line, exactly like `transcript.jsonl` — but its
@@ -204,9 +214,13 @@ export class RunDiskStore implements RunStore {
 		await atomicWriteJson(join(dir, 'run.json'), meta)
 	}
 
-	async writeMessages(run: Run): Promise<void> {
+	async writeMessages(run: Run, throughEventSeq: number): Promise<void> {
 		const dir = this.requireInit()
-		await atomicWriteJson(join(dir, 'messages.json'), run.messages)
+		await atomicWriteJson(join(dir, 'messages.json'), {
+			format: 'namzu.run-message-snapshot.v1',
+			throughEventSeq,
+			messages: run.messages,
+		})
 	}
 
 	async writeReport(content: string): Promise<string> {
@@ -389,8 +403,15 @@ export async function readRunEventsIn(
 	}
 
 	const sinceSeq = options?.sinceSeq ?? 0
+	const strict = options?.integrity === 'strict'
+	if (strict && raw.length > 0 && !raw.endsWith('\n')) {
+		throw new Error(
+			`Invalid run transcript in ${join(runDir, 'transcript.jsonl')}: final record is not newline-terminated`,
+		)
+	}
 	const events: PersistedRunEvent[] = []
 	let position = 0
+	let previousSeq = 0
 
 	for (const line of raw.split('\n')) {
 		if (line.length === 0) continue
@@ -399,12 +420,31 @@ export async function readRunEventsIn(
 		let parsed: Record<string, unknown>
 		try {
 			parsed = JSON.parse(line) as Record<string, unknown>
-		} catch {
+		} catch (error) {
+			if (strict) {
+				throw new Error(
+					`Invalid run transcript in ${join(runDir, 'transcript.jsonl')}: record ${position} is not valid JSON`,
+					{ cause: error },
+				)
+			}
+			continue
+		}
+		if (parsed === null || typeof parsed !== 'object' || typeof parsed.type !== 'string') {
+			if (strict) {
+				throw new Error(
+					`Invalid run transcript in ${join(runDir, 'transcript.jsonl')}: record ${position} is not an event object`,
+				)
+			}
 			continue
 		}
-		if (parsed === null || typeof parsed !== 'object' || typeof parsed.type !== 'string') continue
 
 		const seq = typeof parsed.seq === 'number' ? parsed.seq : position
+		if (strict && (!Number.isSafeInteger(seq) || seq <= 0 || seq !== previousSeq + 1)) {
+			throw new Error(
+				`Invalid run transcript in ${join(runDir, 'transcript.jsonl')}: record ${position} has sequence ${String(seq)}, expected ${previousSeq + 1}`,
+			)
+		}
+		previousSeq = seq
 		if (seq <= sinceSeq) continue
 
 		events.push({
@@ -421,6 +461,57 @@ export async function readRunEventsIn(
 	return events
 }
 
+/**
+ * Read a run's surviving message snapshot without binding a store to it.
+ *
+ * Binding a {@link RunDiskStore} creates the directory, which would turn a
+ * missing run into an apparently empty one. This helper performs no writes.
+ */
+export async function readRunMessagesIn(runDir: string): Promise<RunMessageSnapshot> {
+	let raw: string
+	try {
+		raw = await readFile(join(runDir, 'messages.json'), 'utf-8')
+	} catch (err) {
+		if (isFileNotFound(err)) return { kind: 'unavailable', reason: 'not-persisted' }
+		throw err
+	}
+
+	let parsed: unknown
+	try {
+		parsed = JSON.parse(raw)
+	} catch {
+		throw new Error(
+			`Invalid run message snapshot in ${join(runDir, 'messages.json')}: invalid JSON`,
+		)
+	}
+
+	// The pre-boundary format was the message array itself. Preserve access to
+	// those bytes, but do not manufacture the event-log boundary it never held.
+	if (Array.isArray(parsed)) {
+		return { kind: 'legacy-unverified', messages: parsed as Message[] }
+	}
+
+	if (
+		parsed === null ||
+		typeof parsed !== 'object' ||
+		(parsed as Record<string, unknown>).format !== 'namzu.run-message-snapshot.v1' ||
+		typeof (parsed as Record<string, unknown>).throughEventSeq !== 'number' ||
+		!Number.isSafeInteger((parsed as Record<string, unknown>).throughEventSeq) ||
+		((parsed as Record<string, unknown>).throughEventSeq as number) < 0 ||
+		!Array.isArray((parsed as Record<string, unknown>).messages)
+	) {
+		throw new Error(
+			`Invalid run message snapshot in ${join(runDir, 'messages.json')}: expected a versioned snapshot`,
+		)
+	}
+
+	return {
+		kind: 'available',
+		throughEventSeq: (parsed as Record<string, unknown>).throughEventSeq as number,
+		messages: (parsed as Record<string, unknown>).messages as Message[],
+	}
+}
+
 /**
  * Terminate a transcript whose last line was cut off mid-write.
  *