@namzu/sdk 2.0.0 → 3.0.0

package/src/store/__tests__/schema-adoption.test.ts

@@ -0,0 +1,118 @@
+import { mkdtempSync, rmSync } from 'node:fs'
+import { readFile, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+
+import type { ProjectId, SessionId, TenantId, ThreadId, UserId } from '../../types/ids/index.js'
+import { createUserMessage } from '../../types/message/index.js'
+import type { ActorRef } from '../../types/session/actor.js'
+import { DiskSessionStore } from '../session/disk.js'
+
+/**
+ * The primitive is unit-tested next door. What matters here is that a store
+ * actually goes through it — a version that is stamped but never checked,
+ * or checked but never stamped, is the same silent read it replaces.
+ */
+
+const TENANT = 'tnt_schema' as TenantId
+const THREAD = 'thd_schema' as ThreadId
+
+const actor = (): ActorRef => ({ kind: 'user', userId: 'usr_a' as UserId, tenantId: TENANT })
+
+let rootDir: string
+let store: DiskSessionStore
+
+beforeEach(() => {
+	rootDir = mkdtempSync(join(tmpdir(), 'namzu-schema-'))
+	store = new DiskSessionStore({ rootDir })
+})
+
+afterEach(() => {
+	rmSync(rootDir, { recursive: true, force: true })
+})
+
+async function seed(): Promise<{ projectId: ProjectId; sessionId: SessionId }> {
+	const project = await store.createProject({ tenantId: TENANT, name: 'p' }, TENANT)
+	const session = await store.createSession(
+		{ threadId: THREAD, projectId: project.id, currentActor: actor() },
+		TENANT,
+	)
+	return { projectId: project.id, sessionId: session.id }
+}
+
+const sessionPath = (projectId: ProjectId, sessionId: SessionId) =>
+	join(rootDir, 'projects', projectId, 'sessions', sessionId, 'session.json')
+
+describe('what lands on disk', () => {
+	it('stamps a session record', async () => {
+		const { projectId, sessionId } = await seed()
+		const raw = JSON.parse(await readFile(sessionPath(projectId, sessionId), 'utf-8'))
+		expect(raw.schemaVersion).toBe(1)
+	})
+
+	it('stamps every line of the append-only message log', async () => {
+		const { projectId, sessionId } = await seed()
+		await store.appendMessage(sessionId, createUserMessage('one'), TENANT)
+		await store.appendMessage(sessionId, createUserMessage('two'), TENANT)
+
+		const log = await readFile(
+			join(rootDir, 'projects', projectId, 'sessions', sessionId, 'messages.jsonl'),
+			'utf-8',
+		)
+		const lines = log.split('\n').filter((l) => l.length > 0)
+		expect(lines).toHaveLength(2)
+		// An append-only log is written by many builds over its lifetime, so
+		// its lines can legitimately differ in version — each carries its own.
+		for (const line of lines) expect(JSON.parse(line).schemaVersion).toBe(1)
+	})
+})
+
+describe('what comes back off disk', () => {
+	it('reads a legacy record that predates the stamp', async () => {
+		const { projectId, sessionId } = await seed()
+		const path = sessionPath(projectId, sessionId)
+		const record = JSON.parse(await readFile(path, 'utf-8'))
+		record.schemaVersion = undefined
+		await writeFile(path, JSON.stringify(record), 'utf-8')
+
+		// Every file written before this mechanism existed looks exactly
+		// like this, and has to keep opening.
+		expect(await store.getSession(sessionId, TENANT)).not.toBeNull()
+	})
+
+	it('refuses a record from a build that knew more than this one', async () => {
+		const { projectId, sessionId } = await seed()
+		const path = sessionPath(projectId, sessionId)
+		const record = JSON.parse(await readFile(path, 'utf-8'))
+		record.schemaVersion = 99
+		record.somethingThisBuildDoesNotKnow = { important: true }
+		await writeFile(path, JSON.stringify(record), 'utf-8')
+
+		// Reading it would drop the unknown field, and the next write would
+		// destroy it. A refusal is recoverable by upgrading.
+		await expect(store.getSession(sessionId, TENANT)).rejects.toThrow(/schema version 99/)
+	})
+
+	it('refuses a message line from the future rather than skipping it', async () => {
+		const { projectId, sessionId } = await seed()
+		await store.appendMessage(sessionId, createUserMessage('fine'), TENANT)
+
+		const path = join(rootDir, 'projects', projectId, 'sessions', sessionId, 'messages.jsonl')
+		const line = JSON.parse((await readFile(path, 'utf-8')).trim())
+		await writeFile(path, `${JSON.stringify({ ...line, schemaVersion: 42 })}\n`, 'utf-8')
+
+		// Silently dropping a message the build cannot read would hand the
+		// model a conversation with a hole in it.
+		await expect(store.loadMessages(sessionId, TENANT)).rejects.toThrow(/schema version 42/)
+	})
+
+	it('still round-trips a normal write and read', async () => {
+		const { sessionId } = await seed()
+		await store.appendMessage(sessionId, createUserMessage('hello'), TENANT)
+		await store.appendMessage(sessionId, createUserMessage('world'), TENANT)
+
+		const loaded = await store.loadMessages(sessionId, TENANT)
+		expect(loaded.map((m) => m.content)).toEqual(['hello', 'world'])
+	})
+})

package/src/store/__tests__/schema.test.ts

@@ -0,0 +1,119 @@
+import { describe, expect, it } from 'vitest'
+
+import {
+	INITIAL_SCHEMA_VERSION,
+	SchemaVersionError,
+	defineSchema,
+	migrate,
+	stamp,
+} from '../schema.js'
+
+/**
+ * Every persisted read was `JSON.parse(raw) as T`. A record from an older
+ * build arrived with fields missing and nothing said so; a record from a
+ * NEWER build was read partially and, once written back, permanently lost
+ * whatever this build did not understand.
+ */
+
+const V1 = defineSchema({ kind: 'thing', current: 1, migrations: {} })
+
+const V3 = defineSchema({
+	kind: 'thing',
+	current: 3,
+	migrations: {
+		1: (r) => ({ ...r, addedInV2: true }),
+		2: (r) => ({ ...r, addedInV3: true }),
+	},
+})
+
+describe('reading an older record', () => {
+	it('treats an unstamped record as the first version', () => {
+		// Every file written before this mechanism existed IS version 1 —
+		// by definition, not by assumption.
+		const migrated = migrate<Record<string, unknown>>(V3, { name: 'legacy' })
+		expect(migrated.addedInV2).toBe(true)
+		expect(migrated.addedInV3).toBe(true)
+		expect(migrated.schemaVersion).toBe(3)
+	})
+
+	it('runs only the steps that are missing', () => {
+		const migrated = migrate<Record<string, unknown>>(V3, { schemaVersion: 2, name: 'x' })
+		expect(migrated.addedInV2).toBeUndefined()
+		expect(migrated.addedInV3).toBe(true)
+	})
+
+	it('leaves a current record alone apart from the stamp', () => {
+		const migrated = migrate<Record<string, unknown>>(V3, { schemaVersion: 3, name: 'x' })
+		expect(migrated).toEqual({ schemaVersion: 3, name: 'x' })
+	})
+})
+
+describe('reading a newer record', () => {
+	it('refuses rather than reading it partially', () => {
+		expect(() => migrate(V1, { schemaVersion: 2, name: 'from the future' })).toThrow(
+			SchemaVersionError,
+		)
+	})
+
+	it('says what it found and what it understands', () => {
+		try {
+			migrate(V1, { schemaVersion: 7 })
+			expect.unreachable()
+		} catch (err) {
+			const e = err as SchemaVersionError
+			expect(e.found).toBe(7)
+			expect(e.supported).toBe(1)
+			// A refusal is recoverable by upgrading. A partial read that
+			// gets written back is not.
+			expect(e.message).toContain('Upgrade')
+		}
+	})
+})
+
+describe('declaring a schema', () => {
+	it('rejects a gap in the migration chain at declaration time', () => {
+		// Discovered here, not in production by a user whose session will
+		// not open.
+		expect(() => defineSchema({ kind: 'gappy', current: 3, migrations: { 1: (r) => r } })).toThrow(
+			/no migration from version 2 to 3/,
+		)
+	})
+
+	it('rejects a version below the first one', () => {
+		expect(() => defineSchema({ kind: 'x', current: 0, migrations: {} })).toThrow(/integer >= 1/)
+	})
+
+	it('accepts a schema that has never migrated', () => {
+		expect(() =>
+			defineSchema({ kind: 'x', current: INITIAL_SCHEMA_VERSION, migrations: {} }),
+		).not.toThrow()
+	})
+})
+
+describe('stamping', () => {
+	it('marks a record with the version this build writes', () => {
+		expect(stamp(V3, { a: 1 })).toEqual({ a: 1, schemaVersion: 3 })
+	})
+
+	it('leaves an array untouched', () => {
+		// There is nowhere on an array to put a field that survives
+		// `JSON.stringify`, and wrapping it would change every file already
+		// written.
+		const list = [{ a: 1 }, { a: 2 }]
+		expect(stamp(V3, list)).toEqual(list)
+	})
+
+	it('round-trips through JSON', () => {
+		const written = JSON.stringify(stamp(V1, { name: 'x' }))
+		expect(migrate<Record<string, unknown>>(V1, JSON.parse(written))).toEqual({
+			name: 'x',
+			schemaVersion: 1,
+		})
+	})
+
+	it('passes a non-object through both ways', () => {
+		expect(stamp(V3, null)).toBeNull()
+		expect(migrate(V3, null)).toBeNull()
+		expect(migrate(V3, [1, 2])).toEqual([1, 2])
+	})
+})

package/src/store/memory/disk.ts

@@ -1,4 +1,4 @@
-import { mkdir, readFile, rename, unlink, writeFile } from 'node:fs/promises'
+import { mkdir, readFile, unlink } from 'node:fs/promises'
 import { join } from 'node:path'
 import type { MemoryId } from '../../types/ids/index.js'
 import type {
@@ -9,10 +9,22 @@ import type {
 	MemorySearchResult,
 	MemoryStore,
 } from '../../types/memory/index.js'
+import { atomicWriteFile } from '../../utils/atomic-write.js'
 import { generateMemoryId } from '../../utils/id.js'
 import { type Logger, getRootLogger } from '../../utils/logger.js'
+import { defineSchema, migrate, stamp } from '../schema.js'
 import { InMemoryMemoryIndex } from './index.js'
 
+/**
+ * This store's on-disk format, versioned as a unit — which is how a
+ * migration would actually be written and shipped, and it keeps every call
+ * site free of schema plumbing.
+ *
+ * Bump `current` and add the migration for the step you are leaving when
+ * the shape changes.
+ */
+const SCHEMA = defineSchema({ kind: 'memory-store', current: 1, migrations: {} })
+
 export interface DiskMemoryStoreConfig {
 	baseDir: string
 	logger?: Logger
@@ -48,7 +60,7 @@ export class DiskMemoryStore implements MemoryStore {
 
 		try {
 			const raw = await readFile(this.indexPath, 'utf-8')
-			const entries = JSON.parse(raw) as MemoryIndexEntry[]
+			const entries = migrate<MemoryIndexEntry[]>(SCHEMA, JSON.parse(raw))
 			this.index.rebuild(entries)
 			this.log.info('Memory index loaded', { count: entries.length })
 		} catch (err) {
@@ -105,7 +117,7 @@ export class DiskMemoryStore implements MemoryStore {
 
 		try {
 			const raw = await readFile(this.contentPath(id), 'utf-8')
-			return JSON.parse(raw) as MemoryContent
+			return migrate<MemoryContent>(SCHEMA, JSON.parse(raw))
 		} catch {
 			this.log.warn('Failed to read memory content', { memoryId: id })
 			return undefined
@@ -141,7 +153,7 @@ export class DiskMemoryStore implements MemoryStore {
 		) {
 			try {
 				const raw = await readFile(this.contentPath(id), 'utf-8')
-				const existingContent = JSON.parse(raw) as MemoryContent
+				const existingContent = migrate<MemoryContent>(SCHEMA, JSON.parse(raw))
 
 				const updatedContent: MemoryContent = {
 					...existingContent,
@@ -189,17 +201,6 @@ export class DiskMemoryStore implements MemoryStore {
 	}
 }
 
-async function atomicWriteFile(filePath: string, content: string): Promise<void> {
-	const tempPath = `${filePath}.tmp`
-	try {
-		await writeFile(tempPath, content, 'utf-8')
-		await rename(tempPath, filePath)
-	} catch (err) {
-		await unlink(tempPath).catch(() => undefined)
-		throw err
-	}
-}
-
 async function atomicWriteJson(filePath: string, value: unknown): Promise<void> {
-	await atomicWriteFile(filePath, JSON.stringify(value, null, 2))
+	await atomicWriteFile(filePath, JSON.stringify(stamp(SCHEMA, value), null, 2))
 }

package/src/store/run/disk.ts

@@ -1,8 +1,28 @@
-import { appendFile, mkdir, readFile, readdir, rename, unlink, writeFile } from 'node:fs/promises'
+import { appendFile, mkdir, readFile, readdir, unlink } from 'node:fs/promises'
 import { join } from 'node:path'
 import type { CheckpointId, IterationCheckpoint } from '../../types/hitl/index.js'
 import type { Run, RunEvent, RunStoreConfig } from '../../types/run/index.js'
+import { atomicWriteFile } from '../../utils/atomic-write.js'
 import { type Logger, getRootLogger } from '../../utils/logger.js'
+import { defineSchema, migrate, stamp } from '../schema.js'
+
+/**
+ * This store's on-disk format, versioned as a unit — which is how a
+ * migration would actually be written and shipped, and it keeps every call
+ * site free of schema plumbing.
+ *
+ * Bump `current` and add the migration for the step you are leaving when
+ * the shape changes.
+ */
+const SCHEMA = defineSchema({ kind: 'run-store', current: 1, migrations: {} })
+
+/** One finished tool call, recovered from the transcript. */
+export interface CompletedToolRecord {
+	readonly toolUseId: string
+	readonly toolName: string
+	readonly result: string
+	readonly isError: boolean
+}
 
 export class RunDiskStore {
 	private baseDir: string
@@ -44,6 +64,63 @@ export class RunDiskStore {
 		await appendFile(join(dir, 'transcript.jsonl'), line, 'utf-8')
 	}
 
+	/**
+	 * Every tool call this run has already finished, keyed by `toolUseId`.
+	 *
+	 * A batch's results are pushed onto the history only once the WHOLE
+	 * batch settles, so a hard kill part-way through loses every result
+	 * that had already come back — and the resumed run re-executes those
+	 * calls. For a `write_file` that is waste; for a payment or an email it
+	 * is a second one.
+	 *
+	 * Nothing new has to be written to make that recoverable: the executor
+	 * already awaits a `tool_completed` event per tool, inline, carrying the
+	 * id, the name, the result and the error flag, and the transcript
+	 * already persists it. The record was durable all along and simply
+	 * never read back.
+	 */
+	async readCompletedTools(): Promise<Map<string, CompletedToolRecord>> {
+		const dir = this.requireInit()
+		const completed = new Map<string, CompletedToolRecord>()
+
+		let raw: string
+		try {
+			raw = await readFile(join(dir, 'transcript.jsonl'), 'utf-8')
+		} catch (error) {
+			if ((error as NodeJS.ErrnoException).code === 'ENOENT') return completed
+			throw error
+		}
+
+		for (const line of raw.split('\n')) {
+			if (line.length === 0) continue
+			let event: Record<string, unknown>
+			try {
+				event = JSON.parse(line) as Record<string, unknown>
+			} catch {
+				// A torn last line is the normal shape of a file that was
+				// being appended to when the process died — which is exactly
+				// the case this method exists for. Skip it; every whole line
+				// before it is still good.
+				continue
+			}
+			if (event.type !== 'tool_completed') continue
+			const toolUseId = event.toolUseId
+			const toolName = event.toolName
+			if (typeof toolUseId !== 'string' || typeof toolName !== 'string') continue
+
+			// Last write wins: a retried tool emits one event per attempt and
+			// the final one is what actually answered the call.
+			completed.set(toolUseId, {
+				toolUseId,
+				toolName,
+				result: typeof event.result === 'string' ? event.result : '',
+				isError: event.isError === true,
+			})
+		}
+
+		return completed
+	}
+
 	async writeRunMeta(run: Run): Promise<void> {
 		const dir = this.requireInit()
 
@@ -87,14 +164,21 @@ export class RunDiskStore {
 		const dir = this.requireInit()
 		const cpDir = join(dir, 'checkpoints')
 		await mkdir(cpDir, { recursive: true })
-		await atomicWriteJson(join(cpDir, `${checkpoint.id}.json`), checkpoint)
+		// Stamped, not written bare. Unstamped is read as version 1 by
+		// definition, which is correct only while version 1 is the only
+		// version there has ever been — the moment a second one exists, an
+		// unstamped file written by the newer build is read by the older one
+		// as if it were the older shape, and the refusal that exists to
+		// prevent exactly that never fires. The stamp is what gives the
+		// migration chain something to hang on.
+		await atomicWriteJson(join(cpDir, `${checkpoint.id}.json`), stamp(SCHEMA, checkpoint))
 	}
 
 	async readCheckpoint(checkpointId: CheckpointId): Promise<IterationCheckpoint | null> {
 		const dir = this.requireInit()
 		try {
 			const content = await readFile(join(dir, 'checkpoints', `${checkpointId}.json`), 'utf-8')
-			return JSON.parse(content) as IterationCheckpoint
+			return parseCheckpoint(content, `${checkpointId}.json`)
 		} catch (err) {
 			if (isFileNotFound(err)) return null
 			throw err
@@ -109,12 +193,20 @@ export class RunDiskStore {
 			const checkpoints: IterationCheckpoint[] = []
 			for (const file of files) {
 				if (!file.endsWith('.json')) continue
-				try {
-					const content = await readFile(join(cpDir, file), 'utf-8')
-					checkpoints.push(JSON.parse(content) as IterationCheckpoint)
-				} catch {
-					this.log.warn(`Failed to parse checkpoint file: ${file}`)
-				}
+				// An unreadable checkpoint used to be logged and skipped, so
+				// this returned a silently short list that four callers treat
+				// as complete. A missing NEWEST checkpoint quietly resumes
+				// from an older point and re-runs a whole iteration of tool
+				// calls; a missing PARKED one reports "not parked" and drops
+				// an approval a human already granted, because the file is
+				// the only durable record of a park. Pruning under-deletes
+				// too: a file the keep-count cannot see is immortal.
+				//
+				// The by-id read next door was already strict. Two read paths
+				// disagreeing about whether damage matters is how the lenient
+				// one gets trusted.
+				const content = await readFile(join(cpDir, file), 'utf-8')
+				checkpoints.push(parseCheckpoint(content, file))
 			}
 			return checkpoints.sort((a, b) => a.createdAt - b.createdAt)
 		} catch (err) {
@@ -144,7 +236,7 @@ export class RunDiskStore {
 		try {
 			const indexPath = join(baseDir, 'index.json')
 			const content = await readFile(indexPath, 'utf-8')
-			return JSON.parse(content)
+			return migrate(SCHEMA, JSON.parse(content))
 		} catch (err) {
 			if (isFileNotFound(err)) return []
 			throw err
@@ -168,7 +260,7 @@ export class RunDiskStore {
 
 			try {
 				const content = await readFile(indexPath, 'utf-8')
-				index = JSON.parse(content)
+				index = migrate(SCHEMA, JSON.parse(content))
 			} catch (err) {
 				if (!isFileNotFound(err)) throw err
 			}
@@ -199,19 +291,74 @@ export class RunDiskStore {
 	}
 }
 
-async function atomicWriteFile(filePath: string, content: string): Promise<void> {
-	const tempPath = `${filePath}.tmp`
-	try {
-		await writeFile(tempPath, content, 'utf-8')
-		await rename(tempPath, filePath)
-	} catch (err) {
-		await unlink(tempPath).catch(() => undefined)
-		throw err
-	}
+async function atomicWriteJson(filePath: string, value: unknown): Promise<void> {
+	await atomicWriteFile(filePath, JSON.stringify(stamp(SCHEMA, value), null, 2))
 }
 
-async function atomicWriteJson(filePath: string, value: unknown): Promise<void> {
-	await atomicWriteFile(filePath, JSON.stringify(value, null, 2))
+/**
+ * Parse a checkpoint, checking it is one.
+ *
+ * Both read paths were `JSON.parse(content) as IterationCheckpoint` — a
+ * cast, not a check, so `{}` passed both and failed much later at the
+ * point of use, where the message names a missing property rather than a
+ * damaged file. The fields checked here are the ones the resume path
+ * dereferences immediately; the rest are optional and their absence is
+ * survivable.
+ */
+/** A finite number, not `NaN` and not `Infinity`. */
+function isCount(value: unknown): value is number {
+	return typeof value === 'number' && Number.isFinite(value)
+}
+
+/**
+ * The budget fields a resume restores before its first iteration.
+ *
+ * Checked because a resume DEREFERENCES them: a run recalled at $4.80 of a
+ * $5 cap whose `costInfo` came back malformed continues with `NaN`
+ * budgets, which compare false against every limit — so the guard that
+ * exists to stop it silently never stops it. That is a run that looks
+ * healthy and has lost its cap, which is worse than one that refuses to
+ * resume.
+ */
+function hasUsableBudgets(record: Partial<IterationCheckpoint>): boolean {
+	const usage = record.tokenUsage as Record<string, unknown> | undefined
+	const cost = record.costInfo as Record<string, unknown> | undefined
+	const guard = record.guardState as Record<string, unknown> | undefined
+	if (!usage || !cost || !guard) return false
+	return (
+		isCount(usage.promptTokens) &&
+		isCount(usage.completionTokens) &&
+		isCount(usage.totalTokens) &&
+		isCount(cost.totalCost) &&
+		isCount(guard.iterationCount) &&
+		isCount(guard.elapsedMs)
+	)
+}
+
+function parseCheckpoint(content: string, file: string): IterationCheckpoint {
+	const parsed = migrate<unknown>(SCHEMA, JSON.parse(content))
+	const record = parsed as Partial<IterationCheckpoint> | null
+
+	if (
+		record === null ||
+		typeof record !== 'object' ||
+		typeof record.id !== 'string' ||
+		typeof record.iteration !== 'number' ||
+		typeof record.createdAt !== 'number' ||
+		!Array.isArray(record.messages)
+	) {
+		throw new Error(
+			`Checkpoint file "${file}" is not a usable checkpoint: it parsed as JSON but is missing the fields a resume needs (id, iteration, createdAt, messages). Refusing rather than resuming from it.`,
+		)
+	}
+
+	if (!hasUsableBudgets(record)) {
+		throw new Error(
+			`Checkpoint file "${file}" has malformed budget state (tokenUsage, costInfo, guardState). A resume restores these before its first iteration, so reading them as NaN or undefined produces a run that compares false against every limit and never stops. Refusing rather than resuming without a cap.`,
+		)
+	}
+
+	return record as IterationCheckpoint
 }
 
 function isFileNotFound(err: unknown): boolean {