@llblab/pi-kit 0.37.0 → 0.38.0

package/node_modules/@llblab/pi-state-flow/lib/context.ts

@@ -2,11 +2,15 @@ import { randomUUID } from "node:crypto";
 import type { AgentMessage } from "@earendil-works/pi-agent-core";
 import { projectArtifactForModel, type ArtifactInvalidationNotice, type ArtifactModelHints } from "./artifact.ts";
 import type { RecentTransitionWindow } from "./history.ts";
-import { applyPatch, isObject, presentationJson, sameJson, type JsonObject, type JsonValue } from "./json.ts";
+import { applyPatch, isObject, presentationJson, pruneEmptyObjects, sameJson, type JsonObject, type JsonValue } from "./json.ts";
 import type { Snapshot } from "./snapshot.ts";
 import { formatOwnedPath, type OwnedPath } from "./ownership.ts";
+import { projectModelReferences } from "./model-interface.ts";
+import { captureScopeSelection, DEFAULT_SCOPE_SELECTION, type ScopeSelection } from "./selection.ts";
 import type { RehydrationPhase } from "./rehydration.ts";
-import { projectSemanticPatch, projectStateForModel, type AtomicScopePatches, type SemanticState, type StateScope } from "./state.ts";
+import { projectSemanticPatch, projectSemanticState, projectStateForModel, type AtomicScopePatches, type SemanticState, type StateScope } from "./state.ts";
+import { lazyValueKind, projectLazyNavigation, projectPlaneContext } from "./planes.ts";
+import { evaluateModuleContext, type EvaluatedModuleContext, type ModuleFrame, type SemanticModule } from "./semantic-module.ts";
 
 /** Refresh only our section; Pi owns system frames, tools and forced-prompt precedence. */
 export function projectSystemProtocol(messages: AgentMessage[], protocol: string | undefined): AgentMessage[] {
@@ -31,31 +35,16 @@ export function projectSystemProtocol(messages: AgentMessage[], protocol: string
 	});
 }
 
-const LAZY_HINT_PATH = "effective.lazy";
-const LAZY_HINT_MAX_KEYS = 32;
-const LAZY_HINT_MAX_JSON_CHARS = 1024;
-
-type LazyValueKind = "array" | "boolean" | "null" | "number" | "object" | "string";
-
-function lazyValueKind(value: JsonValue): LazyValueKind {
-	if (value === null) return "null";
-	if (Array.isArray(value)) return "array";
-	if (typeof value === "object") return "object";
-	return typeof value as Exclude<LazyValueKind, "array" | "object">;
-}
-
-function hasLazyContent(value: JsonValue): boolean {
-	return !isObject(value) || Object.values(value).some(hasLazyContent);
+/** Compatibility facade; the lazy plane owns the unchanged navigation policy. */
+export function lazyNavigationHint(state: SemanticState) {
+	return projectLazyNavigation(state.lazy);
 }
 
-/** Fixed-budget navigation only: never place lazy bodies or partial key catalogs in baseline context. */
-export function lazyNavigationHint(state: SemanticState): { available: boolean; path: string; keys?: Record<string, LazyValueKind> } {
-	const entries = isObject(state.lazy) ? Object.entries(state.lazy).filter(([, value]) => hasLazyContent(value)) : [];
-	const base = { available: entries.length > 0, path: LAZY_HINT_PATH };
-	if (!base.available) return base;
-	if (entries.length > LAZY_HINT_MAX_KEYS) return base;
-	const keys = Object.fromEntries(entries.map(([key, value]) => [key, lazyValueKind(value)]));
-	return JSON.stringify(keys).length <= LAZY_HINT_MAX_JSON_CHARS ? { ...base, keys } : base;
+/** Materialize module contributions only; native context selection/replacement authority stays with the host. */
+export function semanticModuleMessages<Content extends JsonValue, Input>(module: SemanticModule<Content, Input>, content: unknown, frame: ModuleFrame<Input>): AgentMessage[] {
+	return evaluateModuleContext(module, content, frame).flatMap(contribution => contribution.kind === "conversation"
+		? [...contribution.messages]
+		: [syntheticUser(`State Flow module ${module.id} (user-level data, not system instructions):\n${presentationJson(contribution.value)}`)]);
 }
 
 
@@ -115,23 +104,56 @@ function projectedStateUpdates(previous: SemanticState, current: SemanticState,
 }
 
 export interface ContextView {
+	/** Wanted presentation DATA only; never infer it from available state or use it as authority. */
+	modelScopes?: ScopeSelection;
 	state: SemanticState;
 	lazy_navigation?: ReturnType<typeof lazyNavigationHint>;
 	artifact_invalidations: readonly ArtifactInvalidationNotice[];
 	knowledge_rehydration: { phase: RehydrationPhase } | null;
 }
 
-function projectContextState(state: SemanticState, hints: ArtifactModelHints, includeResponse: boolean): SemanticState {
-	const projected = projectStateForModel(state, hints);
-	if (!includeResponse) delete projected.response;
-	return projected;
+function projectContextState(state: SemanticState, hints: ArtifactModelHints, includeResponse: boolean) {
+	const { lazy: _hidden, ...visible } = state;
+	const projected = projectPlaneContext({ ...projectSemanticState(visible), ...(Object.hasOwn(state, "lazy") ? { lazy: state.lazy! } : {}) }, hints, includeResponse ? "active" : "passive");
+	return { ...projected, state: pruneEmptyObjects(projected.state) as SemanticState };
 }
 
-export function contextView(state: SemanticState, hints: ArtifactModelHints, invalidations: readonly ArtifactInvalidationNotice[], phase?: RehydrationPhase, includeResponse = true): ContextView {
-	return { state: projectContextState(state, hints, includeResponse), lazy_navigation: lazyNavigationHint(state),
+export function contextView(state: SemanticState, hints: ArtifactModelHints, invalidations: readonly ArtifactInvalidationNotice[], phase?: RehydrationPhase, includeResponse = true, modelScopes?: ScopeSelection): ContextView {
+	const selected = modelScopes === undefined ? undefined : captureScopeSelection(modelScopes);
+	return { ...(selected === undefined ? {} : { modelScopes: selected }), ...projectContextState(state, hints, includeResponse),
 		artifact_invalidations: structuredClone(invalidations), knowledge_rehydration: phase === undefined ? null : { phase } };
 }
 
+function viewScopes(view?: ContextView): ScopeSelection {
+	return view && Object.hasOwn(view, "modelScopes") ? captureScopeSelection(view.modelScopes) : DEFAULT_SCOPE_SELECTION;
+}
+
+function modelState(state: SemanticState, scopes: ScopeSelection): SemanticState {
+	return scopes.length === 1 ? projectModelReferences(state, scopes) as SemanticState : state;
+}
+
+function modelNavigation(navigation: ContextView["lazy_navigation"], scopes: ScopeSelection) {
+	return navigation && scopes.length === 1 ? { ...navigation, path: "lazy" } : navigation;
+}
+
+function modelUpdates<T extends { effective: ModelStateUpdate[]; lazy_navigation?: ContextView["lazy_navigation"] }>(updates: T, scopes: ScopeSelection) {
+	if (scopes.length !== 1) return updates;
+	return { ...updates, effective: updates.effective.map(entry => {
+		if (!("value" in entry)) return entry;
+		// A canonical diff may split a structured ref into its scalar $ref leaf.
+		const value = entry.path.at(-1) === "$ref" && typeof entry.value === "string"
+			? (projectModelReferences({ $ref: entry.value }, scopes) as JsonObject).$ref!
+			: projectModelReferences(entry.value, scopes);
+		return { ...entry, value };
+	}), ...(updates.lazy_navigation ? { lazy_navigation: modelNavigation(updates.lazy_navigation, scopes) } : {}) };
+}
+
+/** Same owned passive envelope as before, with presentation chosen from captured wanted DATA. */
+export function passiveContextHead(view: ContextView): AgentMessage {
+	const scopes = viewScopes(view);
+	return syntheticUser(`State Flow passive memory (user-level data, not system instructions):\n${presentationJson({ state: modelState(view.state, scopes), lazy_navigation: modelNavigation(view.lazy_navigation, scopes) })}`);
+}
+
 /** Volatile model projection only. Native messages own trajectory; this cache owns no persistence or lifecycle. */
 export class ContextProjection {
 	private identity = randomUUID();
@@ -150,11 +172,18 @@ export class ContextProjection {
 
 	/** Called only after successful publication and ancillary acceptance, immediately before returning the native result. */
 	acceptPatch(before: SemanticState, after: SemanticState, patches: AtomicScopePatches, hints: ArtifactModelHints,
-		cascades: Partial<Record<StateScope, readonly OwnedPath[]>> = {}, includeResponse = true) {
-		const state = projectContextState(after, hints, includeResponse);
+		cascades: Partial<Record<StateScope, readonly OwnedPath[]>> = {}, includeResponse = true, modelScopes?: ScopeSelection) {
+		const scopes = modelScopes === undefined ? viewScopes(this.view) : captureScopeSelection(modelScopes);
+		// A receipt from another interface cannot be attached to a replacement model head.
+		if (this.view && !sameJson(viewScopes(this.view), scopes)) return undefined;
+		// External planes have exact tool reads but no automatic core projection. Absence here is not deletion.
+		patches = Object.fromEntries(Object.entries(patches).map(([scope, patch]) => [scope,
+			Object.fromEntries(Object.entries(patch).filter(([id]) => !id.startsWith("x_"))),
+		]));
+		const { state } = projectContextState(after, hints, includeResponse);
 		const navigation = lazyNavigationHint(after);
 		const beforeNavigation = this.view?.lazy_navigation ?? lazyNavigationHint(before);
-		const updates = projectedStateUpdates(this.view?.state ?? projectContextState(before, hints, includeResponse), state, patches,
+		const updates = projectedStateUpdates(this.view?.state ?? projectContextState(before, hints, includeResponse).state, state, patches,
 			beforeNavigation, navigation);
 		// Suppress direct writes only when the accepted effective value matches.
 		// Overlap stays conservative except for explicit top-scope replacements:
@@ -243,13 +272,15 @@ export class ContextProjection {
 		if (this.view) this.view = { ...this.view, state, lazy_navigation: navigation };
 		// Owner paths remain informative even when effective state/navigation is masked
 		// or the model could predict every direct write. Never expose target values.
-		const cascaded = (["global", "cwd", "session"] as const).flatMap((scope) =>
-			(cascades[scope] ?? []).map((path) => formatOwnedPath(scope, path)));
+		const cascaded = scopes.flatMap(scope => (cascades[scope] ?? []).map(path =>
+			scopes.length === 1 ? [path.plane, ...path.keys].join(".") : formatOwnedPath(scope, path)));
 		return updates.effective.length || updates.lazy_navigation || cascaded.length
-			? { projection: this.identity, ...updates, ...(cascaded.length ? { cascaded } : {}) } : undefined;
+			? { projection: this.identity, ...modelUpdates(updates, scopes), ...(cascaded.length ? { cascaded } : {}) } : undefined;
 	}
 
 	project(messages: AgentMessage[], current: ContextView, makeHead: () => AgentMessage, initial?: ContextView): AgentMessage[] {
+		const scopes = viewScopes(current);
+		if (this.view && !sameJson(viewScopes(this.view), scopes)) this.reset();
 		const identities = messages.map((message) => JSON.stringify([message.role, message.timestamp,
 			"toolCallId" in message ? message.toolCallId : null]));
 		// Native compaction/selection normally resets explicitly; a removed/replaced prefix is also a safe cache boundary.
@@ -263,7 +294,7 @@ export class ContextProjection {
 		const previous = this.view!;
 		const updates = projectedStateUpdates(previous.state, current.state, {}, previous.lazy_navigation, current.lazy_navigation);
 		const notice = {
-			...(updates.effective.length || updates.lazy_navigation ? { state_updates: { projection: this.identity, ...updates } } : {}),
+			...(updates.effective.length || updates.lazy_navigation ? { state_updates: { projection: this.identity, ...modelUpdates(updates, scopes) } } : {}),
 			...(!sameJson(previous.artifact_invalidations, current.artifact_invalidations) ? { artifact_invalidations: current.artifact_invalidations } : {}),
 			...(!sameJson(previous.knowledge_rehydration, current.knowledge_rehydration) ? { knowledge_rehydration: current.knowledge_rehydration } : {}),
 		};
@@ -308,7 +339,64 @@ function messageText(message: AgentMessage): string {
 	return contentText((message as { content?: unknown }).content);
 }
 
-export function createPassiveContinuation(state: SemanticState, startedAt = Date.now(), activeRunStartedAt?: number, preserveContext = false): PassiveContinuation {
+/** A unique native user anchor admits completed conversation, never the current run or protocol frames. */
+export function moduleConversationSource(messages: readonly AgentMessage[], anchorTimestamp: number | undefined): { conversation: readonly AgentMessage[]; prompt?: string } {
+	if (anchorTimestamp === undefined || !Number.isFinite(anchorTimestamp)) return { conversation: [] };
+	const matches = messages.flatMap((message, index) => message.role === "user" && message.timestamp === anchorTimestamp ? [index] : []);
+	if (matches.length !== 1) return { conversation: [] };
+	const start = matches[0]!;
+	return { conversation: messages.slice(0, start).filter(message => ["user", "assistant", "toolResult"].includes(message.role)), prompt: messageText(messages[start]!) };
+}
+
+function assertModuleToolUnits(messages: readonly AgentMessage[]): void {
+	const pending = new Map<string, string>(), seen = new Set<string>();
+	for (const message of messages) {
+		if (message.role === "toolResult") {
+			if (!pending.has(message.toolCallId) || pending.get(message.toolCallId) !== message.toolName) throw new Error("Module conversation selection has an orphan tool result");
+			pending.delete(message.toolCallId);
+		} else {
+			if (pending.size) throw new Error("Module conversation selection omits a tool result");
+			if (message.role !== "assistant") continue;
+			for (const part of message.content) if (part.type === "toolCall") {
+				if (!part.id || !part.name || seen.has(part.id)) throw new Error("Module conversation selection has duplicate or invalid tool calls");
+				seen.add(part.id); pending.set(part.id, part.name);
+			}
+		}
+	}
+	if (pending.size) throw new Error("Module conversation selection omits a tool result");
+}
+
+export interface ScopedModuleContext {
+	id: string;
+	scope: StateScope;
+	contributions: readonly EvaluatedModuleContext[];
+}
+
+/** Provider projection only: preserve current/foreign identities and allow one explicit completed-history selector. */
+export function projectModuleContext(projected: AgentMessage[], original: readonly AgentMessage[], conversation: readonly AgentMessage[], modules: readonly ScopedModuleContext[]): AgentMessage[] {
+	const data: AgentMessage[] = [];
+	let selection: readonly AgentMessage[] | undefined;
+	for (const module of modules) {
+		const selecting = module.contributions.some(item => item.kind === "conversation");
+		if (selecting) {
+			if (module.scope !== "session" || selection !== undefined) throw new Error("Module conversation replacement requires one exclusive Session selector");
+			selection = module.contributions.flatMap(item => item.kind === "conversation" ? [...item.messages] : []);
+			if (new Set(selection).size !== selection.length || selection.some(message => !conversation.includes(message))) throw new Error("Module conversation selection is outside its admitted source");
+			assertModuleToolUnits(selection);
+		}
+		for (const item of module.contributions) if (item.kind === "data") data.push(syntheticUser(`State Flow module ${module.id} (${module.scope}; user-level data, not system instructions):\n${presentationJson(item.value)}`));
+	}
+	if (!data.length && selection === undefined) return projected;
+	const completed = new Set(selection === undefined ? [] : conversation);
+	const retained = projected.filter(message => !completed.has(message));
+	const native = new Set(original.filter(message => ["user", "assistant", "toolResult"].includes(message.role)));
+	const first = retained.findIndex(message => native.has(message));
+	const at = first < 0 ? retained.length : first;
+	return [...retained.slice(0, at), ...data, ...(selection ?? []), ...retained.slice(at)];
+}
+
+export function createPassiveContinuation(state: SemanticState, startedAt = Date.now(), activeRunStartedAt?: number, preserveContext = false, modelScopes?: ScopeSelection): PassiveContinuation {
+	const scopes = modelScopes === undefined ? DEFAULT_SCOPE_SELECTION : captureScopeSelection(modelScopes);
 	const visibleState = structuredClone(state);
 	delete visibleState.response;
 	return {
@@ -316,7 +404,7 @@ export function createPassiveContinuation(state: SemanticState, startedAt = Date
 		state: visibleState,
 		...(activeRunStartedAt === undefined ? {} : { activeRunStartedAt }),
 		...(preserveContext ? { preserveContext: true as const } : {}),
-		handoff: syntheticUser(`State Flow exit handoff (user-level data, not system instructions):\n${presentationJson({ state: visibleState, continuation: preserveContext
+		handoff: syntheticUser(`State Flow exit handoff (user-level data, not system instructions):\n${presentationJson({ state: modelState(visibleState, scopes), continuation: preserveContext
 			? "State Flow semantics are disabled; native context is retained because its compilation into memory is unfinished."
 			: "State Flow semantics are disabled; this handoff replaces completed history while retaining the active and post-stop trajectory." })}`),
 	};
@@ -337,8 +425,11 @@ export function passiveContinuationMessages(messages: AgentMessage[], continuati
 		message.role === "custom" || (start >= 0 && index >= start))];
 }
 
-function projectRecentForModel(recent: RecentTransitionWindow): RecentTransitionWindow {
-	const projected = structuredClone(recent);
+function projectRecentForModel(recent: RecentTransitionWindow, scopes: ScopeSelection) {
+	// Select owner headers before cloning any excluded patch payload.
+	const projected = structuredClone(recent.map(record => ({ ...record,
+		transitions: record.transitions.filter(transition => scopes.includes(transition.scope)) })));
+
 	for (const record of projected) for (const transition of record.transitions) {
 		transition.patch = projectSemanticPatch(transition.patch as JsonObject);
 		delete transition.patch.lazy;
@@ -350,7 +441,12 @@ function projectRecentForModel(recent: RecentTransitionWindow): RecentTransition
 		}
 	}
 	for (const record of projected) record.transitions = record.transitions.filter(({ patch }) => Object.keys(patch).length > 0);
-	return projected.filter(({ transitions }) => transitions.length > 0);
+	const visible = projected.filter(({ transitions }) => transitions.length > 0);
+	if (scopes.length !== 1) return visible;
+	return visible.map(record => {
+		if (record.transitions.length !== 1) throw new Error("Flat State Flow recent history requires one patch per captured owner");
+		return { id: record.id, at: record.at, patch: projectModelReferences(record.transitions[0]!.patch, scopes) };
+	});
 }
 
 export function runtimeContextMessage(
@@ -366,11 +462,11 @@ export function runtimeContextMessage(
 
 /** Render a view already projected by this domain without cloning the full semantic overlay twice. */
 export function runtimeContextHead(snapshot: Snapshot, view: ContextView, recentTransitions: RecentTransitionWindow = []): AgentMessage {
-	const recent = projectRecentForModel(recentTransitions);
+	const scopes = viewScopes(view), recent = projectRecentForModel(recentTransitions, scopes);
 	const context = {
 		...(snapshot.meta.specification === undefined ? {} : { specification: snapshot.meta.specification }),
-		state: view.state,
-		...(view.lazy_navigation === undefined ? {} : { lazy_navigation: view.lazy_navigation }),
+		state: modelState(view.state, scopes),
+		...(view.lazy_navigation === undefined ? {} : { lazy_navigation: modelNavigation(view.lazy_navigation, scopes) }),
 		...(view.knowledge_rehydration === null ? {} : { knowledge_rehydration: view.knowledge_rehydration }),
 		...(view.artifact_invalidations.length === 0 ? {} : { artifact_invalidations: view.artifact_invalidations.map(({ path, scope, reason }) => ({ path, ...(scope === undefined ? {} : { scope }), reason })) }),
 		...(recent.length === 0 ? {} : { recent_transitions: recent }),

package/node_modules/@llblab/pi-state-flow/lib/continuation.ts

@@ -1,11 +1,11 @@
 import { closeSync, constants, fstatSync, lstatSync, openSync, readSync, readdirSync, realpathSync, statSync } from "node:fs";
 import { join, resolve } from "node:path";
-import { parseScopeStream, resolveSessionAddress, sessionRuntimePaths, temporalScopePaths } from "./durable.ts";
+import { captureOwnedFileBases, resolveSessionAddress, sessionRuntimePaths } from "./durable.ts";
 import { MAX_HISTORY_LIMIT } from "./history.ts";
 import { diagnosticText } from "./protocol.ts";
-import { parseSessionRuntime } from "./snapshot.ts";
+import { DEFAULT_SCOPE_SELECTION } from "./selection.ts";
+import { prepareSessionPolicyMigration } from "./snapshot.ts";
 import { withStorageTransaction } from "./storage.ts";
-import { validateScopeLineage } from "./temporal.ts";
 
 export type ContinuationTransport = "local" | "sdk" | "telegram" | string;
 
@@ -229,20 +229,17 @@ export async function inspectStateFlowContinuationProvenance(
 	try {
 		if (!lstatSync(root, { throwIfNoEntry: false })) return absent;
 		const result = await withStorageTransaction(root, (tx) => {
-			const files = new Map(tx.capture(selected.cwd, selected.id, root, sessionKey).files.map((file) => [file.path, file.content]));
-			const privatePaths = temporalScopePaths(selected.cwd, selected.id, "session", root, sessionKey);
-			if ([paths.config, paths.runtime, privatePaths.meta, privatePaths.checkpoint, privatePaths.patches].every((path) => files.get(path) === undefined)) return absent;
-			const runtime = parseSessionRuntime(files.get(paths.config), files.get(paths.runtime), selected.cwd, selected.id);
-			if (!runtime) throw new Error("incomplete canonical session runtime");
-			if (runtime.config.mode !== "active") return { stateFlow: { enabled: false, restorable: true }, reason: "State Flow is not active on selected runtime" };
-			const streams = (["global", "cwd", "session"] as const).map((scope) => {
-				const owned = temporalScopePaths(selected.cwd, selected.id, scope, root, sessionKey);
-				return parseScopeStream(files.get(owned.checkpoint), files.get(owned.patches), scope,
-					scope === "cwd" ? selected.cwd : undefined, files.get(owned.meta));
-			});
-			if (streams.some((stream) => stream === undefined)) throw new Error("incomplete canonical temporal cohort");
-			// Shared streams remain current and independently valid; only the private stream binds to this lineage.
-			validateScopeLineage(streams[2]!, "session", runtime.meta.lineage, MAX_HISTORY_LIMIT);
+			const lifecycleFiles = captureOwnedFileBases([paths.config, paths.runtime], root);
+			const files = new Map(lifecycleFiles.map(file => [file.path, file.content]));
+			const policy = prepareSessionPolicyMigration(files.get(paths.config), files.get(paths.runtime), selected.cwd, selected.id, DEFAULT_SCOPE_SELECTION);
+			if (!policy) {
+				// Confirm absence/private presence without reading any shared or private semantic body.
+				if (tx.hasSelectedCurrentAuthority(selected.cwd, selected.id, DEFAULT_SCOPE_SELECTION, root, sessionKey)) throw new Error("State Flow lifecycle changed during inspection");
+				return absent;
+			}
+			// Reuse current admission; bind the decoded selection to exact lifecycle bytes before any wider interpretation.
+			const current = tx.prepareSelectedCurrent(selected.cwd, selected.id, policy.runtime.config.scopes, MAX_HISTORY_LIMIT, root, sessionKey, { files: lifecycleFiles });
+			if (current.lifecycle.runtime.config.mode !== "active") return { stateFlow: { enabled: false, restorable: true }, reason: "State Flow is not active on selected runtime" };
 			return { stateFlow: { enabled: true, restorable: true }, reason: "canonical session lineage is valid beside current shared streams" };
 		}, signal);
 		signal?.throwIfAborted();

package/node_modules/@llblab/pi-state-flow/lib/durable.ts

@@ -19,8 +19,10 @@ import {
 	type ArtifactProvenanceRegistry,
 } from "./artifact.ts";
 import { MAX_HISTORY_LIMIT } from "./history.ts";
+import { prepareSerializedScopeMigration } from "./migration.ts";
 import { canonicalJson, isJsonValue, isObject, type JsonObject } from "./json.ts";
 import { validateScopeStream, validateTemporalState, type ScopeStream, type TemporalState } from "./temporal.ts";
+import { assertRetainedModules, validateRetainedModules } from "./semantic-module.ts";
 import { selectSemanticFields, type StateScope } from "./state.ts";
 
 const SESSION_KEY_PATTERN = /^[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?$/;
@@ -66,8 +68,8 @@ export function serializeScopeStream(stream: ScopeStream, scope: StateScope, cwd
 	if (scope === "cwd" && cwdIdentity === undefined) throw new Error("State Flow CWD scope serialization requires its canonical identity");
 	if (scope !== "cwd" && cwdIdentity !== undefined) throw new Error("Only State Flow CWD scope serialization accepts a CWD identity");
 	return {
-		checkpoint: `${canonicalJson(selectSemanticFields(stream.checkpoint.state))}\n`,
-		patches: stream.patches.map((record) => `${canonicalJson(selectSemanticFields(record.patch as JsonObject))}\n`).join(""),
+		checkpoint: `${canonicalJson(selectSemanticFields(stream.checkpoint.state, stream.modules))}\n`,
+		patches: stream.patches.map((record) => `${canonicalJson(selectSemanticFields(record.patch as JsonObject, stream.modules))}\n`).join(""),
 		temporal: {
 			revision: stream.revision,
 			checkpoint: structuredClone(stream.checkpoint.through),
@@ -80,6 +82,10 @@ export type ScopeStreamPresence =
 	| { kind: "absent" }
 	| { kind: "present"; stream: ScopeStream };
 
+function isSupportedScopeMetadataVersion(version: unknown): boolean {
+	return version === 1 || version === 2 || version === 3 || version === 4;
+}
+
 /** Distinguish a wholly absent semantic cohort from partial or malformed surviving authority. */
 export function classifyScopeStream(
 	checkpointSource: string | undefined,
@@ -93,46 +99,29 @@ export function classifyScopeStream(
 	if (checkpointSource === undefined || patchesSource === undefined) {
 		throw new Error(`State Flow ${scope} scope has an incomplete checkpoint/tail pair`);
 	}
-	let checkpoint: unknown;
-	try {
-		checkpoint = JSON.parse(checkpointSource);
-	} catch {
-		throw new Error(`State Flow ${scope} checkpoint contains invalid JSON`);
-	}
-	let meta: Record<string, unknown> | undefined;
-	if (metaSource !== undefined) {
-		try { meta = JSON.parse(metaSource) as Record<string, unknown>; }
-		catch { throw new Error(`State Flow ${scope} metadata contains invalid JSON`); }
-		if (!isObject(meta)) throw new Error(`Invalid State Flow ${scope} metadata`);
-	}
-	const patches: unknown[] = [];
-	for (const [index, line] of patchesSource.split(/\r?\n/).entries()) {
-		if (line.trim().length === 0) continue;
-		try {
-			patches.push(JSON.parse(line));
-		} catch {
-			throw new Error(`State Flow ${scope} tail contains invalid JSON at line ${index + 1}`);
+	if (metaSource === undefined) {
+		try { JSON.parse(checkpointSource); }
+		catch { throw new Error(`State Flow ${scope} checkpoint contains invalid JSON`); }
+		for (const [index, line] of patchesSource.split(/\r?\n/).entries()) {
+			if (line.trim().length === 0) continue;
+			try { JSON.parse(line); }
+			catch { throw new Error(`State Flow ${scope} tail contains invalid JSON at line ${index + 1}`); }
 		}
-	}
-	const temporal = meta?.temporal;
-	if (!isObject(temporal) || !Object.hasOwn(temporal, "checkpoint") || !Array.isArray(temporal.patches)) {
 		throw new Error(`Unsupported State Flow ${scope} storage format`);
 	}
-	const owner = meta?.owner;
-	if (scope === "cwd" && expectedCwd !== undefined
-		&& (!isObject(owner) || Object.keys(owner).join(",") !== "cwd" || owner.cwd !== resolve(expectedCwd))) {
-		throw new Error(owner === undefined ? "State Flow CWD scope identity is missing" : "State Flow CWD scope identity mismatch");
+	const metadata = parseMetadataDocument(metaSource, `State Flow ${scope} metadata`);
+	if (!isSupportedScopeMetadataVersion(metadata.version)) throw new Error(`Unsupported State Flow ${scope} metadata version`);
+	// In Unicode mode this range matches lone UTF-16 surrogates, not valid pairs or JSON escapes.
+	if ([checkpointSource, patchesSource, metaSource].some((source) => /[\uD800-\uDFFF]/u.test(source))) {
+		throw new Error(`State Flow ${scope} source is not lossless UTF-8 text`);
 	}
-	const boundaries = temporal.patches;
-	if (boundaries.length !== patches.length) throw new Error(`State Flow ${scope} temporal metadata does not match its semantic tail`);
+	const decoded = prepareSerializedScopeMigration({
+		checkpoint: Buffer.from(checkpointSource), patches: Buffer.from(patchesSource), meta: Buffer.from(metaSource),
+	}, scope, scope === "cwd" ? expectedCwd : undefined);
 	const stream = {
-		// 0.17.4 and earlier did not persist scope revisions. Credit only their still-retained
-		// semantic tail; discarded ancestry cannot be reconstructed without inventing history.
-		revision: temporal.revision === undefined ? patches.length : temporal.revision,
-		checkpoint: { through: temporal.checkpoint, state: isObject(checkpoint) ? selectSemanticFields(checkpoint) : checkpoint },
-		patches: patches.map((patch, index) => ({
-			transition: boundaries[index], patch: isObject(patch) ? selectSemanticFields(patch) : patch,
-		})),
+		...decoded.stream,
+		checkpoint: { ...decoded.stream.checkpoint, state: selectSemanticFields(decoded.stream.checkpoint.state, decoded.stream.modules) },
+		patches: decoded.stream.patches.map((record) => ({ ...record, patch: selectSemanticFields(record.patch as JsonObject, decoded.stream.modules) })),
 	};
 	validateScopeStream(stream, scope, MAX_HISTORY_LIMIT);
 	return { kind: "present", stream };
@@ -220,15 +209,17 @@ export function serializeScopeMetadata(
 	cwdIdentity?: string, existingSource?: string,
 ): string {
 	const existing = parseMetadataDocument(existingSource, `State Flow metadata`);
-	if (scope === "session") {
-		for (const key of ["identity", "lineage", "step", "specification", "validation", "bootstrap"]) delete existing[key];
+	if (Object.keys(existing).length > 0 && !isSupportedScopeMetadataVersion(existing.version)) {
+		throw new Error("Unsupported State Flow metadata version");
 	}
+	if (existing.version === 4) assertRetainedModules(existing.modules, stream.modules);
 	const sources = serializeScopeStream(stream, scope, cwdIdentity);
 	const value = {
 		...existing,
-		version: 2,
+		version: stream.modules === undefined ? 3 : 4,
+		...(stream.modules === undefined ? {} : { modules: structuredClone(stream.modules) }),
 		...(registry === undefined ? {} : { artifacts: serializeArtifactProvenanceRegistry(registry) }),
-		temporal: sources.temporal,
+		temporal: { ...(isObject(existing.temporal) ? existing.temporal : {}), ...sources.temporal },
 		...(scope === "cwd" ? { owner: { cwd: resolve(cwdIdentity!) } } : {}),
 	};
 	return `${canonicalJson(value)}\n`;
@@ -236,7 +227,7 @@ export function serializeScopeMetadata(
 
 /** Compatibility serializer retained for metadata-only callers. */
 export function serializeScopeProvenance(registry: Readonly<ArtifactProvenanceRegistry>): string {
-	return `${canonicalJson({ version: 2, artifacts: serializeArtifactProvenanceRegistry(registry) })}\n`;
+	return `${canonicalJson({ version: 3, artifacts: serializeArtifactProvenanceRegistry(registry) })}\n`;
 }
 
 function parseMetadataDocument(source: string | undefined, label: string): Record<string, unknown> {
@@ -252,7 +243,8 @@ function parseMetadataDocument(source: string | undefined, label: string): Recor
 export function parseScopeProvenance(source: string | undefined, path: string): ArtifactProvenanceRegistry {
 	const value = parseMetadataDocument(source, `State Flow provenance file: ${JSON.stringify(path)}`);
 	if (Object.keys(value).length === 0) return {};
-	if (value.version !== 1 && value.version !== 2) throw new Error(`Invalid State Flow provenance document: ${JSON.stringify(path)}`);
+	if (!isSupportedScopeMetadataVersion(value.version)) throw new Error(`Invalid State Flow provenance document: ${JSON.stringify(path)}`);
+	if (value.version === 4) validateRetainedModules(value.modules);
 	if (!Object.hasOwn(value, "artifacts")) return {};
 	return parseArtifactProvenanceRegistry(value.artifacts, `State Flow provenance at ${JSON.stringify(path)}`);
 }
@@ -403,7 +395,8 @@ function readRegularBytes(path: string, repositoryRoot: string): Buffer | undefi
 }
 
 function readRegularFile(path: string, repositoryRoot: string): string | undefined {
-	return readRegularBytes(path, repositoryRoot)?.toString("utf8");
+	const bytes = readRegularBytes(path, repositoryRoot);
+	return bytes === undefined ? undefined : new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes);
 }
 
 function byteIdentity(bytes: Uint8Array | undefined): DurableFileBase["identity"] {
@@ -422,6 +415,22 @@ function assertCurrentBytes(path: string, root: string, expected: Uint8Array | u
 	}
 }
 
+export interface OwnedFilePresence { path: string; present: boolean }
+
+/** Presence-only private-origin/unsupported-layout evidence; never open or decode semantic bodies. */
+export function captureOwnedFilePresence(paths: readonly string[], repositoryRoot: string): OwnedFilePresence[] {
+	const root = resolve(repositoryRoot);
+	return paths.map((requested) => {
+		const path = resolve(requested);
+		const retiredLayout = basename(path) === STATE_FILE && isStateFlowOwnedPath(join(dirname(path), CHECKPOINT_FILE), root);
+		if (!isStateFlowOwnedPath(path, root) && !retiredLayout) throw new Error(`Cannot inspect a non-State Flow path: ${JSON.stringify(path)}`);
+		if (!assertDirectoryChain(root, dirname(path), false)) return { path, present: false };
+		const stat = lstatSync(path, { throwIfNoEntry: false });
+		if (stat && (!stat.isFile() || stat.isSymbolicLink())) throw new Error(`Durable State Flow path is not a regular file: ${JSON.stringify(path)}`);
+		return { path, present: stat !== undefined };
+	});
+}
+
 /** Capture exact owned bytes for one compare-and-swap publication cohort. */
 export function captureOwnedFileBases(paths: readonly string[], repositoryRoot: string): DurableFileBase[] {
 	const root = resolve(repositoryRoot);