@graftland/hmr 0.1.0 → 0.2.0-beta.0

package/src/internal/changes.ts

@@ -0,0 +1,199 @@
+// @effect-diagnostics-next-line nodeBuiltinImport:off -- watcher paths use host-platform syntax without adding a Path service requirement
+import { resolve } from "node:path"
+import type { FileVersion } from "./modules"
+import type { WatchOptions } from "../index"
+import { Base64 } from "effect/encoding"
+import { fileURLToPath, pathToFileURL } from "node:url"
+import {
+	Array,
+	Cause,
+	Exit,
+	Effect,
+	FileSystem,
+	HashMap,
+	Ref,
+	Queue,
+	Option,
+	Stream,
+	String,
+} from "effect"
+
+export type WithFiles = <A, E, R>(
+	use: (files: HashMap.HashMap<string, FileVersion>) => Effect.Effect<A, E, R>,
+) => Effect.Effect<A, E, R>
+
+/** Observe the producer after its resource scope closes, before merge can discard its Exit. */
+const observeSource = <A, E, R>(source: Stream.Stream<A, E, R>) =>
+	Stream.callback<A, E, R>(
+		(queue) =>
+			source.pipe(
+				Stream.runForEachArray((values) => Queue.offerAll(queue, values)),
+				Effect.scoped,
+				Effect.onExit((exit) =>
+					Exit.isFailure(exit) &&
+					Cause.hasInterrupts(exit.cause) &&
+					!Cause.hasInterruptsOnly(exit.cause)
+						? Effect.logError(
+								"Native watcher failed during cancellation",
+								exit.cause,
+							)
+						: Effect.void,
+				),
+				Effect.andThen(Queue.end(queue)),
+			),
+		{ bufferSize: 16 },
+	)
+
+/** Reconcile loaded modules because native watchers can silently miss events. */
+export const changes = (
+	withFiles: WithFiles,
+	paths: ReadonlyArray<string>,
+	options?: WatchOptions,
+) =>
+	Stream.unwrap(
+		Effect.gen(function* () {
+			if (paths.length === 0) return Stream.empty
+			const fs = yield* FileSystem.FileSystem
+			const roots = yield* Effect.forEach(paths, (path) =>
+				Effect.gen(function* () {
+					const canonical = yield* fs.realPath(path)
+					const info = yield* fs.stat(canonical)
+					return {
+						path: canonical,
+						urls: [
+							pathToFileURL(canonical).href,
+							pathToFileURL(resolve(path)).href,
+						],
+						directory: info.type === "Directory",
+					}
+				}),
+			)
+			const accepted = (path: string) => options?.filter?.(path) ?? true
+			const watched = (url: string) =>
+				accepted(fileURLToPath(url)) &&
+				Array.some(roots, ({ urls }) =>
+					Array.some(urls, (root) => {
+						const directory = String.endsWith("/")(root) ? root : `${root}/`
+						return (
+							fileURLToPath(url) === fileURLToPath(root) ||
+							(options?.recursive === true
+								? String.startsWith(directory)(url)
+								: new URL(".", url).href === directory)
+						)
+					}),
+				)
+
+			const snapshot = withFiles((loaded) =>
+				Effect.forEach(
+					Array.filter(Array.fromIterable(loaded), ([url]) => watched(url)),
+					([url, baseline]) =>
+						Effect.gen(function* () {
+							const resolution =
+								baseline.resolution === undefined
+									? undefined
+									: yield* fs.realPath(fileURLToPath(url)).pipe(
+											Effect.map((path) => pathToFileURL(path).href),
+											Effect.catchIf(
+												(error) => error.reason._tag === "NotFound",
+												() => Effect.undefined,
+											),
+										)
+							const content = yield* fs.readFile(fileURLToPath(url)).pipe(
+								Effect.map(Base64.encode),
+								// Deletion is a change; other I/O failures belong to the consumer.
+								Effect.catchIf(
+									(error) => error.reason._tag === "NotFound",
+									() => Effect.undefined,
+								),
+							)
+							return { url, baseline, content, resolution }
+						}),
+				),
+			)
+			// An observation acknowledges disk bytes only for the import version
+			// it reconciled. A new attempt must be checked against its own bytes.
+			const observed = yield* Ref.make(
+				HashMap.empty<
+					string,
+					{
+						readonly baseline: FileVersion
+						readonly content: string | undefined
+						readonly resolution: string | undefined
+					}
+				>(),
+			)
+			return Stream.mergeAll(
+				[
+					...Array.map(roots, (root) =>
+						fs.watch(root.path, options).pipe(
+							observeSource,
+							Stream.filter((event) =>
+								accepted(
+									root.directory ? resolve(root.path, event.path) : root.path,
+								),
+							),
+							Stream.map(() => true),
+						),
+					),
+					Stream.tick("1 second").pipe(
+						Stream.drop(1),
+						Stream.map(() => false),
+					),
+				],
+				{ concurrency: "unbounded" },
+			).pipe(
+				// One consumer serializes snapshots from events and polling. A native
+				// event also advances the baseline, avoiding a second reload on the next tick.
+				Stream.mapEffect((event) =>
+					snapshot.pipe(
+						Effect.flatMap((current) =>
+							Ref.modify(observed, (before) => [
+								event ||
+									Array.some(
+										current,
+										({ url, baseline, content, resolution }) => {
+											const expected = Option.match(HashMap.get(before, url), {
+												onNone: () => ({
+													content: baseline.content,
+													resolution: baseline.resolution,
+												}),
+												onSome: (seen) => {
+													const same =
+														seen.baseline.generation === baseline.generation &&
+														seen.baseline.content === baseline.content &&
+														seen.baseline.resolution === baseline.resolution
+													return {
+														// Failed attempts without source bytes keep the last
+														// observation acknowledged until another save.
+														content:
+															baseline.content === undefined || same
+																? seen.content
+																: baseline.content,
+														resolution: same
+															? seen.resolution
+															: baseline.resolution,
+													}
+												},
+											})
+											return (
+												expected.content !== content ||
+												expected.resolution !== resolution
+											)
+										},
+									),
+								HashMap.fromIterable(
+									Array.map(
+										current,
+										({ url, baseline, content, resolution }) =>
+											[url, { baseline, content, resolution }] as const,
+									),
+								),
+							]),
+						),
+					),
+				),
+				Stream.filter(Boolean),
+				Stream.map(() => undefined),
+			)
+		}),
+	)

package/src/internal/modules.ts

@@ -1,7 +1,16 @@
 // Node's synchronous module hooks have no Effect equivalent.
+import { Base64 } from "effect/encoding"
 import { registerHooks } from "node:module"
-import { fileURLToPath } from "node:url"
-import { HashSet, Option, Predicate, Schema, String as Str } from "effect"
+import { fileURLToPath, pathToFileURL } from "node:url"
+import {
+	Array,
+	HashMap,
+	HashSet,
+	Option,
+	Predicate,
+	Schema,
+	String,
+} from "effect"
 
 export class RestartRequired extends Schema.TaggedError<
 	RestartRequired,
@@ -10,7 +19,7 @@ export class RestartRequired extends Schema.TaggedError<
 
 // Only a missing entry is disposable; missing dependencies still fail the reload.
 export const isMissingEntry = (url: string, cause: unknown): boolean =>
-	Str.startsWith("file:")(url) &&
+	String.startsWith("file:")(url) &&
 	Predicate.hasProperty(cause, "code") &&
 	((cause.code === "ERR_MODULE_NOT_FOUND" &&
 		Predicate.hasProperty(cause, "url") &&
@@ -23,22 +32,144 @@ export const isMissingEntry = (url: string, cause: unknown): boolean =>
 let sessions = 0
 
 export interface Generation {
-	readonly id: string
-	unsupported: HashSet.HashSet<string>
+	readonly id: number
+}
+
+/** Bytes actually read by a particular native import attempt. */
+export interface FileVersion {
+	readonly generation: number
+	readonly content: string | undefined
+	readonly resolution?: string
+}
+
+export interface Attempt {
+	readonly generation: Generation
+	readonly url: string
+	readonly request?: string
+}
+
+interface Dependency extends Attempt {
+	readonly parent: string
+}
+
+/** Resolve local requests lexically; Node's cached physical target is a separate fact. */
+const fileRequest = (
+	specifier: string,
+	parent: string,
+): Option.Option<string> => {
+	if (
+		!Array.some(["file:", "./", "../", "/"], (prefix) =>
+			String.startsWith(prefix)(specifier),
+		)
+	)
+		return Option.none()
+	const url = new URL(specifier, parent)
+	return url.protocol === "file:" ? Option.some(url.href) : Option.none()
 }
 
 /** A scoped Node hook. Mutable bindings bridge synchronous hooks to Effect. */
 export const modules = (base: string, include: (url: string) => boolean) => {
 	const key = `graft-hmr-${++sessions}`
 	const prefix = `${key}:`
+	/** What `request` produces: the generation that imports a specifier. */
+	const decodeRequest = Schema.decodeUnknownOption(
+		Schema.TemplateLiteralParser([
+			prefix,
+			Schema.String.check(Schema.isPattern(/^\d+$/)),
+			":",
+			Schema.String,
+		]),
+	)
 	let revision = 0
-	let recording: Generation | undefined
-	let reloading = false
+	let importing:
+		{ readonly generation: Generation; readonly reload: boolean } | undefined
+	// Dependency facts outlive an import operation: apply may import modules later.
+	let dependencies = HashMap.empty<string, HashSet.HashSet<string>>()
+	let edges = HashMap.empty<
+		string,
+		HashMap.HashMap<string, HashMap.HashMap<string, Dependency>>
+	>()
+	const link = (
+		id: string,
+		parent: string,
+		child: string,
+		request?: string,
+	) => {
+		const graph = Option.getOrElse(
+			HashMap.get(edges, id),
+			HashMap.empty<string, HashMap.HashMap<string, Dependency>>,
+		)
+		const children = Option.getOrElse(
+			HashMap.get(graph, parent),
+			HashMap.empty<string, Dependency>,
+		)
+		edges = HashMap.set(
+			edges,
+			id,
+			HashMap.set(
+				graph,
+				parent,
+				HashMap.set(children, request ?? child, {
+					generation: { id: Number(id) },
+					parent,
+					url: child,
+					...(request === undefined ? {} : { request }),
+				}),
+			),
+		)
+	}
+	const reachable = (id: string, roots: ReadonlyArray<string>) => {
+		const graph = Option.getOrElse(
+			HashMap.get(edges, id),
+			HashMap.empty<string, HashMap.HashMap<string, Dependency>>,
+		)
+		let seen = HashSet.empty<string>()
+		let pending = roots
+		while (Array.isReadonlyArrayNonEmpty(pending)) {
+			const [url, ...rest] = pending
+			pending = rest
+			if (HashSet.has(seen, url)) continue
+			seen = HashSet.add(seen, url)
+			pending = Array.appendAll(
+				pending,
+				Array.map(
+					HashMap.toValues(
+						Option.getOrElse(
+							HashMap.get(graph, url),
+							HashMap.empty<string, Dependency>,
+						),
+					),
+					(edge) => edge.url,
+				),
+			)
+		}
+		return seen
+	}
+	// Both checks and polling derive resolution facts from the same owned graph.
+	const resolutionsFor = (attempt: Attempt): ReadonlyArray<Attempt> => {
+		const id = String.String(attempt.generation.id)
+		const used = reachable(id, [attempt.url])
+		const graph = Option.getOrElse(
+			HashMap.get(edges, id),
+			HashMap.empty<string, HashMap.HashMap<string, Dependency>>,
+		)
+		return [
+			...(attempt.request === undefined ? [] : [attempt]),
+			...Array.flatMap(HashMap.toValues(graph), (children) =>
+				Array.filter(
+					HashMap.toValues(children),
+					(edge) =>
+						edge.request !== undefined && HashSet.has(used, edge.parent),
+				),
+			),
+		]
+	}
+
+	let sources = HashMap.empty<string, HashMap.HashMap<string, string>>()
 	let resolved: string | undefined
 
 	const prepare = (): Generation => ({
-		id: String(++revision),
-		unsupported: HashSet.empty(),
+		id: ++revision,
 	})
 	const clean = (url: string): string => {
 		const value = new URL(url)
@@ -51,48 +182,105 @@ export const modules = (base: string, include: (url: string) => boolean) => {
 		format === "commonjs" ||
 		format === "commonjs-typescript" ||
 		format === "addon" ||
-		Str.endsWith(".node")(new URL(url).pathname)
+		String.endsWith(".node")(new URL(url).pathname)
+	const requireRestart = (url: string, id: string) => {
+		dependencies = HashMap.set(
+			dependencies,
+			id,
+			HashSet.add(
+				Option.getOrElse(HashMap.get(dependencies, id), HashSet.empty<string>),
+				url,
+			),
+		)
+		if (
+			importing !== undefined &&
+			String.String(importing.generation.id) === id &&
+			importing.reload
+		)
+			throw RestartRequired.make({ url })
+	}
+
 	const record = (
 		url: string,
 		id: string,
 		format: string | null | undefined,
 	) => {
-		if (recording?.id !== id) return
-		if (unsupported(url, format)) {
-			recording.unsupported = HashSet.add(recording.unsupported, url)
-			if (reloading) throw RestartRequired.make({ url })
-		}
+		if (unsupported(url, format)) requireRestart(url, id)
 	}
 
 	const hooks = registerHooks({
 		resolve(specifier, context, nextResolve) {
-			const entry = Str.startsWith(prefix)(specifier)
-			const separator = !entry
-				? 0
-				: prefix.length +
-					Option.getOrThrow(
-						Str.indexOf(":")(Str.slice(prefix.length)(specifier)),
-					)
-			const id = entry
-				? Str.slice(prefix.length, separator)(specifier)
-				: version(context.parentURL)
-			if (id === undefined || id === null)
-				return nextResolve(specifier, context)
-			const result = nextResolve(
-				entry ? Str.slice(separator + 1)(specifier) : specifier,
-				{
-					...context,
-					parentURL: entry ? base : clean(context.parentURL!),
-				},
-			)
+			const request = decodeRequest(specifier)
+			const entry = Option.isSome(request)
+			const id = Option.match(request, {
+				onNone: () => version(context.parentURL),
+				onSome: ([, id]) => id,
+			})
+			if (id === null) return nextResolve(specifier, context)
+			const resolve = () =>
+				nextResolve(
+					Option.match(request, {
+						onNone: () => specifier,
+						onSome: ([, , , target]) => target,
+					}),
+					{
+						...context,
+						parentURL: entry ? base : clean(context.parentURL!),
+					},
+				)
+			const result = (() => {
+				try {
+					return resolve()
+				} catch (cause) {
+					// Node reports the missing file even though resolution did not complete.
+					if (
+						Predicate.hasProperty(cause, "url") &&
+						typeof cause.url === "string" &&
+						String.startsWith("file:")(cause.url)
+					) {
+						if (entry) resolved = cause.url
+						else if (context.parentURL !== undefined && include(cause.url))
+							link(
+								id,
+								clean(context.parentURL),
+								cause.url,
+								Option.getOrUndefined(
+									fileRequest(specifier, clean(context.parentURL)),
+								),
+							)
+					}
+					throw cause
+				}
+			})()
 			if (entry) resolved = result.url
+			// Acquiring node:module permits require calls that Node can satisfy
+			// without hooks. Record this capability before any cache can hide them.
 			if (
-				!Str.startsWith("file:")(result.url) ||
+				!entry &&
+				result.url === "node:module" &&
+				context.parentURL !== undefined
+			) {
+				link(id, clean(context.parentURL), result.url)
+				requireRestart(result.url, id)
+			}
+			if (
+				!String.startsWith("file:")(result.url) ||
 				(!entry && !include(result.url))
 			)
 				return result
-			record(result.url, id, result.format)
-			if (unsupported(result.url, result.format)) return result
+			if (!entry && context.parentURL !== undefined)
+				link(
+					id,
+					clean(context.parentURL),
+					result.url,
+					Option.getOrUndefined(
+						fileRequest(specifier, clean(context.parentURL)),
+					),
+				)
+			const required = Array.contains(context.conditions, "require")
+			if (required) requireRestart(result.url, id)
+			else record(result.url, id, result.format)
+			if (required || unsupported(result.url, result.format)) return result
 			const url = new URL(result.url)
 			url.searchParams.set(key, id)
 			return { ...result, url: url.href }
@@ -103,21 +291,126 @@ export const modules = (base: string, include: (url: string) => boolean) => {
 			const original = clean(url)
 			record(original, id, context.format)
 			const result = nextLoad(original, context)
+			const source = result.source
+			if (source !== undefined && source !== null) {
+				const bytes =
+					typeof source === "string"
+						? source
+						: ArrayBuffer.isView(source)
+							? new Uint8Array(
+									source.buffer,
+									source.byteOffset,
+									source.byteLength,
+								)
+							: new Uint8Array(source)
+				sources = HashMap.set(
+					sources,
+					id,
+					HashMap.set(
+						Option.getOrElse(
+							HashMap.get(sources, id),
+							HashMap.empty<string, string>,
+						),
+						original,
+						Base64.encode(bytes),
+					),
+				)
+			}
 			record(original, id, result.format)
 			return result
 		},
 	})
 	return {
 		prepare,
+		fileRequest: (specifier: string) => fileRequest(specifier, base),
+		resolutions: (attempts: ReadonlyArray<Attempt>) =>
+			Array.flatMap(attempts, resolutionsFor),
+		files: (attempts: ReadonlyArray<Attempt>) =>
+			Array.reduce(
+				attempts,
+				HashMap.empty<string, FileVersion>(),
+				(files, { generation, url, request }) => {
+					const contents = Option.getOrElse(
+						HashMap.get(sources, String.String(generation.id)),
+						HashMap.empty<string, string>,
+					)
+					const unversioned = Option.getOrElse(
+						HashMap.get(dependencies, String.String(generation.id)),
+						HashSet.empty<string>,
+					)
+					const tracked = Array.reduce(
+						Array.fromIterable(reachable(String.String(generation.id), [url])),
+						files,
+						(files, url) => {
+							if (
+								!String.startsWith("file:")(url) ||
+								HashSet.has(unversioned, url)
+							)
+								return files
+							// A retained failed lease must not override a newer import
+							// of the same file just because its owner is visited last.
+							if (
+								Option.exists(
+									HashMap.get(files, url),
+									(current) =>
+										current.generation > generation.id ||
+										(current.generation === generation.id &&
+											current.resolution !== undefined),
+								)
+							)
+								return files
+							return HashMap.set(files, url, {
+								generation: generation.id,
+								content: Option.getOrUndefined(HashMap.get(contents, url)),
+							})
+						},
+					)
+					return Array.reduce(
+						resolutionsFor({
+							generation,
+							url,
+							...(request === undefined ? {} : { request }),
+						}),
+						tracked,
+						(files, edge) => {
+							if (
+								edge.request === undefined ||
+								HashSet.has(unversioned, edge.url) ||
+								Option.exists(
+									HashMap.get(files, edge.request),
+									(file) => file.generation > generation.id,
+								)
+							)
+								return files
+							return HashMap.set(files, edge.request, {
+								generation: generation.id,
+								content: Option.getOrUndefined(HashMap.get(contents, edge.url)),
+								resolution: pathToFileURL(fileURLToPath(edge.url)).href,
+							})
+						},
+					)
+				},
+			),
+		unsupported: (generation: Generation, roots?: ReadonlyArray<string>) => {
+			const found = Option.getOrElse(
+				HashMap.get(dependencies, String.String(generation.id)),
+				HashSet.empty<string>,
+			)
+			if (roots === undefined) return found
+			const used = reachable(String.String(generation.id), roots)
+			return HashSet.filter(found, (url) => HashSet.has(used, url))
+		},
 		start: (generation: Generation, reload = false) => {
-			recording = generation
-			reloading = reload
+			importing = { generation, reload }
 		},
 		discard: () => {
-			recording = undefined
+			importing = undefined
+		},
+		request: (specifier: string) => {
+			resolved = undefined
+			return `${prefix}${importing!.generation.id}:${specifier}`
 		},
-		request: (specifier: string) => `${prefix}${recording!.id}:${specifier}`,
-		resolved: () => resolved,
+		resolved: () => Option.fromUndefinedOr(resolved),
 		close: () => hooks.deregister(),
 	}
 }