@graftland/plugin-remote 0.0.0-stage → 0.2.0-beta.2

package/src/internal/runner/Multiplex.ts

@@ -0,0 +1,120 @@
+import {
+	Cause,
+	Effect,
+	Fiber,
+	Option,
+	Queue,
+	Sink,
+	Stream,
+	TxQueue,
+} from "effect"
+import type { Socket } from "effect/socket"
+import { TransportError } from "../../Errors"
+import type { Channel as ByteChannel } from "../port"
+import * as FailureGroup from "../FailureGroup"
+import * as Framing from "./Framing"
+
+export interface Peer {
+	readonly await: Effect.Effect<never, TransportError>
+	readonly fail: (error: TransportError) => Effect.Effect<void>
+	readonly end: Effect.Effect<void, TransportError>
+	readonly send: (
+		channel: Framing.Channel,
+		payload: Uint8Array,
+	) => Effect.Effect<void, TransportError | Framing.FrameError>
+	readonly take: (
+		channel: Framing.Channel,
+	) => Effect.Effect<Uint8Array, TransportError>
+}
+/** Ports own admission and ACK backpressure. This layer only frames and separates session/instance bytes. */
+export const make = Effect.fnUntraced(function* (
+	channel: ByteChannel,
+): Effect.fn.Return<Peer, never, import("effect").Scope.Scope> {
+	const group = yield* FailureGroup.make<TransportError>()
+	const incoming = {
+		0: yield* Queue.unbounded<Uint8Array>(),
+		1: yield* Queue.unbounded<Uint8Array>(),
+	}
+	const outgoing = {
+		0: yield* TxQueue.unbounded<Uint8Array, Cause.Done>(),
+		1: yield* TxQueue.unbounded<Uint8Array, Cause.Done>(),
+	}
+	const next = Effect.tx(
+		Effect.gen(function* () {
+			const session = yield* TxQueue.poll(outgoing[0])
+			if (Option.isSome(session)) return session.value
+			const instance = yield* TxQueue.poll(outgoing[1])
+			if (Option.isSome(instance)) return instance.value
+			if (
+				(yield* TxQueue.isDone(outgoing[0])) &&
+				(yield* TxQueue.isDone(outgoing[1]))
+			)
+				return yield* Cause.done()
+			return yield* Effect.txRetry
+		}),
+	)
+	const writing = yield* group.run(
+		Stream.fromEffectRepeat(next).pipe(
+			Stream.run(channel.output),
+			Effect.asVoid,
+		),
+	)
+	const decoder = Framing.decoder()
+	yield* group.run(
+		channel.input.pipe(
+			Stream.runForEach((bytes) =>
+				Effect.fromResult(decoder.decode(bytes)).pipe(
+					Effect.flatMap((frames) =>
+						Effect.forEach(
+							frames,
+							(frame) => Queue.offer(incoming[frame.channel], frame.payload),
+							{ discard: true },
+						),
+					),
+				),
+			),
+			Effect.andThen(decoder.finish),
+			Effect.andThen(Effect.fail(Framing.EndOfStream.make({}))),
+			Effect.mapError((error) =>
+				error._tag === "TransportError"
+					? error
+					: TransportError.make({ cause: Cause.fail(error) }),
+			),
+		),
+	)
+	return {
+		await: group.await,
+		fail: group.fail,
+		end: Effect.tx(
+			TxQueue.end(outgoing[0]).pipe(Effect.andThen(TxQueue.end(outgoing[1]))),
+		).pipe(Effect.andThen(Fiber.join(writing))),
+		send: (id, payload) =>
+			group.race(
+				Framing.encode({ channel: id, payload }).pipe(
+					Effect.flatMap((bytes) => TxQueue.offer(outgoing[id], bytes)),
+					Effect.asVoid,
+				),
+			),
+		take: (id) => group.race(Queue.take(incoming[id])),
+	}
+})
+export const fromSocket = Effect.fnUntraced(function* (socket: Socket.Socket) {
+	const reader = yield* socket.reader
+	const writer = yield* socket.writer
+	return yield* make({
+		input: Stream.fromEffectRepeat(reader.pull).pipe(
+			Stream.flattenIterable,
+			Stream.map((bytes) =>
+				typeof bytes === "string" ? new TextEncoder().encode(bytes) : bytes,
+			),
+			Stream.mapError((error) =>
+				TransportError.make({ cause: Cause.fail(error) }),
+			),
+		),
+		output: Sink.forEach((bytes: Uint8Array) => writer.write(bytes)).pipe(
+			Sink.mapError((error) =>
+				TransportError.make({ cause: Cause.fail(error) }),
+			),
+		),
+	})
+})

package/src/internal/runner/Packages.ts

@@ -0,0 +1,178 @@
+import {
+	Array,
+	Cause,
+	Effect,
+	FileSystem,
+	Order,
+	Path,
+	Schema,
+	Stream,
+	String,
+} from "effect"
+import type { PlatformError } from "effect"
+import { ChildProcess, ChildProcessSpawner } from "effect/process"
+// @effect-diagnostics-next-line nodeBuiltinImport:off -- WebCrypto has no incremental digest API; package files are hashed as streams without buffering the directory.
+import { createHash } from "node:crypto"
+import process from "node:process"
+import { fileURLToPath } from "node:url"
+import { PackageError, type CodeIdentity } from "./model"
+
+const Manifest = Schema.Struct({ name: Schema.String, version: Schema.String })
+export interface Expected {
+	readonly version?: string
+	readonly digest?: string
+}
+export const verify = (actual: CodeIdentity, expected: Expected = {}) =>
+	expected.version !== undefined && actual.version !== expected.version
+		? Effect.fail(
+				PackageError.make({
+					detail: `Package version mismatch: expected ${expected.version}, received ${actual.version}`,
+				}),
+			)
+		: expected.digest !== undefined && actual.digest !== expected.digest
+			? Effect.fail(
+					PackageError.make({
+						detail: `Package digest mismatch: expected ${expected.digest}, received ${actual.digest}`,
+					}),
+				)
+			: Effect.void
+
+/** A path-sensitive SHA-256 of sorted entries and bytes, excluding every node_modules subtree. */
+export const digest = Effect.fnUntraced(function* (directory: string) {
+	const fs = yield* FileSystem.FileSystem
+	const path = yield* Path.Path
+	const hash = yield* Effect.sync(() =>
+		createHash("sha256").update("graft-package-sha256-v1\0"),
+	)
+	const visit = (
+		relative: string,
+	): Effect.Effect<void, PackageError | PlatformError.PlatformError> =>
+		Effect.gen(function* () {
+			const filename = path.join(directory, relative)
+			const info = yield* fs.stat(filename)
+			if (info.type !== "File" && info.type !== "Directory")
+				return yield* PackageError.make({
+					detail: `Unsupported package entry: ${relative}`,
+				})
+			const name = new TextEncoder().encode(relative)
+			yield* Effect.sync(() => {
+				hash.update(
+					`${info.type}:${name.byteLength}:${info.type === "File" ? info.size : 0}:`,
+				)
+				hash.update(name)
+			})
+			if (info.type === "File") {
+				yield* fs.stream(filename).pipe(
+					Stream.runForEach((bytes) =>
+						Effect.sync(() => {
+							hash.update(bytes)
+						}),
+					),
+				)
+			} else {
+				const entries = yield* fs.readDirectory(filename)
+				for (const entry of Array.sort(entries, Order.String)) {
+					if (entry === "node_modules") continue
+					yield* visit(relative === "" ? entry : `${relative}/${entry}`)
+				}
+			}
+		})
+	yield* visit("")
+	return yield* Effect.sync(() => hash.digest("hex"))
+})
+
+/**
+ * Resolve using Node's import conditions without importing executable package code.
+ * The helper's cwd is the installation root and it inherits no NODE_OPTIONS/loaders.
+ * The installation directory and its package exports are trusted.
+ */
+export const resolve = Effect.fnUntraced(
+	function* (root: string, specifier: string, includeDigest: boolean = false) {
+		const segments = String.split(specifier, "/")
+		const scoped = String.startsWith("@")(specifier)
+		const name = Array.join(Array.take(segments, scoped ? 2 : 1), "/")
+		const fs = yield* FileSystem.FileSystem
+		const path = yield* Path.Path
+		const installation = yield* fs.realPath(root)
+		const directory = yield* fs.realPath(
+			path.join(installation, "node_modules", name),
+		)
+		const manifest = yield* fs
+			.readFileString(path.join(directory, "package.json"))
+			.pipe(
+				Effect.flatMap(Schema.decodeEffect(Schema.fromJsonString(Manifest))),
+			)
+		const resolved = yield* resolveEntry(installation, specifier)
+		const filename = yield* Effect.try({
+			try: () => fileURLToPath(resolved),
+			catch: (error) =>
+				PackageError.make({
+					detail: "The package export is not a file URL",
+					cause: Cause.die(error),
+				}),
+		})
+		const entry = yield* fs.realPath(filename)
+		const identity: CodeIdentity = {
+			...manifest,
+			...(includeDigest ? { digest: yield* digest(directory) } : {}),
+		}
+		return { directory, entry, identity }
+	},
+	Effect.catchTags({
+		PlatformError: (error) =>
+			Effect.fail(
+				PackageError.make({
+					detail: "Reading the installed package failed",
+					cause: Cause.fail(error),
+				}),
+			),
+		SchemaError: (error) =>
+			Effect.fail(
+				PackageError.make({
+					detail: "Reading the package manifest failed",
+					cause: Cause.fail(error),
+				}),
+			),
+	}),
+)
+
+export const resolveEntry = (installation: string, specifier: string) =>
+	Effect.gen(function* () {
+		const spawner = yield* ChildProcessSpawner.ChildProcessSpawner
+		return yield* Effect.scoped(
+			Effect.gen(function* () {
+				const child = yield* spawner.spawn(
+					ChildProcess.make(
+						process.execPath,
+						[
+							"--input-type=module",
+							"--eval",
+							"process.stdout.write(import.meta.resolve(process.argv[1]))",
+							"--",
+							specifier,
+						],
+						{
+							cwd: installation,
+							env: {},
+							stdin: "ignore",
+							stdout: "pipe",
+							stderr: "pipe",
+						},
+					),
+				)
+				const [code, stdout, stderr] = yield* Effect.all(
+					[
+						child.exitCode,
+						child.stdout.pipe(Stream.decodeText(), Stream.mkString),
+						child.stderr.pipe(Stream.decodeText(), Stream.mkString),
+					],
+					{ concurrency: "unbounded" },
+				)
+				if (code !== 0)
+					return yield* PackageError.make({
+						detail: `Package resolution failed: ${stderr}`,
+					})
+				return stdout
+			}),
+		)
+	})

package/src/internal/runner/ProcessObservation.ts

@@ -0,0 +1,63 @@
+import {
+	Array,
+	Cause,
+	Effect,
+	Exit,
+	Fiber,
+	Ref,
+	Schedule,
+	Stream,
+	String,
+} from "effect"
+import type { PlatformError } from "effect"
+import type { ChildProcessSpawner } from "effect/process"
+import * as FailureGroup from "../FailureGroup"
+
+/** Time allowed for a direct execution unit or launcher to exit after input EOF. */
+export const stdinExitTimeout = "1 second"
+
+/** Readers outlive the failure monitor so forced termination still captures its diagnostic tail. */
+export const observe = Effect.fnUntraced(function* (
+	child: ChildProcessSpawner.ChildProcessHandle,
+	options: { readonly drainStdout?: boolean } = {},
+) {
+	const failures = yield* FailureGroup.make<PlatformError.PlatformError>()
+	const tail = yield* Ref.make("")
+	// Command transports own stdout as a protocol stream.
+	const stdout = yield* (
+		options.drainStdout === false ? Effect.void : Stream.runDrain(child.stdout)
+	).pipe(Effect.forkScoped)
+	const stderr = yield* child.stderr.pipe(
+		Stream.decodeText(),
+		Stream.runForEach((text) =>
+			Ref.update(tail, (old) => String.takeRight(old + text, 8192)),
+		),
+		Effect.forkScoped,
+	)
+	yield* failures.run(Fiber.join(stdout))
+	yield* failures.run(Fiber.join(stderr))
+	const exited = yield* Effect.gen(function* () {
+		const result = yield* Effect.exit(child.exitCode)
+		// exitCode can fail before exit on an input error; isRunning follows Node's exit event.
+		yield* child.isRunning.pipe(
+			Effect.repeat({
+				schedule: Schedule.spaced("25 millis"),
+				while: (running) => running,
+			}),
+		)
+		const err = yield* Fiber.await(stderr)
+		const out = yield* Fiber.await(stdout)
+		return {
+			code: Exit.isSuccess(result) ? result.value : null,
+			signal: null,
+			stderr: yield* Ref.get(tail),
+			cause: Array.reduce(
+				[Exit.asVoid(result), err, out],
+				Cause.empty,
+				(cause: Cause.Cause<PlatformError.PlatformError>, exit) =>
+					Exit.isFailure(exit) ? Cause.combine(cause, exit.cause) : cause,
+			),
+		}
+	}).pipe(Effect.forkScoped)
+	return { exited: Fiber.join(exited), failure: failures.await }
+})

package/src/internal/runner/Processes.ts

@@ -0,0 +1,109 @@
+import { Cause, Effect, Option, Schedule, Stream, String } from "effect"
+import { ChildProcess, ChildProcessSpawner } from "effect/process"
+import process from "node:process"
+import {
+	ExecutionError,
+	TerminationUnconfirmed,
+	type ProcessIdentity,
+	type Termination,
+} from "./model"
+
+const inspect = (pid: number) =>
+	Effect.scoped(
+		Effect.gen(function* () {
+			const spawner = yield* ChildProcessSpawner.ChildProcessSpawner
+			const child = yield* spawner.spawn(
+				ChildProcess.make(
+					"ps",
+					["-p", `${pid}`, "-o", "lstart=", "-o", "stat="],
+					{
+						env: { ...process.env, LC_ALL: "C" },
+						stdin: "ignore",
+					},
+				),
+			)
+			const [code, output, stderr] = yield* Effect.all(
+				[
+					child.exitCode,
+					child.stdout.pipe(Stream.decodeText(), Stream.mkString),
+					child.stderr.pipe(Stream.decodeText(), Stream.mkString),
+				],
+				{ concurrency: "unbounded" },
+			)
+			if (code === 1 && String.trim(output) === "") return Option.none()
+			const [, born, status] =
+				/^(.{24})\s+(\S+)$/m.exec(String.trim(output)) ?? []
+			if (code !== 0 || born === undefined || status === undefined)
+				return yield* ExecutionError.make({
+					detail: "Cannot inspect process identity: " + stderr,
+				})
+			if (String.startsWith("Z")(status)) return Option.none()
+			return Option.some({ born })
+		}),
+	).pipe(
+		Effect.mapError((error) =>
+			error._tag === "ExecutionError"
+				? error
+				: ExecutionError.make({
+						detail: "Inspecting process identity failed",
+						cause: Cause.fail(error),
+					}),
+		),
+	)
+
+export const identity = (pid: number) =>
+	Effect.gen(function* () {
+		const current = yield* inspect(pid)
+		if (Option.isNone(current))
+			return yield* ExecutionError.make({
+				detail: "The child exited before its identity could be recorded",
+			})
+		return { pid, start: current.value }
+	})
+
+// The OS may discard argv before reporting a zombie; PID and start time remain the identity.
+export const matches = (child: ProcessIdentity) =>
+	Effect.gen(function* () {
+		const current = yield* inspect(child.pid)
+		if (Option.isNone(current) || current.value.born !== child.start.born)
+			return false
+		return true
+	})
+
+export const recover = (child: ProcessIdentity) =>
+	Effect.gen(function* () {
+		if (!(yield* matches(child)))
+			return {
+				kind: "Exited",
+				exit: { code: null, signal: null },
+				cause: Cause.empty,
+			} satisfies Termination
+		// The PID can still be reused between this final check and kill; an external supervisor covers descendants.
+		const sent = yield* Effect.exit(
+			Effect.try({
+				try: () => process.kill(child.pid, "SIGKILL"),
+				catch: (cause) =>
+					ExecutionError.make({
+						detail: "Signaling the orphan failed",
+						cause: Cause.die(cause),
+					}),
+			}),
+		)
+		const confirmed = yield* matches(child).pipe(
+			Effect.repeat({
+				schedule: Schedule.spaced("50 millis"),
+				while: (value) => value,
+			}),
+			Effect.timeoutOption("10 seconds"),
+		)
+		if (Option.isNone(confirmed))
+			return yield* TerminationUnconfirmed.make({
+				detail: "The orphan did not exit after recovery",
+				cause: sent._tag === "Failure" ? sent.cause : Cause.empty,
+			})
+		return {
+			kind: "Killed",
+			exit: { code: null, signal: "SIGKILL" },
+			cause: sent._tag === "Failure" ? sent.cause : Cause.empty,
+		} satisfies Termination
+	})

package/src/internal/runner/Retention.ts

@@ -0,0 +1,49 @@
+import { Array, HashMap, Option } from "effect"
+import { keyOf, type SessionKey, type SessionRecord } from "./model"
+
+/** A terminal high-water record fences every older Open, even after its request id is forgotten. */
+export const retain = (
+	records: ReadonlyArray<SessionRecord>,
+): ReadonlyArray<SessionRecord> => {
+	const highWater = Array.reduce(
+		records,
+		HashMap.empty<SessionKey, number>(),
+		(water, record) =>
+			HashMap.set(
+				water,
+				keyOf(record.handle.key),
+				Math.max(
+					Option.getOrElse(
+						HashMap.get(water, keyOf(record.handle.key)),
+						() => 0,
+					),
+					record.handle.generation,
+				),
+			),
+	)
+	return Array.filter(
+		records,
+		(record) =>
+			record.phase !== "Stopped" ||
+			record.handle.generation ===
+				Option.getOrUndefined(HashMap.get(highWater, keyOf(record.handle.key))),
+	)
+}
+/** Apply the same retention rule to both in-memory stores. */
+export const save = (
+	records: HashMap.HashMap<string, SessionRecord>,
+	record: SessionRecord,
+) =>
+	HashMap.fromIterable(
+		Array.map(
+			retain(
+				Array.fromIterable(
+					HashMap.values(HashMap.set(records, record.handle.token, record)),
+				),
+			),
+			(record): readonly [string, SessionRecord] => [
+				record.handle.token,
+				record,
+			],
+		),
+	)

package/src/internal/runner/Store.ts

@@ -0,0 +1,160 @@
+import {
+	Array,
+	Cause,
+	Layer,
+	Effect,
+	FileSystem,
+	HashMap,
+	HashSet,
+	Path,
+	Ref,
+	Schema,
+	Semaphore,
+	String,
+	type PlatformError,
+} from "effect"
+import { Digest, SecureRandom } from "effect/crypto"
+import { Key, SessionRecord, Store, StateStorageError } from "./model"
+import * as Retention from "./Retention"
+
+/** Injectable store used by deterministic supervisor tests. */
+export const memory = Effect.gen(function* () {
+	const records = yield* Ref.make(HashMap.empty<string, SessionRecord>())
+	return Store.of({
+		list: Ref.get(records).pipe(
+			Effect.map((records) => Array.fromIterable(HashMap.values(records))),
+		),
+		save: (record) => Ref.update(records, (all) => Retention.save(all, record)),
+	})
+})
+
+/**
+ * Atomically replaced records retain unfinished sessions and each key's high-water mark.
+ * Older terminal request ids expire; Open fences their retries with expectedGeneration.
+ * Requires exclusive directory ownership; it is not a multi-writer database.
+ */
+export const directory = (root: string) =>
+	Effect.gen(function* () {
+		const fs = yield* FileSystem.FileSystem
+		const path = yield* Path.Path
+		const digest = yield* Digest.Digest
+		const json = Schema.fromJsonString(Schema.toCodecJson(SessionRecord))
+		const random = yield* SecureRandom.SecureRandom
+		const storage = <A, R>(
+			effect: Effect.Effect<
+				A,
+				PlatformError.PlatformError | Schema.SchemaError | StateStorageError,
+				R
+			>,
+		) =>
+			effect.pipe(
+				Effect.mapError((error) =>
+					error._tag === "StateStorageError"
+						? error
+						: StateStorageError.make({
+								detail: "Runner state storage failed",
+								cause: Cause.fail(error),
+							}),
+				),
+			)
+		yield* storage(
+			fs.makeDirectory(root, { recursive: true }).pipe(Effect.uninterruptible),
+		)
+		const mutex = yield* Semaphore.make(1)
+		const read = Effect.gen(function* () {
+			const files = yield* fs.readDirectory(root)
+			const rows = yield* Effect.forEach(
+				Array.filter(files, String.endsWith(".json")),
+				(file) =>
+					fs.readFileString(path.join(root, file)).pipe(
+						Effect.flatMap(Schema.decodeEffect(json)),
+						Effect.map((record) => ({ file, record })),
+					),
+			)
+			return rows
+		})
+		const compact = Effect.scoped(
+			Effect.gen(function* () {
+				const rows = yield* read
+				const retained = Retention.retain(Array.map(rows, (row) => row.record))
+				const tokens = HashSet.fromIterable(
+					Array.map(retained, (record) => record.handle.token),
+				)
+				const obsolete = Array.filter(
+					rows,
+					(row) => !HashSet.has(tokens, row.record.handle.token),
+				)
+				yield* Effect.forEach(
+					obsolete,
+					(row) =>
+						fs.remove(path.join(root, row.file)).pipe(Effect.uninterruptible),
+					{ discard: true },
+				)
+				if (obsolete.length > 0) {
+					const directory = yield* fs.open(root, { flag: "r" })
+					yield* directory.sync.pipe(Effect.uninterruptible)
+				}
+			}),
+		)
+		yield* storage(compact)
+		return Store.of({
+			list: storage(
+				mutex.withPermit(
+					read.pipe(Effect.map((rows) => Array.map(rows, (row) => row.record))),
+				),
+			),
+			save: (record) =>
+				storage(
+					mutex.withPermit(
+						Effect.scoped(
+							Effect.gen(function* () {
+								const encoded = yield* Schema.encodeEffect(json)(record)
+								const address = yield* Schema.encodeEffect(
+									Schema.fromJsonString(
+										Schema.Struct({ key: Key, generation: Schema.Int }),
+									),
+								)({
+									key: record.handle.key,
+									generation: record.handle.generation,
+								})
+								const name = yield* digest.digestHex(
+									Digest.sha256,
+									new TextEncoder().encode(address),
+								)
+								const nonce = yield* random.uuidV4()
+								const temporary = path.join(root, `.${name}.${nonce}.tmp`)
+								const destination = path.join(root, name + ".json")
+								yield* Effect.acquireRelease(
+									fs.open(temporary, { flag: "wx", mode: 0o600 }),
+									() =>
+										fs.remove(temporary, { force: true }).pipe(Effect.orDie),
+								).pipe(
+									Effect.flatMap((file) =>
+										file
+											.writeAll(new TextEncoder().encode(encoded))
+											.pipe(
+												Effect.uninterruptible,
+												Effect.andThen(file.sync.pipe(Effect.uninterruptible)),
+											),
+									),
+								)
+								// Keep rename and directory synchronization together if the caller is canceled.
+								yield* Effect.uninterruptible(
+									Effect.gen(function* () {
+										yield* fs.rename(temporary, destination)
+										const directory = yield* fs.open(root, { flag: "r" })
+										yield* directory.sync.pipe(Effect.uninterruptible)
+									}),
+								)
+								// The new high-water file is durable before any old record is removed.
+								yield* compact
+							}),
+						),
+					),
+				),
+		})
+	})
+
+export const layerMemory = Layer.effect(Store, memory)
+export const layerDirectory = (root: string) =>
+	Layer.effect(Store, directory(root))