@graftland/plugin-remote 0.0.0-stage → 0.2.0-beta.2

package/src/internal/runner/Watchdog.ts

@@ -0,0 +1,83 @@
+import { Cause, Deferred, Effect, Schema } from "effect"
+import * as FailureGroup from "../FailureGroup"
+import { Worker } from "node:worker_threads"
+import { ExecutionError } from "./model"
+import { Notice } from "./WatchdogProtocol"
+
+export const start = Effect.gen(function* () {
+	const ready = yield* Deferred.make<void>()
+	const begin = yield* Deferred.make<void>()
+	const stop = yield* Deferred.make<void>()
+	const worker = yield* Effect.acquireRelease(
+		Effect.try({
+			try: () =>
+				new Worker(
+					new URL(
+						import.meta.resolve("@graftland/plugin-remote/internal/watchdog"),
+					),
+				),
+			catch: (cause) =>
+				ExecutionError.make({
+					detail: "Starting the child watchdog failed",
+					cause: Cause.die(cause),
+				}),
+		}),
+		(worker) => Effect.promise(() => worker.terminate()).pipe(Effect.asVoid),
+	)
+	const listeners = yield* FailureGroup.make<ExecutionError>()
+	const run = yield* listeners.runtime<never>()
+	const message = (value: unknown) =>
+		run(
+			Schema.decodeUnknownEffect(Notice)(value).pipe(
+				Effect.mapError((error) =>
+					ExecutionError.make({
+						detail: "Invalid watchdog notice",
+						cause: Cause.fail(error),
+					}),
+				),
+				Effect.flatMap((notice) =>
+					Deferred.succeed(
+						notice === "ready" ? ready : notice === "start" ? begin : stop,
+						undefined,
+					),
+				),
+				Effect.asVoid,
+			),
+		)
+	const error = (error: Error) =>
+		run(
+			Effect.fail(
+				ExecutionError.make({
+					detail: "Child watchdog failed",
+					cause: Cause.die(error),
+				}),
+			),
+		)
+	const exit = (code: number) =>
+		run(
+			Effect.fail(
+				ExecutionError.make({
+					detail: "Child watchdog exited: " + code,
+				}),
+			),
+		)
+	yield* Effect.acquireRelease(
+		Effect.sync(() => {
+			worker.on("message", message)
+			worker.on("error", error)
+			worker.on("exit", exit)
+		}),
+		() =>
+			Effect.sync(() => {
+				worker.off("message", message)
+				worker.off("error", error)
+				worker.off("exit", exit)
+			}),
+	)
+	const failure = listeners.await
+	yield* Effect.raceFirst(Deferred.await(ready), failure)
+	return {
+		begin: Effect.raceFirst(Deferred.await(begin), failure),
+		stop: Effect.raceFirst(Deferred.await(stop), failure),
+	}
+})

package/src/internal/runner/WatchdogProtocol.ts

@@ -0,0 +1,12 @@
+import { Schema } from "effect"
+
+export const Command = Schema.Union([
+	Schema.Struct({
+		kind: Schema.Literal("renew"),
+		millis: Schema.Finite.check(Schema.isGreaterThan(0)),
+	}),
+	Schema.Struct({ kind: Schema.Literal("start") }),
+	Schema.Struct({ kind: Schema.Literal("stop") }),
+])
+export const CommandJson = Schema.fromJsonString(Command)
+export const Notice = Schema.Literals(["ready", "start", "stop"])

package/src/internal/runner/Wire.ts

@@ -0,0 +1,99 @@
+import { Cause, Effect, Sink, Stream, Schema } from "effect"
+import { Rpc, RpcGroup } from "effect/rpc"
+import { TransportError } from "../../Errors"
+import * as Port from "../port"
+import type * as Multiplex from "./Multiplex"
+import {
+	CodeIdentity,
+	Handle,
+	Limits,
+	PositiveInt,
+	PositiveFinite,
+	SessionRecord,
+	SessionAccessDenied,
+	StaleSession,
+	LeaseExpired,
+	SessionStopped,
+	SessionConflict,
+	StateStorageError,
+	ExecutionError,
+	PackageError,
+	TerminationUnconfirmed,
+	RunnerClosing,
+	StateDirectoryLocked,
+	Termination,
+} from "./model"
+
+/** The RPC boundary is the only place that assembles the runner's error schema. */
+export const SessionError = Schema.Union([
+	SessionAccessDenied,
+	StaleSession,
+	LeaseExpired,
+	SessionStopped,
+	SessionConflict,
+	StateStorageError,
+	ExecutionError,
+	PackageError,
+	TerminationUnconfirmed,
+	RunnerClosing,
+	StateDirectoryLocked,
+])
+export type SessionError = typeof SessionError.Type
+
+export const Session = RpcGroup.make(
+	Rpc.make("HighWater", {
+		payload: { instance: Schema.NonEmptyString },
+		success: Schema.NullOr(PositiveInt),
+		error: SessionError,
+	}),
+	Rpc.make("Open", {
+		payload: {
+			instance: Schema.NonEmptyString,
+			requestId: Schema.NonEmptyString,
+			expectedGeneration: Schema.NullOr(PositiveInt),
+			leaseMillis: Schema.optionalKey(PositiveFinite),
+			limits: Limits,
+		},
+		success: SessionRecord,
+		error: SessionError,
+	}),
+	Rpc.make("Load", {
+		payload: {
+			handle: Handle,
+			specifier: Schema.NonEmptyString,
+			digest: Schema.optionalKey(Schema.Boolean),
+		},
+		success: CodeIdentity,
+		error: SessionError,
+	}),
+	Rpc.make("Renew", {
+		payload: Handle,
+		success: Schema.Finite,
+		error: SessionError,
+	}),
+	Rpc.make("Dispose", {
+		payload: Handle,
+		success: Termination,
+		error: SessionError,
+	}),
+	Rpc.make("Status", {
+		payload: Handle,
+		success: SessionRecord,
+		error: SessionError,
+	}),
+).annotateRpcs(Port.Priority, "control")
+
+export type Peer = Multiplex.Peer
+/** Preserve the port writer order within each byte channel. */
+export const channel = (peer: Peer, id: 0 | 1): Port.Channel => ({
+	input: Stream.fromEffectRepeat(peer.take(id)),
+	output: Sink.forEach((bytes: Uint8Array) =>
+		peer
+			.send(id, bytes)
+			.pipe(
+				Effect.mapError((error) =>
+					TransportError.make({ cause: Cause.fail(error) }),
+				),
+			),
+	),
+})

package/src/internal/runner/accept.ts

@@ -0,0 +1,103 @@
+import { Cause, Effect, Ref, Option, Scope, Semaphore } from "effect"
+import { RpcServer } from "effect/rpc"
+import { Peer } from "../Peer"
+import { TransportError } from "../../Errors"
+import type { Channel } from "../../Transport"
+import type { Supervisor } from "./Supervisor"
+import type { Channels } from "./Execution"
+import * as Port from "../port"
+import * as Multiplex from "./Multiplex"
+import * as Wire from "./Wire"
+import { SessionAccessDenied, SessionConflict, type Handle } from "./model"
+
+export const accept = (
+	supervisor: typeof Supervisor.Service,
+	execution: typeof Channels.Service,
+	channel: Channel,
+) =>
+	Effect.gen(function* () {
+		const identity = yield* Peer
+		const scope = yield* Effect.scope
+		const mux = yield* Multiplex.make(channel)
+		const peer = yield* Port.make(Wire.channel(mux, 0), {
+			session: "session-v1",
+			serve: { session: Wire.Session },
+			call: {},
+		})
+		const attached = yield* Ref.make(Option.none<string>())
+		const loading = yield* Semaphore.make(1)
+		const authorize = (handle: Handle) =>
+			handle.key.identity === identity
+				? Effect.void
+				: Effect.fail(
+						SessionAccessDenied.make({
+							detail: "The session belongs to another authenticated identity",
+						}),
+					)
+		const handlers = yield* Wire.Session.toHandlers({
+			HighWater: ({ instance }) => supervisor.highWater({ identity, instance }),
+			Open: ({
+				instance,
+				requestId,
+				expectedGeneration,
+				leaseMillis,
+				limits,
+			}) =>
+				supervisor.open({
+					key: { identity, instance },
+					requestId,
+					expectedGeneration,
+					...(leaseMillis === undefined ? {} : { lease: leaseMillis }),
+					limits,
+				}),
+			Load: ({ handle, ...request }) =>
+				Effect.gen(function* () {
+					yield* authorize(handle)
+					const attachment = yield* Ref.get(attached)
+					if (Option.isSome(attachment) && attachment.value !== handle.token)
+						return yield* SessionConflict.make({
+							detail: "A connection can attach only one execution",
+						})
+					const code = yield* supervisor.load(handle, request)
+					if (Option.isNone(attachment)) {
+						const child = yield* execution
+							.peer(handle.token)
+							.pipe(Scope.provide(scope))
+						yield* Ref.set(attached, Option.some(handle.token))
+						// A child EOF must not destroy the session channel before Dispose can acknowledge exit.
+						yield* Effect.all(
+							[
+								Effect.forever(
+									Effect.flatMap(mux.take(1), (bytes) => child.send(1, bytes)),
+								),
+								Effect.forever(
+									Effect.flatMap(child.take(1), (bytes) => mux.send(1, bytes)),
+								),
+							],
+							{ concurrency: "unbounded" },
+						).pipe(
+							Effect.catchCause((cause) =>
+								Cause.hasInterruptsOnly(cause)
+									? Effect.void
+									: child.fail(TransportError.make({ cause })),
+							),
+							Effect.forkIn(scope),
+						)
+					}
+					return code
+				}).pipe(loading.withPermit),
+			Renew: (handle) =>
+				authorize(handle).pipe(Effect.andThen(supervisor.renew(handle))),
+			Dispose: (handle) =>
+				authorize(handle).pipe(Effect.andThen(supervisor.dispose(handle))),
+			Status: (handle) =>
+				authorize(handle).pipe(Effect.andThen(supervisor.status(handle))),
+		})
+		return yield* RpcServer.make(Wire.Session, {
+			disableFatalDefects: true,
+		}).pipe(
+			Effect.provideContext(handlers),
+			Effect.provideService(RpcServer.Protocol, peer.serverProtocol("session")),
+			Effect.raceFirst(mux.await),
+		)
+	})

package/src/internal/runner/command.ts

@@ -0,0 +1,15 @@
+import { Effect } from "effect"
+import process from "node:process"
+import { fileURLToPath } from "node:url"
+
+/** Resolve the executable through the installed package export, never a relative dist path. */
+export const command = (override?: readonly [string, ...string[]]) =>
+	Effect.sync(
+		(): readonly [string, ...string[]] =>
+			override ?? [
+				process.execPath,
+				fileURLToPath(
+					import.meta.resolve("@graftland/plugin-remote/internal/bin"),
+				),
+			],
+	)

package/src/internal/runner/connectionEnded.ts

@@ -0,0 +1,28 @@
+import { Array, Cause, Effect, Schema } from "effect"
+import { Socket } from "effect/socket"
+import { TransportError } from "../../Errors"
+import { EndOfStream } from "./Framing"
+import { Peer } from "../Peer"
+
+// Multiplex preserves the socket reason inside TransportError; only an entirely normal close is silent.
+export const closedNormally = (cause: Cause.Cause<unknown>): boolean =>
+	cause.reasons.length > 0 &&
+	Array.every(
+		cause.reasons,
+		(reason) =>
+			reason._tag === "Fail" &&
+			(Schema.is(TransportError)(reason.error)
+				? closedNormally(reason.error.cause)
+				: Schema.is(EndOfStream)(reason.error) ||
+					(Socket.isSocketError(reason.error) &&
+						reason.error.reason._tag === "SocketCloseError" &&
+						reason.error.reason.code === 1000)),
+	)
+export const connectionEnded = (cause: Cause.Cause<unknown>) =>
+	Cause.hasInterruptsOnly(cause) || closedNormally(cause)
+		? Effect.void
+		: Effect.flatMap(Peer, (identity) =>
+				Effect.logWarning("Remote host connection failed", cause).pipe(
+					Effect.annotateLogs({ "remote.peer": identity }),
+				),
+			)

package/src/internal/runner/model.ts

@@ -0,0 +1,364 @@
+import { Limits, PositiveInt } from "../Limits"
+export { Limits, PositiveInt, PositiveFinite, defaults } from "../Limits"
+import {
+	Array,
+	Cause,
+	Context,
+	Data,
+	Duration,
+	Effect,
+	Option,
+	Schema,
+} from "effect"
+import type { Scope } from "effect"
+
+const diagnosticCause = Schema.Cause(
+	Schema.Defect({ includeStack: true }),
+	Schema.Defect({ includeStack: true }),
+)
+export class PackageError extends Schema.TaggedError<
+	PackageError,
+	{ readonly brand: unique symbol }
+>()("PackageError", {
+	detail: Schema.String,
+	cause: Schema.optionalKey(diagnosticCause),
+}) {
+	override get message() {
+		return this.detail
+	}
+}
+export class SessionAccessDenied extends Schema.TaggedError<
+	SessionAccessDenied,
+	{ readonly brand: unique symbol }
+>()("SessionAccessDenied", {
+	detail: Schema.String,
+	cause: Schema.optionalKey(diagnosticCause),
+}) {
+	override get message() {
+		return this.detail
+	}
+}
+export class StaleSession extends Schema.TaggedError<
+	StaleSession,
+	{ readonly brand: unique symbol }
+>()("StaleSession", {
+	detail: Schema.String,
+	cause: Schema.optionalKey(diagnosticCause),
+}) {
+	override get message() {
+		return this.detail
+	}
+}
+export class LeaseExpired extends Schema.TaggedError<
+	LeaseExpired,
+	{ readonly brand: unique symbol }
+>()("LeaseExpired", {
+	detail: Schema.String,
+	cause: Schema.optionalKey(diagnosticCause),
+}) {
+	override get message() {
+		return this.detail
+	}
+}
+export class SessionStopped extends Schema.TaggedError<
+	SessionStopped,
+	{ readonly brand: unique symbol }
+>()("SessionStopped", {
+	detail: Schema.String,
+	cause: Schema.optionalKey(diagnosticCause),
+}) {
+	override get message() {
+		return this.detail
+	}
+}
+export class SessionConflict extends Schema.TaggedError<
+	SessionConflict,
+	{ readonly brand: unique symbol }
+>()("SessionConflict", {
+	detail: Schema.String,
+	cause: Schema.optionalKey(diagnosticCause),
+}) {
+	override get message() {
+		return this.detail
+	}
+}
+export class StateStorageError extends Schema.TaggedError<
+	StateStorageError,
+	{ readonly brand: unique symbol }
+>()("StateStorageError", {
+	detail: Schema.String,
+	cause: Schema.optionalKey(diagnosticCause),
+}) {
+	override get message() {
+		return this.detail
+	}
+}
+export class ExecutionError extends Schema.TaggedError<
+	ExecutionError,
+	{ readonly brand: unique symbol }
+>()("ExecutionError", {
+	detail: Schema.String,
+	cause: Schema.optionalKey(diagnosticCause),
+}) {
+	override get message() {
+		return this.detail
+	}
+}
+export class TerminationUnconfirmed extends Schema.TaggedError<
+	TerminationUnconfirmed,
+	{ readonly brand: unique symbol }
+>()("TerminationUnconfirmed", {
+	detail: Schema.String,
+	cause: Schema.optionalKey(diagnosticCause),
+}) {
+	override get message() {
+		return this.detail
+	}
+}
+export class RunnerClosing extends Schema.TaggedError<
+	RunnerClosing,
+	{ readonly brand: unique symbol }
+>()("RunnerClosing", {
+	detail: Schema.String,
+	cause: Schema.optionalKey(diagnosticCause),
+}) {
+	override get message() {
+		return this.detail
+	}
+}
+export class StateDirectoryLocked extends Schema.TaggedError<
+	StateDirectoryLocked,
+	{ readonly brand: unique symbol }
+>()("StateDirectoryLocked", {
+	detail: Schema.String,
+	cause: Schema.optionalKey(diagnosticCause),
+}) {
+	override get message() {
+		return this.detail
+	}
+}
+export const Key = Schema.Struct({
+	identity: Schema.NonEmptyString,
+	instance: Schema.NonEmptyString,
+})
+export type Key = typeof Key.Type
+/** Structural keys keep identity namespaces separate without delimiter conventions. */
+export class SessionKey extends Data.Class<Key> {}
+export const keyOf = (key: Key) => new SessionKey(key)
+export const negotiate = (local: Limits, peer: Limits): Limits => ({
+	control: Math.min(local.control, peer.control),
+	data: Math.min(local.data, peer.data),
+	log: Math.min(local.log, peer.log),
+})
+export const Handle = Schema.Struct({
+	key: Key,
+	token: Schema.NonEmptyString,
+	generation: PositiveInt,
+})
+export type Handle = typeof Handle.Type
+export const ProcessIdentity = Schema.Struct({
+	pid: Schema.Int,
+	start: Schema.Struct({ born: Schema.String }),
+})
+export type ProcessIdentity = typeof ProcessIdentity.Type
+export const ExitStatus = Schema.Struct({
+	stderr: Schema.optionalKey(Schema.String),
+	cause: Schema.optionalKey(diagnosticCause),
+	code: Schema.NullOr(Schema.Int),
+	signal: Schema.NullOr(Schema.String),
+})
+export type ExitStatus = typeof ExitStatus.Type
+export const CodeIdentity = Schema.Struct({
+	name: Schema.String,
+	version: Schema.String,
+	digest: Schema.optionalKey(Schema.String),
+})
+export type CodeIdentity = typeof CodeIdentity.Type
+export const Load = Schema.Struct({
+	specifier: Schema.String,
+	digest: Schema.optionalKey(Schema.Boolean),
+})
+export type Load = typeof Load.Type
+export const Termination = Schema.Struct({
+	kind: Schema.Literals(["Exited", "Killed"]),
+	exit: ExitStatus,
+	cause: diagnosticCause,
+})
+export type Termination = typeof Termination.Type
+const Fields = {
+	version: Schema.Literal(1),
+	handle: Handle,
+	requestId: Schema.String,
+	leaseMillis: Schema.Finite,
+	limits: Limits,
+}
+export const OpenRecord = Schema.Struct({
+	...Fields,
+	phase: Schema.Literal("Open"),
+})
+export const Launch = Schema.Union([
+	Schema.TaggedStruct("Starting", {}),
+	Schema.TaggedStruct("Registered", { child: ProcessIdentity }),
+])
+export const LoadingRecord = Schema.Struct({
+	...Fields,
+	phase: Schema.Literal("Loading"),
+	load: Load,
+	launch: Launch,
+})
+export const LoadedRecord = Schema.Struct({
+	...Fields,
+	phase: Schema.Literal("Loaded"),
+	load: Load,
+	child: ProcessIdentity,
+	code: CodeIdentity,
+})
+export const ActiveRecord = Schema.Union([
+	OpenRecord,
+	LoadingRecord,
+	LoadedRecord,
+])
+export type ActiveRecord = typeof ActiveRecord.Type
+export const StopReason = Schema.Union([
+	...Array.map(
+		[
+			"Disposed",
+			"Expired",
+			"Superseded",
+			"Recovery",
+			"Shutdown",
+			"Exited",
+		] as const,
+		(tag) => Schema.TaggedStruct(tag, {}),
+	),
+	Schema.TaggedStruct("LoadFailed", { cause: diagnosticCause }),
+])
+export type StopReason = typeof StopReason.Type
+export const StoppingRecord = Schema.Struct({
+	...Fields,
+	phase: Schema.Literal("Stopping"),
+	previous: ActiveRecord,
+	reason: StopReason,
+})
+export const StoppedRecord = Schema.Struct({
+	...Fields,
+	phase: Schema.Literal("Stopped"),
+	previous: ActiveRecord,
+	reason: StopReason,
+	termination: Termination,
+})
+export const SessionRecord = Schema.Union([
+	OpenRecord,
+	LoadingRecord,
+	LoadedRecord,
+	StoppingRecord,
+	StoppedRecord,
+])
+export type SessionRecord = typeof SessionRecord.Type
+export const childOf = (
+	record: SessionRecord,
+): Option.Option<ProcessIdentity> => {
+	switch (record.phase) {
+		case "Open":
+			return Option.none()
+		case "Loading":
+			return record.launch._tag === "Registered"
+				? Option.some(record.launch.child)
+				: Option.none()
+		case "Loaded":
+			return Option.some(record.child)
+		case "Stopping":
+		case "Stopped":
+			return childOf(record.previous)
+	}
+}
+export const stoppingRecord = (
+	previous: ActiveRecord,
+	reason: StopReason,
+): typeof StoppingRecord.Type => ({
+	version: previous.version,
+	handle: previous.handle,
+	requestId: previous.requestId,
+	leaseMillis: previous.leaseMillis,
+	limits: previous.limits,
+	phase: "Stopping",
+	previous,
+	reason,
+})
+export const stoppedRecord = (
+	previous: ActiveRecord,
+	reason: StopReason,
+	termination: Termination,
+): typeof StoppedRecord.Type => ({
+	...stoppingRecord(previous, reason),
+	phase: "Stopped",
+	termination,
+})
+
+export class Store extends Context.Service<
+	Store,
+	{
+		readonly list: Effect.Effect<
+			ReadonlyArray<SessionRecord>,
+			StateStorageError
+		>
+		readonly save: (
+			record: SessionRecord,
+		) => Effect.Effect<void, StateStorageError>
+	}
+>()("@graftland/plugin-remote/internal/runner/model/Store") {}
+export type ExecutionFailure =
+	ExecutionError | PackageError | TerminationUnconfirmed
+export interface Unit {
+	readonly code: CodeIdentity
+	/** The supervisor persists identity and renews the lease before releasing the child startup gate. */
+	readonly start: Effect.Effect<void, ExecutionFailure>
+	readonly identity: ProcessIdentity
+	readonly exited: Effect.Effect<ExitStatus, ExecutionFailure>
+	readonly interrupt: Effect.Effect<void, ExecutionFailure>
+	readonly kill: Effect.Effect<void, ExecutionFailure>
+	readonly renew: (millis: number) => Effect.Effect<void, ExecutionFailure>
+}
+export class Execution extends Context.Service<
+	Execution,
+	{
+		readonly spawn: (
+			handle: Handle,
+			request: Load,
+			limits: Limits,
+		) => Effect.Effect<Unit, ExecutionFailure, Scope.Scope>
+		readonly recover: (
+			child: ProcessIdentity,
+		) => Effect.Effect<Termination, ExecutionFailure>
+	}
+>()("@graftland/plugin-remote/internal/runner/model/Execution") {}
+export class RemoteTerminationUnconfirmed extends Schema.TaggedError<
+	RemoteTerminationUnconfirmed,
+	{ readonly brand: unique symbol }
+>()("RemoteTerminationUnconfirmed", { handle: Handle }) {}
+/** Losing contact never becomes a successful stop merely because a lease duration passed. */
+export const awaitTermination = <E, R>(
+	handle: Handle,
+	confirmation: Effect.Effect<Termination, E, R>,
+	leaseMillis: number,
+	marginMillis: number,
+) =>
+	Effect.gen(function* () {
+		const [elapsed, result] = yield* Effect.exit(
+			Effect.timeoutOption(confirmation, leaseMillis + marginMillis),
+		).pipe(Effect.timed)
+		if (result._tag === "Success") {
+			if (Option.isSome(result.value)) return result.value.value
+			return yield* RemoteTerminationUnconfirmed.make({ handle })
+		}
+		// Confirmation and the disconnected wait share one deadline.
+		yield* Effect.sleep(
+			Math.max(0, leaseMillis + marginMillis - Duration.toMillis(elapsed)),
+		)
+		return yield* Effect.failCause(
+			Cause.combine(
+				result.cause,
+				Cause.fail(RemoteTerminationUnconfirmed.make({ handle })),
+			),
+		)
+	})