@graftland/plugin-remote 0.0.0-stage → 0.3.0

package/src/internal/serve.ts

@@ -0,0 +1,117 @@
+import { findSubtree, isSubtreeActive } from "./forkTree"
+import { Graft, Plugin, type Fork } from "@graftland/graft"
+import {
+	ConfigDirectory,
+	Include,
+	Loader,
+	ModuleResolver,
+} from "@graftland/loader"
+import {
+	Array,
+	Cause,
+	Deferred,
+	Effect,
+	Exit,
+	Logger,
+	Option,
+	Path,
+	Stream,
+} from "effect"
+import * as Importer from "./importer"
+
+import { ServeLifecycle, StartupError, make } from "./ServeLifecycle"
+import { cleanupCause } from "./cleanup"
+export { StartupError } from "./ServeLifecycle"
+
+const check = (
+	trees: ReadonlyArray<Fork.ForkTree>,
+): Effect.Effect<void, StartupError> =>
+	Effect.forEach(
+		trees,
+		(tree) => {
+			if (tree.state._tag === "Failed")
+				return Effect.fail(
+					StartupError.make({
+						detail: `Runner entry ${tree.name} failed`,
+						cause: tree.state.cause,
+					}),
+				)
+			return check(tree.children)
+		},
+		{ discard: true },
+	)
+const ready = Plugin.make({
+	name: "remote-ready",
+	inject: [Loader.Loader],
+	apply: Effect.fnUntraced(function* () {
+		const loader = yield* Loader.Loader
+		// A failed authenticator can leave its listener child Pending while the outer plugin is Applying.
+		yield* loader.ready.pipe(
+			Effect.raceFirst(
+				Graft.forks.pipe(
+					Stream.map((trees) => findSubtree(trees, loader.root.id)),
+					Stream.filter(Option.isSome),
+					Stream.runForEach((tree) => check([tree.value])),
+				),
+			),
+		)
+		// Pending can still be reacting to a provider published during startup.
+		// Observe the loader's subtree, excluding this startup observer itself.
+		yield* Graft.forks.pipe(
+			Stream.map((trees) => findSubtree(trees, loader.root.id)),
+			Stream.filter(Option.isSome),
+			Stream.map((tree) => tree.value),
+			Stream.tap((tree) => check([tree])),
+			Stream.filter(isSubtreeActive),
+			Stream.runHead,
+		)
+	}),
+})
+
+/** Loader logs child cleanup errors; the supervisor's release signal also reaches the command exit. */
+const close = (entries: Fork.Fork, lifecycle: typeof ServeLifecycle.Service) =>
+	Effect.gen(function* () {
+		const disposed = yield* Effect.exit(entries.dispose)
+		const released = yield* Effect.exit(Deferred.await(lifecycle.released))
+		const cause = Array.reduce(
+			[disposed, released],
+			Cause.empty,
+			(cause: Cause.Cause<unknown>, exit: Exit.Exit<void, unknown>) =>
+				Exit.isFailure(exit) ? Cause.combine(cause, exit.cause) : cause,
+		)
+		if (cause.reasons.length > 0)
+			return yield* Effect.failCause(cleanupCause(cause))
+	})
+
+const load = Effect.fnUntraced(function* (config: string, directory: string) {
+	const lifecycle = yield* make
+	yield* Graft.provide(ServeLifecycle, lifecycle)
+	const importer = yield* Importer.make(directory)
+	const resolver = yield* ModuleResolver.make({ base: directory, importer })
+	yield* Graft.provide(ModuleResolver.ModuleResolver, resolver)
+	const entries = yield* Graft.plugin(Include.Include, {
+		path: config,
+		watch: false,
+	})
+	yield* entries.await
+	const startup = yield* Graft.plugin(ready)
+	yield* startup.await
+	return yield* Deferred.await(lifecycle.ended).pipe(
+		Effect.ensuring(close(entries, lifecycle)),
+	)
+})
+export const run = Effect.fnUntraced(function* (config: string) {
+	const path = yield* Path.Path
+	const file = path.resolve(config)
+	const directory = path.dirname(file)
+	return yield* load(file, directory).pipe(
+		Effect.mapError((error) =>
+			StartupError.make({
+				detail: "Starting the runner failed",
+				cause: Cause.fail(error),
+			}),
+		),
+		Effect.provideService(ConfigDirectory, Option.some(directory)),
+		Effect.provideService(Logger.LogToStderr, true),
+	)
+})

package/src/internal/services.ts

@@ -0,0 +1,74 @@
+import { Array, Context, Effect, Record } from "effect"
+import { Graft } from "@graftland/graft"
+import { RpcClient, RpcServer } from "effect/rpc"
+import type { Scope } from "effect"
+import type { Rpc, RpcClientError, RpcGroup } from "effect/rpc"
+import type * as Port from "./port"
+
+type Peer = Port.Port
+
+/** RPC types stay inside the descriptor; Graft supplies its dynamic activation context. */
+export interface Service<R = never, ClientR = never> {
+	readonly key: Context.Key<unknown, unknown>
+	readonly group: Pick<RpcGroup.RpcGroup<Rpc.Any>, "requests">
+	readonly serve: (
+		port: Pick<Peer, "serverProtocol">,
+	) => Effect.Effect<never, never, R>
+	readonly publish: (
+		port: Pick<Peer, "clientProtocol">,
+	) => Effect.Effect<
+		void,
+		Graft.ServiceConflict,
+		Graft.Graft | Scope.Scope | ClientR
+	>
+}
+export const service = <
+	I,
+	S,
+	R extends Rpc.Any,
+	H extends RpcGroup.HandlersFrom<R>,
+>(
+	key: Context.Key<I, S>,
+	group: RpcGroup.RpcGroup<R>,
+	mapping: {
+		readonly toHandlers: (value: S) => H
+		readonly fromClient: (
+			client: RpcClient.RpcClient<R, RpcClientError.RpcClientError>,
+		) => S
+	},
+): Service<
+	RpcGroup.HandlersServices<R, H> | Rpc.Middleware<R> | Rpc.ServicesServer<R>,
+	Rpc.MiddlewareClient<R>
+> => ({
+	key,
+	group,
+	serve: Effect.fnUntraced(function* (port) {
+		// Graft admits the activation from this descriptor's key before serving it.
+		const context = yield* Effect.context<never>()
+		const value = yield* Effect.fromOption(
+			Context.getOption(context, key),
+		).pipe(Effect.orDie)
+		const handlers = yield* group.toHandlers(mapping.toHandlers(value))
+		return yield* RpcServer.make(group, { disableFatalDefects: true }).pipe(
+			Effect.provideContext(handlers),
+			Effect.provideService(RpcServer.Protocol, port.serverProtocol(key.key)),
+		)
+	}),
+	publish: Effect.fnUntraced(function* (port) {
+		const client = yield* RpcClient.make(group).pipe(
+			Effect.provideService(RpcClient.Protocol, port.clientProtocol(key.key)),
+		)
+		yield* Graft.provide(key, mapping.fromClient(client))
+	}),
+})
+export const endpoints = (services: ReadonlyArray<Service>) =>
+	Record.fromEntries(
+		Array.map(
+			services,
+			(service) =>
+				[service.key.key, service.group] satisfies readonly [
+					string,
+					Service["group"],
+				],
+		),
+	)

package/src/internal/socketChannel.ts

@@ -0,0 +1,64 @@
+import { Cause, Effect, Exit, Sink, Stream } from "effect"
+import { Socket } from "effect/socket"
+import { TransportError } from "../Errors"
+import { FrameError, isCompleteFrame, decoder, encode } from "./runner/Framing"
+
+/** WebSocket boundaries are a wire compatibility detail, not Channel chunk boundaries. */
+export const fromSocket = Effect.fnUntraced(function* (socket: Socket.Socket) {
+	const reader = yield* socket.reader
+	const writer = yield* socket.writer
+	return {
+		input: Stream.fromEffectRepeat(reader.pull).pipe(
+			Stream.catchTag("SocketError", (error) =>
+				error.reason._tag === "SocketCloseError" && error.reason.code === 1000
+					? Stream.empty
+					: Stream.fail(error),
+			),
+			Stream.flattenIterable,
+			Stream.mapEffect((bytes) => {
+				if (typeof bytes === "string")
+					return Effect.fail(
+						FrameError.make({
+							detail: "Runner WebSocket messages must be binary",
+						}),
+					)
+				// WebSocket messages preserve one complete framing frame, including its header.
+				if (!isCompleteFrame(bytes))
+					return Effect.fail(
+						FrameError.make({
+							detail: "Invalid runner frame header or payload length",
+						}),
+					)
+				return Effect.succeed(bytes)
+			}),
+			Stream.mapError((error) =>
+				TransportError.make({ cause: Cause.fail(error) }),
+			),
+		),
+		output: Sink.suspend(() => {
+			const frames = decoder()
+			return Sink.forEach((bytes: Uint8Array) =>
+				Effect.fromResult(frames.decode(bytes)).pipe(
+					Effect.flatMap((values) =>
+						Effect.forEach(
+							values,
+							(frame) => encode(frame).pipe(Effect.flatMap(writer.write)),
+							{ discard: true },
+						),
+					),
+				),
+			).pipe(
+				Sink.onExit((exit) =>
+					Exit.isSuccess(exit)
+						? frames.finish.pipe(
+								Effect.andThen(writer.write(new Socket.CloseEvent(1000))),
+							)
+						: Effect.void,
+				),
+				Sink.mapError((error) =>
+					TransportError.make({ cause: Cause.fail(error) }),
+				),
+			)
+		}),
+	}
+})

package/src/internal/stdioChannel.ts

@@ -0,0 +1,25 @@
+import { NodeSink } from "@effect/platform-node"
+import { Cause, Effect, Stdio, Stream } from "effect"
+// @effect-diagnostics-next-line nodeBuiltinImport:off -- Node's extra inherited descriptor carries protocol output separately from console output.
+import { createWriteStream } from "node:fs"
+import { TransportError } from "../Errors"
+import type * as Port from "./port"
+
+export const bytes = Effect.gen(function* () {
+	const stdio = yield* Stdio.Stdio
+	const output = yield* Effect.sync(() =>
+		createWriteStream("", { fd: 3, autoClose: false }),
+	)
+	return {
+		input: stdio.stdin.pipe(
+			Stream.mapError((error) =>
+				TransportError.make({ cause: Cause.fail(error) }),
+			),
+		),
+		output: NodeSink.fromWritable({
+			evaluate: () => output,
+			onError: (cause) => TransportError.make({ cause: Cause.die(cause) }),
+			endOnDone: false,
+		}),
+	} satisfies Port.Channel
+})

package/src/internal/supervisorPlugin.ts

@@ -0,0 +1,67 @@
+import { Graft, Plugin } from "@graftland/graft"
+import {
+	Context,
+	Effect,
+	FileSystem,
+	Layer,
+	Option,
+	Path,
+	Schema,
+	Scope,
+} from "effect"
+import { Digest, SecureRandom } from "effect/crypto"
+import { ChildProcessSpawner } from "effect/process"
+import { Supervisor } from "./Supervisor"
+import { PositiveDuration } from "./config"
+import { ConfigDirectory } from "@graftland/loader"
+import * as Lock from "./runner/Lock"
+import * as Stores from "./runner/Store"
+import * as Execution from "./runner/Execution"
+import * as Engine from "./runner/Supervisor"
+import { accept } from "./runner/accept"
+import * as Lifecycle from "./ServeLifecycle"
+
+export const Config = Schema.Struct({
+	root: Schema.String,
+	state: Schema.String,
+	lease: Schema.optionalKey(PositiveDuration),
+	grace: Schema.optionalKey(PositiveDuration),
+})
+export default Plugin.make({
+	name: "remote-supervisor",
+	config: Config,
+	inject: [
+		Plugin.optional(Lifecycle.ServeLifecycle),
+		FileSystem.FileSystem,
+		Path.Path,
+		ChildProcessSpawner.ChildProcessSpawner,
+		SecureRandom.SecureRandom,
+		Digest.Digest,
+	],
+	apply: Effect.fnUntraced(function* (options) {
+		const lifecycle = yield* Effect.serviceOption(Lifecycle.ServeLifecycle)
+		const resources = yield* Lifecycle.resources(lifecycle)
+		const path = yield* Path.Path
+		const directory = yield* ConfigDirectory.pipe(
+			Effect.map(Option.getOrElse(() => ".")),
+		)
+		const state = path.resolve(directory, options.state)
+		yield* Lock.acquire(state).pipe(Scope.provide(resources))
+		const services = yield* Layer.build(
+			Engine.layer(options).pipe(
+				Layer.provideMerge(
+					Layer.merge(
+						Stores.layerDirectory(state),
+						Execution.layer(path.resolve(directory, options.root)),
+					),
+				),
+			),
+		).pipe(Scope.provide(resources))
+		const engine = Context.get(services, Engine.Supervisor)
+		const execution = Context.get(services, Execution.Channels)
+		yield* Plugin.fork(engine.await)
+		yield* Graft.provide(Supervisor, {
+			accept: (channel) => accept(engine, execution, channel),
+		})
+	}),
+})

package/src/internal/transportOptions.ts

@@ -0,0 +1,17 @@
+import type { Duration } from "effect"
+export interface WebsocketOptions {
+	readonly url: string
+	readonly tls?: {
+		readonly ca?: string
+		readonly servername?: string
+	}
+	readonly openTimeout?: Duration.Input
+}
+export interface UnixOptions {
+	readonly path: string
+}
+export interface CommandOptions {
+	readonly argv: readonly [string, ...string[]]
+	readonly cwd?: string
+	readonly env?: Readonly<Record<string, string>>
+}

package/src/internal/watchdogMain.ts

@@ -0,0 +1,72 @@
+/** Executable worker: fd 4 and the lease timer remain independent of the plugin thread. */
+import { NodeRuntime, NodeStream } from "@effect/platform-node"
+import { Effect, FiberHandle, Ref, Schema, Stream } from "effect"
+import { Socket } from "node:net"
+import { parentPort } from "node:worker_threads"
+import process from "node:process"
+import { CommandJson, Notice } from "./runner/WatchdogProtocol"
+
+// The runner machine is trusted. A suspended machine cannot enforce a deadline until it resumes.
+// Killing this live process has no PID lookup; external supervisor kills retain a small PID reuse window.
+const die = Effect.sync(() => process.kill(process.pid, "SIGKILL")).pipe(
+	Effect.andThen(Effect.never),
+)
+const notify = (notice: typeof Notice.Type) =>
+	Schema.encodeEffect(Notice)(notice).pipe(
+		Effect.flatMap((message) =>
+			Effect.sync(() => parentPort?.postMessage(message)),
+		),
+	)
+const now = Effect.sync(() => performance.now())
+const expire = (deadline: number): Effect.Effect<never> =>
+	Effect.gen(function* () {
+		const remaining = deadline - (yield* now)
+		if (remaining <= 0) return yield* die
+		yield* Effect.sleep(Math.min(remaining, 2147483647))
+		return yield* expire(deadline)
+	})
+const main = Effect.scoped(
+	Effect.gen(function* () {
+		const lease = yield* Ref.make({
+			deadline: (yield* now) + 30000,
+			stopped: false,
+		})
+		const timer = yield* FiberHandle.make<never, never>()
+		yield* FiberHandle.run(timer, expire((yield* Ref.get(lease)).deadline))
+		const input = yield* Effect.acquireRelease(
+			Effect.sync(() => new Socket({ fd: 4, readable: true, writable: false })),
+			(socket) =>
+				Effect.sync(() => {
+					socket.destroy()
+				}),
+		)
+		yield* notify("ready")
+		yield* NodeStream.fromReadable({ evaluate: () => input }).pipe(
+			Stream.decodeText(),
+			Stream.splitLines,
+			Stream.mapEffect((line) => Schema.decodeEffect(CommandJson)(line)),
+			Stream.runForEach((command) =>
+				Effect.gen(function* () {
+					if (command.kind === "start") return yield* notify("start")
+					if (command.kind === "stop") {
+						yield* Ref.update(lease, (state) => ({ ...state, stopped: true }))
+						return yield* notify("stop")
+					}
+					const current = yield* Ref.get(lease)
+					const time = yield* now
+					if (current.stopped || time >= current.deadline) return yield* die
+					const deadline = time + command.millis
+					yield* Ref.set(lease, { deadline, stopped: false })
+					yield* FiberHandle.run(timer, expire(deadline))
+				}),
+			),
+		)
+		return yield* die
+	}),
+).pipe(
+	Effect.catchCause((cause) =>
+		Effect.logError("Watchdog failed", cause).pipe(Effect.andThen(die)),
+	),
+)
+
+NodeRuntime.runMain(main)

package/src/internal/websocket.ts

@@ -0,0 +1,81 @@
+import { Array, Effect, Layer, Redacted, Schema } from "effect"
+import { Socket } from "effect/socket"
+import WS from "ws"
+import type { ClientCertificate } from "../Auth"
+import type { Properties } from "../Transport"
+interface PreparedCredentials {
+	readonly certificate?: ClientCertificate
+	readonly authorization?: Redacted.Redacted<string>
+}
+import type { WebsocketOptions } from "./transportOptions"
+import { headerSize, maxPayload } from "./runner/Framing"
+
+export const protocol = "graft-remote.v1"
+export const path = "/graft"
+export const payloadLimit = maxPayload + headerSize
+export const Url = Schema.URLFromString.check(
+	Schema.makeFilter(
+		(url) => url.protocol === "wss:" || url.protocol === "ws:",
+		{
+			expected: "a ws:// or wss:// URL",
+		},
+	),
+)
+
+export const constructor = (
+	tls: WebsocketOptions["tls"],
+	credentials: PreparedCredentials = {},
+) =>
+	Layer.succeed(Socket.WebSocketConstructor, (url, options) => {
+		const settings =
+			typeof options === "object" && !Array.isArray(options)
+				? options
+				: { protocols: options }
+		return new WS(url, settings.protocols, {
+			...tls,
+			...(credentials.certificate === undefined
+				? {}
+				: {
+						cert: credentials.certificate.cert,
+						key: Redacted.value(credentials.certificate.key),
+					}),
+			headers: {
+				...settings.headers,
+				...(credentials.authorization === undefined
+					? {}
+					: { authorization: Redacted.value(credentials.authorization) }),
+			},
+			perMessageDeflate: false,
+			maxPayload: payloadLimit,
+			rejectUnauthorized: true,
+			minVersion: "TLSv1.3",
+		})
+	})
+
+export const open = Effect.fnUntraced(function* (
+	options: WebsocketOptions,
+	credentials: PreparedCredentials = {},
+) {
+	const url = yield* Schema.decodeEffect(Url)(options.url)
+	if (credentials.certificate !== undefined && url.protocol !== "wss:")
+		yield* Schema.decodeEffect(
+			Url.check(
+				Schema.makeFilter((url) => url.protocol === "wss:", {
+					expected: "wss:// for mutual TLS",
+				}),
+			),
+		)(options.url)
+	const address = url.pathname === "/" ? new URL(path, url).href : url.href
+	const services = yield* Layer.build(constructor(options.tls, credentials))
+	return yield* Socket.makeWebSocket(address, {
+		protocols: protocol,
+		openTimeout: options.openTimeout ?? "10 seconds",
+	}).pipe(Effect.provideContext(services))
+})
+
+export const properties = (url: URL): Properties => ({
+	protocol: url.protocol === "wss:" ? "wss" : "ws",
+	encrypted: url.protocol === "wss:",
+	network: true,
+	carries: url.protocol === "wss:" ? ["header", "certificate"] : ["header"],
+})

package/src/internal/websocketServer.ts

@@ -0,0 +1,152 @@
+import { NodeHttpServer, NodeHttpServerRequest } from "@effect/platform-node"
+import {
+	Array,
+	Cause,
+	Deferred,
+	Effect,
+	Option,
+	Redacted,
+	Scope,
+	String,
+} from "effect"
+import { HttpServerRequest, HttpServerResponse } from "effect/http"
+import type { Socket } from "effect/socket"
+import { TLSSocket } from "node:tls"
+import { AuthenticationFailed, TransportError } from "../Errors"
+import type { Accept, Handshake, Serving } from "../Transport"
+import * as Channel from "./socketChannel"
+import { path, protocol, payloadLimit } from "./websocket"
+
+const rejection = (error: AuthenticationFailed) =>
+	Effect.logWarning(
+		"Remote host authentication failed",
+		{
+			reason: error.reason,
+		},
+		...(error.cause === undefined ? [] : [error.cause]),
+	).pipe(
+		Effect.as(
+			HttpServerResponse.empty({
+				status: 401,
+				headers:
+					error.challenge === undefined
+						? {}
+						: { "www-authenticate": error.challenge },
+			}),
+		),
+	)
+
+const handshake: Effect.Effect<
+	Handshake,
+	never,
+	HttpServerRequest.HttpServerRequest
+> = Effect.gen(function* () {
+	const request = yield* HttpServerRequest.HttpServerRequest
+	const { socket } = NodeHttpServerRequest.toIncomingMessage(request)
+	// Only a TLS-verified peer is certificate evidence; forwarded headers are not.
+	const peer =
+		socket instanceof TLSSocket && socket.authorized
+			? Option.some(socket.getPeerCertificate())
+			: Option.none()
+	return {
+		authorization: Option.map(
+			Option.fromUndefinedOr(request.headers["authorization"]),
+			Redacted.make,
+		),
+		certificate: Option.map(peer, (certificate) => ({
+			der: certificate.raw,
+			subjectAltName: Option.fromUndefinedOr(certificate.subjectaltname),
+		})),
+	}
+})
+
+const open = Effect.fnUntraced(function* (
+	createServer: Parameters<typeof NodeHttpServer.make>[0],
+	options: Pick<NodeHttpServer.Options, "host" | "port" | "path">,
+) {
+	// The lazily created ws server belongs here, so serving scopes close before ws.close waits.
+	const resources = yield* Effect.acquireRelease(Scope.make(), Scope.close)
+	const ready = yield* Deferred.make<void>()
+	const server = yield* NodeHttpServer.make(createServer, {
+		...options,
+		disablePreemptiveShutdown: true,
+		websocket: {
+			perMessageDeflate: false,
+			maxPayload: payloadLimit,
+			handleProtocols: (protocols) =>
+				protocols.has(protocol) ? protocol : false,
+		},
+	}).pipe(Scope.provide(resources))
+	return {
+		address: server.address,
+		ready: Deferred.await(ready),
+		run: <A, E, R>(
+			accept: Effect.Effect<
+				(socket: Socket.Socket) => Effect.Effect<A, E, R>,
+				AuthenticationFailed,
+				HttpServerRequest.HttpServerRequest
+			>,
+		) =>
+			server
+				.serve(
+					Effect.gen(function* () {
+						const request = yield* HttpServerRequest.HttpServerRequest
+						if (request.url !== path)
+							return HttpServerResponse.empty({ status: 404 })
+						if (
+							String.toLowerCase(request.headers["upgrade"] ?? "") !==
+								"websocket" ||
+							!Array.some(
+								String.split(
+									request.headers["sec-websocket-protocol"] ?? "",
+									",",
+								),
+								(value) => String.trim(value) === protocol,
+							)
+						)
+							return HttpServerResponse.empty({ status: 400 })
+						const session = yield* Effect.result(accept)
+						if (session._tag === "Failure")
+							return yield* rejection(session.failure)
+						// Authentication finishes before upgrade; session work is owned by the request scope.
+						yield* request.upgrade.pipe(
+							Effect.flatMap(session.success),
+							Effect.interruptible,
+							Effect.scoped,
+						)
+						return HttpServerResponse.empty()
+					}),
+				)
+				.pipe(
+					Effect.andThen(Deferred.succeed(ready, undefined)),
+					Effect.andThen(Effect.never),
+				),
+	}
+})
+
+export const bind = Effect.fnUntraced(function* (
+	createServer: Parameters<typeof NodeHttpServer.make>[0],
+	options: Pick<NodeHttpServer.Options, "host" | "port" | "path">,
+) {
+	const server = yield* open(createServer, options)
+	return {
+		address: server.address,
+		ready: server.ready,
+		serve: (accept: Accept) =>
+			server
+				.run(
+					handshake.pipe(
+						Effect.flatMap(accept),
+						Effect.map(
+							(session) => (socket: Socket.Socket) =>
+								Channel.fromSocket(socket).pipe(Effect.flatMap(session)),
+						),
+					),
+				)
+				.pipe(
+					Effect.mapError((error) =>
+						TransportError.make({ cause: Cause.fail(error) }),
+					),
+				),
+	} satisfies Serving & { readonly address: typeof server.address }
+})

package/src/internal/workerBootstrap.ts

@@ -0,0 +1,6 @@
+import { Schema } from "effect"
+
+export const Bootstrap = Schema.Struct({
+	contractModule: Schema.String,
+	session: Schema.String,
+})