@graftland/plugin-remote 0.0.0-stage → 0.3.0

package/src/bin.ts

@@ -0,0 +1,145 @@
+#!/usr/bin/env node
+/** Runs local children and WebSocket supervisors, and issues their deployment certificates. */
+import { Graft } from "@graftland/graft"
+import { NodeRuntime } from "@effect/platform-node"
+import { Console, Array, Effect, Layer, Logger, Option, Schema } from "effect"
+import { Argument, CliError, Command, Flag } from "effect/cli"
+import process from "node:process"
+import { pathToFileURL } from "node:url"
+import { platform } from "./internal/platform"
+import { bytes } from "./internal/stdioChannel"
+import { load, run } from "./internal/childRuntime"
+import { Bootstrap } from "./internal/runner/Bootstrap"
+import * as Watchdog from "./internal/runner/Watchdog"
+import * as Serve from "./internal/serve"
+import * as Certificates from "./internal/Certificates"
+
+const serve = Command.make(
+	"serve",
+	{ config: Flag.String("config") },
+	({ config }) => Serve.run(config),
+)
+const certificates = Command.make(
+	"certificates",
+	{
+		out: Flag.String("out").pipe(
+			Flag.withDescription("Directory for the CA and issued certificates"),
+		),
+		server: Flag.String("server").pipe(
+			Flag.atLeast(0),
+			Flag.withDescription(
+				"Runner DNS name or IP address; repeat for additional SANs",
+			),
+		),
+		identity: Flag.String("identity").pipe(
+			Flag.atLeast(0),
+			Flag.withDescription(
+				"Host identity; repeat to issue multiple client certificates",
+			),
+		),
+		days: Flag.Int("days").pipe(
+			Flag.withDefault(365),
+			Flag.filter(
+				(days) => days > 0,
+				() => "Days must be a positive integer",
+			),
+		),
+		force: Flag.Boolean("force").pipe(
+			Flag.withDefault(false),
+			Flag.withDescription(
+				"Replace requested leaf certificates, keeping the existing CA",
+			),
+		),
+	},
+	(options) =>
+		Certificates.generate(options).pipe(
+			Effect.andThen(Console.log(`Certificates written to ${options.out}`)),
+			Effect.catchTag("CertificateExists", (error) =>
+				CliError.UserError.make({ cause: error, userMessage: error.message }),
+			),
+		),
+)
+const child = Command.make(
+	"child",
+	{
+		contract: Argument.String("contract"),
+		session: Argument.String("session"),
+		bootstrap: Argument.String("bootstrap").pipe(
+			Argument.withSchema(Schema.fromJsonString(Bootstrap)),
+		),
+	},
+	({ contract, session, bootstrap }) =>
+		Effect.gen(function* () {
+			const watchdog = yield* Watchdog.start
+			// The executable owns stdin; it must not retain the process after a supervisor stop.
+			yield* Effect.addFinalizer(() =>
+				Effect.sync(() => {
+					process.stdin.destroy()
+				}),
+			)
+			const begin = yield* Effect.raceFirst(
+				watchdog.begin.pipe(Effect.as(true)),
+				watchdog.stop.pipe(Effect.as(false)),
+			)
+			if (!begin) return
+			const definition = yield* load(pathToFileURL(contract).href)
+			const channel = yield* bytes
+			return yield* run(definition, channel, session, bootstrap, watchdog.stop)
+		}),
+)
+const cli = Command.make(
+	"graft-remote",
+	{
+		contract: Argument.String("contract").pipe(Argument.optional),
+		session: Argument.String("session").pipe(Argument.withDefault("local")),
+	},
+	({ contract, session }) =>
+		Effect.gen(function* () {
+			if (Option.isNone(contract))
+				return yield* CliError.UserError.make({
+					cause: "Missing contract module",
+					userMessage:
+						"A contract module or the child/serve/certificates subcommand is required",
+				})
+			const definition = yield* load(contract.value)
+			const channel = yield* bytes
+			return yield* run(definition, channel, session)
+		}),
+).pipe(Command.withSubcommands([child, serve, certificates]))
+
+NodeRuntime.runMain(
+	Effect.scoped(
+		Effect.gen(function* () {
+			const context = yield* Layer.build(
+				Layer.provideMerge(Graft.layer, platform),
+			)
+			return yield* Command.run(cli, { version: "0.0.0" }).pipe(
+				Effect.provideContext(context),
+			)
+		}),
+	).pipe(
+		(effect) =>
+			Effect.withFiber((main) =>
+				effect.pipe(
+					Effect.tapCause((cause) =>
+						// NodeRuntime uses this fiber's id; a self-interrupt here is indistinguishable, but worker cleanup is not.
+						Array.every(
+							cause.reasons,
+							(reason) =>
+								reason._tag === "Interrupt" && reason.fiberId === main.id,
+						) ||
+						Array.every(
+							cause.reasons,
+							(reason) =>
+								reason._tag === "Fail" && CliError.isCliError(reason.error),
+						)
+							? Effect.void
+							: Effect.logError(cause),
+					),
+				),
+			),
+		Effect.provideService(Logger.LogToStderr, true),
+	),
+	// Report inside the stderr context, including failures from resource cleanup.
+	{ disableErrorReporting: true },
+)

package/src/host/auth/bearer.ts

@@ -0,0 +1,10 @@
+import type { Graft, Plugin } from "@graftland/graft"
+import { Config, plugin } from "../../internal/auth/tokenCredentials"
+export { Config }
+
+const capability: Plugin.Plugin<
+	typeof Config.Type,
+	typeof Config.Encoded,
+	Graft.ServiceConflict
+> = plugin("remote-bearer-credentials")
+export default capability

package/src/host/auth/jwt.ts

@@ -0,0 +1,10 @@
+import type { Graft, Plugin } from "@graftland/graft"
+import { Config, plugin } from "../../internal/auth/tokenCredentials"
+export { Config }
+
+const capability: Plugin.Plugin<
+	typeof Config.Type,
+	typeof Config.Encoded,
+	Graft.ServiceConflict
+> = plugin("remote-jwt-credentials")
+export default capability

package/src/host/auth/mtls.ts

@@ -0,0 +1,48 @@
+import { Plugin, Graft } from "@graftland/graft"
+import {
+	Cause,
+	Effect,
+	FileSystem,
+	Option,
+	Path,
+	Redacted,
+	Schema,
+} from "effect"
+import { CredentialsError } from "../../Errors"
+import { ConfigDirectory } from "@graftland/loader"
+import { Credentials, type Prepared } from "../../Auth"
+
+export const Config = Schema.Struct({ cert: Schema.String, key: Schema.String })
+export default Plugin.make({
+	name: "remote-certificate-credentials",
+	config: Config,
+	inject: [FileSystem.FileSystem, Path.Path],
+	apply: Effect.fnUntraced(function* (config) {
+		const fs = yield* FileSystem.FileSystem
+		const path = yield* Path.Path
+		const directory = yield* ConfigDirectory.pipe(
+			Effect.map(Option.getOrElse(() => ".")),
+		)
+		yield* Graft.provide(Credentials, {
+			kind: "Certificate",
+			// File credentials are read for each connection, not cached on activation.
+			read: Effect.all({
+				cert: fs.readFileString(path.resolve(directory, config.cert)),
+				key: fs
+					.readFileString(path.resolve(directory, config.key))
+					.pipe(Effect.map(Redacted.make)),
+			}).pipe(
+				Effect.mapBoth({
+					onSuccess: (
+						value,
+					): Extract<Prepared, { readonly _tag: "Certificate" }> => ({
+						_tag: "Certificate",
+						value,
+					}),
+					onFailure: (error) =>
+						CredentialsError.make({ cause: Cause.fail(error) }),
+				}),
+			),
+		})
+	}),
+})

package/src/host/auth/none.ts

@@ -0,0 +1,13 @@
+import { Plugin, Graft } from "@graftland/graft"
+import { Effect, Schema } from "effect"
+import { Credentials } from "../../Auth"
+
+export default Plugin.make({
+	name: "remote-none-credentials",
+	config: Schema.Struct({}),
+	apply: () =>
+		Graft.provide(Credentials, {
+			kind: "None",
+			read: Effect.succeed<{ readonly _tag: "None" }>({ _tag: "None" }),
+		}),
+})

package/src/host/command.ts

@@ -0,0 +1,131 @@
+import type { CommandOptions } from "../internal/transportOptions"
+import { Plugin, Graft } from "@graftland/graft"
+import {
+	Array,
+	Cause,
+	Effect,
+	Option,
+	Path,
+	Schema,
+	Scope,
+	Sink,
+	Stream,
+} from "effect"
+import { ChildProcess, ChildProcessSpawner } from "effect/process"
+import { Connector } from "../Transport"
+import { ConfigDirectory } from "@graftland/loader"
+import { TransportError } from "../Errors"
+import { cleanupCause } from "../internal/cleanup"
+import * as ProcessObservation from "../internal/runner/ProcessObservation"
+
+export class CommandExited extends Schema.TaggedError<
+	CommandExited,
+	{ readonly brand: unique symbol }
+>()("CommandExited", {
+	code: Schema.NullOr(Schema.Int),
+	stderr: Schema.String,
+	cause: Schema.Cause(Schema.Unknown, Schema.Unknown),
+}) {}
+
+/** The launcher exit describes this channel, never the remote execution's termination. */
+export const make = (options: CommandOptions) =>
+	Effect.map(ChildProcessSpawner.ChildProcessSpawner, (spawner) =>
+		Connector.of({
+			properties: {
+				protocol: "command",
+				encrypted: false,
+				network: false,
+				carries: [],
+			},
+			connect: Effect.fnUntraced(
+				function* () {
+					const scope = yield* Effect.scope
+					const readers = yield* Scope.fork(scope)
+					const child = yield* spawner.spawn(
+						ChildProcess.make(options.argv[0], Array.drop(options.argv, 1), {
+							...options,
+							stdin: "pipe",
+							stdout: "pipe",
+							stderr: "pipe",
+							forceKillAfter: "1 second",
+						}),
+					)
+					const observation = yield* ProcessObservation.observe(child, {
+						drainStdout: false,
+					}).pipe(Scope.provide(readers))
+					const exited = observation.exited.pipe(
+						Effect.map((status) => CommandExited.make(status)),
+					)
+					yield* Effect.addFinalizer(() =>
+						Stream.empty.pipe(
+							Stream.run(child.stdin),
+							Effect.andThen(exited),
+							Effect.flatMap((status) =>
+								status.code === 0 && status.cause.reasons.length === 0
+									? Effect.void
+									: Effect.fail(status),
+							),
+							Effect.interruptible,
+							Effect.timeout(ProcessObservation.stdinExitTimeout),
+							Effect.catchCause((cause) =>
+								Effect.failCause(cleanupCause(cause)),
+							),
+						),
+					)
+					return {
+						input: child.stdout.pipe(
+							Stream.concat(
+								Stream.fromEffect(
+									exited.pipe(
+										Effect.flatMap((status) =>
+											status.code === 0 && status.cause.reasons.length === 0
+												? Effect.void
+												: Effect.fail(status),
+										),
+									),
+								).pipe(Stream.drain),
+							),
+							Stream.interruptWhen(observation.failure),
+							Stream.mapError((error) =>
+								TransportError.make({ cause: Cause.fail(error) }),
+							),
+						),
+						output: child.stdin.pipe(
+							Sink.mapError((error) =>
+								TransportError.make({ cause: Cause.fail(error) }),
+							),
+						),
+					}
+				},
+				Effect.mapError((error) =>
+					TransportError.make({ cause: Cause.fail(error) }),
+				),
+			),
+		}),
+	)
+export const Config = Schema.Struct({
+	argv: Schema.NonEmptyArray(Schema.String),
+	cwd: Schema.optionalKey(Schema.String),
+	env: Schema.optionalKey(Schema.Record(Schema.String, Schema.String)),
+})
+export default Plugin.make({
+	name: "remote-command-connector",
+	config: Config,
+	inject: [ChildProcessSpawner.ChildProcessSpawner, Path.Path],
+	apply: Effect.fnUntraced(function* (config) {
+		const path = yield* Path.Path
+		const directory = yield* ConfigDirectory
+		const connector = yield* make({
+			...config,
+			...(config.cwd === undefined
+				? {}
+				: {
+						cwd: path.resolve(
+							Option.getOrElse(directory, () => "."),
+							config.cwd,
+						),
+					}),
+		})
+		yield* Graft.provide(Connector, connector)
+	}),
+})

package/src/host/local.ts

@@ -0,0 +1,41 @@
+import { Plugin, type Graft } from "@graftland/graft"
+import { Array, Effect, Scope } from "effect"
+import { ChildProcess, ChildProcessSpawner } from "effect/process"
+import { LocalTransport } from "../internal/directTransport"
+import { command } from "../internal/runner/command"
+import * as Local from "../internal/host/local"
+
+const make = Effect.map(ChildProcessSpawner.ChildProcessSpawner, (spawner) =>
+	LocalTransport.of({
+		start: Effect.fnUntraced(function* (options) {
+			const scope = yield* Effect.scope
+			// Readers must outlive the process finalizer to capture its final diagnostics.
+			const readers = yield* Scope.fork(scope)
+			const argv = yield* command(options.command)
+			const child = yield* spawner.spawn(
+				ChildProcess.make(
+					argv[0],
+					[...Array.drop(argv, 1), options.contractModule, options.session],
+					{
+						stdin: "pipe",
+						stdout: "pipe",
+						stderr: "pipe",
+						additionalFds: { fd3: { type: "output" } },
+						forceKillAfter: "1 second",
+					},
+				),
+			)
+			return yield* Local.observe(child).pipe(Scope.provide(readers))
+		}),
+	}),
+)
+const plugin: Plugin.Plugin<void, void, Graft.ServiceConflict> = Plugin.service(
+	LocalTransport,
+	make,
+	{
+		name: "remote-local-transport",
+		inject: [ChildProcessSpawner.ChildProcessSpawner],
+	},
+)
+
+export default plugin

package/src/host/unix.ts

@@ -0,0 +1,65 @@
+import type { UnixOptions } from "../internal/transportOptions"
+import { Plugin, Graft } from "@graftland/graft"
+import { ConfigDirectory } from "@graftland/loader"
+import { Cause, Effect, Layer, Option, Path, Redacted, Schema } from "effect"
+import { Socket } from "effect/socket"
+import WS from "ws"
+import type { Prepared } from "../Auth"
+import { Connector, type ConnectorApi } from "../Transport"
+import { TransportError } from "../Errors"
+import { path, protocol, payloadLimit } from "../internal/websocket"
+import * as Channel from "../internal/socketChannel"
+
+export const properties: ConnectorApi["properties"] = {
+	protocol: "unix",
+	encrypted: false,
+	network: false,
+	carries: ["header"],
+}
+const constructor = (credentials: Prepared) =>
+	Layer.succeed(
+		Socket.WebSocketConstructor,
+		(url) =>
+			new WS(url, protocol, {
+				headers:
+					credentials._tag === "Authorization"
+						? { authorization: Redacted.value(credentials.value) }
+						: {},
+				perMessageDeflate: false,
+				maxPayload: payloadLimit,
+			}),
+	)
+export const make = (options: UnixOptions): ConnectorApi => ({
+	properties,
+	connect: Effect.fnUntraced(
+		function* (credentials) {
+			const services = yield* Layer.build(constructor(credentials))
+			const socket = yield* Socket.makeWebSocket(
+				`ws+unix://${options.path}:${path}`,
+			).pipe(Effect.provideContext(services))
+			return yield* Channel.fromSocket(socket)
+		},
+		Effect.mapError((error) =>
+			TransportError.make({ cause: Cause.fail(error) }),
+		),
+	),
+})
+export const Config = Schema.Struct({ path: Schema.String })
+export default Plugin.make({
+	name: "remote-unix-connector",
+	config: Config,
+	inject: [Path.Path],
+	apply: Effect.fnUntraced(function* (config) {
+		const path = yield* Path.Path
+		const directory = yield* ConfigDirectory
+		yield* Graft.provide(
+			Connector,
+			make({
+				path: path.resolve(
+					Option.getOrElse(directory, () => "."),
+					config.path,
+				),
+			}),
+		)
+	}),
+})

package/src/host/websocket.ts

@@ -0,0 +1,66 @@
+import type { WebsocketOptions } from "../internal/transportOptions"
+import { Plugin, Graft } from "@graftland/graft"
+import { Cause, Effect, FileSystem, Option, Path, Schema } from "effect"
+import { Connector, type ConnectorApi } from "../Transport"
+import { TransportError } from "../Errors"
+import { ConfigDirectory } from "@graftland/loader"
+import * as Websocket from "../internal/websocket"
+import * as Channel from "../internal/socketChannel"
+
+/** Configured Connector; certificate paths belong to the enclosing configuration. */
+export const Config = Schema.Struct({
+	url: Websocket.Url,
+	ca: Schema.optionalKey(Schema.String),
+	servername: Schema.optionalKey(Schema.String),
+})
+export const make = (
+	options: WebsocketOptions,
+	properties: ConnectorApi["properties"],
+): ConnectorApi => ({
+	properties,
+	connect: (prepared) =>
+		Websocket.open(
+			options,
+			prepared._tag === "Certificate"
+				? { certificate: prepared.value }
+				: prepared._tag === "Authorization"
+					? { authorization: prepared.value }
+					: {},
+		).pipe(
+			Effect.flatMap(Channel.fromSocket),
+			Effect.mapError((error) =>
+				TransportError.make({ cause: Cause.fail(error) }),
+			),
+		),
+})
+export default Plugin.make({
+	name: "remote-websocket-connector",
+	config: Config,
+	inject: [FileSystem.FileSystem, Path.Path],
+	apply: Effect.fnUntraced(function* (config) {
+		const fs = yield* FileSystem.FileSystem
+		const path = yield* Path.Path
+		const directory = yield* ConfigDirectory.pipe(
+			Effect.map(Option.getOrElse(() => ".")),
+		)
+		const ca =
+			config.ca === undefined
+				? undefined
+				: yield* fs.readFileString(path.resolve(directory, config.ca))
+		yield* Graft.provide(
+			Connector,
+			make(
+				{
+					url: config.url.href,
+					tls: {
+						...(ca === undefined ? {} : { ca }),
+						...(config.servername === undefined
+							? {}
+							: { servername: config.servername }),
+					},
+				},
+				Websocket.properties(config.url),
+			),
+		)
+	}),
+})

package/src/host/worker.ts

@@ -0,0 +1,90 @@
+import { Plugin, type Graft } from "@graftland/graft"
+import { Cause, Deferred, Effect, FiberSet, Option, Schema } from "effect"
+import { Worker } from "node:worker_threads"
+import { WorkerTransport } from "../internal/directTransport"
+import { ProcessExited, TransportError } from "../Errors"
+import * as Channel from "../internal/workerChannel"
+import * as FailureGroup from "../internal/FailureGroup"
+import { Bootstrap } from "../internal/workerBootstrap"
+import { cleanupCause } from "../internal/cleanup"
+
+const start: typeof WorkerTransport.Service.start = Effect.fnUntraced(
+	function* (options) {
+		const data = yield* Schema.encodeEffect(Bootstrap)(options).pipe(
+			Effect.mapError((error) =>
+				TransportError.make({ cause: Cause.fail(error) }),
+			),
+		)
+		const exited = yield* Deferred.make<number>()
+		const failures = yield* FailureGroup.make<TransportError>()
+		const callbacks = yield* FiberSet.make()
+		const run = yield* FiberSet.runtime(callbacks)()
+		const onExit = (code: number) => run(Deferred.succeed(exited, code))
+		const onError = (error: Error) =>
+			run(failures.fail(TransportError.make({ cause: Cause.die(error) })))
+		const worker = yield* Effect.acquireRelease(
+			Effect.try({
+				try: () => {
+					const worker = new Worker(
+						new URL(
+							import.meta.resolve("@graftland/plugin-remote/internal/worker"),
+						),
+						{
+							workerData: data,
+							...(options.execArgv === undefined
+								? {}
+								: { execArgv: [...options.execArgv] }),
+						},
+					)
+					worker.on("exit", onExit)
+					worker.on("error", onError)
+					return worker
+				},
+				catch: (cause) => TransportError.make({ cause: Cause.die(cause) }),
+			}),
+			(worker) =>
+				Effect.tryPromise({
+					try: () => worker.terminate(),
+					catch: (cause) => TransportError.make({ cause: Cause.die(cause) }),
+				}).pipe(
+					Effect.andThen(Deferred.await(exited)),
+					Effect.ensuring(
+						Effect.sync(() => {
+							worker.off("exit", onExit)
+							worker.off("error", onError)
+						}),
+					),
+					Effect.catchCause((cause) => Effect.failCause(cleanupCause(cause))),
+				),
+		)
+		const threadId = worker.threadId
+		const channel = yield* Channel.make(worker)
+		return {
+			channel,
+			pid: Option.none(),
+			threadId,
+			failure: failures.await,
+			exited: Deferred.await(exited).pipe(
+				Effect.flatMap((exitCode) =>
+					failures.cause.pipe(
+						Effect.map((statusCause) =>
+							ProcessExited.make({
+								pid: Option.none(),
+								threadId,
+								exitCode,
+								stderr: "",
+								statusCause,
+							}),
+						),
+					),
+				),
+			),
+		}
+	},
+)
+const plugin: Plugin.Plugin<void, void, Graft.ServiceConflict> = Plugin.service(
+	WorkerTransport,
+	Effect.succeed({ start }),
+	{ name: "remote-thread-transport" },
+)
+export default plugin