@stringsync/splates 0.0.53 → 0.0.55

package/worker-relay.test.ts

@@ -1,233 +0,0 @@
-import {
-	afterEach,
-	beforeEach,
-	describe,
-	expect,
-	it,
-	setDefaultTimeout,
-} from "bun:test";
-import { chmod } from "node:fs/promises";
-import { basename, dirname, join } from "node:path";
-import { type DevServer, dev } from "@stringsync/splates-server";
-import { printed } from "@stringsync/splates-server/agents/testing";
-import { relayDocument } from "@stringsync/splates-server/records";
-import { FsStorage } from "@stringsync/splates-server/storage";
-import { Testing } from "@stringsync/splates-server/testing";
-import type { RemoteWorker } from "@stringsync/splates-server/workers";
-import {
-	DevWorkers,
-	UnixWorkerSocket,
-	WORKER_KEY,
-	WorkerDaemon,
-	workerCreds,
-} from "@stringsync/splates-server/workers";
-import { FakeSecretStore } from "webappwiz/creds/testing";
-import { color, MemoryLogger } from "webappwiz/log";
-import { NodePs, OpenPortProvider } from "webappwiz/system";
-import { Duration } from "webappwiz/time";
-
-const CLI = join(import.meta.dirname, "index.ts");
-
-// Real git, a real coordinator and worker, and a fresh CLI process per
-// command, as in the e2e tests.
-setDefaultTimeout(90_000);
-
-describe("WorkerRelay", () => {
-	let env: Testing;
-	let server: DevServer;
-	let workers: DevWorkers;
-	let daemon: WorkerDaemon;
-	/** The worker, as the coordinator holds it. */
-	let worker: RemoteWorker;
-	/** A worker on another machine: a home of its own, and a clone with no `.git/splates`. */
-	let home: string;
-	let clone: string;
-	let bin: string;
-	let relay: string;
-
-	beforeEach(async () => {
-		env = await Testing.open();
-		await env.commit(
-			env.root,
-			"splates.config.ts",
-			`export default { preMerge: "true", remote: "origin" };\n`,
-			"config",
-		);
-		env.config.remote = "origin";
-		const base = dirname(env.root);
-		// The central remote, which only the coordinator moves trunk on.
-		const remote = join(base, "remote.git");
-		await env.gitCli(base, "init", "--quiet", "--bare", "-b", "main", remote);
-		await env.gitCli(env.root, "remote", "add", "origin", remote);
-		await env.gitCli(env.root, "push", "--quiet", "origin", "main");
-		const socket = join(base, "worker.sock");
-		const log = new MemoryLogger();
-		workers = new DevWorkers(log);
-		server = await dev(env, {
-			ports: OpenPortProvider.any(),
-			workers,
-			workerSocket: socket,
-		});
-		home = join(base, "worker", "home");
-		clone = join(base, "worker", "repo");
-		await env.gitCli(base, "clone", "--quiet", remote, clone);
-		await env.gitCli(clone, "config", "user.email", "splates@example.com");
-		await env.gitCli(clone, "config", "user.name", "splates");
-		await env.gitCli(clone, "config", "commit.gpgsign", "false");
-		// `splates add` runs no install of its own here.
-		bin = join(base, "worker", "bin");
-		await env.fs.mkdir(bin);
-		await env.fs.write(join(bin, "bunx"), "#!/bin/sh\nexit 0\n");
-		await chmod(join(bin, "bunx"), 0o755);
-		const joined = new Promise<RemoteWorker>((resolve) =>
-			workers.registry.events.on(
-				"joined",
-				(each) => each.id === "remote" && resolve(each),
-			),
-		);
-		daemon = new WorkerDaemon(
-			{ where: socket, dial: () => UnixWorkerSocket.connect(socket) },
-			{
-				id: "remote",
-				slots: 2,
-				ps: new NodePs(),
-				fs: env.fs,
-				log,
-				backoff: { min: Duration.ms(20), max: Duration.ms(100) },
-				relay: relayDocument(new FsStorage(home, env.fs)),
-			},
-		);
-		void daemon.run();
-		worker = await joined;
-		relay = JSON.parse(await env.fs.read(`${home}/relay.json`)).url;
-	});
-
-	afterEach(async () => {
-		await daemon.disposeAsync();
-		await server.disposeAsync();
-		await env.disposeAsync();
-	});
-
-	/** Runs the CLI in `cwd` on the worker, as an agent there would with `agentEnv`. */
-	const splates = async (
-		cwd: string,
-		args: string[],
-		agentEnv: Record<string, string> = {
-			SPLATES_DEV: relay,
-			SPLATES_PROJECT: basename(env.root),
-		},
-	): Promise<string> => {
-		const { exitCode, stdout, stderr } = await env.ps.spawnCapture(
-			["bun", CLI, ...args],
-			{
-				cwd,
-				env: {
-					PATH: `${bin}:${process.env.PATH}`,
-					SPLATES_HOME: home,
-					...agentEnv,
-				},
-			},
-		);
-		if (exitCode !== 0) {
-			throw new Error(`splates ${args.join(" ")}: ${stderr}`);
-		}
-		return color.strip(stdout);
-	};
-
-	/** What an agent the worker starts finds as `SPLATES_DEV`, told the coordinator's own. */
-	const agentDev = (): Promise<string> =>
-		printed(
-			worker.start(["sh", "-c", 'printf "$SPLATES_DEV"'], clone, {
-				SPLATES_DEV: "http://127.0.0.1:1",
-			}),
-		);
-
-	it("points an agent the worker starts at the relay", async () => {
-		expect(await agentDev()).toBe(relay);
-	});
-
-	it("finds the relay and the project by hand, with no env", async () => {
-		await splates(clone, ["task", "add", "make it so", "--slug", "so"], {});
-
-		expect(await env.tasks.all()).toMatchObject([{ subject: "make it so" }]);
-	});
-
-	it("reaches the coordinator directly by hand once no worker daemon runs", async () => {
-		await daemon.disposeAsync();
-		// As a daemon dialing a coordinator elsewhere leaves it once stopped.
-		await relayDocument(new FsStorage(home, env.fs)).write({
-			worker: "remote",
-			coordinator: `http://127.0.0.1:${server.port}`,
-			keyFile: true,
-		});
-		await env.fs.write(
-			`${home}/worker.env`,
-			"SPLATES_WORKER_ACCESS_ID=client.access\nSPLATES_WORKER_ACCESS_SECRET=secret\n",
-		);
-
-		await splates(clone, ["task", "add", "make it so", "--slug", "so"], {});
-
-		expect(worker.connected).toBe(false);
-		expect(await env.tasks.all()).toMatchObject([{ subject: "make it so" }]);
-	});
-
-	// scry-ignore simple-test-setup: minting a token and saving the creds it gets are what this test checks, and no other test here does either.
-	it("joins with the token `worker add` mints, over the coordinator's own routes", async () => {
-		const coordinator = `http://127.0.0.1:${server.port}`;
-		const added = await env.ps.spawnCapture(["bun", CLI, "worker", "add"], {
-			cwd: env.root,
-			env: { SPLATES_DEV: coordinator },
-		});
-		const token = /splj_\S+/.exec(color.strip(added.stdout + added.stderr));
-		expect(token).not.toBeNull();
-
-		const store = new FakeSecretStore();
-		const { key } = await workerCreds("box", {
-			coordinator,
-			slots: 1,
-			id: "box",
-			join: token?.[0],
-			log: new MemoryLogger(),
-			ps: new NodePs(),
-			fs: env.fs,
-			store,
-		});
-
-		expect(await store.get(WORKER_KEY)).toBe(key);
-		expect(workers.records.verify("box", key)).toBe(true);
-	});
-
-	it("takes a task from add to merge on the coordinator's records", async () => {
-		await env.tasks.add("make it so", null);
-		relay = await agentDev();
-		await splates(clone, ["lane", "add", "first"]);
-		await splates(clone, ["add", "alpha", "--goal", "land it", "--task", "1"]);
-		const tree = (await splates(clone, ["path", "alpha"])).trim();
-
-		expect(await env.fs.exists(join(clone, ".git", "splates"))).toBe(false);
-		expect((await env.service.find("alpha")).state).toMatchObject({
-			status: "working",
-			worktree: tree,
-		});
-		expect(await env.tasks.find(1)).toMatchObject({ takenBy: "alpha" });
-
-		await splates(tree, ["claim", "alpha"]);
-		await env.fs.write(join(clone, "..", "shot.png"), "png");
-		await splates(tree, ["attach", join(clone, "..", "shot.png")]);
-
-		await env.commit(tree, "work.txt", "done\n", "Do the work");
-		await env.finish(tree);
-		await splates(tree, ["merge"]);
-
-		// Landed by the coordinator, on the remote and in its own checkout.
-		const remote = join(dirname(env.root), "remote.git");
-		expect(await env.gitCli(remote, "log", "-1", "--format=%s", "main")).toBe(
-			"Do the work",
-		);
-		expect(await env.gitCli(env.root, "log", "-1", "--format=%s")).toBe(
-			"Do the work",
-		);
-		expect((await env.service.find("alpha")).status).toBe("removed");
-		expect(await env.tasks.all()).toEqual([]);
-	});
-});

package/worker.ts

@@ -1,300 +0,0 @@
-import { splatesHome } from "@stringsync/splates-server/models";
-import type { SplatesDeps } from "@stringsync/splates-server/server";
-import {
-	addWorker,
-	drainWorker,
-	FileLogger,
-	listWorkers,
-	removeWorker,
-	setWorkerSlots,
-	WORKER_JOIN,
-	wakeWorker,
-	workerJoin,
-	workerRecipe,
-	workerRun,
-} from "@stringsync/splates-server/workers";
-import { cli } from "webappwiz/cmd";
-import { CompositeLogger } from "webappwiz/log";
-import { z } from "zod";
-import { splatesWaiting } from "./waiting";
-
-/** `splates worker`, which the root cli mounts after `splates slots`. */
-export const splatesWorker = cli<SplatesDeps>("worker").description(
-	"the machines that run tasks for `splates server`: each dials out to it and runs what it is given, agents, terminals and the git and process reads around them, on its own machine",
-);
-
-splatesWorker
-	.command("run")
-	.description(
-		"run a worker on this machine until stopped: it dials `splates server`, by default the one on this machine through a socket in ~/.splates only you may open, and dials again whenever the link drops, while what it runs carries on; it stops reading a process's output while the server is away too long, so nothing is lost. Once its agents have moved off it, a server on another build has it restart on that build, through `bunx @stringsync/splates@<build>`, as it does when it refuses this worker for another protocol; one that says no build has it exit `outdated`",
-	)
-	.option("server", z.string(), {
-		default: "",
-		description:
-			"the `splates server` to dial: its URL, whose /api/workers it opens, or unix:<path>; by default this machine's",
-	})
-	.option("coordinator", z.string(), {
-		default: "",
-		description:
-			"the old name for --server, still read for workers installed before it, and passed to a build older than it",
-	})
-	.option("slots", z.coerce.number().int().min(0), {
-		default: 0,
-		description:
-			"how many tasks this worker takes at once; by default one for each core and 4 GB of memory it has, whichever is fewer, as a VM such as WSL2 sees them",
-	})
-	.option("id", z.string(), {
-		default: "",
-		description:
-			"what the server knows this worker by, kept across restarts so it picks up where it was; by default the id it joined as, kept beside its key, else the machine's host name",
-	})
-	.option("join", z.string(), {
-		default: "",
-		description:
-			"a join token from `splates worker add` on the server, spent to join it before dialing; the key it answers is kept in this machine's system store, or read from SPLATES_WORKER_KEY on one without",
-	})
-	.option("ephemeral", z.boolean(), {
-		default: false,
-		description:
-			"keep the key --join answers in memory alone, for a machine destroyed rather than restarted, as one a worker provider makes",
-	})
-	.option("key-file", z.boolean(), {
-		default: false,
-		description:
-			"keep the key --join answers in ~/.splates/worker.env, which only this user may read, rather than the system store, for a machine whose store is locked or missing at boot, as WSL's is with no one signed in; read it from there on each run after",
-	})
-	.option("quiet", z.boolean(), {
-		default: false,
-		description:
-			"say only what went wrong, not each time it joins or loses its server, as the worker `splates server` starts beside itself does; all of it still goes to worker.log in ~/.splates, the one before it kept as worker.log.1",
-	})
-	.action((opts, ctx) =>
-		workerRun({
-			coordinator: opts.server || opts.coordinator,
-			slots: opts.slots,
-			id: opts.id,
-			join: opts.join || ctx.ps.env(WORKER_JOIN) || "",
-			ephemeral: opts.ephemeral,
-			keyFile: opts["key-file"],
-			log: opts.quiet ? quietLog(ctx) : ctx.log,
-			ps: ctx.ps,
-			fs: ctx.fs,
-		}),
-	);
-
-splatesWorker
-	.command("join")
-	.description(
-		"spend a join token from `splates worker add` and keep the key it answers, as `splates worker run --join` does, then exit, for an installer that leaves running the worker to a service",
-	)
-	.option("server", z.string(), {
-		default: "",
-		description: "the `splates server` to join: its URL",
-	})
-	.option("coordinator", z.string(), {
-		default: "",
-		description: "the old name for --server",
-	})
-	.option("join", z.string(), {
-		default: "",
-		description:
-			"the join token `splates worker add` printed there; by default read from SPLATES_WORKER_JOIN, which keeps it out of the process list other users of the machine can read",
-	})
-	.option("id", z.string(), {
-		default: "",
-		description:
-			"what the server knows this worker by, kept beside its key so a change of host name leaves it the same; by default the machine's host name",
-	})
-	.option("key-file", z.boolean(), {
-		default: false,
-		description:
-			"keep the key in ~/.splates/worker.env, which only this user may read, rather than the system store, for `splates worker run --key-file` to read",
-	})
-	.action((opts, ctx) =>
-		workerJoin({
-			coordinator: opts.server || opts.coordinator,
-			join: opts.join || ctx.ps.env(WORKER_JOIN) || "",
-			id: opts.id,
-			keyFile: opts["key-file"],
-			log: ctx.log,
-			ps: ctx.ps,
-			fs: ctx.fs,
-		}),
-	);
-
-splatesWorker
-	.command("recipe")
-	.description(
-		"print the shell script, run as root on a fresh Debian or Ubuntu machine, that installs what a Linux worker needs: bun, git, the shared browser and this build of splates, for the user `splates`. It is what a provider's snapshot is built from, and what the Windows installer runs in WSL",
-	)
-	.action((_opts, ctx) => workerRecipe({ log: ctx.log }));
-
-splatesWorker
-	.command("list")
-	.description(
-		"list the workers the running `splates server` has: kind (local over its socket, registered otherwise), platform, build, protocol, slots used and total, health with when it was last heard, tasks and $/hour, then any it refused for speaking another protocol",
-	)
-	.option("dev", z.string(), {
-		default: "",
-		description:
-			"the `splates server` to ask, a URL such as http://localhost:4310; by default the one an agent runs under, from SPLATES_DEV",
-	})
-	.option("json", z.boolean(), { default: false, description: "emit JSON" })
-	.action((opts, ctx) =>
-		listWorkers({
-			dev: opts.dev,
-			json: opts.json,
-			log: ctx.log,
-			fs: ctx.fs,
-			ps: ctx.ps,
-			send: ctx.fetch,
-		}),
-	);
-
-const dev = {
-	default: "",
-	description:
-		"the `splates server` to ask, a URL such as http://localhost:4310; by default the one an agent runs under, from SPLATES_DEV",
-};
-
-splatesWorker
-	.command("add")
-	.description(
-		"add a worker to the running `splates server`: prints a join token, single use and good for 15 minutes, for `splates worker run --join <token>` on the machine to join, which gets a key of its own that `splates worker remove` revokes; with --local N, makes this machine's own worker run N agents at once and take new ones again, adding it back if it was removed; with --provider, has the `workerProvider` in splates.config.ts make one, booted from the snapshot splates builds for it, which joins on its own and whose machine `splates worker remove` destroys",
-	)
-	.option("local", z.coerce.number().int().min(0), {
-		default: 0,
-		description: "this machine's own worker, with this many slots",
-	})
-	.option("provider", z.boolean(), {
-		default: false,
-		description:
-			"have the server's worker provider make the worker, of its cheapest type unless --type says",
-	})
-	.option("type", z.string(), {
-		default: "",
-		description:
-			"the worker provider's type to make, as it names it; implies --provider",
-	})
-	.option("dev", z.string(), dev)
-	.action((opts, ctx) =>
-		addWorker({
-			local: opts.local,
-			provider: opts.provider,
-			type: opts.type,
-			dev: opts.dev,
-			log: ctx.log,
-			fs: ctx.fs,
-			ps: ctx.ps,
-			send: ctx.fetch,
-		}),
-	);
-
-splatesWorker
-	.command("drain")
-	.description(
-		"take no new agents on a worker, letting those on it run to their end, but for one asking you something in a task, which saves its task and lets its slot go, its question kept for you to answer and carry it on elsewhere; --undo takes new ones again",
-	)
-	.arg("id", z.string(), {
-		description: "the worker, as `splates worker list` names it",
-	})
-	.option("undo", z.boolean(), {
-		default: false,
-		description: "take new agents again",
-	})
-	.option("dev", z.string(), dev)
-	.action((opts, ctx) =>
-		drainWorker({
-			id: opts.id,
-			undo: opts.undo,
-			dev: opts.dev,
-			log: ctx.log,
-			fs: ctx.fs,
-			ps: ctx.ps,
-			send: ctx.fetch,
-		}),
-	);
-
-splatesWorker
-	.command("wake")
-	.description(
-		"wake a worker asleep at home, a Windows PC that joined in WSL2: a worker awake on its network sends the Wake-on-LAN packet. Placement wakes one by itself when agents queue, before making a worker",
-	)
-	.arg("id", z.string(), {
-		description: "the worker, as `splates worker list` names it",
-	})
-	.option("dev", z.string(), dev)
-	.action((opts, ctx) =>
-		wakeWorker({
-			id: opts.id,
-			dev: opts.dev,
-			log: ctx.log,
-			fs: ctx.fs,
-			ps: ctx.ps,
-			send: ctx.fetch,
-		}),
-	);
-
-splatesWorker
-	.command("slots")
-	.description(
-		"set how many agents a worker runs at once, from now on, over what it started with",
-	)
-	.arg("id", z.string(), {
-		description: "the worker, as `splates worker list` names it",
-	})
-	.arg("slots", z.coerce.number().int().min(1), {
-		description: "how many at once",
-	})
-	.option("dev", z.string(), dev)
-	.action((opts, ctx) =>
-		setWorkerSlots({
-			id: opts.id,
-			slots: opts.slots,
-			dev: opts.dev,
-			log: ctx.log,
-			fs: ctx.fs,
-			ps: ctx.ps,
-			send: ctx.fetch,
-		}),
-	);
-
-splatesWorker
-	.command("remove")
-	.description(
-		"remove a worker, revoking its key: one with agents on it drains and is removed once they end; --force ends them now, frees their slots, and places each again on another worker, to carry on its task from the checkpoint it last pushed. This machine's own worker, `local`, stops taking agents, its agents running to their end, and leaves this machine only coordinating until `splates worker add --local <slots>`; it takes no --force",
-	)
-	.arg("id", z.string(), {
-		description: "the worker, as `splates worker list` names it",
-	})
-	.option("force", z.boolean(), {
-		default: false,
-		description: "end the agents on it now rather than wait for them",
-	})
-	.option("dev", z.string(), dev)
-	.action((opts, ctx) =>
-		removeWorker({
-			id: opts.id,
-			force: opts.force,
-			dev: opts.dev,
-			log: ctx.log,
-			fs: ctx.fs,
-			ps: ctx.ps,
-			send: ctx.fetch,
-		}),
-	);
-
-splatesWorker.mount("queue", splatesWaiting);
-
-/**
- * What a quiet worker says: what went wrong on the terminal, and everything
- * in `worker.log` in the splates home, which is all a worker `splates
- * server` started leaves of how it joined and lost its server.
- */
-function quietLog(ctx: SplatesDeps) {
-	const file = new FileLogger(ctx.fs, `${splatesHome(ctx.ps)}/worker.log`);
-	return new CompositeLogger([
-		{ info: () => {}, error: (...said) => ctx.log.error(...said) },
-		file,
-	]);
-}