@stringsync/splates 0.0.52 → 0.0.53

package/creds.ts

@@ -0,0 +1,144 @@
+import { DEFAULT_TIMEOUT, PROVIDERS } from "@stringsync/splates-server/creds";
+import { fail } from "@stringsync/splates-server/exit";
+import { repository } from "@stringsync/splates-server/records";
+import type { SplatesDeps } from "@stringsync/splates-server/server";
+import { untilAfter } from "@stringsync/splates-server/tasks";
+import { cli } from "webappwiz/cmd";
+import { color } from "webappwiz/log";
+import { Duration } from "webappwiz/time";
+import { z } from "zod";
+import { here } from "./flags";
+
+/** `splates creds`, which the root cli mounts after `splates lane`. */
+export const splatesCreds = cli<SplatesDeps>("creds").description(
+	"creds an agent needs a person for, such as an API token: request one saying what it is for, with instructions for getting it, which the Creds page of `splates server` shows for the person to paste, then wait for it; the value goes into the system's store (the macOS Keychain, Windows Credential Manager, a Linux secret service), never to the agent, and code reads it through webappwiz/creds",
+);
+
+splatesCreds
+	.command("request")
+	.use(repository({ needs: "own" }))
+	.description(
+		"ask a person for a cred, saying why it is needed and what it will be used for, with markdown instructions for getting it; returns at once, and does nothing when the store already keeps it unless --replace; run from a worktree, it records the task asking",
+	)
+	.arg("name", z.string(), {
+		description: "what it is kept as, an environment variable's name",
+	})
+	.option("purpose", z.string(), {
+		default: "",
+		description:
+			"why it is needed and what it will be used for, in a sentence the person reads before getting it",
+	})
+	.option("instructions", z.string(), {
+		default: "",
+		description:
+			"how to get it, in markdown: where to go, what to pick, what to paste",
+	})
+	.option("device", z.boolean(), {
+		default: false,
+		description:
+			"keep it for every project on this device, as for a person's own account, rather than this project alone",
+	})
+	.option("provider", z.optional(z.enum(PROVIDERS)), {
+		description:
+			"who keeps it: `system`, the machine's own store, or `ssm`, AWS Parameter Store, where that is offered",
+	})
+	.option("replace", z.boolean(), {
+		default: false,
+		description:
+			"ask again although one is kept, since the kept one does not work; --purpose and --instructions left off are those of the last request",
+	})
+	.action(async (opts, ctx) => {
+		const scope = opts.device ? "device" : "project";
+		const provider = opts.provider ?? "system";
+		const request = await ctx.journal.record(
+			"creds request",
+			null,
+			async () =>
+				ctx.creds.request(opts.name, {
+					scope,
+					provider,
+					purpose: opts.purpose,
+					instructions: opts.instructions,
+					task: await here(ctx),
+					replace: opts.replace,
+				}),
+			(asked) => ({ name: opts.name, asked: asked !== null }),
+		);
+		if (request === null) {
+			ctx.log.info(
+				`${color.green("kept")} ${opts.name} in ${ctx.creds.label(scope, provider)}: nothing to ask`,
+			);
+			return;
+		}
+		ctx.log.info(
+			`${color.yellow("requested")} ${opts.name}  ${color.dim(request.purpose)}`,
+		);
+		ctx.log.error(
+			`It shows on the Creds page of \`splates server\`. \`splates creds wait ${opts.name}\` until it is saved.`,
+		);
+	});
+
+splatesCreds
+	.command("wait")
+	.use(repository({ needs: "own" }))
+	.description(
+		"block until a person saves the cred requested, or dismisses the request (`dismissed`); gives up with `timeout` rather than waiting forever",
+	)
+	.arg("name", z.string(), { description: "the cred requested" })
+	.option("timeout-secs", z.coerce.number(), {
+		default: DEFAULT_TIMEOUT.secs,
+		description: "how long to wait before giving up",
+	})
+	.action(async (opts, ctx) => {
+		const timeout = Duration.secs(opts["timeout-secs"]);
+		await ctx.journal.waiting(
+			"creds wait",
+			null,
+			{ name: opts.name, until: untilAfter(timeout) },
+			async () => {
+				const ended = await ctx.creds.wait(opts.name, { timeout });
+				if (ended === "dismissed") {
+					fail(
+						"dismissed",
+						`${opts.name} was dismissed: carry on without it, or ask the person why`,
+						{ name: opts.name },
+					);
+				}
+			},
+		);
+		ctx.log.info(`${color.green("saved")} ${opts.name}`);
+	});
+
+splatesCreds
+	.command("list")
+	.use(repository({ needs: "read" }))
+	.description(
+		"the creds requested and still open, with what each is for, and every cred the project's store and the device's keep, by name; never a value",
+	)
+	.option("json", z.boolean(), { default: false, description: "emit JSON" })
+	.action(async (opts, ctx) => {
+		const open = (await ctx.creds.requests()).filter(
+			(each) => each.status === "open",
+		);
+		const kept = await ctx.creds.kept();
+		if (opts.json) {
+			ctx.log.info(JSON.stringify({ requested: open, kept }, null, 2));
+			return;
+		}
+		for (const each of open) {
+			ctx.log.info(
+				`${color.yellow("requested")} ${each.name}  ${color.dim(each.scope)}${each.task === null ? "" : color.dim(`  by ${each.task}`)}`,
+			);
+			if (each.purpose !== "") {
+				ctx.log.info(`          ${each.purpose}`);
+			}
+		}
+		for (const each of kept) {
+			ctx.log.info(
+				`${color.green("kept")}      ${each.name}  ${color.dim(`${each.provider} ${each.scope}`)}${each.savedBy === null ? "" : color.dim(`  saved by ${each.savedBy}`)}`,
+			);
+		}
+		if (open.length === 0 && kept.length === 0) {
+			ctx.log.info(color.dim("no creds requested or kept"));
+		}
+	});

package/dev/dev.ts

@@ -0,0 +1,86 @@
+import type { SplatesDeps } from "@stringsync/splates-server/server";
+import type { Cli } from "webappwiz/cmd";
+import { z } from "zod";
+import { evalSkill } from "./eval";
+import { fix } from "./fix";
+import { PROOFS, prove } from "./prove";
+import { ship } from "./ship";
+import { test } from "./test";
+
+/** Adds `splates dev`, the commands for working on this repository, to `program`. */
+export function mountDev(program: Cli<SplatesDeps>): void {
+	// Only `checkout.ts`, the entry `bin/splates` runs, calls this, so the
+	// published cli never carries these: a release bundles what its `bin` and
+	// `exports` entries import, and `index.ts` imports none of this.
+	const dev = program
+		.group("dev")
+		.description("work on the splates repository");
+
+	dev
+		.command("eval")
+		.description(
+			"compare versions of the splates skill by running agents on it; uses tokens billed to an API key or counted against a Claude plan, so run it only when asked",
+		)
+		.rest("args", z.string(), {
+			description:
+				"[fast|list|report] [--probe id] [--target agent:model] [--parallel n], or explicit e2e [run|report]",
+		})
+		.allowUnknownOption()
+		.action((opts, { ps }) => evalSkill({ ...opts, ps }));
+
+	dev
+		.command("fix")
+		.description("format, check, and typecheck the repository")
+		.option("check", z.boolean(), {
+			default: false,
+			description: "report problems without writing fixes (for CI)",
+		})
+		.option("code-only", z.boolean(), {
+			default: false,
+			description: "run Scry code rules without credentials or model calls",
+		})
+		.action(({ "code-only": codeOnly, ...opts }, { log, ps }) =>
+			fix({ ...opts, codeOnly, log, ps }),
+		);
+
+	dev
+		.command("prove")
+		.description(
+			"prove splates on Linux end to end with Docker, nothing of yours used: `coordinator` runs the server in a Debian container as the user splates, as the AWS box will, with this machine as its worker, and drives a task, a restart, an archive and a cred through it; `worker` runs a container as a worker of a server here. Read docker/prove-<which>.sh for what each checks",
+		)
+		.arg("proof", z.enum(PROOFS), {
+			description: "coordinator or worker",
+		})
+		.rest("args", z.string(), {
+			description:
+				"handed to the script: a port, and for `worker` --agent <id> --model <model> to hand the task to a roster agent",
+		})
+		.allowUnknownOption()
+		.action(({ proof, args }, { log, ps }) => prove({ proof, args, log, ps }));
+
+	dev
+		.command("ship")
+		.description("release the package, with its skill stamped at its version")
+		.action((_opts, { log, fs, ps }) => ship({ log, fs, ps }));
+
+	dev
+		.command("test")
+		.description("run the package's tests; eval harness tests are separate")
+		.option("evals", z.boolean(), {
+			default: false,
+			description: "run only free eval harness tests (no model calls)",
+		})
+		.option("all", z.boolean(), {
+			default: false,
+			description: "run the package's and the eval harness's tests together",
+		})
+		.option("linux", z.boolean(), {
+			default: false,
+			description:
+				"run them on Linux, in a Docker container built from the checkout, as the box and the workers run; needs Docker",
+		})
+		.rest("files", z.string(), {
+			description: "run only the test files whose path has one of these",
+		})
+		.action((opts, { log, ps }) => test({ ...opts, log, ps }));
+}

package/dev/eval.ts

@@ -0,0 +1,21 @@
+import { NodePs, type Ps } from "webappwiz/system";
+import { ROOT } from "./root";
+
+export interface EvalOptions {
+	/** What splates's evals entry is given: fast selection, report, or explicit e2e, then its arguments. */
+	args: string[];
+	ps?: Ps;
+}
+
+/** Run the focused suite by default; legacy end-to-end evaluation requires `e2e`. */
+export async function evalSkill(opts: EvalOptions): Promise<void> {
+	const ps = opts.ps ?? new NodePs();
+	const args = opts.args.length ? opts.args : ["fast"];
+	const { exitCode } = await ps.spawn(
+		["bun", "evals/splates-skill/index.ts", ...args],
+		{ cwd: ROOT },
+	);
+	if (exitCode !== 0) {
+		throw new Error("The evals failed");
+	}
+}

package/dev/fix.ts

@@ -0,0 +1,63 @@
+import { ConsoleLogger, color, type Logger } from "webappwiz/log";
+import { NodePs, type Ps } from "webappwiz/system";
+import { ROOT } from "./root";
+
+export interface FixOptions {
+	/** Report problems without writing fixes, as CI wants it. */
+	check: boolean;
+	/** Check only code rules, without credentials or model calls. */
+	codeOnly?: boolean;
+	/** Skip Scry when checking a release that was already reviewed. */
+	skipScry?: boolean;
+	log?: Logger;
+	ps?: Ps;
+}
+
+/**
+ * Runs biome, the type checker, then scry on what changed since main, in
+ * that order. Scry leaves off each model its budget no longer covers, so a
+ * spent budget narrows the check rather than fail it.
+ */
+export async function fix(opts: FixOptions): Promise<void> {
+	const log = opts.log ?? new ConsoleLogger();
+	const ps = opts.ps ?? new NodePs();
+
+	// Every step says how it went and the first failure ends the run, so the
+	// verdict a person reads last is the one that stopped them.
+	const step = (name: string, ok: boolean, failure: string): void => {
+		log.info(`${name}: ${ok ? color.green("success") : color.red("failed")}`);
+		if (!ok) {
+			throw new Error(failure);
+		}
+	};
+
+	const flags = opts.check ? [] : ["--write", "--unsafe"];
+	const biome = await ps.spawn(["bunx", "biome", "check", ...flags, "."], {
+		cwd: ROOT,
+	});
+	step("biome", biome.exitCode === 0, "Biome check failed");
+
+	const typecheck = await ps.spawn(["bunx", "tsc", "--noEmit"], {
+		cwd: ROOT,
+	});
+	step("typecheck", typecheck.exitCode === 0, "Typechecking failed");
+
+	if (opts.skipScry) {
+		return;
+	}
+
+	const scry = await ps.spawn(
+		[
+			"bunx",
+			"@webappwiz/cli",
+			"scry",
+			"check",
+			"--since",
+			"main",
+			"--disable-over-budget",
+			...(opts.codeOnly ? ["--disable", "som", "--disable", "llm"] : []),
+		],
+		{ cwd: ROOT },
+	);
+	step("scry", scry.exitCode === 0, "Scry check failed");
+}

package/dev/index.ts

@@ -0,0 +1 @@
+export { mountDev } from "./dev";

package/dev/prove.ts

@@ -0,0 +1,38 @@
+import { join } from "node:path";
+import { ConsoleLogger, type Logger } from "webappwiz/log";
+import { NodePs, type Ps } from "webappwiz/system";
+import { ROOT } from "./root";
+
+/** The proofs, each a script in `docker/`. */
+export const PROOFS = ["coordinator", "worker"] as const;
+export type Proof = (typeof PROOFS)[number];
+
+export interface ProveOptions {
+	proof: Proof;
+	/** What the script takes after its name, such as a port or `--agent`. */
+	args?: string[];
+	ps?: Ps;
+	log?: Logger;
+}
+
+/**
+ * Runs a proof of splates on Linux end to end: `coordinator` runs the server
+ * in a Docker container with this machine as its worker, and `worker` the
+ * inverse. Each prints what it checked and fails on the first thing wrong.
+ */
+export async function prove(opts: ProveOptions): Promise<void> {
+	const ps = opts.ps ?? new NodePs();
+	const log = opts.log ?? new ConsoleLogger();
+	log.info(`proving the ${opts.proof} on Linux; needs Docker`);
+	const { exitCode } = await ps.spawn(
+		[
+			"sh",
+			join(ROOT, "docker", `prove-${opts.proof}.sh`),
+			...(opts.args ?? []),
+		],
+		{ cwd: ROOT },
+	);
+	if (exitCode !== 0) {
+		throw new Error(`The ${opts.proof} proof failed`);
+	}
+}

package/dev/root.ts

@@ -0,0 +1,7 @@
+import { join } from "node:path";
+
+/**
+ * The checkout this `splates dev` runs from, which `bin/_run` picks as the one you
+ * are standing in: a git worktree's own copy, so testing it tests your work.
+ */
+export const ROOT = join(import.meta.dir, "../../..");

package/dev/ship.ts

@@ -0,0 +1,46 @@
+import { ConsoleLogger, type Logger } from "webappwiz/log";
+import { releases } from "webappwiz/ship";
+import { type Fs, NodePs, type Ps } from "webappwiz/system";
+import { fix } from "./fix";
+
+/**
+ * The agent skill this repository ships. Projects install it from the
+ * release's tag with `bunx skills add`, so stamping it is what has a
+ * project's copy say which release it came from.
+ */
+const SKILL = "skills/splates/SKILL.md";
+
+export interface ShipOptions {
+	log?: Logger;
+	fs?: Fs;
+	ps?: Ps;
+	// scry-ignore objects-over-callbacks: the platform's own prompt() is the
+	// dependency here, and it is a bare function
+	prompt?: (message: string) => string | null;
+}
+
+/**
+ * Releases the package, always a patch, with its skill stamped at the same
+ * version. A release that died is finished first, at the version it already
+ * holds.
+ */
+export async function ship(opts: ShipOptions = {}): Promise<void> {
+	const log = opts.log ?? new ConsoleLogger();
+	const ps = opts.ps ?? new NodePs();
+	// The build compiles without checking types, so a typecheck is the only
+	// gate there is: run it before anything is stamped or pushed. The suite is
+	// not run here; whoever ships is expected to have run it already.
+	await fix({ check: true, skipScry: true, log, ps });
+
+	const declared = releases.lockstep(
+		await releases.workspace({ fs: opts.fs, ps }),
+		releases.skill(SKILL, { fs: opts.fs }),
+	);
+	await declared.release({
+		bump: "patch",
+		prompt: opts.prompt,
+		log,
+		fs: opts.fs,
+		ps,
+	});
+}

package/dev/test.ts

@@ -0,0 +1,154 @@
+import { availableParallelism } from "node:os";
+import {
+	type CoreClaims,
+	coreClaimsOfMachine,
+	heavyJobsOfMachine,
+} from "@stringsync/splates-server/server";
+import { ConsoleLogger, type Logger } from "webappwiz/log";
+import { type HeavyCap, NodeFs, NodePs, type Ps } from "webappwiz/system";
+import { Duration } from "webappwiz/time";
+import { ROOT } from "./root";
+
+export interface TestOptions {
+	/** Run only the free eval harness tests. */
+	evals?: boolean;
+	/** Run the package's and the eval harness's tests together. */
+	all?: boolean;
+	/**
+	 * Run the tests on Linux, in a Docker container built from the checkout
+	 * (`docker/test.Dockerfile`), as the coordinator's box and the workers
+	 * run them.
+	 */
+	linux?: boolean;
+	/** Run only the test files whose path has one of these, as `bun test` filters. */
+	files?: string[];
+	/**
+	 * Test files run at once; half the cores when left out, fewer (down to half
+	 * of that) when the machine's other heavy jobs hold some of its cores.
+	 */
+	workers?: number;
+	ps?: Ps;
+	log?: Logger;
+	/** The machine's heavy-job slots, one of which the run waits for. */
+	heavy?: HeavyCap;
+	/** The machine's cores, which the run claims its workers from. */
+	cores?: CoreClaims;
+	/** How long the run may take before it is stopped. */
+	deadline?: Duration;
+}
+
+/**
+ * How long a test run may hold its heavy slot. Tests should not hang this
+ * long, even under load.
+ */
+// At its fewest workers, a quarter of 16 cores, with load near 40, the suite
+// took 111s, its time spread over hundreds of files rather than a few slow
+// ones, and 140s at 8 workers beside two other runs: 3 minutes left a gate on
+// a busier machine too little headroom.
+export const TEST_DEADLINE = Duration.mins(6);
+
+/** The image `--linux` builds from the checkout and runs the tests in. */
+export const LINUX_IMAGE = "splates-test";
+
+export async function test(opts: TestOptions): Promise<void> {
+	const ps = opts.ps ?? new NodePs();
+	const log = opts.log ?? new ConsoleLogger();
+
+	if (opts.evals && opts.all) {
+		throw new Error("choose either --evals or --all");
+	}
+	const filter = opts.all
+		? []
+		: opts.evals
+			? ["evals/"]
+			: ["--path-ignore-patterns=evals/**"];
+
+	// a worker per file; suites opt into within-file concurrency themselves,
+	// since most still share a fixture built in beforeEach. Half the cores, not
+	// bun's one per core: several agents test at once, and a worker per core
+	// each starves their git-spawning suites past the 5s timeout. Three
+	// overlapping runs on 16 cores failed 20 to 30 tests each at 16 workers and
+	// 0 to 2 at 8, finishing sooner too; one run alone is about 1.5s slower.
+	// Even so, those suites run many times slower under load than alone, so
+	// each test gets 30s rather than 5s.
+	const want =
+		opts.workers ?? Math.max(1, Math.floor(availableParallelism() / 2));
+	// one of the machine's few heavy-job slots, as splates's `preMerge` takes,
+	// so agents testing at once queue rather than run out of memory; run from
+	// a `preMerge` hook, it runs in the slot the merge already holds
+	const heavy = opts.heavy ?? heavyJobsOfMachine({ fs: new NodeFs(), ps, log });
+	// `splates merge` names where its gate wants a JUnit report, to queue a
+	// task for each slow test; bun keeps printing its own output beside it.
+	// Not on Linux, where the path would be the container's.
+	const report = opts.linux ? undefined : ps.env("SPLATES_TEST_REPORT");
+	await using slot = await heavy.take({ what: "splates dev test" });
+	// a worker is a core claimed: when other heavy jobs hold the machine's
+	// cores the run waits (five minutes at most) for half its workers, rather
+	// than starve the server and the agents; asked for workers by number, it
+	// waits for exactly those. Run in a merge's gate, it takes the cores the
+	// merge claimed, first in line, and claims none of its own
+	const claims =
+		opts.cores ?? coreClaimsOfMachine({ fs: new NodeFs(), ps, log });
+	await using claim = await claims.claim({
+		what: "splates dev test",
+		want,
+		min: opts.workers === undefined ? Math.max(1, Math.floor(want / 2)) : want,
+	});
+	const workers = claim.cores;
+	const bunTest = [
+		"bun",
+		"test",
+		"--pass-with-no-tests",
+		`--parallel=${workers}`,
+		"--timeout",
+		"30000",
+		...filter,
+		...(report ? ["--reporter=junit", `--reporter-outfile=${report}`] : []),
+		...(opts.files ?? []),
+	];
+	// The image is built in the slot, since an install is a heavy job, but
+	// ahead of the deadline: the first build pulls the base image and
+	// installs every dependency, minutes that are not the tests'.
+	if (opts.linux) {
+		const build = await ps.spawn(
+			[
+				"docker",
+				"build",
+				"-f",
+				"docker/test.Dockerfile",
+				"-t",
+				LINUX_IMAGE,
+				".",
+			],
+			{ cwd: ROOT, env: slot.env },
+		);
+		if (build.exitCode !== 0) {
+			throw new Error(
+				"Building the Linux test image failed: is Docker running?",
+			);
+		}
+	}
+	// bun's per-test timeout cannot stop a test spinning in a synchronous
+	// loop, and such a run held its slot for ten hours while every add, merge
+	// and conversation on the machine queued behind it. Ending bun's runner
+	// ends its workers too; `docker run` passes the signal on to the
+	// container's bun, and `--init` reaps what the tests spawn under it.
+	const deadline = opts.deadline ?? TEST_DEADLINE;
+	const signal = AbortSignal.timeout(deadline.ms);
+	const argv = opts.linux
+		? ["docker", "run", "--rm", "--init", LINUX_IMAGE, ...bunTest]
+		: bunTest;
+	const { exitCode } = await ps
+		.spawn(argv, { cwd: ROOT, env: slot.env, signal })
+		.catch((error: unknown) => {
+			if (signal.aborted) {
+				throw new Error(
+					`Tests ran past ${deadline.mins} minutes, so they were stopped: tests should not hang this long, even under load; a test may be stuck in a loop`,
+				);
+			}
+			throw error;
+		});
+	if (exitCode !== 0) {
+		throw new Error("Tests failed");
+	}
+}