@cldmv/gh-broker 0.7.0 → 0.8.0

package/src/verbs/ghcr/config.ts

@@ -0,0 +1,142 @@
+/**
+ *
+ *	@Project: @cldmv/gh-broker
+ *	@Filename: /src/verbs/ghcr/config.ts
+ *	@Date: 2026-10-31T10:57:30-07:00 (1793469450)
+ *	@Author: Nate Corcoran <CLDMV>
+ *	@Email: <Shinrai@users.noreply.github.com>
+ *	-----
+ *	@Last modified by: Nate Corcoran <CLDMV> (Shinrai@users.noreply.github.com)
+ *	@Last modified time: 2026-11-01T00:40:55-07:00 (1793518855)
+ *	-----
+ *	@Copyright: Copyright (c) 2013-2026 Catalyzed Motivation Inc. All rights reserved.
+ *
+ */
+
+/**
+ * Config block for the GHCR verbs (`ghcr_push`, `ghcr_push_status`, `ghcr_tags`).
+ *
+ * GHCR is a separate credential and a separate policy from the GitHub API routing in
+ * `config.owners`: the registry does not accept fine-grained PATs, so it needs a classic
+ * PAT with `write:packages`, and that token must only ever reach `ghcr.io`. Keeping it in
+ * its own block (rather than a third ref on `owners.<o>`) keeps the existing owner
+ * credentials untouched and lets the installer treat the whole block as operator data, set with
+ * `--ghcr-*` flags or a `--config` file (see deploy/lib/config-merge.mjs).
+ *
+ *   "ghcr": {
+ *     "dropDir": "/srv/gh-broker/ghcr-drop",
+ *     "protectReleaseTags": true,
+ *     "owners": {
+ *       "your-org": { "username": "your-login", "credential": "cred:your-org-packages", "images": ["your-image", "your-image-worker-*"] }
+ *     }
+ *   }
+ *
+ * Every key is checked strictly (unknown keys are refused) because this is a security
+ * policy: a typo such as `image` for `images` must fail loudly, not silently allow nothing
+ * or everything.
+ */
+
+import { isAbsolute, normalize } from "node:path";
+
+export interface GhcrOwnerConfig {
+	/** GitHub login the classic PAT belongs to; sent as the Basic-auth user to ghcr.io/token. */
+	username: string;
+	/** "cred:<name>" ref for the classic PAT (write:packages). */
+	credential: string;
+	/** Image names under ghcr.io/<owner>/ that may be pushed or listed. `*` matches within one path segment. */
+	images: string[];
+}
+
+export interface GhcrConfig {
+	/** The only directory archives are read from. Absolute; the installer creates it. */
+	dropDir: string;
+	/** Refuse to overwrite an existing tag shaped like a release (`1.2.3` / `v1.2.3`). */
+	protectReleaseTags: boolean;
+	/** Largest archive accepted, in bytes. */
+	maxArchiveBytes: number;
+	owners: Record<string, GhcrOwnerConfig>;
+}
+
+export const DEFAULT_MAX_ARCHIVE_BYTES = 16 * 1024 ** 3;
+const TOP_KEYS = new Set(["dropDir", "protectReleaseTags", "maxArchiveBytes", "owners"]);
+const OWNER_KEYS = new Set(["username", "credential", "images"]);
+/** Directories that must never be a drop dir: system state, other users' homes, secrets. */
+const FORBIDDEN_ROOTS = ["/etc", "/proc", "/sys", "/dev", "/run", "/boot", "/root", "/home", "/tmp", "/usr", "/bin", "/sbin", "/lib", "/lib64", "/var/tmp"];
+/** One image-name glob: lowercase OCI path components, `*` allowed, `/` separates segments. */
+const IMAGE_GLOB = /^[a-z0-9*](?:[a-z0-9._*-]*[a-z0-9*])?(?:\/[a-z0-9*](?:[a-z0-9._*-]*[a-z0-9*])?)*$/;
+const OWNER_NAME = /^[a-z0-9](?:[a-z0-9-]{0,38})$/;
+
+const fail = (msg: string): never => {
+	throw new Error(`config.ghcr${msg}`);
+};
+
+function strictKeys(o: Record<string, unknown>, allowed: Set<string>, where: string): void {
+	const extra = Object.keys(o).filter((k) => !allowed.has(k));
+	if (extra.length) fail(`${where}: unknown key(s) ${extra.join(", ")} — expected ${[...allowed].join(", ")}`);
+}
+
+const isObject = (v: unknown): v is Record<string, unknown> => typeof v === "object" && v !== null && !Array.isArray(v);
+
+export function parseGhcrConfig(raw: unknown): GhcrConfig {
+	if (!isObject(raw)) return fail(": must be an object");
+	strictKeys(raw, TOP_KEYS, "");
+
+	const dropDir = raw.dropDir;
+	if (typeof dropDir !== "string" || !isAbsolute(dropDir)) fail('.dropDir: must be an absolute path, e.g. "/srv/gh-broker/ghcr-drop"');
+	const dir = normalize(dropDir as string).replace(/\/+$/, "");
+	if (!dir || dir !== (dropDir as string).replace(/\/+$/, "")) fail(".dropDir: must be a normalized path (no ., .. or doubled slashes)");
+	if (dir.split("/").filter(Boolean).length < 2) fail(".dropDir: must be at least two levels deep, e.g. /srv/gh-broker/ghcr-drop");
+	for (const root of FORBIDDEN_ROOTS) {
+		if (dir === root || dir.startsWith(`${root}/`)) fail(`.dropDir: may not be under ${root}`);
+	}
+
+	if (raw.protectReleaseTags !== undefined && typeof raw.protectReleaseTags !== "boolean") fail(".protectReleaseTags: must be a boolean");
+	let maxArchiveBytes = DEFAULT_MAX_ARCHIVE_BYTES;
+	if (raw.maxArchiveBytes !== undefined) {
+		const m = raw.maxArchiveBytes;
+		if (typeof m !== "number" || !Number.isSafeInteger(m) || m <= 0) fail(".maxArchiveBytes: must be a positive integer");
+		maxArchiveBytes = m as number;
+	}
+
+	if (!isObject(raw.owners) || Object.keys(raw.owners).length === 0) fail(".owners: must be an object with at least one owner");
+	const owners: Record<string, GhcrOwnerConfig> = {};
+	for (const [name, v] of Object.entries(raw.owners as Record<string, unknown>)) {
+		const owner = name.toLowerCase();
+		if (!OWNER_NAME.test(owner)) fail(`.owners: "${name}" is not a valid GitHub owner name`);
+		if (!isObject(v)) fail(`.owners.${name}: must be an object`);
+		const o = v as Record<string, unknown>;
+		strictKeys(o, OWNER_KEYS, `.owners.${name}`);
+		if (typeof o.username !== "string" || !/^[A-Za-z0-9](?:[A-Za-z0-9-]{0,38})$/.test(o.username)) fail(`.owners.${name}.username: must be the GitHub login that owns the PAT`);
+		if (typeof o.credential !== "string" || !/^cred:[A-Za-z0-9._-]+$/.test(o.credential) || /^cred:\.\.?$/.test(o.credential)) {
+			fail(`.owners.${name}.credential: must be a "cred:<name>" ref`);
+		}
+		if (!Array.isArray(o.images) || o.images.length === 0) fail(`.owners.${name}.images: must be a non-empty list of image names or globs`);
+		for (const img of o.images as unknown[]) {
+			if (typeof img !== "string" || img.length > 255 || !IMAGE_GLOB.test(img)) {
+				fail(`.owners.${name}.images: "${String(img)}" is not a valid image name or glob (lowercase, [a-z0-9._-], "/" between segments, "*" within a segment)`);
+			}
+		}
+		owners[owner] = { username: o.username as string, credential: o.credential as string, images: [...(o.images as string[])] };
+	}
+
+	return { dropDir: dir, protectReleaseTags: raw.protectReleaseTags !== false, maxArchiveBytes, owners };
+}
+
+/** Every "cred:<name>" ref the ghcr block references, for eager validation at startup. */
+export function ghcrCredRefs(cfg: GhcrConfig | undefined): string[] {
+	return cfg ? Object.values(cfg.owners).map((o) => o.credential) : [];
+}
+
+/** Compile an image glob to an anchored regex: `*` matches `[a-z0-9._-]*` (never `/`). */
+export function imageGlobToRegExp(glob: string): RegExp {
+	const body = glob
+		.split("*")
+		.map((part) => part.replace(/[.\\^$+?()[\]{}|-]/g, "\\$&"))
+		.join("[a-z0-9._-]*");
+	return new RegExp(`^${body}$`);
+}
+
+/** True when `image` (a name under the owner, e.g. "hoot-worker-a") is on the owner's allowlist. */
+export function imageAllowed(owner: GhcrOwnerConfig, image: string): boolean {
+	return owner.images.some((g) => imageGlobToRegExp(g).test(image));
+}

package/src/verbs/ghcr/jobs.ts

@@ -0,0 +1,141 @@
+/**
+ *
+ *	@Project: @cldmv/gh-broker
+ *	@Filename: /src/verbs/ghcr/jobs.ts
+ *	@Date: 2026-10-31T11:08:29-07:00 (1793470109)
+ *	@Author: Nate Corcoran <CLDMV>
+ *	@Email: <Shinrai@users.noreply.github.com>
+ *	-----
+ *	@Last modified by: Nate Corcoran <CLDMV> (Shinrai@users.noreply.github.com)
+ *	@Last modified time: 2026-10-31T11:13:29-07:00 (1793470409)
+ *	-----
+ *	@Copyright: Copyright (c) 2013-2026 Catalyzed Motivation Inc. All rights reserved.
+ *
+ */
+
+/**
+ * In-memory queue for GHCR push jobs.
+ *
+ * Images are 1–3 GB, far longer than one MCP call should block, so `ghcr_push` enqueues a
+ * job and returns its id; `ghcr_push_status` reads it back. Jobs run one at a time (a
+ * push saturates the uplink; two at once only make both slower) in submission order.
+ *
+ * Bounded on purpose: at most `maxPending` queued-or-running jobs (each holds an open
+ * archive descriptor), and only the newest `keepFinished` finished jobs are remembered.
+ * State lives in memory only, so a broker restart forgets every job; a push that was
+ * running is simply not completed (the registry discards an unfinished upload, and no
+ * tag moves until the final manifest PUT).
+ */
+
+import { randomUUID } from "node:crypto";
+import type { PushProgress, PushResult } from "./push.ts";
+
+export type JobState = "queued" | "running" | "succeeded" | "failed";
+
+export interface JobMeta {
+	owner: string;
+	/** e.g. "ghcr.io/cldmv/hoot" */
+	image: string;
+	tag: string;
+	archive: string;
+	archiveBytes: number;
+	format: string;
+}
+
+export interface Job extends JobMeta {
+	job: string;
+	state: JobState;
+	progress?: PushProgress;
+	result?: PushResult;
+	error?: string;
+	createdAt: string;
+	startedAt?: string;
+	finishedAt?: string;
+}
+
+export interface JobHooks {
+	/** Does the work. Report progress through `progress`; resolve with the result or throw. */
+	run(progress: (p: PushProgress) => void): Promise<PushResult>;
+	/** Always called once the job ends (success, failure, or never started). */
+	finally?(job: Job): void;
+	/** Turns a thrown error into the message stored on the job (must already be secret-free). */
+	describeError?(err: unknown): string;
+}
+
+export class QueueFull extends Error {}
+
+export class JobQueue {
+	private readonly jobs = new Map<string, Job>();
+	private readonly pending: Array<{ job: Job; hooks: JobHooks }> = [];
+	private running = false;
+	private readonly maxPending: number;
+	private readonly keepFinished: number;
+
+	constructor({ maxPending = 8, keepFinished = 50 }: { maxPending?: number; keepFinished?: number } = {}) {
+		this.maxPending = maxPending;
+		this.keepFinished = keepFinished;
+	}
+
+	/** Queued + running jobs. */
+	activeCount(): number {
+		return this.pending.length + (this.running ? 1 : 0);
+	}
+
+	submit(meta: JobMeta, hooks: JobHooks): Job {
+		if (this.activeCount() >= this.maxPending) throw new QueueFull(`ghcr_push: ${this.activeCount()} pushes are already queued or running — wait for one to finish (ghcr_push_status)`);
+		const job: Job = { ...meta, job: randomUUID(), state: "queued", createdAt: new Date().toISOString() };
+		this.jobs.set(job.job, job);
+		this.pending.push({ job, hooks });
+		const snapshot = { ...job }; // as submitted, before pump() may flip it to running
+		void this.pump();
+		return snapshot;
+	}
+
+	get(id: string): Job | undefined {
+		const j = this.jobs.get(id);
+		return j ? structuredClone(j) : undefined;
+	}
+
+	list(owner: string): Job[] {
+		return [...this.jobs.values()].filter((j) => j.owner === owner).map((j) => structuredClone(j));
+	}
+
+	/** Resolves once nothing is queued or running (for tests and orderly shutdown). */
+	async idle(): Promise<void> {
+		while (this.activeCount() > 0) await new Promise((r) => setTimeout(r, 5));
+	}
+
+	private async pump(): Promise<void> {
+		if (this.running) return;
+		const next = this.pending.shift();
+		if (!next) return;
+		this.running = true;
+		const { job, hooks } = next;
+		job.state = "running";
+		job.startedAt = new Date().toISOString();
+		try {
+			job.result = await hooks.run((p) => {
+				job.progress = p;
+			});
+			job.state = "succeeded";
+		} catch (err) {
+			job.state = "failed";
+			job.error = hooks.describeError ? hooks.describeError(err) : "push failed";
+		} finally {
+			job.finishedAt = new Date().toISOString();
+			try {
+				hooks.finally?.(structuredClone(job));
+			} catch {
+				/* a cleanup/audit hook must not wedge the queue */
+			}
+			this.running = false;
+			this.prune();
+			void this.pump();
+		}
+	}
+
+	private prune(): void {
+		const finished = [...this.jobs.values()].filter((j) => j.state === "succeeded" || j.state === "failed");
+		for (const j of finished.slice(0, Math.max(0, finished.length - this.keepFinished))) this.jobs.delete(j.job);
+	}
+}

package/src/verbs/ghcr/push.ts

@@ -0,0 +1,192 @@
+/**
+ *
+ *	@Project: @cldmv/gh-broker
+ *	@Filename: /src/verbs/ghcr/push.ts
+ *	@Date: 2026-10-31T11:00:50-07:00 (1793469650)
+ *	@Author: Nate Corcoran <CLDMV>
+ *	@Email: <Shinrai@users.noreply.github.com>
+ *	-----
+ *	@Last modified by: Nate Corcoran <CLDMV> (Shinrai@users.noreply.github.com)
+ *	@Last modified time: 2026-10-31T11:13:29-07:00 (1793470409)
+ *	-----
+ *	@Copyright: Copyright (c) 2013-2026 Catalyzed Motivation Inc. All rights reserved.
+ *
+ */
+
+/**
+ * Pushes one planned image (see archive.ts) to a registry repository.
+ *
+ * Phases:
+ *  1. preparing — for every layer the archive holds uncompressed, gzip it once just to
+ *     learn the compressed digest and size. This costs a second pass of CPU, but it means
+ *     the blob `HEAD` check works for those layers too (a re-push of an unchanged image
+ *     uploads nothing), and nothing has to be buffered or written to disk: the broker's
+ *     unit has no writable scratch space.
+ *  2. uploading — config then layers: `HEAD` each blob and skip it when present,
+ *     otherwise stream it. Compressed-on-push layers are gzipped again with the same
+ *     settings; zlib is deterministic, and the registry rejects the upload if the bytes
+ *     don't match the digest computed in phase 1.
+ *  3. manifest — re-check a protected release tag (another push may have landed it while
+ *     this one waited in the queue), then `PUT` the manifest under the tag.
+ */
+
+import { createReadStream } from "node:fs";
+import { createHash } from "node:crypto";
+import { createGzip } from "node:zlib";
+import { PassThrough, Writable, type Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { MEDIA, type BlobPlan, type ImagePlan } from "./archive.ts";
+import type { Registry } from "./registry.ts";
+
+export const RELEASE_TAG = /^v?\d+\.\d+\.\d+$/;
+const GZIP_LEVEL = 6;
+
+export type Phase = "queued" | "preparing" | "uploading" | "manifest" | "done";
+
+export interface PushProgress {
+	phase: Phase;
+	layersTotal: number;
+	blobsUploaded: number;
+	blobsSkipped: number;
+	/** Bytes of every blob the image references, as stored in the registry. Known once preparing ends. */
+	bytesTotal: number;
+	bytesSent: number;
+}
+
+export interface PushResult {
+	digest: string;
+	mediaType: string;
+	/** Total size of config + layers in the registry. */
+	size: number;
+	blobsUploaded: number;
+	blobsSkipped: number;
+}
+
+interface Descriptor {
+	mediaType: string;
+	digest: string;
+	size: number;
+}
+
+export class PushRefused extends Error {}
+
+function memberStream(fd: number, plan: BlobPlan): Readable {
+	if (plan.member.size === 0) {
+		const empty = new PassThrough();
+		empty.end();
+		return empty;
+	}
+	return createReadStream("", { fd, start: plan.member.offset, end: plan.member.offset + plan.member.size - 1, autoClose: false });
+}
+
+function gzipStream(fd: number, plan: BlobPlan): Readable {
+	const src = memberStream(fd, plan);
+	const gz = createGzip({ level: GZIP_LEVEL });
+	src.on("error", (e) => gz.destroy(e));
+	return src.pipe(gz);
+}
+
+async function gzipDigest(fd: number, plan: BlobPlan): Promise<{ digest: string; size: number }> {
+	const hash = createHash("sha256");
+	let size = 0;
+	await pipeline(
+		gzipStream(fd, plan),
+		new Writable({
+			write(chunk: Buffer, _enc, cb) {
+				hash.update(chunk);
+				size += chunk.length;
+				cb();
+			}
+		})
+	);
+	return { digest: `sha256:${hash.digest("hex")}`, size };
+}
+
+const sha256 = (b: Buffer): string => `sha256:${createHash("sha256").update(b).digest("hex")}`;
+
+export async function pushImage(
+	registry: Registry,
+	fd: number,
+	plan: ImagePlan,
+	tag: string,
+	opts: { protectReleaseTags: boolean; onProgress?: (p: PushProgress) => void }
+): Promise<PushResult> {
+	const progress: PushProgress = { phase: "preparing", layersTotal: plan.layers.length, blobsUploaded: 0, blobsSkipped: 0, bytesTotal: 0, bytesSent: 0 };
+	const report = () => opts.onProgress?.({ ...progress });
+	report();
+
+	const layers: Descriptor[] = [];
+	for (const l of plan.layers) {
+		if (l.compress) {
+			const { digest, size } = await gzipDigest(fd, l);
+			layers.push({ mediaType: l.mediaType, digest, size });
+		} else {
+			layers.push({ mediaType: l.mediaType, digest: l.sourceDigest ?? (await rawDigest(fd, l)), size: l.sourceSize });
+		}
+	}
+	const config: Descriptor = { mediaType: plan.config.mediaType, digest: plan.config.digest, size: plan.config.sourceSize };
+	progress.bytesTotal = config.size + layers.reduce((n, d) => n + d.size, 0);
+	progress.phase = "uploading";
+	report();
+
+	const blobs: Array<{ desc: Descriptor; plan: BlobPlan }> = [{ desc: config, plan: plan.config }, ...layers.map((desc, i) => ({ desc, plan: plan.layers[i]! }))];
+	const seen = new Set<string>();
+	for (const { desc, plan: p } of blobs) {
+		if (seen.has(desc.digest) || (await registry.blobExists(desc.digest))) {
+			seen.add(desc.digest);
+			progress.blobsSkipped++;
+			report();
+			continue;
+		}
+		seen.add(desc.digest);
+		await registry.uploadBlob(desc.digest, desc.size, () => (p.compress ? gzipStream(fd, p) : memberStream(fd, p)), (n) => {
+			progress.bytesSent += n;
+			report();
+		});
+		progress.blobsUploaded++;
+		report();
+	}
+
+	progress.phase = "manifest";
+	report();
+	let bytes: Buffer;
+	let mediaType: string;
+	if (plan.verbatimManifest) {
+		bytes = plan.verbatimManifest.bytes;
+		mediaType = plan.verbatimManifest.mediaType;
+	} else {
+		mediaType = MEDIA.ociManifest;
+		const cfg = { ...config, mediaType: config.mediaType === MEDIA.dockerConfig ? MEDIA.ociConfig : config.mediaType };
+		bytes = Buffer.from(
+			JSON.stringify({
+				schemaVersion: 2,
+				mediaType,
+				config: cfg,
+				layers: layers.map((l) => ({ ...l, mediaType: l.mediaType === MEDIA.dockerLayerGzip ? MEDIA.ociLayerGzip : l.mediaType })),
+				...(plan.annotations ? { annotations: plan.annotations } : {})
+			})
+		);
+	}
+	if (opts.protectReleaseTags && RELEASE_TAG.test(tag) && (await registry.manifestExists(tag))) {
+		throw new PushRefused(`ghcr_push: tag ${tag} already exists on ${registry.repository} and release tags are protected — push a new version instead`);
+	}
+	await registry.putManifest(tag, mediaType, bytes);
+	progress.phase = "done";
+	report();
+	return { digest: sha256(bytes), mediaType, size: progress.bytesTotal, blobsUploaded: progress.blobsUploaded, blobsSkipped: progress.blobsSkipped };
+}
+
+/** sha256 of a member streamed from the archive (docker-archive layers that are already compressed). */
+async function rawDigest(fd: number, plan: BlobPlan): Promise<string> {
+	const hash = createHash("sha256");
+	await pipeline(
+		memberStream(fd, plan),
+		new Writable({
+			write(chunk: Buffer, _enc, cb) {
+				hash.update(chunk);
+				cb();
+			}
+		})
+	);
+	return `sha256:${hash.digest("hex")}`;
+}