@edgehero/pi-dispatch 2.0.0 → 3.0.0

package/src/project-id.mjs

@@ -0,0 +1,17 @@
+/**
+ * The project id rule (issue #499, INT-PROJECTS-FILE-CONTRACT), in a module with NO imports. `run-history.mjs` checks a
+ * record's `project` against it, and the admin loads run-history inside pi, so the record module must not drag the
+ * projects parser's graph (config.mjs and its fs, os and child_process) in with it: the rule `model-ref.mjs` keeps for
+ * the same reader. `projects.mjs` re-exports both names.
+ */
+
+/**
+ * A project id: lowercase, 1 to 32 characters, free of `:`, `#` and `/`, so it can enter a run record and a Valkey key
+ * without escaping.
+ */
+export const PROJECT_ID_RE = /^[a-z0-9][a-z0-9-]{0,31}$/;
+
+/** Is this a well-formed project id? The record path's charset check. */
+export function isProjectId(value) {
+	return typeof value === "string" && PROJECT_ID_RE.test(value);
+}

package/src/projects.mjs

@@ -0,0 +1,238 @@
+/**
+ * Projects (issue #499, INT-PROJECTS-FILE-CONTRACT): a named group of repos and folders. One `projects.json` of
+ * `{ version: 1, projects: [{ id, name?, members }] }`. A job whose scope is a member belongs to that project, and the
+ * project's id is written into the job's run record (`project`), so spend can later be read and capped per project.
+ *
+ * This module is pure and fs-injectable, on the scoped-limits.mjs pattern: `parseProjects` validates the file TEXT
+ * fail-loud, `loadProjects` layers the one fs read on top, and `projectOf` is what the pickup gate and the record path
+ * consume. The worker holds the parsed list in a watched ref with a last-good copy (start.mjs).
+ *
+ * `version` is REQUIRED and a newer one is refused: which project a scope belongs to decides which cap applies to it
+ * (issue #499 part B), so this is a money file, and a field an old build silently drops could widen a cap.
+ *
+ * `name` is display text for the panel. It never enters a record or a log line: the record carries the `id`, which is
+ * charset-checked, and so stays free of personal data by construction. For the same reason no error message here
+ * quotes a name, and an invalid-JSON error does not quote the parser's message, which carries file text.
+ *
+ * Custom: projects validated inline per scoped-limits.mjs precedent; zod not in deps
+ */
+
+import { existsSync as fsExistsSync, readFileSync as fsReadFileSync } from "node:fs";
+import { isAbsolute, resolve } from "node:path";
+import { configError } from "./config.mjs";
+import { fingerprint } from "./fingerprint.mjs";
+import { hash16 } from "./fleet-lease.mjs";
+import { PROJECT_ID_RE, isProjectId } from "./project-id.mjs";
+import { parseScopeString, qualifiedScopeOf } from "./pause-windows.mjs";
+import { canonicalScope } from "./scoped-limits.mjs";
+
+/** The highest schema version this build reads. A file declaring a higher one is refused loudly. */
+export const PROJECTS_VERSION = 1;
+
+// The id rule lives in an import-free module, so the run record can check an id without this module's graph.
+export { PROJECT_ID_RE, isProjectId };
+
+/** The longest `name` accepted, in UTF-16 code units. A display label, not a document. */
+const NAME_MAX = 120;
+
+/**
+ * Parse, validate and normalize the projects file TEXT. Returns the normalized list: each project rebuilt as an
+ * explicit `{ id, name, members }` literal (`name` null when absent, members in their stored spelling, unknown fields
+ * dropped by the operator-file policy). Throws `configError` on anything malformed. `path` is for messages only.
+ *
+ * Members use the scope grammar of `parseScopeString` (issue #498):
+ *   - a forge-qualified scope (`github:acme/web`), stored as `<kind>:<repo>`;
+ *   - an absolute folder (`/srv/shop-tools`), stored resolved, the spelling `canonicalScope` gives a local job.
+ * A bare `owner/name` is refused: it names that repo on every forge, and this file has no legacy to keep. A relative
+ * folder, a drive path on a host where it is not absolute, `*` and globs are refused too: each would be a member no
+ * job's scope can ever equal.
+ *
+ * Refused across the file: a duplicate id, a scope claimed by two projects (both ids named, one project per scope),
+ * a scope listed twice in one project, and an empty `members`.
+ */
+export function parseProjects(text, path) {
+	try {
+		return parseProjectsText(text, path);
+	} catch (error) {
+		// Every refusal ESCAPED at the source (PR #569's review): a folder member may hold a C1 or bidi character, and this
+		// message reaches the worker's `projects_reload_invalid` log line, doctor and an admin tool's error. JSON quoting
+		// keeps C0 visible but leaves C1, bidi and zero-width characters raw.
+		if (error?.piDispatchConfig === true) error.message = escapeControls(error.message);
+		throw error;
+	}
+}
+
+/**
+ * The characters `escapeControls` writes out, the SAME set the admin panel's `escapeInterpreted` escapes (PR #569's
+ * second review; `admin/test/projects.test.mjs` compares the two over every code point): the controls (C0, DEL, C1),
+ * every format character (bidi controls and isolates, zero-width and invisible ones, the tag block) but the two joiners
+ * that compose (U+200C, U+200D), the line and paragraph separators, every blank that is not U+0020 but draws one
+ * column (the no-break and fixed-width spaces, U+2800), the Hangul fillers, and the unassigned code points the
+ * terminal draws as nothing (U+2065, U+FFF0-U+FFF8, the special-purpose plane outside its variation selectors).
+ * U+3000 is kept, as the panel keeps it: it draws two columns, an ordinary full-width space.
+ */
+const ESCAPED = /(?![\u200c\u200d\u3000\u{e0100}-\u{e01ef}])[\p{Cc}\p{Cf}\p{Zl}\p{Zp}\p{Zs}\u2800\u115f\u1160\u3164\uffa0\u2065\ufff0-\ufff8\u{e0000}-\u{e0fff}]/gu;
+
+/**
+ * Text with the characters that change what a reader SEES written out as `\\u{XXXX}` (`ESCAPED`), U+0020 kept. For a
+ * message about operator text that reaches a log line or a terminal. The worker's twin of the panel's
+ * `escapeInterpreted`; it imports nothing.
+ */
+export function escapeControls(text) {
+	return String(text ?? "").replace(ESCAPED, (ch) => (ch === " " ? ch : `\\u{${ch.codePointAt(0).toString(16).toUpperCase().padStart(4, "0")}}`));
+}
+
+function parseProjectsText(text, path) {
+	let parsed;
+	try {
+		parsed = JSON.parse(text);
+	} catch (error) {
+		// Not the parser's own message: it quotes the file's text around the fault, and that text may be a `name`.
+		const at = /position (\d+)/.exec(String(error?.message))?.[1];
+		throw configError(`projects file is not valid JSON${at === undefined ? "" : ` (at character ${at})`}: ${path}`);
+	}
+	if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
+		throw configError(`projects file must be an object with "version" and "projects": ${path}`);
+	}
+	const version = parsed.version;
+	if (!Number.isInteger(version) || version < 1) {
+		throw configError(`projects file must have "version": ${PROJECTS_VERSION} (an integer >= 1): ${path}`);
+	}
+	if (version > PROJECTS_VERSION) {
+		throw configError(`projects file written by a newer pi-dispatch (version ${version}; this build understands ${PROJECTS_VERSION}): ${path}`);
+	}
+	if (!Array.isArray(parsed.projects)) throw configError(`projects file must have a "projects" array: ${path}`);
+	const projects = parsed.projects.map((entry, index) => normalizeProject(entry, index, path));
+	const ids = new Map();
+	const owners = new Map();
+	projects.forEach((project, index) => {
+		if (ids.has(project.id)) {
+			throw configError(`project at index ${index}: duplicate id "${project.id}" (first at index ${ids.get(project.id)}): ${path}`);
+		}
+		ids.set(project.id, index);
+		for (const member of project.members) {
+			const owner = owners.get(member);
+			// One project per scope: a job in two projects would have two project ledgers and an unclear refusal.
+			if (owner !== undefined) {
+				throw configError(`project at index ${index}: ${JSON.stringify(member)} is claimed by both "${owner}" and "${project.id}" (a scope belongs to one project): ${path}`);
+			}
+			owners.set(member, project.id);
+		}
+	});
+	return projects;
+}
+
+function normalizeProject(entry, index, path) {
+	const at = `project at index ${index}`;
+	if (entry === null || typeof entry !== "object" || Array.isArray(entry)) throw configError(`${at}: must be an object: ${path}`);
+	if (!isProjectId(entry.id)) {
+		throw configError(`${at}: id must match ${PROJECT_ID_RE.source} (lowercase letters, digits and "-", 1 to 32 characters, starting with a letter or digit): ${path}`);
+	}
+	const id = entry.id;
+	let name = null;
+	if (entry.name !== undefined && entry.name !== null) {
+		// The name's own text is never echoed: it is free text, and this message can reach a log line.
+		if (typeof entry.name !== "string" || entry.name.trim() === "" || entry.name.length > NAME_MAX || /[\u0000-\u001f\u007f]/.test(entry.name)) {
+			throw configError(`${at} ("${id}"): name, when given, must be a string of 1 to ${NAME_MAX} characters with no control characters: ${path}`);
+		}
+		name = entry.name.trim();
+	}
+	if (!Array.isArray(entry.members) || entry.members.length === 0) {
+		throw configError(`${at} ("${id}"): members must be a non-empty array of scopes (github:owner/name or an absolute folder): ${path}`);
+	}
+	const members = [];
+	entry.members.forEach((raw, m) => {
+		const member = normalizeMember(raw, `${at} ("${id}") member ${m}`, path);
+		if (members.includes(member)) throw configError(`${at} ("${id}") member ${m}: ${JSON.stringify(member)} is listed twice: ${path}`);
+		members.push(member);
+	});
+	return { id, name, members };
+}
+
+/** One member, in its stored spelling, or the refusal naming where it sits. */
+function normalizeMember(raw, at, path) {
+	if (typeof raw !== "string" || raw.trim() === "") throw configError(`${at}: must be a non-empty string: ${path}`);
+	const trimmed = raw.trim().normalize("NFC");
+	if (trimmed.includes("*")) throw configError(`${at}: members match exactly; a scope containing "*" is refused (no globs): ${path}`);
+	let form;
+	try {
+		form = parseScopeString(trimmed);
+	} catch (error) {
+		throw configError(`${at}: ${error.message}: ${path}`);
+	}
+	if (form.type === "qualified") return `${form.kind}:${form.repo}`;
+	if (form.type === "bare") {
+		throw configError(`${at}: ${JSON.stringify(trimmed)} is a bare repo, which names that repo on every forge; write it with its forge, such as github:${trimmed}, or give an absolute folder: ${path}`);
+	}
+	// `local`: platform-native isAbsolute, so a drive path on a POSIX host is refused rather than kept as a member no
+	// job here can have (scoped limits keep such a row verbatim and inert; a new file need not).
+	if (!isAbsolute(trimmed)) throw configError(`${at}: ${JSON.stringify(trimmed)} is not an absolute path on this host: ${path}`);
+	return resolve(trimmed);
+}
+
+/**
+ * Load and validate the projects file named by `config.projectsFile`. Returns `[]` when it is unset (no projects, a
+ * valid deployment). An empty string is a value, so it reaches `existsSync` and is refused, as the scoped-limits key's
+ * is. `readFileSync`/`existsSync` are injectable for tests.
+ */
+export function loadProjects(config, { readFileSync = fsReadFileSync, existsSync = fsExistsSync } = {}) {
+	const path = config.projectsFile;
+	if (path === null || path === undefined) return [];
+	if (!existsSync(path)) throw configError(`projects file does not exist: ${escapeControls(path)}`);
+	return parseProjects(readFileSync(path, "utf8"), path);
+}
+
+/**
+ * The id of the project this job belongs to, or null. `job` is job data (`kind`, `repo`, `folder`). A forge job is
+ * matched by its forge-qualified scope (`github:acme/web`), never by its bare repo, so a GitHub job and a Forgejo job
+ * for `acme/web` can sit in different projects. A local job is matched by its resolved folder (`canonicalScope`), the
+ * spelling a member is stored in, so `/srv/shop-tools/` matches the member `/srv/shop-tools`.
+ */
+export function projectOf(job, projects) {
+	if (!Array.isArray(projects) || projects.length === 0) return null;
+	const scope = memberScopeOf(job);
+	if (typeof scope !== "string" || scope === "") return null;
+	for (const project of projects) {
+		if (Array.isArray(project?.members) && project.members.includes(scope)) return project.id;
+	}
+	return null;
+}
+
+/**
+ * The scope a job is matched to a member by (`projectOf`): its forge-qualified scope, or for a local job its resolved
+ * folder. When `projectOf` found a project, this IS the member it matched, in the stored spelling a priorities plan
+ * names by `scopeRef` (issue #504 part B), so the repo share a plan gave it needs no second matching rule.
+ */
+export function memberScopeOf(job) {
+	return job?.kind === "local" ? canonicalScope(job) : qualifiedScopeOf(job);
+}
+
+/**
+ * What `fpProjects` hashes (issue #499 part C, INT-HOST-REGISTRY-CONTRACT), exported so a test can read it: one entry
+ * per project, sorted by id, as `{ id, members }` where `members` is the sorted 16-hex hash of each member. Never a
+ * `name`, which is free text, and never a member in clear: a folder member is a host path and a repo member a
+ * repository name, and the registry's content rule keeps both out of a Valkey value, even inside a digest's input.
+ * The id is charset-checked operator text, the same admissibility a run record gives it.
+ *
+ * Membership is what the comparison is about: two hosts that put one scope in two projects, or a scope in a project on
+ * one host and in none on the other, record different projects and count the scope against different project rows.
+ * A `name` decides nothing, so renaming the display text on one host is not a disagreement.
+ */
+export function projectsFingerprintInput(projects) {
+	return (Array.isArray(projects) ? projects : [])
+		.filter((p) => isProjectId(p?.id))
+		.map((p) => ({ id: p.id, members: (Array.isArray(p.members) ? p.members : []).map((m) => hash16(String(m))).sort() }))
+		.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+}
+
+/**
+ * The 16-hex fingerprint `fpProjects` of a host's live projects (`fingerprint.mjs`). It never abstains: a host with no
+ * projects file publishes the fingerprint of no projects, because a host that puts a repo in no project while a peer
+ * puts it in one is the disagreement worth seeing.
+ */
+export function projectsFingerprint(projects) {
+	return fingerprint(projectsFingerprintInput(projects));
+}
+
+/** The fingerprint of no projects: how a reader tells whether projects are in use anywhere on the fleet. */
+export const EMPTY_PROJECTS_FINGERPRINT = projectsFingerprint([]);

package/src/provider-key.mjs

@@ -27,15 +27,40 @@
 // worker/test/provider-key.test.mjs -- never against a second copy of a table.
 export const OAUTH_KEY_RE = /_OAUTH_TOKEN$/;
 
+// The second fact of the same kind, added with the pi 0.99.1 bump (issue #509): a variable pi reads for a
+// provider that is NOT an API key, because pi sends it as a different header. At 0.99.1 pi lists
+// ANTHROPIC_AUTH_TOKEN FIRST for `anthropic` (pi-ai/dist/env-api-keys.js getApiKeyEnvVars), and its
+// resolver (pi-ai/dist/providers/anthropic.js) sends it as `Authorization: Bearer <value>` AHEAD of both
+// ANTHROPIC_OAUTH_TOKEN and ANTHROPIC_API_KEY. Without this rule `apiKeyVariable` picked it: an auth.json
+// API key would have been written into the container under the bearer name, so pi would send it as
+// `Authorization: Bearer` rather than as the `x-api-key` an API key travels in, and doctor would have told
+// an operator to set the bearer variable.
+// pi's own `getEnvApiKey` skips it for the same reason, which is the evidence that the distinction is real
+// rather than ours. A suffix rule, for OAUTH_KEY_RE's reason: a set naming one variable would silently
+// bless the next provider's bearer variable. Pinned against pi in worker/test/provider-key.test.mjs.
+export const BEARER_KEY_RE = /_AUTH_TOKEN$/;
+
+/**
+ * Why a variable pi reads is not an API key, or null when it is one. The two answers need different
+ * advice, so the caller is told which: an OAuth token is a subscription login that expires, a bearer token
+ * is a credential pi sends in another header and reads BEFORE the API key.
+ */
+export function nonApiKeyKind(name) {
+	if (OAUTH_KEY_RE.test(name)) return "oauth";
+	if (BEARER_KEY_RE.test(name)) return "bearer";
+	return null;
+}
+
 /**
  * The api-key variable to write and to name, given pi's candidate list for a provider in pi's own
- * precedence order. Never the OAuth token, whatever that precedence says: pi returns
- * ANTHROPIC_OAUTH_TOKEN first, "set your subscription login" is wrong advice for an unattended service,
- * and an api-key credential written under that name would be a value whose variable lies about what it
- * is. Falls back to the first candidate only for a provider with no non-OAuth variable at all, which is
- * no provider pi has today; `null` for a provider pi reads no key variable for, which is the caller's
- * cue to refuse rather than to guess.
+ * precedence order. Never the OAuth token and never the bearer token, whatever that precedence says: pi
+ * returns ANTHROPIC_AUTH_TOKEN then ANTHROPIC_OAUTH_TOKEN before ANTHROPIC_API_KEY, "set your subscription
+ * login" is wrong advice for an unattended service, and an api-key credential written under either name
+ * would be a value whose variable lies about what it is (under the bearer name pi would also send it in
+ * the wrong header). Falls back to the first candidate only for a provider with no API-key variable at
+ * all, which is no provider pi has today; `null` for a provider pi reads no key variable for, which is the
+ * caller's cue to refuse rather than to guess.
  */
 export function apiKeyVariable(candidates) {
-	return candidates.find((name) => !OAUTH_KEY_RE.test(name)) ?? candidates[0] ?? null;
+	return candidates.find((name) => nonApiKeyKind(name) === null) ?? candidates[0] ?? null;
 }