@davidvornholt/standards 0.18.0 → 0.20.0

package/src/creds-store-github.ts

@@ -0,0 +1,121 @@
+import { isNonEmptyString, isRecord } from './github-settings-parse';
+
+export type GithubBrokerApp = {
+  readonly owner: string | null;
+  readonly appId: number;
+  readonly slug: string;
+  readonly htmlUrl: string;
+  readonly clientId: string;
+  readonly privateKey: string;
+};
+
+export const isGithubBrokerApp = (value: unknown): value is GithubBrokerApp =>
+  isRecord(value) &&
+  (value.owner === null || isNonEmptyString(value.owner)) &&
+  typeof value.appId === 'number' &&
+  Number.isInteger(value.appId) &&
+  isNonEmptyString(value.slug) &&
+  isNonEmptyString(value.htmlUrl) &&
+  isNonEmptyString(value.clientId) &&
+  isNonEmptyString(value.privateKey);
+
+const parseGithubApp = (
+  raw: unknown,
+  owner: string | null,
+): GithubBrokerApp => {
+  if (!isRecord(raw)) {
+    throw new Error('invalid github: run `standards creds login github`');
+  }
+  const app = {
+    owner,
+    appId: raw.app_id,
+    slug: raw.slug,
+    htmlUrl: raw.html_url,
+    clientId: raw.client_id,
+    privateKey: raw.private_key,
+  };
+  if (!isGithubBrokerApp(app)) {
+    throw new Error('invalid github: run `standards creds login github`');
+  }
+  return app;
+};
+
+export const validateGithubApps = (
+  apps: ReadonlyArray<GithubBrokerApp>,
+  path: string,
+): ReadonlyArray<GithubBrokerApp> => {
+  const owners = new Set<string>();
+  const appIds = new Set<number>();
+  for (const app of apps) {
+    if (!isGithubBrokerApp(app)) {
+      throw new Error(
+        `${path}: invalid GitHub App; run \`standards creds login github\``,
+      );
+    }
+    if (app.owner === null) {
+      if (apps.length !== 1) {
+        throw new Error(
+          `${path}: the ownerless legacy GitHub App must be migrated before another App can be stored`,
+        );
+      }
+    } else {
+      const owner = app.owner.toLowerCase();
+      if (owners.has(owner)) {
+        throw new Error(
+          `${path}: duplicate GitHub owner ${app.owner}; keep one broker App per account`,
+        );
+      }
+      owners.add(owner);
+    }
+    if (appIds.has(app.appId)) {
+      throw new Error(
+        `${path}: duplicate GitHub App id ${app.appId}; keep each App once`,
+      );
+    }
+    appIds.add(app.appId);
+  }
+  return apps;
+};
+
+export const parseGithubApps = (
+  raw: unknown,
+  path: string,
+): ReadonlyArray<GithubBrokerApp> => {
+  if (raw === undefined || raw === null) {
+    return [];
+  }
+  if (!Array.isArray(raw)) {
+    return [parseGithubApp(raw, null)];
+  }
+  const apps = raw.map((entry) => {
+    if (!(isRecord(entry) && isNonEmptyString(entry.owner))) {
+      throw new Error(
+        'invalid github owner: run `standards creds login github`',
+      );
+    }
+    return parseGithubApp(entry, entry.owner);
+  });
+  return validateGithubApps(apps, path);
+};
+
+const appDocument = (
+  app: GithubBrokerApp,
+): Readonly<Record<string, unknown>> => ({
+  ...(app.owner === null ? {} : { owner: app.owner }),
+  // biome-ignore lint/style/useNamingConvention: The persisted broker YAML contract uses snake_case.
+  app_id: app.appId,
+  slug: app.slug,
+  // biome-ignore lint/style/useNamingConvention: The persisted broker YAML contract uses snake_case.
+  html_url: app.htmlUrl,
+  // biome-ignore lint/style/useNamingConvention: The persisted broker YAML contract uses snake_case.
+  client_id: app.clientId,
+  // biome-ignore lint/style/useNamingConvention: The persisted broker YAML contract uses snake_case.
+  private_key: app.privateKey,
+});
+
+export const githubStoreDocument = (
+  apps: ReadonlyArray<GithubBrokerApp>,
+): unknown =>
+  apps.length === 1 && apps[0]?.owner === null
+    ? appDocument(apps[0])
+    : apps.map(appDocument);

package/src/creds-store-lock-holder.ts

@@ -0,0 +1,98 @@
+import { randomUUID } from 'node:crypto';
+import { mkdir, rename, rmdir, unlink, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import type { Worker } from 'node:worker_threads';
+import { startBrokerLockLease } from './creds-store-lock-lease';
+import { isRecord } from './github-settings-parse';
+
+const OWNER_ONLY_DIR_MODE = 0o700;
+const OWNER_ONLY_FILE_MODE = 0o600;
+const HOLDER_PREFIX = 'holder-';
+const HOLDER_SUFFIX = '.json';
+const LEASE_DIVISOR = 3;
+
+export type HeldBrokerLock = {
+  readonly holderPath: string;
+  readonly lease: Worker;
+  readonly lockPath: string;
+};
+
+const isErrorCode = (error: unknown, code: string): boolean =>
+  isRecord(error) && error.code === code;
+
+const cleanupCandidate = async (
+  candidatePath: string,
+  holderPath: string,
+): Promise<void> => {
+  await unlink(holderPath).catch(() => undefined);
+  await rmdir(candidatePath).catch(() => undefined);
+};
+
+export const tryAcquireBrokerLock = async (
+  lockPath: string,
+  staleMs: number,
+): Promise<HeldBrokerLock | null> => {
+  const generation = randomUUID();
+  const candidatePath = `${lockPath}.pending-${generation}`;
+  const candidateHolderPath = join(
+    candidatePath,
+    `${HOLDER_PREFIX}${generation}${HOLDER_SUFFIX}`,
+  );
+  await mkdir(candidatePath, { mode: OWNER_ONLY_DIR_MODE });
+  try {
+    await writeFile(candidateHolderPath, JSON.stringify({ generation }), {
+      flag: 'wx',
+      mode: OWNER_ONLY_FILE_MODE,
+    });
+    try {
+      await rename(candidatePath, lockPath);
+    } catch (error) {
+      if (isErrorCode(error, 'EEXIST') || isErrorCode(error, 'ENOTEMPTY')) {
+        await cleanupCandidate(candidatePath, candidateHolderPath);
+        return null;
+      }
+      throw error;
+    }
+  } catch (error) {
+    await cleanupCandidate(candidatePath, candidateHolderPath);
+    throw error;
+  }
+  const holderPath = join(
+    lockPath,
+    `${HOLDER_PREFIX}${generation}${HOLDER_SUFFIX}`,
+  );
+  try {
+    const lease = startBrokerLockLease(
+      holderPath,
+      Math.max(1, Math.floor(staleMs / LEASE_DIVISOR)),
+    );
+    return { holderPath, lease, lockPath };
+  } catch (error) {
+    await unlink(holderPath).catch(() => undefined);
+    await rmdir(lockPath).catch(() => undefined);
+    throw error;
+  }
+};
+
+export const releaseBrokerLock = async ({
+  holderPath,
+  lease,
+  lockPath,
+}: HeldBrokerLock): Promise<void> => {
+  await lease.terminate();
+  try {
+    await unlink(holderPath);
+  } catch (error) {
+    if (isErrorCode(error, 'ENOENT')) {
+      return;
+    }
+    throw error;
+  }
+  try {
+    await rmdir(lockPath);
+  } catch (error) {
+    if (!isErrorCode(error, 'ENOENT')) {
+      throw error;
+    }
+  }
+};

package/src/creds-store-lock-inspection.ts

@@ -0,0 +1,125 @@
+import { readdir, readFile, rmdir, stat, unlink } from 'node:fs/promises';
+import { join } from 'node:path';
+import { isRecord } from './github-settings-parse';
+
+const HOLDER_PREFIX = 'holder-';
+const HOLDER_SUFFIX = '.json';
+
+export type BrokerLockAvailability = 'blocked' | 'live' | 'retry';
+
+const isErrorCode = (error: unknown, code: string): boolean =>
+  isRecord(error) && error.code === code;
+
+type ExistingGeneration =
+  | { readonly kind: 'missing' }
+  | { readonly kind: 'unknown' }
+  | {
+      readonly entryPath: string | null;
+      readonly kind: 'incomplete';
+      readonly modifiedAt: number;
+    }
+  | {
+      readonly entryPath: string;
+      readonly kind: 'valid';
+      readonly modifiedAt: number;
+    };
+
+const directoryModifiedAt = async (
+  lockPath: string,
+): Promise<number | null> => {
+  try {
+    return (await stat(lockPath)).mtimeMs;
+  } catch {
+    return null;
+  }
+};
+
+const readExistingGeneration = async (
+  lockPath: string,
+): Promise<ExistingGeneration> => {
+  let names: ReadonlyArray<string>;
+  try {
+    names = await readdir(lockPath);
+  } catch (error) {
+    return isErrorCode(error, 'ENOENT')
+      ? { kind: 'missing' }
+      : { kind: 'unknown' };
+  }
+  if (names.length === 0) {
+    const modifiedAt = await directoryModifiedAt(lockPath);
+    return modifiedAt === null
+      ? { kind: 'missing' }
+      : { entryPath: null, kind: 'incomplete', modifiedAt };
+  }
+  const entryName = names.length === 1 ? names[0] : undefined;
+  if (entryName === undefined) {
+    return { kind: 'unknown' };
+  }
+  const entryPath = join(lockPath, entryName);
+  let modifiedAt: number;
+  try {
+    modifiedAt = (await stat(entryPath)).mtimeMs;
+  } catch {
+    return { kind: 'unknown' };
+  }
+  const incomplete = {
+    entryPath,
+    kind: 'incomplete' as const,
+    modifiedAt,
+  };
+  if (
+    !(entryName.startsWith(HOLDER_PREFIX) && entryName.endsWith(HOLDER_SUFFIX))
+  ) {
+    return incomplete;
+  }
+  try {
+    const decoded: unknown = JSON.parse(await readFile(entryPath, 'utf8'));
+    const generation = entryName.slice(
+      HOLDER_PREFIX.length,
+      -HOLDER_SUFFIX.length,
+    );
+    return isRecord(decoded) && decoded.generation === generation
+      ? { entryPath, kind: 'valid', modifiedAt }
+      : incomplete;
+  } catch {
+    return incomplete;
+  }
+};
+
+const removeObservedGeneration = async (
+  lockPath: string,
+  entryPath: string | null,
+): Promise<boolean> => {
+  if (entryPath !== null) {
+    try {
+      await unlink(entryPath);
+    } catch (error) {
+      return isErrorCode(error, 'ENOENT');
+    }
+  }
+  try {
+    await rmdir(lockPath);
+    return true;
+  } catch (error) {
+    return isErrorCode(error, 'ENOENT');
+  }
+};
+
+export const inspectBrokerLock = async (
+  lockPath: string,
+  staleMs: number,
+): Promise<BrokerLockAvailability> => {
+  const generation = await readExistingGeneration(lockPath);
+  if (generation.kind === 'missing') {
+    return 'retry';
+  }
+  if (generation.kind === 'unknown') {
+    return 'blocked';
+  }
+  if (Date.now() - generation.modifiedAt < staleMs) {
+    return generation.kind === 'valid' ? 'live' : 'blocked';
+  }
+  return (await removeObservedGeneration(lockPath, generation.entryPath))
+    ? 'retry'
+    : 'blocked';
+};

package/src/creds-store-lock-lease.ts

@@ -0,0 +1,32 @@
+import { utimes } from 'node:fs/promises';
+import {
+  isMainThread,
+  parentPort,
+  Worker,
+  workerData,
+} from 'node:worker_threads';
+
+type LeaseWorkerData = {
+  readonly holderPath: string;
+  readonly intervalMs: number;
+};
+
+if (!isMainThread) {
+  const { holderPath, intervalMs } = workerData as LeaseWorkerData;
+  const timer = setInterval(
+    () => utimes(holderPath, new Date(), new Date()).catch(() => undefined),
+    intervalMs,
+  );
+  parentPort?.on('close', () => clearInterval(timer));
+}
+
+export const startBrokerLockLease = (
+  holderPath: string,
+  intervalMs: number,
+): Worker => {
+  const lease = new Worker(import.meta.url, {
+    workerData: { holderPath, intervalMs } satisfies LeaseWorkerData,
+  });
+  lease.unref();
+  return lease;
+};

package/src/creds-store-lock.ts

@@ -1,12 +1,17 @@
-// Broker store lock: a mkdir-based mutex serializing read-modify-write
-// cycles on broker.yaml. Store writes are sub-second, so a lock directory
-// older than the staleness window can only belong to a dead process and is
-// broken automatically; the timeout error names the directory and the manual
-// remediation for the pathological rest.
+// Broker store lock: a fully initialized generation is atomically renamed into
+// place before its operation starts. A worker-thread lease remains live while
+// synchronous SOPS/Nix blocks the main event loop. Cleanup unlinks the exact
+// generation token before removing the directory, so an old owner cannot
+// remove a replacement.
 
-import { mkdir, rmdir, stat } from 'node:fs/promises';
+import { mkdir } from 'node:fs/promises';
 import { dirname } from 'node:path';
-import { isRecord } from './github-settings-parse';
+import {
+  type HeldBrokerLock,
+  releaseBrokerLock,
+  tryAcquireBrokerLock,
+} from './creds-store-lock-holder';
+import { inspectBrokerLock } from './creds-store-lock-inspection';
 
 const OWNER_ONLY_DIR_MODE = 0o700;
 
@@ -23,59 +28,30 @@ const DEFAULT_STALE_MS = 30_000;
 const sleep = (ms: number): Promise<void> =>
   new Promise((resolve) => setTimeout(resolve, ms));
 
-const tryAcquire = async (lockPath: string): Promise<boolean> => {
-  try {
-    await mkdir(lockPath, { mode: OWNER_ONLY_DIR_MODE });
-    return true;
-  } catch (error) {
-    if (isRecord(error) && error.code === 'EEXIST') {
-      return false;
-    }
-    throw error;
-  }
-};
-
-// True when the contender may immediately retry: the lock vanished, or it
-// was stale and this process removed it (losing the removal race to another
-// contender also lands here — the retry's mkdir arbitrates).
-const breakIfStale = async (
-  lockPath: string,
-  staleMs: number,
-): Promise<boolean> => {
-  try {
-    const info = await stat(lockPath);
-    if (Date.now() - info.mtimeMs < staleMs) {
-      return false;
-    }
-    await rmdir(lockPath);
-    return true;
-  } catch {
-    return true;
-  }
-};
-
 // One acquisition attempt: acquired, or cleared a stale/vanished lock (retry
-// immediately), or waited one retry slot; a live lock past the deadline
-// throws with the remediation hint.
+// immediately), or waited one retry slot. Proven-live generations do not
+// consume the recovery deadline; incomplete or ambiguous generations do.
 const attemptAcquire = async (
   lockPath: string,
   staleMs: number,
   retryMs: number,
   deadline: number,
-): Promise<boolean> => {
-  if (await tryAcquire(lockPath)) {
-    return true;
+): Promise<HeldBrokerLock | null> => {
+  const held = await tryAcquireBrokerLock(lockPath, staleMs);
+  if (held !== null) {
+    return held;
   }
-  if (await breakIfStale(lockPath, staleMs)) {
-    return false;
+  const availability = await inspectBrokerLock(lockPath, staleMs);
+  if (availability === 'retry') {
+    return null;
   }
-  if (Date.now() >= deadline) {
+  if (availability === 'blocked' && Date.now() >= deadline) {
     throw new Error(
       `lock timeout: another creds process holds ${lockPath}; if none is running, remove that directory and retry`,
     );
   }
   await sleep(retryMs);
-  return false;
+  return null;
 };
 
 export const withBrokerLock = async <T>(
@@ -89,14 +65,14 @@ export const withBrokerLock = async <T>(
   const lockPath = `${path}.lock`;
   await mkdir(dirname(path), { recursive: true, mode: OWNER_ONLY_DIR_MODE });
   const deadline = Date.now() + timeoutMs;
-  let acquired = false;
-  while (!acquired) {
+  let held: HeldBrokerLock | null = null;
+  while (held === null) {
     // biome-ignore lint/performance/noAwaitInLoops: lock acquisition retries are inherently sequential.
-    acquired = await attemptAcquire(lockPath, staleMs, retryMs, deadline);
+    held = await attemptAcquire(lockPath, staleMs, retryMs, deadline);
   }
   try {
     return await operation();
   } finally {
-    await rmdir(lockPath);
+    await releaseBrokerLock(held);
   }
 };

package/src/creds-store.ts

@@ -6,23 +6,24 @@ import { basename, dirname, join } from 'node:path';
 import process from 'node:process';
 import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
 import { isCloudflareAccountId } from './creds-cloudflare-account';
+import {
+  type GithubBrokerApp,
+  githubStoreDocument,
+  parseGithubApps,
+  validateGithubApps,
+} from './creds-store-github';
 import { withBrokerLock } from './creds-store-lock';
 import { isNonEmptyString, isRecord } from './github-settings-parse';
-export type GithubBrokerApp = {
-  readonly appId: number;
-  readonly slug: string;
-  readonly htmlUrl: string;
-  readonly clientId: string;
-  readonly privateKey: string;
-};
+
+export type { GithubBrokerApp } from './creds-store-github';
 export type CloudflareBrokerAccount = Readonly<
   Record<'accountId' | 'token', string>
 >;
 export type BrokerStore = {
-  readonly github: GithubBrokerApp | null;
+  readonly github: ReadonlyArray<GithubBrokerApp>;
   readonly cloudflare: ReadonlyArray<CloudflareBrokerAccount>;
 };
-export const EMPTY_BROKER_STORE: BrokerStore = { github: null, cloudflare: [] };
+export const EMPTY_BROKER_STORE: BrokerStore = { github: [], cloudflare: [] };
 const OWNER_ONLY_FILE_MODE = 0o600;
 const OWNER_ONLY_DIR_MODE = 0o700;
 const FILE_MODE_MODULUS = 0o1000;
@@ -48,29 +49,6 @@ export const resolveBrokerPath = (): string =>
     process.env.XDG_CONFIG_HOME ?? join(homedir(), '.config'),
     'standards/broker.yaml',
   );
-const parseGithub = (raw: unknown): GithubBrokerApp | null => {
-  if (raw === undefined || raw === null) {
-    return null;
-  }
-  if (
-    !isRecord(raw) ||
-    typeof raw.app_id !== 'number' ||
-    !Number.isInteger(raw.app_id) ||
-    !isNonEmptyString(raw.slug) ||
-    !isNonEmptyString(raw.html_url) ||
-    !isNonEmptyString(raw.client_id) ||
-    !isNonEmptyString(raw.private_key)
-  ) {
-    throw new Error('invalid github: run `standards creds login github`');
-  }
-  return {
-    appId: raw.app_id,
-    slug: raw.slug,
-    htmlUrl: raw.html_url,
-    clientId: raw.client_id,
-    privateKey: raw.private_key,
-  };
-};
 const validateCloudflareAccounts = (
   accounts: ReadonlyArray<CloudflareBrokerAccount>,
   path: string,
@@ -131,7 +109,7 @@ export const readBrokerStore = async (path: string): Promise<BrokerStore> => {
     throw new Error(`${path} must contain a YAML mapping`);
   }
   return {
-    github: parseGithub(raw.github),
+    github: parseGithubApps(raw.github, path),
     cloudflare: parseCloudflare(raw.cloudflare, path),
   };
 };
@@ -142,21 +120,14 @@ export const updateBrokerStore = async (
 ): Promise<void> =>
   withBrokerLock(path, async () => {
     const store = await update(await readBrokerStore(path));
+    validateGithubApps(store.github, path);
     validateCloudflareAccounts(store.cloudflare, path);
     await writeBrokerStoreUnlocked(path, store, sync);
   });
 const storeDocument = (store: BrokerStore): unknown => ({
-  ...(store.github === null
+  ...(store.github.length === 0
     ? {}
-    : {
-        github: {
-          app_id: store.github.appId,
-          slug: store.github.slug,
-          html_url: store.github.htmlUrl,
-          client_id: store.github.clientId,
-          private_key: store.github.privateKey,
-        },
-      }),
+    : { github: githubStoreDocument(store.github) }),
   ...(store.cloudflare.length === 0
     ? {}
     : {

package/src/github-command-shared.ts

@@ -33,7 +33,14 @@ export const optOutEligibilityProblem = (
     : null;
 
 export const ADMIN_VISIBILITY_ADVICE =
-  'Use a token with read access to repository administration (in CI: ci.github_settings_read_token in secrets/ci.yaml), or verify locally with admin auth';
+  'Use a user-scoped token with read access to repository administration, or verify locally with admin auth; a CI token cannot hold that access, so widening what CI reads with is not the fix';
+
+// Merge settings have a different remedy from everything else this advice used
+// to cover. GraphQL serves them to read-only tokens by design, so reaching this
+// message means the fallback request itself failed — pointing the reader at a
+// permission they cannot obtain would send them after the wrong cause.
+export const MERGE_SETTINGS_VISIBILITY_ADVICE =
+  'REST serves repository merge settings only to write-level viewers, and the GraphQL fallback that answers them for a read-only token did not respond, so this is more likely a failed request than a permission gap: re-run the check, and if it keeps failing verify locally with admin auth';
 
 // Declared state the token cannot see is a gate failure, not a pass with a
 // log line: a gate that cannot perform its comparison fails closed. The advice

package/src/github-live-drift.ts

@@ -13,8 +13,8 @@ import {
 } from './github-api';
 import { resolveHiddenBypassActors } from './github-bypass-actors';
 import {
-  ADMIN_VISIBILITY_ADVICE,
   enforceableRepositorySettings,
+  MERGE_SETTINGS_VISIBILITY_ADVICE,
   optOutEligibilityProblem,
   unverifiableProblem,
 } from './github-command-shared';
@@ -35,7 +35,7 @@ import {
 import { type GithubSettings, isRecord } from './github-settings-parse';
 
 const LABEL_VISIBILITY_PROBLEM =
-  'declared labels not visible to this token, so the gate cannot verify them. In CI, use a valid ci.github_settings_read_token from secrets/ci.yaml with read-only "Issues" access (or "Pull requests" read); locally use a token with one of those permissions';
+  'declared labels not visible to this token, so the gate cannot verify them. Label reads need issues: read (or pull-requests: read); the canonical github-settings job already grants issues: read, so in CI this points at a stale synced workflow or an organization policy restricting the workflow token rather than anything to change in the declaration. Locally, use a token with one of those permissions';
 const PERMISSION_DENIAL_MESSAGES: ReadonlySet<string> = new Set([
   'Resource not accessible by integration',
   'Resource not accessible by personal access token',
@@ -87,7 +87,7 @@ const repositoryDrift = async (
     ...unverifiableProblem(
       'repository setting(s)',
       rediff.unverifiable,
-      ADMIN_VISIBILITY_ADVICE,
+      MERGE_SETTINGS_VISIBILITY_ADVICE,
     ),
   ];
 };