@davidvornholt/standards 0.19.0 → 0.21.0

package/src/creds-sops-value.ts

@@ -0,0 +1,89 @@
+// Proving that a stored secret matches the value that was just minted, without
+// that value reaching a return type, a log line, or an error message. Callers
+// destroy the new credential on a proven mismatch and keep it on an
+// unverifiable read, so the two negative answers must stay distinct: anything
+// that is not a leaf we can read as a scalar string is unverifiable.
+
+import { isContainedSopsPath, parseSopsKeyPath } from './creds-sops-structure';
+import { runSops } from './sops-exec';
+
+export type SopsStoredValueVerification =
+  | { readonly ok: true; readonly matches: boolean }
+  | { readonly ok: false; readonly problem: string };
+type SopsStoredValueInput = {
+  readonly consumer: string;
+  readonly rel: string;
+  readonly dottedPath: string;
+  readonly expectedValue: string;
+};
+
+// Extracting a scalar leaf prints the stored value itself, with no quoting and
+// no framing of any kind — probed against sops 3.13.0 and 3.13.3. Every byte of
+// stdout is therefore value, which makes exact equality the only sound
+// comparison: a trailing newline is a difference in the stored secret, not
+// output framing.
+//
+// Once equality has been ruled out, output that parses as JSON is output we
+// cannot read as this leaf's own value. It is a branch or list rather than a
+// scalar; or a typed leaf, which a stored string could equally have produced;
+// or a quoted rendering of exactly the value expected, which is either stray
+// quoting in storage or framing from some sops we have not probed. None of
+// those can be told apart from what was printed, and the wrong answer deletes a
+// credential, so all of them stay unverifiable.
+const isUnreadableOutput = (stdout: string, expectedValue: string): boolean => {
+  if (stdout === '') {
+    return true;
+  }
+  try {
+    const parsed: unknown = JSON.parse(stdout);
+    return typeof parsed !== 'string' || parsed === expectedValue;
+  } catch {
+    // Not JSON at all, which is the normal case: an opaque token value, or a
+    // hex access key ID that reads as a malformed number.
+    return false;
+  }
+};
+
+export const verifySopsStoredValueWith = (
+  runner: typeof runSops,
+  input: SopsStoredValueInput,
+): SopsStoredValueVerification => {
+  const { consumer, rel, dottedPath, expectedValue } = input;
+  const path = parseSopsKeyPath(dottedPath);
+  const problem = `could not verify stored SOPS value at ${dottedPath} in ${rel}`;
+  if (path === null || !isContainedSopsPath(consumer, rel, 'file')) {
+    return { ok: false, problem };
+  }
+  const extract = path
+    .map((segment) => `[${JSON.stringify(segment)}]`)
+    .join('');
+  const result = runner(
+    ['decrypt', '--extract', extract, '--output-type', 'json', rel],
+    consumer,
+  );
+  if (result.status !== 0) {
+    return { ok: false, problem };
+  }
+  // Equality is decided before anything is classified, so a secret whose own
+  // text happens to be valid JSON — an all-digit token — is proven by the bytes
+  // rather than second-guessed by their shape.
+  if (result.stdout === expectedValue) {
+    return { ok: true, matches: true };
+  }
+  return isUnreadableOutput(result.stdout, expectedValue)
+    ? { ok: false, problem }
+    : { ok: true, matches: false };
+};
+
+export const verifySopsStoredValue = (
+  consumer: string,
+  rel: string,
+  dottedPath: string,
+  expectedValue: string,
+): SopsStoredValueVerification =>
+  verifySopsStoredValueWith(runSops, {
+    consumer,
+    rel,
+    dottedPath,
+    expectedValue,
+  });

package/src/creds-sops.ts

@@ -15,7 +15,6 @@ import {
 import {
   inspectSopsStructure,
   isContainedSopsPath,
-  parseSopsKeyPath,
   type SopsShapeFailure,
 } from './creds-sops-structure';
 import { runSops } from './sops-exec';
@@ -36,15 +35,6 @@ export type SopsScalarDestinationResult = SopsScalarStructureResult | ReadError;
 export type SopsWriteResult =
   | { readonly ok: true }
   | { readonly ok: false; readonly problem: string };
-export type SopsStoredValueVerification =
-  | { readonly ok: true; readonly matches: boolean }
-  | { readonly ok: false; readonly problem: string };
-type SopsStoredValueInput = {
-  readonly consumer: string;
-  readonly rel: string;
-  readonly dottedPath: string;
-  readonly expectedValue: string;
-};
 
 export const listEncryptedKeys = (text: string): EncryptedKeysReadResult => {
   const result = inspectSopsStructure(text, true);
@@ -123,49 +113,6 @@ const editorCommand = (): string => {
   return `"${process.execPath}" "${editor}"`;
 };
 
-export const verifySopsStoredValueWith = (
-  runner: typeof runSops,
-  input: SopsStoredValueInput,
-): SopsStoredValueVerification => {
-  const { consumer, rel, dottedPath, expectedValue } = input;
-  const path = parseSopsKeyPath(dottedPath);
-  const problem = `could not verify stored SOPS value at ${dottedPath} in ${rel}`;
-  if (path === null || !isContainedSopsPath(consumer, rel, 'file')) {
-    return { ok: false, problem };
-  }
-  const extract = path
-    .map((segment) => `[${JSON.stringify(segment)}]`)
-    .join('');
-  const result = runner(
-    ['decrypt', '--extract', extract, '--output-type', 'json', rel],
-    consumer,
-  );
-  if (result.status !== 0) {
-    return { ok: false, problem };
-  }
-  try {
-    const stored: unknown = JSON.parse(result.stdout);
-    return typeof stored === 'string'
-      ? { ok: true, matches: stored === expectedValue }
-      : { ok: false, problem };
-  } catch {
-    return { ok: false, problem };
-  }
-};
-
-export const verifySopsStoredValue = (
-  consumer: string,
-  rel: string,
-  dottedPath: string,
-  expectedValue: string,
-): SopsStoredValueVerification =>
-  verifySopsStoredValueWith(runSops, {
-    consumer,
-    rel,
-    dottedPath,
-    expectedValue,
-  });
-
 export const setSopsValues = (
   consumer: string,
   rel: string,

package/src/creds-store-github.ts

@@ -0,0 +1,121 @@
+import { isNonEmptyString, isRecord } from './github-settings-parse';
+
+export type GithubBrokerApp = {
+  readonly owner: string | null;
+  readonly appId: number;
+  readonly slug: string;
+  readonly htmlUrl: string;
+  readonly clientId: string;
+  readonly privateKey: string;
+};
+
+export const isGithubBrokerApp = (value: unknown): value is GithubBrokerApp =>
+  isRecord(value) &&
+  (value.owner === null || isNonEmptyString(value.owner)) &&
+  typeof value.appId === 'number' &&
+  Number.isInteger(value.appId) &&
+  isNonEmptyString(value.slug) &&
+  isNonEmptyString(value.htmlUrl) &&
+  isNonEmptyString(value.clientId) &&
+  isNonEmptyString(value.privateKey);
+
+const parseGithubApp = (
+  raw: unknown,
+  owner: string | null,
+): GithubBrokerApp => {
+  if (!isRecord(raw)) {
+    throw new Error('invalid github: run `standards creds login github`');
+  }
+  const app = {
+    owner,
+    appId: raw.app_id,
+    slug: raw.slug,
+    htmlUrl: raw.html_url,
+    clientId: raw.client_id,
+    privateKey: raw.private_key,
+  };
+  if (!isGithubBrokerApp(app)) {
+    throw new Error('invalid github: run `standards creds login github`');
+  }
+  return app;
+};
+
+export const validateGithubApps = (
+  apps: ReadonlyArray<GithubBrokerApp>,
+  path: string,
+): ReadonlyArray<GithubBrokerApp> => {
+  const owners = new Set<string>();
+  const appIds = new Set<number>();
+  for (const app of apps) {
+    if (!isGithubBrokerApp(app)) {
+      throw new Error(
+        `${path}: invalid GitHub App; run \`standards creds login github\``,
+      );
+    }
+    if (app.owner === null) {
+      if (apps.length !== 1) {
+        throw new Error(
+          `${path}: the ownerless legacy GitHub App must be migrated before another App can be stored`,
+        );
+      }
+    } else {
+      const owner = app.owner.toLowerCase();
+      if (owners.has(owner)) {
+        throw new Error(
+          `${path}: duplicate GitHub owner ${app.owner}; keep one broker App per account`,
+        );
+      }
+      owners.add(owner);
+    }
+    if (appIds.has(app.appId)) {
+      throw new Error(
+        `${path}: duplicate GitHub App id ${app.appId}; keep each App once`,
+      );
+    }
+    appIds.add(app.appId);
+  }
+  return apps;
+};
+
+export const parseGithubApps = (
+  raw: unknown,
+  path: string,
+): ReadonlyArray<GithubBrokerApp> => {
+  if (raw === undefined || raw === null) {
+    return [];
+  }
+  if (!Array.isArray(raw)) {
+    return [parseGithubApp(raw, null)];
+  }
+  const apps = raw.map((entry) => {
+    if (!(isRecord(entry) && isNonEmptyString(entry.owner))) {
+      throw new Error(
+        'invalid github owner: run `standards creds login github`',
+      );
+    }
+    return parseGithubApp(entry, entry.owner);
+  });
+  return validateGithubApps(apps, path);
+};
+
+const appDocument = (
+  app: GithubBrokerApp,
+): Readonly<Record<string, unknown>> => ({
+  ...(app.owner === null ? {} : { owner: app.owner }),
+  // biome-ignore lint/style/useNamingConvention: The persisted broker YAML contract uses snake_case.
+  app_id: app.appId,
+  slug: app.slug,
+  // biome-ignore lint/style/useNamingConvention: The persisted broker YAML contract uses snake_case.
+  html_url: app.htmlUrl,
+  // biome-ignore lint/style/useNamingConvention: The persisted broker YAML contract uses snake_case.
+  client_id: app.clientId,
+  // biome-ignore lint/style/useNamingConvention: The persisted broker YAML contract uses snake_case.
+  private_key: app.privateKey,
+});
+
+export const githubStoreDocument = (
+  apps: ReadonlyArray<GithubBrokerApp>,
+): unknown =>
+  apps.length === 1 && apps[0]?.owner === null
+    ? appDocument(apps[0])
+    : apps.map(appDocument);

package/src/creds-store-lock-holder.ts

@@ -0,0 +1,105 @@
+import { randomUUID } from 'node:crypto';
+import { mkdir, rename, rmdir, unlink, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import type { Worker } from 'node:worker_threads';
+import { startBrokerLockLease } from './creds-store-lock-lease';
+import { isRecord } from './github-settings-parse';
+
+const OWNER_ONLY_DIR_MODE = 0o700;
+const OWNER_ONLY_FILE_MODE = 0o600;
+const HOLDER_PREFIX = 'holder-';
+const HOLDER_SUFFIX = '.json';
+const LEASE_DIVISOR = 3;
+// Both codes mean the lock is no longer this owner's to clean up. ENOENT: it is
+// already gone. ENOTEMPTY: a contender published its own generation into the
+// directory during the moment the unlink left it empty — renaming onto an empty
+// directory succeeds, which is exactly how a waiter takes the lock. Releasing
+// runs from a finally, so throwing here would replace the result of an
+// operation that already succeeded.
+const RELEASABLE_RMDIR_CODES = ['ENOENT', 'ENOTEMPTY'] as const;
+
+export type HeldBrokerLock = {
+  readonly holderPath: string;
+  readonly lease: Worker;
+  readonly lockPath: string;
+};
+
+const isErrorCode = (error: unknown, code: string): boolean =>
+  isRecord(error) && error.code === code;
+
+const cleanupCandidate = async (
+  candidatePath: string,
+  holderPath: string,
+): Promise<void> => {
+  await unlink(holderPath).catch(() => undefined);
+  await rmdir(candidatePath).catch(() => undefined);
+};
+
+export const tryAcquireBrokerLock = async (
+  lockPath: string,
+  staleMs: number,
+): Promise<HeldBrokerLock | null> => {
+  const generation = randomUUID();
+  const candidatePath = `${lockPath}.pending-${generation}`;
+  const candidateHolderPath = join(
+    candidatePath,
+    `${HOLDER_PREFIX}${generation}${HOLDER_SUFFIX}`,
+  );
+  await mkdir(candidatePath, { mode: OWNER_ONLY_DIR_MODE });
+  try {
+    await writeFile(candidateHolderPath, JSON.stringify({ generation }), {
+      flag: 'wx',
+      mode: OWNER_ONLY_FILE_MODE,
+    });
+    try {
+      await rename(candidatePath, lockPath);
+    } catch (error) {
+      if (isErrorCode(error, 'EEXIST') || isErrorCode(error, 'ENOTEMPTY')) {
+        await cleanupCandidate(candidatePath, candidateHolderPath);
+        return null;
+      }
+      throw error;
+    }
+  } catch (error) {
+    await cleanupCandidate(candidatePath, candidateHolderPath);
+    throw error;
+  }
+  const holderPath = join(
+    lockPath,
+    `${HOLDER_PREFIX}${generation}${HOLDER_SUFFIX}`,
+  );
+  try {
+    const lease = startBrokerLockLease(
+      holderPath,
+      Math.max(1, Math.floor(staleMs / LEASE_DIVISOR)),
+    );
+    return { holderPath, lease, lockPath };
+  } catch (error) {
+    await unlink(holderPath).catch(() => undefined);
+    await rmdir(lockPath).catch(() => undefined);
+    throw error;
+  }
+};
+
+export const releaseBrokerLock = async ({
+  holderPath,
+  lease,
+  lockPath,
+}: HeldBrokerLock): Promise<void> => {
+  await lease.terminate();
+  try {
+    await unlink(holderPath);
+  } catch (error) {
+    if (isErrorCode(error, 'ENOENT')) {
+      return;
+    }
+    throw error;
+  }
+  try {
+    await rmdir(lockPath);
+  } catch (error) {
+    if (!RELEASABLE_RMDIR_CODES.some((code) => isErrorCode(error, code))) {
+      throw error;
+    }
+  }
+};

package/src/creds-store-lock-inspection.ts

@@ -0,0 +1,125 @@
+import { readdir, readFile, rmdir, stat, unlink } from 'node:fs/promises';
+import { join } from 'node:path';
+import { isRecord } from './github-settings-parse';
+
+const HOLDER_PREFIX = 'holder-';
+const HOLDER_SUFFIX = '.json';
+
+export type BrokerLockAvailability = 'blocked' | 'live' | 'retry';
+
+const isErrorCode = (error: unknown, code: string): boolean =>
+  isRecord(error) && error.code === code;
+
+type ExistingGeneration =
+  | { readonly kind: 'missing' }
+  | { readonly kind: 'unknown' }
+  | {
+      readonly entryPath: string | null;
+      readonly kind: 'incomplete';
+      readonly modifiedAt: number;
+    }
+  | {
+      readonly entryPath: string;
+      readonly kind: 'valid';
+      readonly modifiedAt: number;
+    };
+
+const directoryModifiedAt = async (
+  lockPath: string,
+): Promise<number | null> => {
+  try {
+    return (await stat(lockPath)).mtimeMs;
+  } catch {
+    return null;
+  }
+};
+
+const readExistingGeneration = async (
+  lockPath: string,
+): Promise<ExistingGeneration> => {
+  let names: ReadonlyArray<string>;
+  try {
+    names = await readdir(lockPath);
+  } catch (error) {
+    return isErrorCode(error, 'ENOENT')
+      ? { kind: 'missing' }
+      : { kind: 'unknown' };
+  }
+  if (names.length === 0) {
+    const modifiedAt = await directoryModifiedAt(lockPath);
+    return modifiedAt === null
+      ? { kind: 'missing' }
+      : { entryPath: null, kind: 'incomplete', modifiedAt };
+  }
+  const entryName = names.length === 1 ? names[0] : undefined;
+  if (entryName === undefined) {
+    return { kind: 'unknown' };
+  }
+  const entryPath = join(lockPath, entryName);
+  let modifiedAt: number;
+  try {
+    modifiedAt = (await stat(entryPath)).mtimeMs;
+  } catch {
+    return { kind: 'unknown' };
+  }
+  const incomplete = {
+    entryPath,
+    kind: 'incomplete' as const,
+    modifiedAt,
+  };
+  if (
+    !(entryName.startsWith(HOLDER_PREFIX) && entryName.endsWith(HOLDER_SUFFIX))
+  ) {
+    return incomplete;
+  }
+  try {
+    const decoded: unknown = JSON.parse(await readFile(entryPath, 'utf8'));
+    const generation = entryName.slice(
+      HOLDER_PREFIX.length,
+      -HOLDER_SUFFIX.length,
+    );
+    return isRecord(decoded) && decoded.generation === generation
+      ? { entryPath, kind: 'valid', modifiedAt }
+      : incomplete;
+  } catch {
+    return incomplete;
+  }
+};
+
+const removeObservedGeneration = async (
+  lockPath: string,
+  entryPath: string | null,
+): Promise<boolean> => {
+  if (entryPath !== null) {
+    try {
+      await unlink(entryPath);
+    } catch (error) {
+      return isErrorCode(error, 'ENOENT');
+    }
+  }
+  try {
+    await rmdir(lockPath);
+    return true;
+  } catch (error) {
+    return isErrorCode(error, 'ENOENT');
+  }
+};
+
+export const inspectBrokerLock = async (
+  lockPath: string,
+  staleMs: number,
+): Promise<BrokerLockAvailability> => {
+  const generation = await readExistingGeneration(lockPath);
+  if (generation.kind === 'missing') {
+    return 'retry';
+  }
+  if (generation.kind === 'unknown') {
+    return 'blocked';
+  }
+  if (Date.now() - generation.modifiedAt < staleMs) {
+    return generation.kind === 'valid' ? 'live' : 'blocked';
+  }
+  return (await removeObservedGeneration(lockPath, generation.entryPath))
+    ? 'retry'
+    : 'blocked';
+};

package/src/creds-store-lock-lease.ts

@@ -0,0 +1,32 @@
+import { utimes } from 'node:fs/promises';
+import {
+  isMainThread,
+  parentPort,
+  Worker,
+  workerData,
+} from 'node:worker_threads';
+
+type LeaseWorkerData = {
+  readonly holderPath: string;
+  readonly intervalMs: number;
+};
+
+if (!isMainThread) {
+  const { holderPath, intervalMs } = workerData as LeaseWorkerData;
+  const timer = setInterval(
+    () => utimes(holderPath, new Date(), new Date()).catch(() => undefined),
+    intervalMs,
+  );
+  parentPort?.on('close', () => clearInterval(timer));
+}
+
+export const startBrokerLockLease = (
+  holderPath: string,
+  intervalMs: number,
+): Worker => {
+  const lease = new Worker(import.meta.url, {
+    workerData: { holderPath, intervalMs } satisfies LeaseWorkerData,
+  });
+  lease.unref();
+  return lease;
+};

package/src/creds-store-lock.ts

@@ -1,12 +1,18 @@
-// Broker store lock: a mkdir-based mutex serializing read-modify-write
-// cycles on broker.yaml. Store writes are sub-second, so a lock directory
-// older than the staleness window can only belong to a dead process and is
-// broken automatically; the timeout error names the directory and the manual
-// remediation for the pathological rest.
+// Broker store lock: a fully initialized generation is atomically renamed into
+// place before its operation starts. A worker-thread lease remains live while
+// synchronous SOPS/Nix blocks the main event loop. Cleanup unlinks the exact
+// generation token before removing the directory, so an old owner cannot
+// remove a replacement — and finding one there instead of an empty directory
+// is a no-op, never a failure of the operation being unwound.
 
-import { mkdir, rmdir, stat } from 'node:fs/promises';
+import { mkdir } from 'node:fs/promises';
 import { dirname } from 'node:path';
-import { isRecord } from './github-settings-parse';
+import {
+  type HeldBrokerLock,
+  releaseBrokerLock,
+  tryAcquireBrokerLock,
+} from './creds-store-lock-holder';
+import { inspectBrokerLock } from './creds-store-lock-inspection';
 
 const OWNER_ONLY_DIR_MODE = 0o700;
 
@@ -23,59 +29,30 @@ const DEFAULT_STALE_MS = 30_000;
 const sleep = (ms: number): Promise<void> =>
   new Promise((resolve) => setTimeout(resolve, ms));
 
-const tryAcquire = async (lockPath: string): Promise<boolean> => {
-  try {
-    await mkdir(lockPath, { mode: OWNER_ONLY_DIR_MODE });
-    return true;
-  } catch (error) {
-    if (isRecord(error) && error.code === 'EEXIST') {
-      return false;
-    }
-    throw error;
-  }
-};
-
-// True when the contender may immediately retry: the lock vanished, or it
-// was stale and this process removed it (losing the removal race to another
-// contender also lands here — the retry's mkdir arbitrates).
-const breakIfStale = async (
-  lockPath: string,
-  staleMs: number,
-): Promise<boolean> => {
-  try {
-    const info = await stat(lockPath);
-    if (Date.now() - info.mtimeMs < staleMs) {
-      return false;
-    }
-    await rmdir(lockPath);
-    return true;
-  } catch {
-    return true;
-  }
-};
-
 // One acquisition attempt: acquired, or cleared a stale/vanished lock (retry
-// immediately), or waited one retry slot; a live lock past the deadline
-// throws with the remediation hint.
+// immediately), or waited one retry slot. Proven-live generations do not
+// consume the recovery deadline; incomplete or ambiguous generations do.
 const attemptAcquire = async (
   lockPath: string,
   staleMs: number,
   retryMs: number,
   deadline: number,
-): Promise<boolean> => {
-  if (await tryAcquire(lockPath)) {
-    return true;
+): Promise<HeldBrokerLock | null> => {
+  const held = await tryAcquireBrokerLock(lockPath, staleMs);
+  if (held !== null) {
+    return held;
   }
-  if (await breakIfStale(lockPath, staleMs)) {
-    return false;
+  const availability = await inspectBrokerLock(lockPath, staleMs);
+  if (availability === 'retry') {
+    return null;
   }
-  if (Date.now() >= deadline) {
+  if (availability === 'blocked' && Date.now() >= deadline) {
     throw new Error(
       `lock timeout: another creds process holds ${lockPath}; if none is running, remove that directory and retry`,
     );
   }
   await sleep(retryMs);
-  return false;
+  return null;
 };
 
 export const withBrokerLock = async <T>(
@@ -89,14 +66,14 @@ export const withBrokerLock = async <T>(
   const lockPath = `${path}.lock`;
   await mkdir(dirname(path), { recursive: true, mode: OWNER_ONLY_DIR_MODE });
   const deadline = Date.now() + timeoutMs;
-  let acquired = false;
-  while (!acquired) {
+  let held: HeldBrokerLock | null = null;
+  while (held === null) {
     // biome-ignore lint/performance/noAwaitInLoops: lock acquisition retries are inherently sequential.
-    acquired = await attemptAcquire(lockPath, staleMs, retryMs, deadline);
+    held = await attemptAcquire(lockPath, staleMs, retryMs, deadline);
   }
   try {
     return await operation();
   } finally {
-    await rmdir(lockPath);
+    await releaseBrokerLock(held);
   }
 };