@davidvornholt/standards 0.19.0 → 0.20.0

package/src/creds-login-github.ts

@@ -1,15 +1,20 @@
 import { randomBytes } from 'node:crypto';
 import { openInBrowser } from './creds-browser';
+import {
+  loadOwnedGithubStore,
+  sameGithubApp,
+  upsertGithubApp,
+} from './creds-github-apps';
 import {
   buildAppManifest,
   convertManifestCode,
   createManifestState,
   MANIFEST_STATE_BYTES,
   manifestFormHtml,
+  resolveGithubAppName,
 } from './creds-login-github-manifest';
-import { BROKER_IDENTITY_NAME } from './creds-naming';
 import {
-  readBrokerStore,
+  type GithubBrokerApp,
   resolveBrokerPath,
   updateBrokerStore,
 } from './creds-store';
@@ -84,50 +89,99 @@ export const waitForCode = (
 };
 export const githubInstallMessage = (installUrl: string): string =>
   `Install it only on the selected repositories that need broker credentials: ${installUrl}`;
-export const runCredsLoginGithub = async (options: {
-  readonly name: string | undefined;
-  readonly org: string | undefined;
-}): Promise<boolean> => {
+
+type GithubLoginDependencies = {
+  readonly waitForCode: typeof waitForCode;
+  readonly convertManifestCode: typeof convertManifestCode;
+  readonly openInBrowser: typeof openInBrowser;
+};
+
+const DEFAULT_LOGIN_DEPENDENCIES: GithubLoginDependencies = {
+  waitForCode,
+  convertManifestCode,
+  openInBrowser,
+};
+
+export const runCredsLoginGithub = async (
+  options: {
+    readonly name: string | undefined;
+    readonly org: string | undefined;
+  },
+  dependencies: GithubLoginDependencies = DEFAULT_LOGIN_DEPENDENCIES,
+): Promise<boolean> => {
+  const resolvedName = resolveGithubAppName(options.name, options.org);
+  if (!resolvedName.ok) {
+    console.error(`standards creds: ${resolvedName.problem}`);
+    return false;
+  }
   const storePath = resolveBrokerPath();
-  const store = await readBrokerStore(storePath);
-  if (store.github !== null) {
-    console.error(
-      `standards creds: a broker GitHub App is already configured (${store.github.htmlUrl}); remove the "github" section from ${storePath} to re-run login`,
-    );
+  const loaded = await loadOwnedGithubStore(storePath);
+  if (!loaded.ok) {
+    console.error(`standards creds: ${loaded.problem}`);
     return false;
   }
   const action =
     options.org === undefined
       ? 'https://github.com/settings/apps/new'
       : `https://github.com/organizations/${options.org}/settings/apps/new`;
-  const name = options.name ?? BROKER_IDENTITY_NAME;
-  const code = await waitForCode((port, state) =>
+  const code = await dependencies.waitForCode((port, state) =>
     manifestFormHtml(
       action,
       JSON.stringify(
-        buildAppManifest(name, `http://127.0.0.1:${port}/callback`),
+        buildAppManifest(
+          resolvedName.value,
+          `http://127.0.0.1:${port}/callback`,
+        ),
       ),
       state,
     ),
   );
-  const conversion = await convertManifestCode(code);
+  const conversion = await dependencies.convertManifestCode(code);
   if (!conversion.ok) {
     console.error(`standards creds: ${conversion.problem}`);
     return false;
   }
   const { app } = conversion;
-  await updateBrokerStore(storePath, (current) => {
-    if (current.github !== null) {
-      throw new Error(
-        `a broker GitHub App was configured while login was in progress (${current.github.htmlUrl}); the newly created App ${app.htmlUrl} was not stored and should be deleted`,
+  const observed = loaded.value.github.find(
+    (entry) => entry.owner?.toLowerCase() === app.owner?.toLowerCase(),
+  );
+  let replaced: GithubBrokerApp | undefined;
+  try {
+    await updateBrokerStore(storePath, (current) => {
+      const currentEntry = current.github.find(
+        (entry) => entry.owner?.toLowerCase() === app.owner?.toLowerCase(),
       );
-    }
-    return { ...current, github: app };
-  });
-  console.log(`standards creds: created App ${app.slug} (${app.htmlUrl})`);
+      const unchanged =
+        observed === undefined
+          ? currentEntry === undefined
+          : currentEntry !== undefined && sameGithubApp(currentEntry, observed);
+      if (!unchanged) {
+        throw new Error(
+          `the broker App for ${app.owner} changed while this App was being created`,
+        );
+      }
+      const upserted = upsertGithubApp(current.github, app);
+      replaced = upserted.replaced ?? undefined;
+      return { ...current, github: upserted.apps };
+    });
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    console.error(
+      `standards creds: the newly created App ${app.htmlUrl} was not stored (${message}); retry login, and delete the new App if it is still absent from \`creds status\``,
+    );
+    return false;
+  }
+  console.log(
+    `standards creds: created App ${app.slug} for ${app.owner} (${app.htmlUrl})`,
+  );
   console.log(`  credentials stored in ${storePath}`);
+  if (replaced !== undefined) {
+    console.log(
+      `  replaced App ${replaced.slug} for ${app.owner}; update its SOPS destinations, verify token minting, then delete the old App`,
+    );
+  }
   const installUrl = `${app.htmlUrl}/installations/new`;
   console.log(githubInstallMessage(installUrl));
-  openInBrowser(installUrl);
+  dependencies.openInBrowser(installUrl);
   return true;
 };

package/src/creds-plan-renew.ts

@@ -7,11 +7,8 @@ import { createAccountToken, deleteAccountToken } from './creds-cloudflare';
 import { resolveTargetRel } from './creds-dest';
 import type { PlannedAction } from './creds-plan-types';
 import { destinationWrites, s3AccessKeyPath, s3PairPaths } from './creds-r2';
-import {
-  inspectSopsScalarDestination,
-  setSopsValues,
-  verifySopsStoredValue,
-} from './creds-sops';
+import { inspectSopsScalarDestination, setSopsValues } from './creds-sops';
+import { verifySopsStoredValue } from './creds-sops-value';
 import type { CloudflareBrokerAccount } from './creds-store';
 
 type RenewAction = Extract<PlannedAction, { readonly kind: 'renew' }>;

package/src/creds-sops-value.ts

@@ -0,0 +1,89 @@
+// Proving that a stored secret matches the value that was just minted, without
+// that value reaching a return type, a log line, or an error message. Callers
+// destroy the new credential on a proven mismatch and keep it on an
+// unverifiable read, so the two negative answers must stay distinct: anything
+// that is not a leaf we can read as a scalar string is unverifiable.
+
+import { isContainedSopsPath, parseSopsKeyPath } from './creds-sops-structure';
+import { runSops } from './sops-exec';
+
+export type SopsStoredValueVerification =
+  | { readonly ok: true; readonly matches: boolean }
+  | { readonly ok: false; readonly problem: string };
+type SopsStoredValueInput = {
+  readonly consumer: string;
+  readonly rel: string;
+  readonly dottedPath: string;
+  readonly expectedValue: string;
+};
+
+// Extracting a scalar leaf prints the stored value itself, with no quoting and
+// no framing of any kind — probed against sops 3.13.0 and 3.13.3. Every byte of
+// stdout is therefore value, which makes exact equality the only sound
+// comparison: a trailing newline is a difference in the stored secret, not
+// output framing.
+//
+// Once equality has been ruled out, output that parses as JSON is output we
+// cannot read as this leaf's own value. It is a branch or list rather than a
+// scalar; or a typed leaf, which a stored string could equally have produced;
+// or a quoted rendering of exactly the value expected, which is either stray
+// quoting in storage or framing from some sops we have not probed. None of
+// those can be told apart from what was printed, and the wrong answer deletes a
+// credential, so all of them stay unverifiable.
+const isUnreadableOutput = (stdout: string, expectedValue: string): boolean => {
+  if (stdout === '') {
+    return true;
+  }
+  try {
+    const parsed: unknown = JSON.parse(stdout);
+    return typeof parsed !== 'string' || parsed === expectedValue;
+  } catch {
+    // Not JSON at all, which is the normal case: an opaque token value, or a
+    // hex access key ID that reads as a malformed number.
+    return false;
+  }
+};
+
+export const verifySopsStoredValueWith = (
+  runner: typeof runSops,
+  input: SopsStoredValueInput,
+): SopsStoredValueVerification => {
+  const { consumer, rel, dottedPath, expectedValue } = input;
+  const path = parseSopsKeyPath(dottedPath);
+  const problem = `could not verify stored SOPS value at ${dottedPath} in ${rel}`;
+  if (path === null || !isContainedSopsPath(consumer, rel, 'file')) {
+    return { ok: false, problem };
+  }
+  const extract = path
+    .map((segment) => `[${JSON.stringify(segment)}]`)
+    .join('');
+  const result = runner(
+    ['decrypt', '--extract', extract, '--output-type', 'json', rel],
+    consumer,
+  );
+  if (result.status !== 0) {
+    return { ok: false, problem };
+  }
+  // Equality is decided before anything is classified, so a secret whose own
+  // text happens to be valid JSON — an all-digit token — is proven by the bytes
+  // rather than second-guessed by their shape.
+  if (result.stdout === expectedValue) {
+    return { ok: true, matches: true };
+  }
+  return isUnreadableOutput(result.stdout, expectedValue)
+    ? { ok: false, problem }
+    : { ok: true, matches: false };
+};
+
+export const verifySopsStoredValue = (
+  consumer: string,
+  rel: string,
+  dottedPath: string,
+  expectedValue: string,
+): SopsStoredValueVerification =>
+  verifySopsStoredValueWith(runSops, {
+    consumer,
+    rel,
+    dottedPath,
+    expectedValue,
+  });

package/src/creds-sops.ts

@@ -15,7 +15,6 @@ import {
 import {
   inspectSopsStructure,
   isContainedSopsPath,
-  parseSopsKeyPath,
   type SopsShapeFailure,
 } from './creds-sops-structure';
 import { runSops } from './sops-exec';
@@ -36,15 +35,6 @@ export type SopsScalarDestinationResult = SopsScalarStructureResult | ReadError;
 export type SopsWriteResult =
   | { readonly ok: true }
   | { readonly ok: false; readonly problem: string };
-export type SopsStoredValueVerification =
-  | { readonly ok: true; readonly matches: boolean }
-  | { readonly ok: false; readonly problem: string };
-type SopsStoredValueInput = {
-  readonly consumer: string;
-  readonly rel: string;
-  readonly dottedPath: string;
-  readonly expectedValue: string;
-};
 
 export const listEncryptedKeys = (text: string): EncryptedKeysReadResult => {
   const result = inspectSopsStructure(text, true);
@@ -123,49 +113,6 @@ const editorCommand = (): string => {
   return `"${process.execPath}" "${editor}"`;
 };
 
-export const verifySopsStoredValueWith = (
-  runner: typeof runSops,
-  input: SopsStoredValueInput,
-): SopsStoredValueVerification => {
-  const { consumer, rel, dottedPath, expectedValue } = input;
-  const path = parseSopsKeyPath(dottedPath);
-  const problem = `could not verify stored SOPS value at ${dottedPath} in ${rel}`;
-  if (path === null || !isContainedSopsPath(consumer, rel, 'file')) {
-    return { ok: false, problem };
-  }
-  const extract = path
-    .map((segment) => `[${JSON.stringify(segment)}]`)
-    .join('');
-  const result = runner(
-    ['decrypt', '--extract', extract, '--output-type', 'json', rel],
-    consumer,
-  );
-  if (result.status !== 0) {
-    return { ok: false, problem };
-  }
-  try {
-    const stored: unknown = JSON.parse(result.stdout);
-    return typeof stored === 'string'
-      ? { ok: true, matches: stored === expectedValue }
-      : { ok: false, problem };
-  } catch {
-    return { ok: false, problem };
-  }
-};
-
-export const verifySopsStoredValue = (
-  consumer: string,
-  rel: string,
-  dottedPath: string,
-  expectedValue: string,
-): SopsStoredValueVerification =>
-  verifySopsStoredValueWith(runSops, {
-    consumer,
-    rel,
-    dottedPath,
-    expectedValue,
-  });
-
 export const setSopsValues = (
   consumer: string,
   rel: string,

package/src/creds-store-github.ts

@@ -0,0 +1,121 @@
+import { isNonEmptyString, isRecord } from './github-settings-parse';
+
+export type GithubBrokerApp = {
+  readonly owner: string | null;
+  readonly appId: number;
+  readonly slug: string;
+  readonly htmlUrl: string;
+  readonly clientId: string;
+  readonly privateKey: string;
+};
+
+export const isGithubBrokerApp = (value: unknown): value is GithubBrokerApp =>
+  isRecord(value) &&
+  (value.owner === null || isNonEmptyString(value.owner)) &&
+  typeof value.appId === 'number' &&
+  Number.isInteger(value.appId) &&
+  isNonEmptyString(value.slug) &&
+  isNonEmptyString(value.htmlUrl) &&
+  isNonEmptyString(value.clientId) &&
+  isNonEmptyString(value.privateKey);
+
+const parseGithubApp = (
+  raw: unknown,
+  owner: string | null,
+): GithubBrokerApp => {
+  if (!isRecord(raw)) {
+    throw new Error('invalid github: run `standards creds login github`');
+  }
+  const app = {
+    owner,
+    appId: raw.app_id,
+    slug: raw.slug,
+    htmlUrl: raw.html_url,
+    clientId: raw.client_id,
+    privateKey: raw.private_key,
+  };
+  if (!isGithubBrokerApp(app)) {
+    throw new Error('invalid github: run `standards creds login github`');
+  }
+  return app;
+};
+
+export const validateGithubApps = (
+  apps: ReadonlyArray<GithubBrokerApp>,
+  path: string,
+): ReadonlyArray<GithubBrokerApp> => {
+  const owners = new Set<string>();
+  const appIds = new Set<number>();
+  for (const app of apps) {
+    if (!isGithubBrokerApp(app)) {
+      throw new Error(
+        `${path}: invalid GitHub App; run \`standards creds login github\``,
+      );
+    }
+    if (app.owner === null) {
+      if (apps.length !== 1) {
+        throw new Error(
+          `${path}: the ownerless legacy GitHub App must be migrated before another App can be stored`,
+        );
+      }
+    } else {
+      const owner = app.owner.toLowerCase();
+      if (owners.has(owner)) {
+        throw new Error(
+          `${path}: duplicate GitHub owner ${app.owner}; keep one broker App per account`,
+        );
+      }
+      owners.add(owner);
+    }
+    if (appIds.has(app.appId)) {
+      throw new Error(
+        `${path}: duplicate GitHub App id ${app.appId}; keep each App once`,
+      );
+    }
+    appIds.add(app.appId);
+  }
+  return apps;
+};
+
+export const parseGithubApps = (
+  raw: unknown,
+  path: string,
+): ReadonlyArray<GithubBrokerApp> => {
+  if (raw === undefined || raw === null) {
+    return [];
+  }
+  if (!Array.isArray(raw)) {
+    return [parseGithubApp(raw, null)];
+  }
+  const apps = raw.map((entry) => {
+    if (!(isRecord(entry) && isNonEmptyString(entry.owner))) {
+      throw new Error(
+        'invalid github owner: run `standards creds login github`',
+      );
+    }
+    return parseGithubApp(entry, entry.owner);
+  });
+  return validateGithubApps(apps, path);
+};
+
+const appDocument = (
+  app: GithubBrokerApp,
+): Readonly<Record<string, unknown>> => ({
+  ...(app.owner === null ? {} : { owner: app.owner }),
+  // biome-ignore lint/style/useNamingConvention: The persisted broker YAML contract uses snake_case.
+  app_id: app.appId,
+  slug: app.slug,
+  // biome-ignore lint/style/useNamingConvention: The persisted broker YAML contract uses snake_case.
+  html_url: app.htmlUrl,
+  // biome-ignore lint/style/useNamingConvention: The persisted broker YAML contract uses snake_case.
+  client_id: app.clientId,
+  // biome-ignore lint/style/useNamingConvention: The persisted broker YAML contract uses snake_case.
+  private_key: app.privateKey,
+});
+
+export const githubStoreDocument = (
+  apps: ReadonlyArray<GithubBrokerApp>,
+): unknown =>
+  apps.length === 1 && apps[0]?.owner === null
+    ? appDocument(apps[0])
+    : apps.map(appDocument);

package/src/creds-store-lock-holder.ts

@@ -0,0 +1,98 @@
+import { randomUUID } from 'node:crypto';
+import { mkdir, rename, rmdir, unlink, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import type { Worker } from 'node:worker_threads';
+import { startBrokerLockLease } from './creds-store-lock-lease';
+import { isRecord } from './github-settings-parse';
+
+const OWNER_ONLY_DIR_MODE = 0o700;
+const OWNER_ONLY_FILE_MODE = 0o600;
+const HOLDER_PREFIX = 'holder-';
+const HOLDER_SUFFIX = '.json';
+const LEASE_DIVISOR = 3;
+
+export type HeldBrokerLock = {
+  readonly holderPath: string;
+  readonly lease: Worker;
+  readonly lockPath: string;
+};
+
+const isErrorCode = (error: unknown, code: string): boolean =>
+  isRecord(error) && error.code === code;
+
+const cleanupCandidate = async (
+  candidatePath: string,
+  holderPath: string,
+): Promise<void> => {
+  await unlink(holderPath).catch(() => undefined);
+  await rmdir(candidatePath).catch(() => undefined);
+};
+
+export const tryAcquireBrokerLock = async (
+  lockPath: string,
+  staleMs: number,
+): Promise<HeldBrokerLock | null> => {
+  const generation = randomUUID();
+  const candidatePath = `${lockPath}.pending-${generation}`;
+  const candidateHolderPath = join(
+    candidatePath,
+    `${HOLDER_PREFIX}${generation}${HOLDER_SUFFIX}`,
+  );
+  await mkdir(candidatePath, { mode: OWNER_ONLY_DIR_MODE });
+  try {
+    await writeFile(candidateHolderPath, JSON.stringify({ generation }), {
+      flag: 'wx',
+      mode: OWNER_ONLY_FILE_MODE,
+    });
+    try {
+      await rename(candidatePath, lockPath);
+    } catch (error) {
+      if (isErrorCode(error, 'EEXIST') || isErrorCode(error, 'ENOTEMPTY')) {
+        await cleanupCandidate(candidatePath, candidateHolderPath);
+        return null;
+      }
+      throw error;
+    }
+  } catch (error) {
+    await cleanupCandidate(candidatePath, candidateHolderPath);
+    throw error;
+  }
+  const holderPath = join(
+    lockPath,
+    `${HOLDER_PREFIX}${generation}${HOLDER_SUFFIX}`,
+  );
+  try {
+    const lease = startBrokerLockLease(
+      holderPath,
+      Math.max(1, Math.floor(staleMs / LEASE_DIVISOR)),
+    );
+    return { holderPath, lease, lockPath };
+  } catch (error) {
+    await unlink(holderPath).catch(() => undefined);
+    await rmdir(lockPath).catch(() => undefined);
+    throw error;
+  }
+};
+
+export const releaseBrokerLock = async ({
+  holderPath,
+  lease,
+  lockPath,
+}: HeldBrokerLock): Promise<void> => {
+  await lease.terminate();
+  try {
+    await unlink(holderPath);
+  } catch (error) {
+    if (isErrorCode(error, 'ENOENT')) {
+      return;
+    }
+    throw error;
+  }
+  try {
+    await rmdir(lockPath);
+  } catch (error) {
+    if (!isErrorCode(error, 'ENOENT')) {
+      throw error;
+    }
+  }
+};

package/src/creds-store-lock-inspection.ts

@@ -0,0 +1,125 @@
+import { readdir, readFile, rmdir, stat, unlink } from 'node:fs/promises';
+import { join } from 'node:path';
+import { isRecord } from './github-settings-parse';
+
+const HOLDER_PREFIX = 'holder-';
+const HOLDER_SUFFIX = '.json';
+
+export type BrokerLockAvailability = 'blocked' | 'live' | 'retry';
+
+const isErrorCode = (error: unknown, code: string): boolean =>
+  isRecord(error) && error.code === code;
+
+type ExistingGeneration =
+  | { readonly kind: 'missing' }
+  | { readonly kind: 'unknown' }
+  | {
+      readonly entryPath: string | null;
+      readonly kind: 'incomplete';
+      readonly modifiedAt: number;
+    }
+  | {
+      readonly entryPath: string;
+      readonly kind: 'valid';
+      readonly modifiedAt: number;
+    };
+
+const directoryModifiedAt = async (
+  lockPath: string,
+): Promise<number | null> => {
+  try {
+    return (await stat(lockPath)).mtimeMs;
+  } catch {
+    return null;
+  }
+};
+
+const readExistingGeneration = async (
+  lockPath: string,
+): Promise<ExistingGeneration> => {
+  let names: ReadonlyArray<string>;
+  try {
+    names = await readdir(lockPath);
+  } catch (error) {
+    return isErrorCode(error, 'ENOENT')
+      ? { kind: 'missing' }
+      : { kind: 'unknown' };
+  }
+  if (names.length === 0) {
+    const modifiedAt = await directoryModifiedAt(lockPath);
+    return modifiedAt === null
+      ? { kind: 'missing' }
+      : { entryPath: null, kind: 'incomplete', modifiedAt };
+  }
+  const entryName = names.length === 1 ? names[0] : undefined;
+  if (entryName === undefined) {
+    return { kind: 'unknown' };
+  }
+  const entryPath = join(lockPath, entryName);
+  let modifiedAt: number;
+  try {
+    modifiedAt = (await stat(entryPath)).mtimeMs;
+  } catch {
+    return { kind: 'unknown' };
+  }
+  const incomplete = {
+    entryPath,
+    kind: 'incomplete' as const,
+    modifiedAt,
+  };
+  if (
+    !(entryName.startsWith(HOLDER_PREFIX) && entryName.endsWith(HOLDER_SUFFIX))
+  ) {
+    return incomplete;
+  }
+  try {
+    const decoded: unknown = JSON.parse(await readFile(entryPath, 'utf8'));
+    const generation = entryName.slice(
+      HOLDER_PREFIX.length,
+      -HOLDER_SUFFIX.length,
+    );
+    return isRecord(decoded) && decoded.generation === generation
+      ? { entryPath, kind: 'valid', modifiedAt }
+      : incomplete;
+  } catch {
+    return incomplete;
+  }
+};
+
+const removeObservedGeneration = async (
+  lockPath: string,
+  entryPath: string | null,
+): Promise<boolean> => {
+  if (entryPath !== null) {
+    try {
+      await unlink(entryPath);
+    } catch (error) {
+      return isErrorCode(error, 'ENOENT');
+    }
+  }
+  try {
+    await rmdir(lockPath);
+    return true;
+  } catch (error) {
+    return isErrorCode(error, 'ENOENT');
+  }
+};
+
+export const inspectBrokerLock = async (
+  lockPath: string,
+  staleMs: number,
+): Promise<BrokerLockAvailability> => {
+  const generation = await readExistingGeneration(lockPath);
+  if (generation.kind === 'missing') {
+    return 'retry';
+  }
+  if (generation.kind === 'unknown') {
+    return 'blocked';
+  }
+  if (Date.now() - generation.modifiedAt < staleMs) {
+    return generation.kind === 'valid' ? 'live' : 'blocked';
+  }
+  return (await removeObservedGeneration(lockPath, generation.entryPath))
+    ? 'retry'
+    : 'blocked';
+};

package/src/creds-store-lock-lease.ts

@@ -0,0 +1,32 @@
+import { utimes } from 'node:fs/promises';
+import {
+  isMainThread,
+  parentPort,
+  Worker,
+  workerData,
+} from 'node:worker_threads';
+
+type LeaseWorkerData = {
+  readonly holderPath: string;
+  readonly intervalMs: number;
+};
+
+if (!isMainThread) {
+  const { holderPath, intervalMs } = workerData as LeaseWorkerData;
+  const timer = setInterval(
+    () => utimes(holderPath, new Date(), new Date()).catch(() => undefined),
+    intervalMs,
+  );
+  parentPort?.on('close', () => clearInterval(timer));
+}
+
+export const startBrokerLockLease = (
+  holderPath: string,
+  intervalMs: number,
+): Worker => {
+  const lease = new Worker(import.meta.url, {
+    workerData: { holderPath, intervalMs } satisfies LeaseWorkerData,
+  });
+  lease.unref();
+  return lease;
+};