@davidvornholt/standards 0.19.0 → 0.21.0

package/src/creds-store.ts

@@ -6,23 +6,24 @@ import { basename, dirname, join } from 'node:path';
 import process from 'node:process';
 import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
 import { isCloudflareAccountId } from './creds-cloudflare-account';
+import {
+  type GithubBrokerApp,
+  githubStoreDocument,
+  parseGithubApps,
+  validateGithubApps,
+} from './creds-store-github';
 import { withBrokerLock } from './creds-store-lock';
 import { isNonEmptyString, isRecord } from './github-settings-parse';
-export type GithubBrokerApp = {
-  readonly appId: number;
-  readonly slug: string;
-  readonly htmlUrl: string;
-  readonly clientId: string;
-  readonly privateKey: string;
-};
+
+export type { GithubBrokerApp } from './creds-store-github';
 export type CloudflareBrokerAccount = Readonly<
   Record<'accountId' | 'token', string>
 >;
 export type BrokerStore = {
-  readonly github: GithubBrokerApp | null;
+  readonly github: ReadonlyArray<GithubBrokerApp>;
   readonly cloudflare: ReadonlyArray<CloudflareBrokerAccount>;
 };
-export const EMPTY_BROKER_STORE: BrokerStore = { github: null, cloudflare: [] };
+export const EMPTY_BROKER_STORE: BrokerStore = { github: [], cloudflare: [] };
 const OWNER_ONLY_FILE_MODE = 0o600;
 const OWNER_ONLY_DIR_MODE = 0o700;
 const FILE_MODE_MODULUS = 0o1000;
@@ -48,29 +49,6 @@ export const resolveBrokerPath = (): string =>
     process.env.XDG_CONFIG_HOME ?? join(homedir(), '.config'),
     'standards/broker.yaml',
   );
-const parseGithub = (raw: unknown): GithubBrokerApp | null => {
-  if (raw === undefined || raw === null) {
-    return null;
-  }
-  if (
-    !isRecord(raw) ||
-    typeof raw.app_id !== 'number' ||
-    !Number.isInteger(raw.app_id) ||
-    !isNonEmptyString(raw.slug) ||
-    !isNonEmptyString(raw.html_url) ||
-    !isNonEmptyString(raw.client_id) ||
-    !isNonEmptyString(raw.private_key)
-  ) {
-    throw new Error('invalid github: run `standards creds login github`');
-  }
-  return {
-    appId: raw.app_id,
-    slug: raw.slug,
-    htmlUrl: raw.html_url,
-    clientId: raw.client_id,
-    privateKey: raw.private_key,
-  };
-};
 const validateCloudflareAccounts = (
   accounts: ReadonlyArray<CloudflareBrokerAccount>,
   path: string,
@@ -131,7 +109,7 @@ export const readBrokerStore = async (path: string): Promise<BrokerStore> => {
     throw new Error(`${path} must contain a YAML mapping`);
   }
   return {
-    github: parseGithub(raw.github),
+    github: parseGithubApps(raw.github, path),
     cloudflare: parseCloudflare(raw.cloudflare, path),
   };
 };
@@ -142,21 +120,14 @@ export const updateBrokerStore = async (
 ): Promise<void> =>
   withBrokerLock(path, async () => {
     const store = await update(await readBrokerStore(path));
+    validateGithubApps(store.github, path);
     validateCloudflareAccounts(store.cloudflare, path);
     await writeBrokerStoreUnlocked(path, store, sync);
   });
 const storeDocument = (store: BrokerStore): unknown => ({
-  ...(store.github === null
+  ...(store.github.length === 0
     ? {}
-    : {
-        github: {
-          app_id: store.github.appId,
-          slug: store.github.slug,
-          html_url: store.github.htmlUrl,
-          client_id: store.github.clientId,
-          private_key: store.github.privateKey,
-        },
-      }),
+    : { github: githubStoreDocument(store.github) }),
   ...(store.cloudflare.length === 0
     ? {}
     : {

package/src/dev-env-compose.ts

@@ -0,0 +1,120 @@
+// Composes the dev env layers — tracked config/dev.yaml, decrypted
+// secrets/dev.yaml, and the gitignored per-machine config/dev.local.yaml —
+// into one workspace-keyed target set. Later layers override earlier ones
+// per env key. A key declared by both tracked shared layers is a placement
+// error: a value is either configuration or a secret, never both.
+
+import type { DevEnvDocument, EnvValues } from './dev-env-document';
+
+export type DevEnvLayer = {
+  readonly source: string;
+  readonly document: DevEnvDocument;
+};
+
+export type ComposedDevEnvTarget = {
+  readonly group: string;
+  readonly workspace: string;
+  readonly env: EnvValues;
+  readonly sources: ReadonlyArray<string>;
+};
+
+export type ComposedDevEnv = {
+  readonly targets: ReadonlyArray<ComposedDevEnvTarget>;
+  readonly problems: ReadonlyArray<string>;
+};
+
+type MutableTarget = {
+  readonly group: string;
+  readonly workspace: string;
+  readonly env: Record<string, string>;
+  readonly sources: Array<string>;
+};
+
+const copyEnv = (source: EnvValues): Record<string, string> => {
+  const copy = Object.create(null) as Record<string, string>;
+  for (const [key, value] of Object.entries(source)) {
+    copy[key] = value;
+  }
+  return copy;
+};
+
+const applyLayer = (
+  merged: Map<string, MutableTarget>,
+  layer: DevEnvLayer,
+): void => {
+  for (const target of layer.document.targets) {
+    const key = `${target.group}.${target.workspace}`;
+    const existing = merged.get(key);
+    if (existing === undefined) {
+      merged.set(key, {
+        group: target.group,
+        workspace: target.workspace,
+        env: copyEnv(target.env),
+        sources: [layer.source],
+      });
+    } else {
+      for (const [envKey, value] of Object.entries(target.env)) {
+        existing.env[envKey] = value;
+      }
+      existing.sources.push(layer.source);
+    }
+  }
+};
+
+const sharedOverlapProblems = (
+  config: DevEnvLayer,
+  secrets: DevEnvLayer,
+): ReadonlyArray<string> => {
+  const secretTargets = new Map(
+    secrets.document.targets.map((target) => [
+      `${target.group}.${target.workspace}`,
+      target,
+    ]),
+  );
+  return config.document.targets.flatMap((target) => {
+    const counterpart = secretTargets.get(
+      `${target.group}.${target.workspace}`,
+    );
+    if (counterpart === undefined) {
+      return [];
+    }
+    return [...target.declaredKeys]
+      .filter((key) => counterpart.declaredKeys.has(key))
+      .map(
+        (key) =>
+          `${target.group}.${target.workspace}.${key} is declared in both ${config.source} and ${secrets.source}; a value is either configuration or a secret, so keep it in exactly one`,
+      );
+  });
+};
+
+export const composeDevEnv = (
+  config: DevEnvLayer | null,
+  secrets: DevEnvLayer,
+  local: DevEnvLayer | null,
+): ComposedDevEnv => {
+  const problems: ReadonlyArray<string> = [
+    ...(config === null ? [] : config.document.problems),
+    ...secrets.document.problems,
+    ...(local === null ? [] : local.document.problems),
+    ...(config === null ? [] : sharedOverlapProblems(config, secrets)),
+  ];
+  const merged = new Map<string, MutableTarget>();
+  for (const layer of [config, secrets, local]) {
+    if (layer !== null) {
+      applyLayer(merged, layer);
+    }
+  }
+  const targets = [...merged.values()]
+    .sort((a, b) =>
+      a.group === b.group
+        ? a.workspace.localeCompare(b.workspace)
+        : a.group.localeCompare(b.group),
+    )
+    .map((target) => ({
+      group: target.group,
+      workspace: target.workspace,
+      env: target.env,
+      sources: target.sources,
+    }));
+  return { targets, problems };
+};

package/src/dev-env-destination-duplicates.ts

@@ -1,19 +1,19 @@
 import { resolve } from 'node:path';
-import type { DevEnvWrite } from './dev-env-destination';
+import type { DevEnvMutation } from './dev-env-destination';
 
 export const duplicateDestinationProblems = (
   root: string,
-  writes: ReadonlyArray<DevEnvWrite>,
+  mutations: ReadonlyArray<DevEnvMutation>,
 ): ReadonlyArray<string> => {
-  const rawDuplicates = writes.flatMap((write, index) => {
-    const firstIndex = writes.findIndex(({ rel }) => rel === write.rel);
+  const rawDuplicates = mutations.flatMap((mutation, index) => {
+    const firstIndex = mutations.findIndex(({ rel }) => rel === mutation.rel);
     return firstIndex < index
-      ? [`${write.rel} is declared more than once`]
+      ? [`${mutation.rel} is declared more than once`]
       : [];
   });
-  const resolved = writes.map((write) => ({
-    write,
-    dest: resolve(root, write.rel),
+  const resolved = mutations.map((mutation) => ({
+    mutation,
+    dest: resolve(root, mutation.rel),
   }));
   const normalizedDuplicates = resolved.flatMap((destination, index) => {
     const firstIndex = resolved.findIndex(
@@ -22,9 +22,9 @@ export const duplicateDestinationProblems = (
     const first = resolved[firstIndex];
     return firstIndex < index &&
       first !== undefined &&
-      first.write.rel !== destination.write.rel
+      first.mutation.rel !== destination.mutation.rel
       ? [
-          `${destination.write.rel} resolves to the same destination as ${first.write.rel}`,
+          `${destination.mutation.rel} resolves to the same destination as ${first.mutation.rel}`,
         ]
       : [];
   });

package/src/dev-env-destination-gitignore.ts

@@ -0,0 +1,22 @@
+import { spawnSync } from 'node:child_process';
+
+// Decrypted values are safe only when git proves it will ignore them.
+export const devEnvGitIgnoreProblem = (
+  consumer: string,
+  rel: string,
+): string | null => {
+  const result = spawnSync('git', ['check-ignore', '-q', '--', rel], {
+    cwd: consumer,
+    encoding: 'utf8',
+  });
+  if (result.error !== undefined || result.status === null) {
+    return `cannot run git to verify ${rel} is gitignored`;
+  }
+  if (result.status === 0) {
+    return null;
+  }
+  if (result.status === 1) {
+    return `${rel} is not gitignored; ignore it before generating dev env files`;
+  }
+  return `cannot verify ${rel} is gitignored (git check-ignore exited ${result.status})`;
+};

package/src/dev-env-destination.ts

@@ -1,14 +1,29 @@
 import { randomUUID } from 'node:crypto';
 import type { Stats } from 'node:fs';
-import { lstat, realpath } from 'node:fs/promises';
+import { realpath } from 'node:fs/promises';
 import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
 import { duplicateDestinationProblems } from './dev-env-destination-duplicates';
+import { devEnvGitIgnoreProblem } from './dev-env-destination-gitignore';
+import {
+  devEnvStatOrNull,
+  matchesDevEnvRemoval,
+} from './dev-env-reconciliation';
 
 export type DevEnvWrite = {
   readonly rel: string;
   readonly content: string;
 };
 
+export type DevEnvRemoval = {
+  readonly rel: string;
+  readonly identity: {
+    readonly device: number;
+    readonly inode: number;
+  };
+};
+
+export type DevEnvMutation = DevEnvWrite | DevEnvRemoval;
+
 type DevEnvPathIdentity = {
   readonly path: string;
   readonly device: number;
@@ -16,7 +31,7 @@ type DevEnvPathIdentity = {
 };
 
 export type DevEnvDestination = {
-  readonly write: DevEnvWrite;
+  readonly mutation: DevEnvMutation;
   readonly dest: string;
   readonly previous: Stats | null;
   readonly temp: string;
@@ -26,6 +41,7 @@ export type DevEnvDestination = {
   readonly realRoot: string;
   backupCreated: boolean;
   committed: boolean;
+  rollbackBlocked: boolean;
 };
 
 type PreflightResult =
@@ -40,27 +56,16 @@ const containedBy = (root: string, candidate: string): boolean => {
   return rel !== '..' && !rel.startsWith(`..${sep}`) && !isAbsolute(rel);
 };
 
-export const devEnvStatOrNull = async (path: string): Promise<Stats | null> => {
-  try {
-    return await lstat(path);
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
-      return null;
-    }
-    throw error;
-  }
-};
-
 const inspectDestination = async (
   root: string,
   realRoot: string,
-  write: DevEnvWrite,
+  mutation: DevEnvMutation,
 ): Promise<string | DevEnvDestination> => {
-  const dest = resolve(root, write.rel);
+  const dest = resolve(root, mutation.rel);
   if (!containedBy(root, dest)) {
-    return `${write.rel} escapes the consumer repository`;
+    return `${mutation.rel} escapes the consumer repository`;
   }
-  const paths = dirname(write.rel)
+  const paths = dirname(mutation.rel)
     .split(sep)
     .map((_segment, index, segments) =>
       join(root, ...segments.slice(0, index + 1)),
@@ -73,19 +78,24 @@ const inspectDestination = async (
         entry === null || !entry.isDirectory() || entry.isSymbolicLink(),
     )
   ) {
-    return `${write.rel} has an unsafe destination directory`;
+    return `${mutation.rel} has an unsafe destination directory`;
   }
   const realParent = await realpath(dirname(dest));
   if (!containedBy(realRoot, realParent)) {
-    return `${write.rel} resolves outside the consumer repository`;
+    return `${mutation.rel} resolves outside the consumer repository`;
   }
   const previous = await devEnvStatOrNull(dest);
   if (previous !== null && (!previous.isFile() || previous.isSymbolicLink())) {
-    return `${write.rel} must be absent or a regular file, not a symlink or other file type`;
+    return `${mutation.rel} must be absent or a regular file, not a symlink or other file type`;
+  }
+  if (
+    !('content' in mutation || (await matchesDevEnvRemoval(dest, mutation)))
+  ) {
+    return `${mutation.rel} changed after planning`;
   }
   const suffix = randomUUID();
   return {
-    write,
+    mutation,
     dest,
     previous,
     temp: join(dirname(dest), `.env.local.standards-${suffix}.tmp`),
@@ -99,32 +109,44 @@ const inspectDestination = async (
     realRoot,
     backupCreated: false,
     committed: false,
+    rollbackBlocked: false,
   };
 };
 
 export const preflightDevEnvDestinations = async (
   consumer: string,
-  writes: ReadonlyArray<DevEnvWrite>,
+  mutations: ReadonlyArray<DevEnvMutation>,
 ): Promise<PreflightResult> => {
   const root = resolve(consumer);
   const realRoot = await realpath(root);
-  const duplicates = duplicateDestinationProblems(root, writes);
+  const duplicates = duplicateDestinationProblems(root, mutations);
   const checked = await Promise.all(
-    writes.map(async (write) => {
+    mutations.map(async (mutation) => {
       try {
-        return await inspectDestination(root, realRoot, write);
+        return await inspectDestination(root, realRoot, mutation);
       } catch (error) {
         const detail = error instanceof Error ? error.message : String(error);
-        return `${write.rel} could not be preflighted: ${detail}`;
+        return `${mutation.rel} could not be preflighted: ${detail}`;
       }
     }),
   );
   const destinations = checked.filter(
     (item): item is DevEnvDestination => typeof item !== 'string',
   );
+  const gitignoreProblems = destinations.flatMap((destination) =>
+    [
+      destination.mutation.rel,
+      relative(root, destination.temp),
+      relative(root, destination.backup),
+    ].flatMap((rel) => {
+      const problem = devEnvGitIgnoreProblem(consumer, rel);
+      return problem === null ? [] : [problem];
+    }),
+  );
   const problems = [
     ...duplicates,
     ...checked.filter((item): item is string => typeof item === 'string'),
+    ...gitignoreProblems,
   ];
   return problems.length > 0
     ? { ok: false, problems }
@@ -158,18 +180,18 @@ export const devEnvParentProblem = async (
       realParent !== destination.realParent ||
       !containedBy(destination.realRoot, realParent)
     ) {
-      return `${destination.write.rel} destination directory changed after preflight`;
+      return `${destination.mutation.rel} destination directory changed after preflight`;
     }
     return null;
   } catch {
-    return `${destination.write.rel} destination directory changed after preflight`;
+    return `${destination.mutation.rel} destination directory changed after preflight`;
   }
 };
 
 export const devEnvDestinationProblems = async (
   consumer: string,
-  writes: ReadonlyArray<DevEnvWrite>,
+  mutations: ReadonlyArray<DevEnvMutation>,
 ): Promise<ReadonlyArray<string>> => {
-  const checked = await preflightDevEnvDestinations(consumer, writes);
+  const checked = await preflightDevEnvDestinations(consumer, mutations);
   return checked.ok ? [] : checked.problems;
 };

package/src/dev-env-document.ts

@@ -12,6 +12,7 @@ export type DevEnvTarget = {
   readonly group: string;
   readonly workspace: string;
   readonly env: EnvValues;
+  readonly declaredKeys: ReadonlySet<string>;
 };
 
 export type DevEnvDocument = {
@@ -31,12 +32,19 @@ type ParsedWorkspaces = {
 const parseWorkspaceEnv = (
   label: string,
   raw: Record<string, unknown>,
-): { readonly env: EnvValues; readonly problems: ReadonlyArray<string> } => {
+): {
+  readonly env: EnvValues;
+  readonly declaredKeys: ReadonlySet<string>;
+  readonly problems: ReadonlyArray<string>;
+} => {
   const problems: Array<string> = [];
-  const entries: Array<readonly [string, string]> = [];
+  const env = Object.create(null) as Record<string, string>;
+  const declaredKeys = new Set<string>();
   for (const [key, value] of Object.entries(raw)) {
     const portableName = PORTABLE_ENV_NAME.test(key);
-    if (!portableName) {
+    if (portableName) {
+      declaredKeys.add(key);
+    } else {
       problems.push(
         `${label} env key ${JSON.stringify(key)} must be a portable environment variable name`,
       );
@@ -48,10 +56,10 @@ const parseWorkspaceEnv = (
         `${label}.${key} cannot be represented losslessly in Bun dotenv syntax`,
       );
     } else if (portableName) {
-      entries.push([key, value]);
+      env[key] = value;
     }
   }
-  return { env: Object.fromEntries(entries), problems };
+  return { env, declaredKeys, problems };
 };
 
 const parseWorkspaces = (
@@ -75,7 +83,12 @@ const parseWorkspaces = (
       );
       problems.push(...parsed.problems);
       if (validWorkspace) {
-        targets.push({ group, workspace, env: parsed.env });
+        targets.push({
+          group,
+          workspace,
+          env: parsed.env,
+          declaredKeys: parsed.declaredKeys,
+        });
       }
     } else {
       problems.push(

package/src/dev-env-dotenv.ts

@@ -1,6 +1,15 @@
 import type { EnvValues } from './dev-env-document';
 import { encodeBunDotenvValue } from './dev-env-dotenv-value';
 
+export const DEV_ENV_GENERATED_HEADER =
+  '# Generated by `bun standards dev-env` (just dev-env-generate). Do not edit.';
+
+export const hasDevEnvGeneratedHeader = (content: string): boolean => {
+  const newline = content.indexOf('\n');
+  const firstLine = newline === -1 ? content : content.slice(0, newline);
+  return firstLine === DEV_ENV_GENERATED_HEADER;
+};
+
 const renderValue = (key: string, value: string): string => {
   const encoded = encodeBunDotenvValue(value);
   if (encoded === null) {
@@ -11,12 +20,12 @@ const renderValue = (key: string, value: string): string => {
 
 export const renderDotenv = (
   sourcePath: string,
-  secretsFile: string,
+  sources: ReadonlyArray<string>,
   env: EnvValues,
 ): string =>
   [
-    '# Generated by `bun standards dev-env` (just dev-env-generate). Do not edit.',
-    `# Source: ${secretsFile} (${sourcePath})`,
+    DEV_ENV_GENERATED_HEADER,
+    `# Sources: ${sources.join(' + ')} (${sourcePath})`,
     ...Object.entries(env).map(([key, value]) => renderValue(key, value)),
     '',
   ].join('\n');