@davidvornholt/standards 0.20.0 → 0.22.0

package/src/dev-env-brokered.ts

@@ -0,0 +1,83 @@
+// Brokered S3 pair references let a plain configuration layer point a
+// workspace env key at a credential pair the creds broker owns in another
+// SOPS target. The secret value then has exactly one home — the broker's
+// ledger — so rotation stays the broker's job and no copy can go stale.
+// A reference is configuration, never a secret, so the SOPS-encrypted
+// secrets layer must not declare one.
+
+import { parseDestination } from './creds-dest';
+import { parseSopsKeyPath } from './creds-sops-structure';
+import { isRecord } from './github-settings-parse';
+
+export const BROKERED_S3_PARTS = [
+  'access_key_id',
+  'secret_access_key',
+] as const;
+
+export type BrokeredS3Part = (typeof BROKERED_S3_PARTS)[number];
+
+export type BrokeredS3Reference = {
+  readonly brokeredS3: string;
+  readonly key: string;
+  readonly part: BrokeredS3Part;
+};
+
+const REFERENCE_PROPERTIES = ['brokeredS3', 'key', 'part'] as const;
+
+export const isBrokeredS3ReferenceShape = (
+  value: unknown,
+): value is Record<string, unknown> => isRecord(value) && 'brokeredS3' in value;
+
+export type BrokeredS3ParseResult =
+  | { readonly ok: true; readonly reference: BrokeredS3Reference }
+  | { readonly ok: false; readonly problems: ReadonlyArray<string> };
+
+const isBrokeredS3Part = (value: unknown): value is BrokeredS3Part =>
+  BROKERED_S3_PARTS.includes(value as BrokeredS3Part);
+
+// Target existence and contained-path resolution are owned by
+// `resolveTargetRel` at resolution time; parsing rejects a target name that
+// could never name a broker destination.
+export const parseBrokeredS3Reference = (
+  label: string,
+  raw: Record<string, unknown>,
+): BrokeredS3ParseResult => {
+  const problems: Array<string> = [];
+  for (const property of Object.keys(raw)) {
+    if (!REFERENCE_PROPERTIES.includes(property as 'brokeredS3')) {
+      problems.push(
+        `${label} brokered S3 pair reference has unknown property ${JSON.stringify(property)}; allowed properties are brokeredS3, key, and part`,
+      );
+    }
+  }
+  const { brokeredS3: target, key, part } = raw;
+  if (
+    typeof target !== 'string' ||
+    parseDestination(`${target}:reference_validation`) === null
+  ) {
+    problems.push(
+      `${label} brokered S3 pair reference needs a valid "brokeredS3" secrets target name`,
+    );
+  }
+  if (typeof key !== 'string' || parseSopsKeyPath(key) === null) {
+    problems.push(
+      `${label} brokered S3 pair reference needs a "key" naming the pair's dotted SOPS key`,
+    );
+  }
+  if (!isBrokeredS3Part(part)) {
+    problems.push(
+      `${label} brokered S3 pair reference needs a "part" of either "access_key_id" or "secret_access_key"`,
+    );
+  }
+  if (problems.length > 0) {
+    return { ok: false, problems };
+  }
+  return {
+    ok: true,
+    reference: {
+      brokeredS3: target as string,
+      key: key as string,
+      part: part as BrokeredS3Part,
+    },
+  };
+};

package/src/dev-env-compose.ts

@@ -0,0 +1,129 @@
+// Composes the dev env layers — tracked config/dev.yaml, decrypted
+// secrets/dev.yaml, and the gitignored per-machine config/dev.local.yaml —
+// into one workspace-keyed target set. Later layers override earlier ones
+// per env key. A key declared by both tracked shared layers is a placement
+// error: a value is either configuration or a secret, never both.
+
+import type {
+  DevEnvDocument,
+  DevEnvValue,
+  EnvValues,
+} from './dev-env-document';
+
+export type DevEnvLayer = {
+  readonly source: string;
+  readonly document: DevEnvDocument;
+};
+
+export type ComposedDevEnvTarget = {
+  readonly group: string;
+  readonly workspace: string;
+  readonly env: EnvValues;
+  readonly sources: ReadonlyArray<string>;
+};
+
+export type ComposedDevEnv = {
+  readonly targets: ReadonlyArray<ComposedDevEnvTarget>;
+  readonly brokeredReferences: ReadonlySet<string>;
+  readonly problems: ReadonlyArray<string>;
+};
+
+type MutableTarget = {
+  readonly group: string;
+  readonly workspace: string;
+  readonly env: Record<string, DevEnvValue>;
+  readonly sources: Array<string>;
+};
+
+const copyEnv = (source: EnvValues): Record<string, DevEnvValue> => {
+  const copy = Object.create(null) as Record<string, DevEnvValue>;
+  for (const [key, value] of Object.entries(source)) {
+    copy[key] = value;
+  }
+  return copy;
+};
+
+const applyLayer = (
+  merged: Map<string, MutableTarget>,
+  layer: DevEnvLayer,
+): void => {
+  for (const target of layer.document.targets) {
+    const key = `${target.group}.${target.workspace}`;
+    const existing = merged.get(key);
+    if (existing === undefined) {
+      merged.set(key, {
+        group: target.group,
+        workspace: target.workspace,
+        env: copyEnv(target.env),
+        sources: [layer.source],
+      });
+    } else {
+      for (const [envKey, value] of Object.entries(target.env)) {
+        existing.env[envKey] = value;
+      }
+      existing.sources.push(layer.source);
+    }
+  }
+};
+
+const sharedOverlapProblems = (
+  config: DevEnvLayer,
+  secrets: DevEnvLayer,
+): ReadonlyArray<string> => {
+  const secretTargets = new Map(
+    secrets.document.targets.map((target) => [
+      `${target.group}.${target.workspace}`,
+      target,
+    ]),
+  );
+  return config.document.targets.flatMap((target) => {
+    const counterpart = secretTargets.get(
+      `${target.group}.${target.workspace}`,
+    );
+    if (counterpart === undefined) {
+      return [];
+    }
+    return [...target.declaredKeys]
+      .filter((key) => counterpart.declaredKeys.has(key))
+      .map(
+        (key) =>
+          `${target.group}.${target.workspace}.${key} is declared in both ${config.source} and ${secrets.source}; a value is either configuration or a secret, so keep it in exactly one`,
+      );
+  });
+};
+
+export const composeDevEnv = (
+  config: DevEnvLayer | null,
+  secrets: DevEnvLayer,
+  local: DevEnvLayer | null,
+): ComposedDevEnv => {
+  const problems: ReadonlyArray<string> = [
+    ...(config === null ? [] : config.document.problems),
+    ...secrets.document.problems,
+    ...(local === null ? [] : local.document.problems),
+    ...(config === null ? [] : sharedOverlapProblems(config, secrets)),
+  ];
+  const merged = new Map<string, MutableTarget>();
+  for (const layer of [config, secrets, local]) {
+    if (layer !== null) {
+      applyLayer(merged, layer);
+    }
+  }
+  const targets = [...merged.values()]
+    .sort((a, b) =>
+      a.group === b.group
+        ? a.workspace.localeCompare(b.workspace)
+        : a.group.localeCompare(b.group),
+    )
+    .map((target) => ({
+      group: target.group,
+      workspace: target.workspace,
+      env: target.env,
+      sources: target.sources,
+    }));
+  return {
+    targets,
+    brokeredReferences: secrets.document.brokeredReferences,
+    problems,
+  };
+};

package/src/dev-env-destination-duplicates.ts

@@ -1,19 +1,19 @@
 import { resolve } from 'node:path';
-import type { DevEnvWrite } from './dev-env-destination';
+import type { DevEnvMutation } from './dev-env-destination';
 
 export const duplicateDestinationProblems = (
   root: string,
-  writes: ReadonlyArray<DevEnvWrite>,
+  mutations: ReadonlyArray<DevEnvMutation>,
 ): ReadonlyArray<string> => {
-  const rawDuplicates = writes.flatMap((write, index) => {
-    const firstIndex = writes.findIndex(({ rel }) => rel === write.rel);
+  const rawDuplicates = mutations.flatMap((mutation, index) => {
+    const firstIndex = mutations.findIndex(({ rel }) => rel === mutation.rel);
     return firstIndex < index
-      ? [`${write.rel} is declared more than once`]
+      ? [`${mutation.rel} is declared more than once`]
       : [];
   });
-  const resolved = writes.map((write) => ({
-    write,
-    dest: resolve(root, write.rel),
+  const resolved = mutations.map((mutation) => ({
+    mutation,
+    dest: resolve(root, mutation.rel),
   }));
   const normalizedDuplicates = resolved.flatMap((destination, index) => {
     const firstIndex = resolved.findIndex(
@@ -22,9 +22,9 @@ export const duplicateDestinationProblems = (
     const first = resolved[firstIndex];
     return firstIndex < index &&
       first !== undefined &&
-      first.write.rel !== destination.write.rel
+      first.mutation.rel !== destination.mutation.rel
       ? [
-          `${destination.write.rel} resolves to the same destination as ${first.write.rel}`,
+          `${destination.mutation.rel} resolves to the same destination as ${first.mutation.rel}`,
         ]
       : [];
   });

package/src/dev-env-destination-gitignore.ts

@@ -0,0 +1,22 @@
+import { spawnSync } from 'node:child_process';
+
+// Decrypted values are safe only when git proves it will ignore them.
+export const devEnvGitIgnoreProblem = (
+  consumer: string,
+  rel: string,
+): string | null => {
+  const result = spawnSync('git', ['check-ignore', '-q', '--', rel], {
+    cwd: consumer,
+    encoding: 'utf8',
+  });
+  if (result.error !== undefined || result.status === null) {
+    return `cannot run git to verify ${rel} is gitignored`;
+  }
+  if (result.status === 0) {
+    return null;
+  }
+  if (result.status === 1) {
+    return `${rel} is not gitignored; ignore it before generating dev env files`;
+  }
+  return `cannot verify ${rel} is gitignored (git check-ignore exited ${result.status})`;
+};

package/src/dev-env-destination.ts

@@ -1,14 +1,29 @@
 import { randomUUID } from 'node:crypto';
 import type { Stats } from 'node:fs';
-import { lstat, realpath } from 'node:fs/promises';
+import { realpath } from 'node:fs/promises';
 import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
 import { duplicateDestinationProblems } from './dev-env-destination-duplicates';
+import { devEnvGitIgnoreProblem } from './dev-env-destination-gitignore';
+import {
+  devEnvStatOrNull,
+  matchesDevEnvRemoval,
+} from './dev-env-reconciliation';
 
 export type DevEnvWrite = {
   readonly rel: string;
   readonly content: string;
 };
 
+export type DevEnvRemoval = {
+  readonly rel: string;
+  readonly identity: {
+    readonly device: number;
+    readonly inode: number;
+  };
+};
+
+export type DevEnvMutation = DevEnvWrite | DevEnvRemoval;
+
 type DevEnvPathIdentity = {
   readonly path: string;
   readonly device: number;
@@ -16,7 +31,7 @@ type DevEnvPathIdentity = {
 };
 
 export type DevEnvDestination = {
-  readonly write: DevEnvWrite;
+  readonly mutation: DevEnvMutation;
   readonly dest: string;
   readonly previous: Stats | null;
   readonly temp: string;
@@ -26,6 +41,7 @@ export type DevEnvDestination = {
   readonly realRoot: string;
   backupCreated: boolean;
   committed: boolean;
+  rollbackBlocked: boolean;
 };
 
 type PreflightResult =
@@ -40,27 +56,16 @@ const containedBy = (root: string, candidate: string): boolean => {
   return rel !== '..' && !rel.startsWith(`..${sep}`) && !isAbsolute(rel);
 };
 
-export const devEnvStatOrNull = async (path: string): Promise<Stats | null> => {
-  try {
-    return await lstat(path);
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
-      return null;
-    }
-    throw error;
-  }
-};
-
 const inspectDestination = async (
   root: string,
   realRoot: string,
-  write: DevEnvWrite,
+  mutation: DevEnvMutation,
 ): Promise<string | DevEnvDestination> => {
-  const dest = resolve(root, write.rel);
+  const dest = resolve(root, mutation.rel);
   if (!containedBy(root, dest)) {
-    return `${write.rel} escapes the consumer repository`;
+    return `${mutation.rel} escapes the consumer repository`;
   }
-  const paths = dirname(write.rel)
+  const paths = dirname(mutation.rel)
     .split(sep)
     .map((_segment, index, segments) =>
       join(root, ...segments.slice(0, index + 1)),
@@ -73,19 +78,24 @@ const inspectDestination = async (
         entry === null || !entry.isDirectory() || entry.isSymbolicLink(),
     )
   ) {
-    return `${write.rel} has an unsafe destination directory`;
+    return `${mutation.rel} has an unsafe destination directory`;
   }
   const realParent = await realpath(dirname(dest));
   if (!containedBy(realRoot, realParent)) {
-    return `${write.rel} resolves outside the consumer repository`;
+    return `${mutation.rel} resolves outside the consumer repository`;
   }
   const previous = await devEnvStatOrNull(dest);
   if (previous !== null && (!previous.isFile() || previous.isSymbolicLink())) {
-    return `${write.rel} must be absent or a regular file, not a symlink or other file type`;
+    return `${mutation.rel} must be absent or a regular file, not a symlink or other file type`;
+  }
+  if (
+    !('content' in mutation || (await matchesDevEnvRemoval(dest, mutation)))
+  ) {
+    return `${mutation.rel} changed after planning`;
   }
   const suffix = randomUUID();
   return {
-    write,
+    mutation,
     dest,
     previous,
     temp: join(dirname(dest), `.env.local.standards-${suffix}.tmp`),
@@ -99,32 +109,44 @@ const inspectDestination = async (
     realRoot,
     backupCreated: false,
     committed: false,
+    rollbackBlocked: false,
   };
 };
 
 export const preflightDevEnvDestinations = async (
   consumer: string,
-  writes: ReadonlyArray<DevEnvWrite>,
+  mutations: ReadonlyArray<DevEnvMutation>,
 ): Promise<PreflightResult> => {
   const root = resolve(consumer);
   const realRoot = await realpath(root);
-  const duplicates = duplicateDestinationProblems(root, writes);
+  const duplicates = duplicateDestinationProblems(root, mutations);
   const checked = await Promise.all(
-    writes.map(async (write) => {
+    mutations.map(async (mutation) => {
       try {
-        return await inspectDestination(root, realRoot, write);
+        return await inspectDestination(root, realRoot, mutation);
       } catch (error) {
         const detail = error instanceof Error ? error.message : String(error);
-        return `${write.rel} could not be preflighted: ${detail}`;
+        return `${mutation.rel} could not be preflighted: ${detail}`;
       }
     }),
   );
   const destinations = checked.filter(
     (item): item is DevEnvDestination => typeof item !== 'string',
   );
+  const gitignoreProblems = destinations.flatMap((destination) =>
+    [
+      destination.mutation.rel,
+      relative(root, destination.temp),
+      relative(root, destination.backup),
+    ].flatMap((rel) => {
+      const problem = devEnvGitIgnoreProblem(consumer, rel);
+      return problem === null ? [] : [problem];
+    }),
+  );
   const problems = [
     ...duplicates,
     ...checked.filter((item): item is string => typeof item === 'string'),
+    ...gitignoreProblems,
   ];
   return problems.length > 0
     ? { ok: false, problems }
@@ -158,18 +180,18 @@ export const devEnvParentProblem = async (
       realParent !== destination.realParent ||
       !containedBy(destination.realRoot, realParent)
     ) {
-      return `${destination.write.rel} destination directory changed after preflight`;
+      return `${destination.mutation.rel} destination directory changed after preflight`;
     }
     return null;
   } catch {
-    return `${destination.write.rel} destination directory changed after preflight`;
+    return `${destination.mutation.rel} destination directory changed after preflight`;
   }
 };
 
 export const devEnvDestinationProblems = async (
   consumer: string,
-  writes: ReadonlyArray<DevEnvWrite>,
+  mutations: ReadonlyArray<DevEnvMutation>,
 ): Promise<ReadonlyArray<string>> => {
-  const checked = await preflightDevEnvDestinations(consumer, writes);
+  const checked = await preflightDevEnvDestinations(consumer, mutations);
   return checked.ok ? [] : checked.problems;
 };

package/src/dev-env-document.ts

@@ -1,21 +1,40 @@
-// Workspace-group keyed dev secrets document: `apps.<name>` and
-// `packages.<name>` map env keys to string values — the shape mirrored in
-// secrets/dev.example.yaml. Parsing gathers every problem instead of failing
-// on the first one so a malformed document is repaired in one pass.
+// Workspace-group keyed dev env document: `apps.<name>` and
+// `packages.<name>` map env keys to values — the shape mirrored in
+// secrets/dev.example.yaml. Plain configuration layers may also declare a
+// brokered S3 pair reference instead of a literal; the secrets layer may
+// not, because a reference is configuration. The secrets layer instead owns
+// the encrypted allowlist that authorizes exact target/key pairs. Parsing
+// gathers every problem instead of failing on the first one so a malformed
+// document is repaired in one pass.
 
+import {
+  type BrokeredS3Reference,
+  isBrokeredS3ReferenceShape,
+  parseBrokeredS3Reference,
+} from './dev-env-brokered';
+import {
+  BROKERED_REFERENCES_KEY,
+  parseBrokeredAllowlist,
+} from './dev-env-brokered-allowlist';
 import { encodeBunDotenvValue } from './dev-env-dotenv-value';
 import { isRecord } from './github-settings-parse';
 
-export type EnvValues = Readonly<Record<string, string>>;
+export type DevEnvValue = string | BrokeredS3Reference;
+
+export type EnvValues = Readonly<Record<string, DevEnvValue>>;
+
+export type DevEnvLayerKind = 'configuration' | 'secrets';
 
 export type DevEnvTarget = {
   readonly group: string;
   readonly workspace: string;
   readonly env: EnvValues;
+  readonly declaredKeys: ReadonlySet<string>;
 };
 
 export type DevEnvDocument = {
   readonly targets: ReadonlyArray<DevEnvTarget>;
+  readonly brokeredReferences: ReadonlySet<string>;
   readonly problems: ReadonlyArray<string>;
 };
 
@@ -28,36 +47,84 @@ type ParsedWorkspaces = {
   readonly problems: ReadonlyArray<string>;
 };
 
+const parseWorkspaceValue = (
+  label: string,
+  key: string,
+  value: unknown,
+  layerKind: DevEnvLayerKind,
+): {
+  readonly value: DevEnvValue | null;
+  readonly problems: ReadonlyArray<string>;
+} => {
+  if (typeof value === 'string') {
+    return encodeBunDotenvValue(value) === null
+      ? {
+          value: null,
+          problems: [
+            `${label}.${key} cannot be represented losslessly in Bun dotenv syntax`,
+          ],
+        }
+      : { value, problems: [] };
+  }
+  if (isBrokeredS3ReferenceShape(value)) {
+    if (layerKind === 'secrets') {
+      return {
+        value: null,
+        problems: [
+          `${label}.${key} is a brokered S3 pair reference; references are configuration and belong in config/dev.yaml or config/dev.local.yaml, not the secrets layer`,
+        ],
+      };
+    }
+    const parsed = parseBrokeredS3Reference(`${label}.${key}`, value);
+    return parsed.ok
+      ? { value: parsed.reference, problems: [] }
+      : { value: null, problems: parsed.problems };
+  }
+  return {
+    value: null,
+    problems: [
+      layerKind === 'configuration'
+        ? `${label}.${key} must be a string value or a brokered S3 pair reference`
+        : `${label}.${key} must be a string value`,
+    ],
+  };
+};
+
 const parseWorkspaceEnv = (
   label: string,
   raw: Record<string, unknown>,
-): { readonly env: EnvValues; readonly problems: ReadonlyArray<string> } => {
+  layerKind: DevEnvLayerKind,
+): {
+  readonly env: EnvValues;
+  readonly declaredKeys: ReadonlySet<string>;
+  readonly problems: ReadonlyArray<string>;
+} => {
   const problems: Array<string> = [];
-  const entries: Array<readonly [string, string]> = [];
+  const env = Object.create(null) as Record<string, DevEnvValue>;
+  const declaredKeys = new Set<string>();
   for (const [key, value] of Object.entries(raw)) {
     const portableName = PORTABLE_ENV_NAME.test(key);
-    if (!portableName) {
+    if (portableName) {
+      declaredKeys.add(key);
+    } else {
       problems.push(
         `${label} env key ${JSON.stringify(key)} must be a portable environment variable name`,
       );
     }
-    if (typeof value !== 'string') {
-      problems.push(`${label}.${key} must be a string value`);
-    } else if (encodeBunDotenvValue(value) === null) {
-      problems.push(
-        `${label}.${key} cannot be represented losslessly in Bun dotenv syntax`,
-      );
-    } else if (portableName) {
-      entries.push([key, value]);
+    const parsed = parseWorkspaceValue(label, key, value, layerKind);
+    problems.push(...parsed.problems);
+    if (portableName && parsed.value !== null) {
+      env[key] = parsed.value;
     }
   }
-  return { env: Object.fromEntries(entries), problems };
+  return { env, declaredKeys, problems };
 };
 
 const parseWorkspaces = (
   source: string,
   group: string,
   workspaces: Record<string, unknown>,
+  layerKind: DevEnvLayerKind,
 ): ParsedWorkspaces => {
   const problems: Array<string> = [];
   const targets: Array<DevEnvTarget> = [];
@@ -72,10 +139,16 @@ const parseWorkspaces = (
       const parsed = parseWorkspaceEnv(
         `${source} "${group}.${workspace}"`,
         env,
+        layerKind,
       );
       problems.push(...parsed.problems);
       if (validWorkspace) {
-        targets.push({ group, workspace, env: parsed.env });
+        targets.push({
+          group,
+          workspace,
+          env: parsed.env,
+          declaredKeys: parsed.declaredKeys,
+        });
       }
     } else {
       problems.push(
@@ -89,22 +162,31 @@ const parseWorkspaces = (
 export const parseDevEnvDocument = (
   raw: unknown,
   source: string,
+  layerKind: DevEnvLayerKind,
 ): DevEnvDocument => {
   if (!isRecord(raw)) {
     return {
       targets: [],
+      brokeredReferences: new Set(),
       problems: [`${source} must decrypt to a YAML object`],
     };
   }
   const problems: Array<string> = [];
   const targets: Array<DevEnvTarget> = [];
+  const brokeredReferences = new Set<string>();
   for (const [group, workspaces] of Object.entries(raw)) {
-    if (!WORKSPACE_GROUPS.includes(group)) {
+    if (group === BROKERED_REFERENCES_KEY) {
+      const parsed = parseBrokeredAllowlist(source, workspaces, layerKind);
+      problems.push(...parsed.problems);
+      for (const entry of parsed.entries) {
+        brokeredReferences.add(entry);
+      }
+    } else if (!WORKSPACE_GROUPS.includes(group)) {
       problems.push(
-        `${source} top-level key "${group}" must be "apps" or "packages"`,
+        `${source} top-level key "${group}" must be "apps" or "packages"${layerKind === 'secrets' ? ` or "${BROKERED_REFERENCES_KEY}"` : ''}`,
       );
     } else if (isRecord(workspaces)) {
-      const parsed = parseWorkspaces(source, group, workspaces);
+      const parsed = parseWorkspaces(source, group, workspaces, layerKind);
       problems.push(...parsed.problems);
       targets.push(...parsed.targets);
     } else {
@@ -113,5 +195,5 @@ export const parseDevEnvDocument = (
       );
     }
   }
-  return { targets, problems };
+  return { targets, brokeredReferences, problems };
 };