@davidvornholt/standards 0.21.0 → 0.22.1

package/src/dev-env-brokered-preserve.ts

@@ -0,0 +1,65 @@
+import { lstatSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import type { DevEnvValue } from './dev-env-document';
+import { preserveRenderedDotenvValues, renderDotenv } from './dev-env-dotenv';
+
+export type DevEnvRunOptions = {
+  readonly preservedBrokeredReferences: ReadonlySet<string>;
+};
+
+type RenderInput = {
+  readonly consumer: string;
+  readonly rel: string;
+  readonly sourcePath: string;
+  readonly sources: ReadonlyArray<string>;
+  readonly resolvedEnv: Readonly<Record<string, string>>;
+  readonly composedEnv: Readonly<Record<string, DevEnvValue>>;
+  readonly preservedReferences: ReadonlySet<string>;
+};
+
+type RenderResult =
+  | { readonly ok: true; readonly content: string }
+  | { readonly ok: false; readonly problem: string };
+
+const preservedKeys = (
+  env: Readonly<Record<string, DevEnvValue>>,
+  references: ReadonlySet<string>,
+): ReadonlyArray<string> =>
+  Object.entries(env).flatMap(([key, value]) =>
+    typeof value !== 'string' &&
+    references.has(`${value.brokeredS3}:${value.key}`)
+      ? [key]
+      : [],
+  );
+
+export const renderDevEnvPreservingUnsafeReferences = (
+  input: RenderInput,
+): RenderResult => {
+  const rendered = renderDotenv(
+    input.sourcePath,
+    input.sources,
+    input.resolvedEnv,
+  );
+  const keys = preservedKeys(input.composedEnv, input.preservedReferences);
+  if (keys.length === 0) {
+    return { ok: true, content: rendered };
+  }
+  let previous: string;
+  try {
+    const path = join(input.consumer, input.rel);
+    const stat = lstatSync(path);
+    if (!stat.isFile() || stat.isSymbolicLink()) {
+      throw new Error('not a regular file');
+    }
+    previous = readFileSync(path, 'utf8');
+  } catch {
+    return {
+      ok: false,
+      problem: `${input.rel} has no readable prior generated values for unsafe brokered references`,
+    };
+  }
+  const preserved = preserveRenderedDotenvValues(previous, rendered, keys);
+  return preserved.ok
+    ? preserved
+    : { ok: false, problem: `${input.rel}: ${preserved.problem}` };
+};

package/src/dev-env-brokered-refresh.ts

@@ -0,0 +1,129 @@
+// After the creds broker writes into a SOPS target, the generated dev env
+// files are stale if any workspace env key references a brokered pair in
+// that target. Regenerating here closes the rotation loop: a renewed pair
+// reaches every generated .env.local without a manual copy step.
+
+import { resolveTargetRelResult } from './creds-target';
+import { runDevEnv } from './dev-env';
+import { parseDevEnvDocument } from './dev-env-document';
+import { readPlainLayer } from './dev-env-plain-layer';
+
+const DEV_PLAIN_LAYERS = ['config/dev.yaml', 'config/dev.local.yaml'] as const;
+
+type LayerReferenceDiscovery = {
+  readonly references: ReadonlyArray<{
+    readonly target: string;
+    readonly key: string;
+  }>;
+  readonly problems: ReadonlyArray<string>;
+};
+
+const layerReferenceTargets = (
+  consumer: string,
+  layerRel: string,
+): LayerReferenceDiscovery => {
+  const layer = readPlainLayer(consumer, layerRel);
+  if (layer.input === null) {
+    return { references: [], problems: layer.problems };
+  }
+  const document = parseDevEnvDocument(
+    layer.input.raw,
+    layerRel,
+    'configuration',
+  );
+  return {
+    references: document.targets.flatMap((target) =>
+      Object.values(target.env).flatMap((value) =>
+        typeof value === 'string'
+          ? []
+          : [{ target: value.brokeredS3, key: value.key }],
+      ),
+    ),
+    problems: [...layer.problems, ...document.problems],
+  };
+};
+
+type BrokeredReferenceDiscovery = {
+  readonly references: ReadonlySet<string>;
+  readonly problems: ReadonlyArray<string>;
+};
+
+export type BrokeredRefreshEvidence = {
+  readonly target: string;
+  readonly key: string;
+  readonly safety: 'verified' | 'unsafe';
+};
+
+const referenceIdentity = (target: string, key: string): string =>
+  `${target}:${key}`;
+
+const discoverBrokeredReferenceRels = (
+  consumer: string,
+): BrokeredReferenceDiscovery => {
+  const references = new Set<string>();
+  const problems: Array<string> = [];
+  for (const layerRel of DEV_PLAIN_LAYERS) {
+    const layer = layerReferenceTargets(consumer, layerRel);
+    problems.push(...layer.problems);
+    for (const reference of layer.references) {
+      const resolved = resolveTargetRelResult(consumer, reference.target);
+      if (resolved.ok) {
+        references.add(referenceIdentity(reference.target, reference.key));
+      } else {
+        problems.push(
+          resolved.kind === 'ambiguous'
+            ? `dev config references ${resolved.problem}`
+            : `dev config references secrets target "${reference.target}", but it is missing or is not a contained regular SOPS file`,
+        );
+      }
+    }
+  }
+  return { references, problems };
+};
+
+export const refreshDevEnvForSopsWrites = async (
+  consumer: string,
+  evidence: ReadonlyArray<BrokeredRefreshEvidence>,
+): Promise<boolean> => {
+  if (evidence.length === 0) {
+    return true;
+  }
+  const discovery = discoverBrokeredReferenceRels(consumer);
+  if (discovery.problems.length > 0) {
+    console.error(
+      `standards creds: cannot determine whether changed credentials require dev env regeneration because the configuration has ${discovery.problems.length} problem(s):`,
+    );
+    console.error(
+      discovery.problems.map((problem) => `  - ${problem}`).join('\n'),
+    );
+    console.error(
+      'standards creds: credential changes were written, but generated .env.local files were not updated; fix the configuration and run `bun standards dev-env`',
+    );
+    return false;
+  }
+  const unsafe = new Set(
+    evidence
+      .filter((entry) => entry.safety === 'unsafe')
+      .map((entry) => referenceIdentity(entry.target, entry.key)),
+  );
+  const verified = new Set(
+    evidence
+      .filter((entry) => entry.safety === 'verified')
+      .map((entry) => referenceIdentity(entry.target, entry.key)),
+  );
+  const changedReferences = [...verified].filter(
+    (reference) =>
+      discovery.references.has(reference) && !unsafe.has(reference),
+  );
+  if (changedReferences.length === 0) {
+    return true;
+  }
+  console.log(
+    'standards creds: regenerating dev env files (a brokered pair referenced by the dev config changed)',
+  );
+  return await runDevEnv(consumer, {
+    preservedBrokeredReferences: new Set(
+      [...unsafe].filter((reference) => discovery.references.has(reference)),
+    ),
+  });
+};

package/src/dev-env-brokered-resolve.ts

@@ -0,0 +1,158 @@
+// Resolves brokered S3 pair references in the composed dev env into literal
+// values by decrypting each referenced SOPS target once. Resolution runs
+// after layer composition so a later layer's literal can override an earlier
+// layer's reference, and a reference that lost the merge is never decrypted.
+
+import { resolveTargetRelResult } from './creds-target';
+import type { BrokeredS3Reference } from './dev-env-brokered';
+import type { ComposedDevEnvTarget } from './dev-env-compose';
+import { encodeBunDotenvValue } from './dev-env-dotenv-value';
+import { isRecord } from './github-settings-parse';
+import { runSops } from './sops-exec';
+
+export type ResolvedDevEnvTarget = {
+  readonly group: string;
+  readonly workspace: string;
+  readonly env: Readonly<Record<string, string>>;
+  readonly sources: ReadonlyArray<string>;
+};
+
+export type ResolvedDevEnv = {
+  readonly targets: ReadonlyArray<ResolvedDevEnvTarget>;
+  readonly problems: ReadonlyArray<string>;
+};
+
+type SopsDocumentResult =
+  | { readonly ok: true; readonly value: unknown }
+  | { readonly ok: false; readonly problem: string };
+
+const decryptSopsJson = (consumer: string, rel: string): SopsDocumentResult => {
+  const result = runSops(['--decrypt', '--output-type', 'json', rel], consumer);
+  if (result.status !== 0) {
+    const detail = result.errorMessage ?? result.stderr.trim();
+    return {
+      ok: false,
+      problem: detail
+        ? `could not decrypt ${rel}: ${detail}`
+        : `could not decrypt ${rel}`,
+    };
+  }
+  try {
+    return { ok: true, value: JSON.parse(result.stdout) as unknown };
+  } catch (error) {
+    return {
+      ok: false,
+      problem: `could not parse decrypted ${rel} as JSON: ${
+        error instanceof Error ? error.message : String(error)
+      }`,
+    };
+  }
+};
+
+const lookupReference = (
+  document: unknown,
+  reference: BrokeredS3Reference,
+): { readonly value: string | null; readonly problem: string | null } => {
+  let node: unknown = document;
+  for (const segment of reference.key.split('.')) {
+    if (!isRecord(node) || node[segment] === undefined) {
+      return {
+        value: null,
+        problem: `has no key "${reference.key}"; mint the pair with \`bun standards creds add cloudflare --s3 --dest ${reference.brokeredS3}:${reference.key}\``,
+      };
+    }
+    node = node[segment];
+  }
+  if (
+    !isRecord(node) ||
+    typeof node.access_key_id !== 'string' ||
+    typeof node.secret_access_key !== 'string'
+  ) {
+    return {
+      value: null,
+      problem: `key "${reference.key}" does not hold a complete brokered S3 pair; both "access_key_id" and "secret_access_key" must be strings`,
+    };
+  }
+  return { value: node[reference.part] as string, problem: null };
+};
+
+export const resolveBrokeredReferences = (
+  consumer: string,
+  targets: ReadonlyArray<ComposedDevEnvTarget>,
+  allowedReferences: ReadonlySet<string>,
+  preservedReferences: ReadonlySet<string> = new Set(),
+): ResolvedDevEnv => {
+  const problems: Array<string> = [];
+  const documents = new Map<string, SopsDocumentResult>();
+  const readDocument = (targetName: string): SopsDocumentResult => {
+    const cached = documents.get(targetName);
+    if (cached !== undefined) {
+      return cached;
+    }
+    const resolved = resolveTargetRelResult(consumer, targetName);
+    const result = resolved.ok
+      ? decryptSopsJson(consumer, resolved.rel)
+      : {
+          ok: false as const,
+          problem:
+            resolved.kind === 'ambiguous'
+              ? resolved.problem
+              : `secrets target "${targetName}" does not exist; create it and mint the pair with \`bun standards creds add cloudflare --s3\``,
+        };
+    documents.set(targetName, result);
+    return result;
+  };
+  const resolveValue = (
+    label: string,
+    reference: BrokeredS3Reference,
+  ): { readonly value: string | null; readonly problem: string | null } => {
+    const allowlistEntry = `${reference.brokeredS3}:${reference.key}`;
+    if (!allowedReferences.has(allowlistEntry)) {
+      return {
+        value: null,
+        problem: `${label}: unauthorized brokered S3 pair; add "${allowlistEntry}" to the encrypted secrets/dev.yaml brokeredReferences allowlist`,
+      };
+    }
+    if (preservedReferences.has(allowlistEntry)) {
+      return { value: '', problem: null };
+    }
+    const document = readDocument(reference.brokeredS3);
+    if (!document.ok) {
+      return { value: null, problem: `${label}: ${document.problem}` };
+    }
+    const lookup = lookupReference(document.value, reference);
+    if (lookup.value === null) {
+      return { value: null, problem: `${label}: ${lookup.problem}` };
+    }
+    if (encodeBunDotenvValue(lookup.value) === null) {
+      return {
+        value: null,
+        problem: `${label}: resolved value cannot be represented losslessly in Bun dotenv syntax`,
+      };
+    }
+    return { value: lookup.value, problem: null };
+  };
+  const resolved = targets.map((target) => {
+    const env = Object.create(null) as Record<string, string>;
+    for (const [key, value] of Object.entries(target.env)) {
+      if (typeof value === 'string') {
+        env[key] = value;
+      } else {
+        const label = `${target.group}.${target.workspace}.${key} reference to secrets target "${value.brokeredS3}"`;
+        const outcome = resolveValue(label, value);
+        if (outcome.value === null) {
+          problems.push(outcome.problem ?? `${label}: unresolved reference`);
+        } else {
+          env[key] = outcome.value;
+        }
+      }
+    }
+    return {
+      group: target.group,
+      workspace: target.workspace,
+      env,
+      sources: target.sources,
+    };
+  });
+  return { targets: resolved, problems };
+};

package/src/dev-env-brokered.ts

@@ -0,0 +1,83 @@
+// Brokered S3 pair references let a plain configuration layer point a
+// workspace env key at a credential pair the creds broker owns in another
+// SOPS target. The secret value then has exactly one home — the broker's
+// ledger — so rotation stays the broker's job and no copy can go stale.
+// A reference is configuration, never a secret, so the SOPS-encrypted
+// secrets layer must not declare one.
+
+import { parseDestination } from './creds-dest';
+import { parseSopsKeyPath } from './creds-sops-structure';
+import { isRecord } from './github-settings-parse';
+
+export const BROKERED_S3_PARTS = [
+  'access_key_id',
+  'secret_access_key',
+] as const;
+
+export type BrokeredS3Part = (typeof BROKERED_S3_PARTS)[number];
+
+export type BrokeredS3Reference = {
+  readonly brokeredS3: string;
+  readonly key: string;
+  readonly part: BrokeredS3Part;
+};
+
+const REFERENCE_PROPERTIES = ['brokeredS3', 'key', 'part'] as const;
+
+export const isBrokeredS3ReferenceShape = (
+  value: unknown,
+): value is Record<string, unknown> => isRecord(value) && 'brokeredS3' in value;
+
+export type BrokeredS3ParseResult =
+  | { readonly ok: true; readonly reference: BrokeredS3Reference }
+  | { readonly ok: false; readonly problems: ReadonlyArray<string> };
+
+const isBrokeredS3Part = (value: unknown): value is BrokeredS3Part =>
+  BROKERED_S3_PARTS.includes(value as BrokeredS3Part);
+
+// Target existence and contained-path resolution are owned by
+// `resolveTargetRel` at resolution time; parsing rejects a target name that
+// could never name a broker destination.
+export const parseBrokeredS3Reference = (
+  label: string,
+  raw: Record<string, unknown>,
+): BrokeredS3ParseResult => {
+  const problems: Array<string> = [];
+  for (const property of Object.keys(raw)) {
+    if (!REFERENCE_PROPERTIES.includes(property as 'brokeredS3')) {
+      problems.push(
+        `${label} brokered S3 pair reference has unknown property ${JSON.stringify(property)}; allowed properties are brokeredS3, key, and part`,
+      );
+    }
+  }
+  const { brokeredS3: target, key, part } = raw;
+  if (
+    typeof target !== 'string' ||
+    parseDestination(`${target}:reference_validation`) === null
+  ) {
+    problems.push(
+      `${label} brokered S3 pair reference needs a valid "brokeredS3" secrets target name`,
+    );
+  }
+  if (typeof key !== 'string' || parseSopsKeyPath(key) === null) {
+    problems.push(
+      `${label} brokered S3 pair reference needs a "key" naming the pair's dotted SOPS key`,
+    );
+  }
+  if (!isBrokeredS3Part(part)) {
+    problems.push(
+      `${label} brokered S3 pair reference needs a "part" of either "access_key_id" or "secret_access_key"`,
+    );
+  }
+  if (problems.length > 0) {
+    return { ok: false, problems };
+  }
+  return {
+    ok: true,
+    reference: {
+      brokeredS3: target as string,
+      key: key as string,
+      part: part as BrokeredS3Part,
+    },
+  };
+};

package/src/dev-env-compose.ts

@@ -4,7 +4,11 @@
 // per env key. A key declared by both tracked shared layers is a placement
 // error: a value is either configuration or a secret, never both.
 
-import type { DevEnvDocument, EnvValues } from './dev-env-document';
+import type {
+  DevEnvDocument,
+  DevEnvValue,
+  EnvValues,
+} from './dev-env-document';
 
 export type DevEnvLayer = {
   readonly source: string;
@@ -20,18 +24,19 @@ export type ComposedDevEnvTarget = {
 
 export type ComposedDevEnv = {
   readonly targets: ReadonlyArray<ComposedDevEnvTarget>;
+  readonly brokeredReferences: ReadonlySet<string>;
   readonly problems: ReadonlyArray<string>;
 };
 
 type MutableTarget = {
   readonly group: string;
   readonly workspace: string;
-  readonly env: Record<string, string>;
+  readonly env: Record<string, DevEnvValue>;
   readonly sources: Array<string>;
 };
 
-const copyEnv = (source: EnvValues): Record<string, string> => {
-  const copy = Object.create(null) as Record<string, string>;
+const copyEnv = (source: EnvValues): Record<string, DevEnvValue> => {
+  const copy = Object.create(null) as Record<string, DevEnvValue>;
   for (const [key, value] of Object.entries(source)) {
     copy[key] = value;
   }
@@ -116,5 +121,9 @@ export const composeDevEnv = (
       env: target.env,
       sources: target.sources,
     }));
-  return { targets, problems };
+  return {
+    targets,
+    brokeredReferences: secrets.document.brokeredReferences,
+    problems,
+  };
 };

package/src/dev-env-document.ts

@@ -1,12 +1,29 @@
-// Workspace-group keyed dev secrets document: `apps.<name>` and
-// `packages.<name>` map env keys to string values — the shape mirrored in
-// secrets/dev.example.yaml. Parsing gathers every problem instead of failing
-// on the first one so a malformed document is repaired in one pass.
+// Workspace-group keyed dev env document: `apps.<name>` and
+// `packages.<name>` map env keys to values — the shape mirrored in
+// secrets/dev.example.yaml. Plain configuration layers may also declare a
+// brokered S3 pair reference instead of a literal; the secrets layer may
+// not, because a reference is configuration. The secrets layer instead owns
+// the encrypted allowlist that authorizes exact target/key pairs. Parsing
+// gathers every problem instead of failing on the first one so a malformed
+// document is repaired in one pass.
 
+import {
+  type BrokeredS3Reference,
+  isBrokeredS3ReferenceShape,
+  parseBrokeredS3Reference,
+} from './dev-env-brokered';
+import {
+  BROKERED_REFERENCES_KEY,
+  parseBrokeredAllowlist,
+} from './dev-env-brokered-allowlist';
 import { encodeBunDotenvValue } from './dev-env-dotenv-value';
 import { isRecord } from './github-settings-parse';
 
-export type EnvValues = Readonly<Record<string, string>>;
+export type DevEnvValue = string | BrokeredS3Reference;
+
+export type EnvValues = Readonly<Record<string, DevEnvValue>>;
+
+export type DevEnvLayerKind = 'configuration' | 'secrets';
 
 export type DevEnvTarget = {
   readonly group: string;
@@ -17,6 +34,7 @@ export type DevEnvTarget = {
 
 export type DevEnvDocument = {
   readonly targets: ReadonlyArray<DevEnvTarget>;
+  readonly brokeredReferences: ReadonlySet<string>;
   readonly problems: ReadonlyArray<string>;
 };
 
@@ -29,16 +47,60 @@ type ParsedWorkspaces = {
   readonly problems: ReadonlyArray<string>;
 };
 
+const parseWorkspaceValue = (
+  label: string,
+  key: string,
+  value: unknown,
+  layerKind: DevEnvLayerKind,
+): {
+  readonly value: DevEnvValue | null;
+  readonly problems: ReadonlyArray<string>;
+} => {
+  if (typeof value === 'string') {
+    return encodeBunDotenvValue(value) === null
+      ? {
+          value: null,
+          problems: [
+            `${label}.${key} cannot be represented losslessly in Bun dotenv syntax`,
+          ],
+        }
+      : { value, problems: [] };
+  }
+  if (isBrokeredS3ReferenceShape(value)) {
+    if (layerKind === 'secrets') {
+      return {
+        value: null,
+        problems: [
+          `${label}.${key} is a brokered S3 pair reference; references are configuration and belong in config/dev.yaml or config/dev.local.yaml, not the secrets layer`,
+        ],
+      };
+    }
+    const parsed = parseBrokeredS3Reference(`${label}.${key}`, value);
+    return parsed.ok
+      ? { value: parsed.reference, problems: [] }
+      : { value: null, problems: parsed.problems };
+  }
+  return {
+    value: null,
+    problems: [
+      layerKind === 'configuration'
+        ? `${label}.${key} must be a string value or a brokered S3 pair reference`
+        : `${label}.${key} must be a string value`,
+    ],
+  };
+};
+
 const parseWorkspaceEnv = (
   label: string,
   raw: Record<string, unknown>,
+  layerKind: DevEnvLayerKind,
 ): {
   readonly env: EnvValues;
   readonly declaredKeys: ReadonlySet<string>;
   readonly problems: ReadonlyArray<string>;
 } => {
   const problems: Array<string> = [];
-  const env = Object.create(null) as Record<string, string>;
+  const env = Object.create(null) as Record<string, DevEnvValue>;
   const declaredKeys = new Set<string>();
   for (const [key, value] of Object.entries(raw)) {
     const portableName = PORTABLE_ENV_NAME.test(key);
@@ -49,14 +111,10 @@ const parseWorkspaceEnv = (
         `${label} env key ${JSON.stringify(key)} must be a portable environment variable name`,
       );
     }
-    if (typeof value !== 'string') {
-      problems.push(`${label}.${key} must be a string value`);
-    } else if (encodeBunDotenvValue(value) === null) {
-      problems.push(
-        `${label}.${key} cannot be represented losslessly in Bun dotenv syntax`,
-      );
-    } else if (portableName) {
-      env[key] = value;
+    const parsed = parseWorkspaceValue(label, key, value, layerKind);
+    problems.push(...parsed.problems);
+    if (portableName && parsed.value !== null) {
+      env[key] = parsed.value;
     }
   }
   return { env, declaredKeys, problems };
@@ -66,6 +124,7 @@ const parseWorkspaces = (
   source: string,
   group: string,
   workspaces: Record<string, unknown>,
+  layerKind: DevEnvLayerKind,
 ): ParsedWorkspaces => {
   const problems: Array<string> = [];
   const targets: Array<DevEnvTarget> = [];
@@ -80,6 +139,7 @@ const parseWorkspaces = (
       const parsed = parseWorkspaceEnv(
         `${source} "${group}.${workspace}"`,
         env,
+        layerKind,
       );
       problems.push(...parsed.problems);
       if (validWorkspace) {
@@ -102,22 +162,31 @@ const parseWorkspaces = (
 export const parseDevEnvDocument = (
   raw: unknown,
   source: string,
+  layerKind: DevEnvLayerKind,
 ): DevEnvDocument => {
   if (!isRecord(raw)) {
     return {
       targets: [],
+      brokeredReferences: new Set(),
       problems: [`${source} must decrypt to a YAML object`],
     };
   }
   const problems: Array<string> = [];
   const targets: Array<DevEnvTarget> = [];
+  const brokeredReferences = new Set<string>();
   for (const [group, workspaces] of Object.entries(raw)) {
-    if (!WORKSPACE_GROUPS.includes(group)) {
+    if (group === BROKERED_REFERENCES_KEY) {
+      const parsed = parseBrokeredAllowlist(source, workspaces, layerKind);
+      problems.push(...parsed.problems);
+      for (const entry of parsed.entries) {
+        brokeredReferences.add(entry);
+      }
+    } else if (!WORKSPACE_GROUPS.includes(group)) {
       problems.push(
-        `${source} top-level key "${group}" must be "apps" or "packages"`,
+        `${source} top-level key "${group}" must be "apps" or "packages"${layerKind === 'secrets' ? ` or "${BROKERED_REFERENCES_KEY}"` : ''}`,
       );
     } else if (isRecord(workspaces)) {
-      const parsed = parseWorkspaces(source, group, workspaces);
+      const parsed = parseWorkspaces(source, group, workspaces, layerKind);
       problems.push(...parsed.problems);
       targets.push(...parsed.targets);
     } else {
@@ -126,5 +195,5 @@ export const parseDevEnvDocument = (
       );
     }
   }
-  return { targets, problems };
+  return { targets, brokeredReferences, problems };
 };