@davidvornholt/standards 0.26.3 → 0.26.4

package/README.md

@@ -265,3 +265,7 @@ The worker accepts only exact issue or draft-PR revisions approved by a user wit
 The [root README](../../README.md) gives the short overview. The [adoption guide](../../docs/adoption.md) and [sync guide](../../docs/sync-and-ownership.md) explain repository ownership and first use.
 
 The poller timers use monotonic intervals: after boot, acknowledgement starts after one minute and work starts after two minutes. Missed downtime intervals are not replayed; each tick reads the current queue. The service budget reserves thirty minutes for scanning and setup in addition to its configured agent budget, with at most twelve watched repositories per poller. Split larger lists across pollers with independent API budgets; unusually slow provider or clone operations can exhaust that overhead and require investigating the service timeout before retrying.
+
+## Source text validation
+
+`bun standards source-text` checks tracked source, configuration, and documentation files as UTF-8 text, regardless of Git’s binary classification. `bun standards check` includes this gate. Raw Unicode control characters are rejected except tab and LF; write control characters in fixtures and composite keys using escaped representations such as `\0` or `\u0000`. Binary asset formats and untracked files are outside this gate. Missing tracked source files and source symlinks fail validation.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@davidvornholt/standards",
-  "version": "0.26.3",
+  "version": "0.26.4",
   "description": "Bootstrap, synchronize, and validate davidvornholt/standards consumers",
   "license": "MIT",
   "repository": {
@@ -162,6 +162,7 @@
     "src/screenshots-publish.ts",
     "src/screenshots-upload.ts",
     "src/sops-exec.ts",
+    "src/source-text.ts",
     "src/structure-check.ts",
     "src/structure-bun-version.ts",
     "src/structure-profile.ts",

package/src/cli.ts

@@ -44,6 +44,7 @@ import {
 } from './managed-path-ownership';
 import { runPollerCommand } from './poller-commands';
 import { runScreenshotsCommand } from './screenshots-commands';
+import { collectSourceTextProblems } from './source-text';
 import { collectStructureProblems } from './structure-check';
 import type { StructureProfile } from './structure-profile';
 import { hasSafeCommand } from './structure-script';
@@ -92,6 +93,7 @@ type Command =
   | 'help'
   | 'init'
   | 'poller'
+  | 'source-text'
   | 'structure'
   | 'sync';
 
@@ -969,6 +971,7 @@ Commands:
   sync        Mirror canonical files from upstream, regenerate the composed Dependabot config, and rewrite the lock
   check       Verify canonical files, extension seams, monorepo structure, and GitHub settings
   doctor      Validate extension seams only
+  source-text Reject raw control characters in tracked source text
   structure   Validate monorepo structure rules only
   dependabot  Verify (--check) or regenerate (--write) the composed .github/dependabot.yml
   dev-env     Compose each workspace's generated .env.local from its three dev layers and authorized broker-owned S3 pair references
@@ -1002,6 +1005,7 @@ const commandFromArg = (arg: string): Command => {
     arg === 'help' ||
     arg === 'init' ||
     arg === 'poller' ||
+    arg === 'source-text' ||
     arg === 'structure' ||
     arg === 'sync'
   ) {
@@ -1209,10 +1213,24 @@ const runStructure = async (
   return true;
 };
 
+const runSourceTextCheck = (consumer: string): boolean => {
+  const problems = collectSourceTextProblems(consumer);
+  if (problems.length > 0) {
+    console.error(`standards source-text: ${problems.length} problem(s):`);
+    console.error(problems.map((problem) => `  - ${problem}`).join('\n'));
+    return false;
+  }
+  console.log(
+    'standards source-text: tracked source text contains no forbidden control characters',
+  );
+  return true;
+};
+
 const runCheckCommand = async (consumer: string): Promise<boolean> => {
   const driftIsClean = await runCheck(consumer);
   const integrationIsValid = await runDoctor(consumer);
   const structureIsValid = await runStructure(consumer, 'consumer');
+  const sourceTextIsValid = runSourceTextCheck(consumer);
   // The GitHub gate activates with the synced declaration and then fails
   // closed: once .github/settings.json exists, an unreachable API or an
   // unreadable origin is a failure, not a skip.
@@ -1220,7 +1238,11 @@ const runCheckCommand = async (consumer: string): Promise<boolean> => {
     ? await runGithubCheckGate(consumer)
     : true;
   return (
-    driftIsClean && integrationIsValid && structureIsValid && githubIsConverged
+    driftIsClean &&
+    integrationIsValid &&
+    structureIsValid &&
+    sourceTextIsValid &&
+    githubIsConverged
   );
 };
 
@@ -1296,6 +1318,7 @@ const runGateCommand = (
     | 'dev-env'
     | 'doctor'
     | 'github'
+    | 'source-text'
     | 'structure',
   consumer: string,
   apply: boolean,
@@ -1313,6 +1336,9 @@ const runGateCommand = (
   if (command === 'doctor') {
     return runDoctor(consumer);
   }
+  if (command === 'source-text') {
+    return Promise.resolve(runSourceTextCheck(consumer));
+  }
   if (command === 'structure') {
     return runStructure(consumer, profile);
   }

package/src/source-text.ts

@@ -0,0 +1,98 @@
+import { execFileSync } from 'node:child_process';
+import { readFileSync } from 'node:fs';
+import { basename, extname, join } from 'node:path';
+import { isContainedPath } from './contained-path';
+
+const sourceExtensions = new Set(
+  '.ts,.tsx,.mts,.cts,.js,.jsx,.mjs,.cjs,.json,.jsonc,.yaml,.yml,.toml,.nix,.sh,.bash,.zsh,.rb,.py,.css,.scss,.html,.xml,.svg,.md,.mdx,.sql,.just'.split(
+    ',',
+  ),
+);
+const sourceFileNames = new Set([
+  'Dockerfile',
+  'Containerfile',
+  'Makefile',
+  'justfile',
+  '.envrc',
+  '.gitignore',
+  '.gitattributes',
+  '.dockerignore',
+  '.editorconfig',
+  '.npmrc',
+]);
+const dotenvFileName = /^\.env(?:\..*)?$/u;
+const forbiddenControl = /[^\P{Cc}\t\n]/u;
+const gitOutputLimit = 16_777_216;
+const hexadecimalRadix = 16;
+const minimumCodePointWidth = 4;
+
+export const inspectSourceText = (
+  path: string,
+  bytes: Uint8Array,
+): string | null => {
+  let text: string;
+  try {
+    text = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
+  } catch {
+    return `${JSON.stringify(path)}: source text must be valid UTF-8`;
+  }
+  const match = forbiddenControl.exec(text);
+  if (!match) {
+    return null;
+  }
+  const point = match[0]
+    .codePointAt(0)
+    ?.toString(hexadecimalRadix)
+    .toUpperCase()
+    .padStart(minimumCodePointWidth, '0');
+  const line = text.slice(0, match.index).split('\n').length;
+  return `${JSON.stringify(path)}:${line}: raw control character U+${point}; use an escaped representation (only tab and LF are allowed)`;
+};
+
+export const collectSourceTextProblems = (
+  root: string,
+): ReadonlyArray<string> => {
+  let tracked: string;
+  try {
+    tracked = execFileSync('git', ['ls-files', '--cached', '-z'], {
+      cwd: root,
+      encoding: 'utf8',
+      maxBuffer: gitOutputLimit,
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+  } catch {
+    return [
+      'Cannot enumerate tracked source files; run the gate from a readable Git working tree.',
+    ];
+  }
+  const problems: Array<string> = [];
+  const paths = new Set(
+    tracked
+      .split('\0')
+      .filter(
+        (path) =>
+          sourceExtensions.has(extname(path)) ||
+          sourceFileNames.has(basename(path)) ||
+          dotenvFileName.test(basename(path)),
+      ),
+  );
+  for (const path of paths) {
+    if (isContainedPath(root, path, 'file')) {
+      try {
+        const problem = inspectSourceText(path, readFileSync(join(root, path)));
+        if (problem !== null) {
+          problems.push(problem);
+        }
+      } catch {
+        problems.push(
+          `${JSON.stringify(path)}: cannot read tracked source text`,
+        );
+      }
+    } else {
+      problems.push(
+        `${JSON.stringify(path)}: tracked source must be a contained regular file`,
+      );
+    }
+  }
+  return problems;
+};

package/src/structure-profile.ts

@@ -31,6 +31,7 @@ const SOURCE_ROOT_EXPECTATIONS: ReadonlyArray<RootScriptExpectation> = [
     name: 'check',
     commands: [
       `${SOURCE_CLI} structure --profile source`,
+      `${SOURCE_CLI} source-text`,
       `${SOURCE_CLI} dependabot --check`,
       `${SOURCE_CLI} github --check`,
       'turbo run lint check-types test --output-logs=errors-only',
@@ -41,6 +42,7 @@ const SOURCE_ROOT_EXPECTATIONS: ReadonlyArray<RootScriptExpectation> = [
     name: 'check:fix',
     commands: [
       `${SOURCE_CLI} structure --profile source`,
+      `${SOURCE_CLI} source-text`,
       `${SOURCE_CLI} dependabot --write`,
       `${SOURCE_CLI} github --check`,
       'turbo run lint:fix check-types test --output-logs=errors-only',