@intentius/chant 0.79.0 → 0.80.0

package/src/config.test.ts

@@ -12,6 +12,7 @@ import {
   environmentEndpoint,
   matchesEnvironmentPattern,
   matchesDeclaredEnvironment,
+  resolveDiscoveryGlobs,
 } from "./config";
 import { writeFileSync, mkdirSync, rmSync } from "fs";
 import { join } from "path";
@@ -448,3 +449,42 @@ describe("resolveKnowledgeDir (#1864, design #1059)", () => {
     expect(resolveKnowledgeDir({ knowledge: {} }, "/proj")).toBe(join("/proj", "knowledge"));
   });
 });
+
+describe("exclude / include discovery globs (#2519)", () => {
+  test("loads string arrays", async () => {
+    writeFileSync(join(TEST_DIR, "chant.config.json"), JSON.stringify({ exclude: ["ops/**"], include: ["ops/keep.ts"] }));
+    const { config } = await loadChantConfig(TEST_DIR);
+    expect(config.exclude).toEqual(["ops/**"]);
+    expect(config.include).toEqual(["ops/keep.ts"]);
+  });
+
+  test.each([
+    ["a non-string entry", { exclude: ["ops/**", 42] }, /exclude\.1: .*expected string/],
+    ["a bare string", { exclude: "ops/**" }, /exclude: .*expected array/],
+    ["an empty pattern", { include: [""] }, /include\.0: /],
+  ])("rejects %s loudly", async (_label, raw, message) => {
+    writeFileSync(join(TEST_DIR, "chant.config.json"), JSON.stringify(raw));
+    await expect(loadChantConfig(TEST_DIR)).rejects.toThrow(message);
+    await expect(resolveDiscoveryGlobs(TEST_DIR)).rejects.toThrow(message);
+  });
+
+  test("resolveDiscoveryGlobs is undefined without exclude, so include alone changes nothing", async () => {
+    expect(await resolveDiscoveryGlobs(TEST_DIR)).toBeUndefined();
+    writeFileSync(join(TEST_DIR, "chant.config.json"), JSON.stringify({ include: ["ops/**"] }));
+    expect(await resolveDiscoveryGlobs(TEST_DIR)).toBeUndefined();
+  });
+
+  test("resolveDiscoveryGlobs anchors the globs at the project config's directory, past a lint fragment", async () => {
+    const src = join(TEST_DIR, "src");
+    mkdirSync(src, { recursive: true });
+    writeFileSync(join(src, "chant.config.json"), JSON.stringify({ rules: { COR001: "off" } }));
+    writeFileSync(join(TEST_DIR, "chant.config.json"), JSON.stringify({ exclude: ["ops"] }));
+    expect(await resolveDiscoveryGlobs(src)).toEqual({ root: TEST_DIR, exclude: ["ops"], include: [] });
+  });
+
+  test("resolveDiscoveryGlobs leaves an unrelated config error to the command's own load", async () => {
+    writeFileSync(join(TEST_DIR, "chant.config.json"), JSON.stringify({ lexicons: [7], exclude: ["ops"] }));
+    await expect(loadChantConfig(TEST_DIR)).rejects.toThrow(/lexicons\.0/);
+    expect(await resolveDiscoveryGlobs(TEST_DIR)).toBeUndefined();
+  });
+});

package/src/config.ts

@@ -10,6 +10,7 @@ import type { BuildParamsConfig } from "./build-params";
 import type { BuildParamProvenance } from "./provenance";
 import { findProjectConfig } from "./project-root";
 import { evaluateProjectConfig } from "./config-sandbox";
+import { lexiconNames, registerLexiconDeclarations, type LexiconDeclaration } from "./lexicon-module";
 
 /**
  * One project-declared environment (chant #1166). Historically always a bare
@@ -125,8 +126,16 @@ const EnvironmentEntrySchema = z.union([
   }),
 ]);
 
+const LexiconEntrySchema = z.union([
+  z.string().min(1),
+  z.object({
+    name: z.string().min(1),
+    module: z.string().min(1),
+  }).strict(),
+]);
+
 export const ChantConfigSchema = z.object({
-  lexicons: z.array(z.string().min(1)).optional(),
+  lexicons: z.array(LexiconEntrySchema).optional(),
   capabilities: z.array(z.string().min(1)).optional(),
   environments: z.array(EnvironmentEntrySchema).optional(),
   sourceDir: z.string().min(1).optional(),
@@ -184,6 +193,8 @@ export const ChantConfigSchema = z.object({
   knowledge: z.object({
     dir: z.string().min(1).optional(),
   }).optional(),
+  exclude: z.array(z.string().min(1)).optional(),
+  include: z.array(z.string().min(1)).optional(),
 }).passthrough();
 
 /**
@@ -192,8 +203,24 @@ export const ChantConfigSchema = z.object({
  * Loaded from `chant.config.ts` (preferred) or `chant.config.json`.
  */
 export interface ChantConfig {
-  /** Lexicon package names to load (e.g. ["aws"]) */
-  lexicons?: string[];
+  /**
+   * Lexicons to load. A bare name (`"aws"`) loads the package
+   * `@intentius/chant-lexicon-aws`. `{ name, module }` (#2520) loads a
+   * project-local lexicon from a module path, resolved against the directory
+   * holding this config; see `./lexicon-module.ts`.
+   *
+   * Once loaded, this holds names only: the loader replaces each
+   * `{ name, module }` entry with its name and records the path in
+   * {@link ChantConfig.lexiconModules}.
+   */
+  lexicons?: LexiconDeclaration[];
+
+  /**
+   * Set by the config loader, never written by hand: each path-declared
+   * lexicon's name mapped to its absolute module path. Absent when every
+   * `lexicons` entry is a bare name.
+   */
+  lexiconModules?: Record<string, string>;
 
   /**
    * Capability plugin package names to load in addition to the built-in
@@ -418,14 +445,43 @@ export interface ChantConfig {
     /** Bundle directory, relative to the project root. Defaults to `"knowledge"`. */
     dir?: string;
   };
+
+  /**
+   * Globs, relative to the directory holding this config, naming files that
+   * source discovery must skip (#2519): `build`, `lint`, `list`, `explain`
+   * and every other command that walks the project for declarations never
+   * import or lint a matching file. A pattern matches a file when it matches
+   * the file's path or the path of any directory above it, so `"ops"` and
+   * `"ops/**"` both skip everything under `ops/`. Matched with picomatch,
+   * dot files included. See `./discovery/files.ts`'s `compileDiscoveryFilter`.
+   */
+  exclude?: string[];
+
+  /**
+   * Globs that re-admit files {@link exclude} would skip. `include` always
+   * wins over `exclude`, and it never narrows discovery on its own: a file
+   * no `exclude` pattern matches is discovered whether or not it matches an
+   * `include`. It does not re-admit what discovery skips unconditionally
+   * (`node_modules`, test files, child projects, the generated and skip
+   * markers).
+   */
+  include?: string[];
 }
 
 /**
  * Resolved project configuration with metadata about how it was loaded.
  */
+/**
+ * A configuration as the loader returns it. `lexicons` holds names only: each
+ * `{ name, module }` entry has been replaced by its name and its path moved
+ * to `lexiconModules` (chant #2520), so a reader of a loaded config never
+ * sees the object form.
+ */
+export type LoadedChantConfig = Omit<ChantConfig, "lexicons"> & { lexicons?: string[] };
+
 export interface ResolvedConfig {
   /** The loaded configuration */
-  config: ChantConfig;
+  config: LoadedChantConfig;
 
   /** Path to the config file that was loaded, or undefined if defaults */
   configPath?: string;
@@ -434,7 +490,7 @@ export interface ResolvedConfig {
 /**
  * Default configuration when no config file exists.
  */
-export const DEFAULT_CHANT_CONFIG: ChantConfig = {};
+export const DEFAULT_CHANT_CONFIG: LoadedChantConfig = {};
 
 /**
  * Load project configuration from a directory.
@@ -499,14 +555,53 @@ export async function loadChantConfig(dir: string): Promise<ResolvedConfig> {
  * config declaring any project-level key still wins where it stands.
  */
 export async function loadChantConfigUpward(startDir: string): Promise<ResolvedConfig> {
+  const { dir, configPath } = findProjectConfigPastFragments(startDir);
+  if (configPath) warnIfFragmentShadowsProjectConfig(configPath);
+  return loadChantConfig(dir);
+}
+
+/** The walk {@link loadChantConfigUpward} makes, lint-only fragments skipped, without its warning. */
+function findProjectConfigPastFragments(startDir: string): { dir: string; configPath?: string } {
   let { dir, configPath } = findProjectConfig(startDir);
   while (configPath && isLintOnlyFragment(configPath)) {
     const parent = dirname(dir);
     if (parent === dir) break;
     ({ dir, configPath } = findProjectConfig(parent));
   }
-  if (configPath) warnIfFragmentShadowsProjectConfig(configPath);
-  return loadChantConfig(dir);
+  return { dir, configPath };
+}
+
+/** A project's discovery globs (#2519) and the directory they are relative to. */
+export interface DiscoveryGlobs {
+  /** The directory holding the config that declared the globs. */
+  root: string;
+  exclude: string[];
+  include: string[];
+}
+
+/**
+ * The `exclude`/`include` globs of the project `startDir` belongs to, found by
+ * the same upward walk as {@link loadChantConfigUpward}. `undefined` when the
+ * project declares no `exclude`, which leaves discovery exactly as it was
+ * before the keys existed: `include` only ever re-admits what `exclude` skips.
+ *
+ * A config that fails to load is left to the command's own config load to
+ * report, so discovery keeps working where it did before, unless the failure
+ * is in `exclude` or `include` themselves: ignoring those would discover the
+ * very files the project asked to skip.
+ */
+export async function resolveDiscoveryGlobs(startDir: string): Promise<DiscoveryGlobs | undefined> {
+  const { dir, configPath } = findProjectConfigPastFragments(startDir);
+  if (!configPath) return undefined;
+  let config: ChantConfig;
+  try {
+    ({ config } = await loadChantConfig(dir));
+  } catch (err) {
+    if (err instanceof InvalidChantConfigError && (err.key === "exclude" || err.key === "include")) throw err;
+    return undefined;
+  }
+  if (!config.exclude || config.exclude.length === 0) return undefined;
+  return { root: dir, exclude: config.exclude, include: config.include ?? [] };
 }
 
 /**
@@ -531,7 +626,7 @@ const LINT_FRAGMENT_KEYS = new Set(["$schema", "extends", "rules", "overrides",
  */
 const isCommentKey = (k: string): boolean => k.startsWith("_");
 
-function isLintOnlyFragment(configPath: string): boolean {
+export function isLintOnlyFragment(configPath: string): boolean {
   if (!configPath.endsWith("chant.config.json")) return false;
   try {
     const parsed = JSON.parse(readFileSync(configPath, "utf-8")) as unknown;
@@ -813,10 +908,20 @@ export function resolveKnowledgeDir(config: ChantConfig, projectPath: string): s
   return join(projectPath, config.knowledge?.dir ?? "knowledge");
 }
 
+/**
+ * A config that failed schema validation. `key` is the top-level key of the
+ * first problem. `name` stays `"Error"`, so the printed error reads as before.
+ */
+export class InvalidChantConfigError extends Error {
+  constructor(message: string, readonly key?: string) {
+    super(message);
+  }
+}
+
 /**
  * Validate and normalize a raw config object into ChantConfig shape.
  */
-function normalizeConfig(raw: Record<string, unknown>, source?: string): ChantConfig {
+function normalizeConfig(raw: Record<string, unknown>, source?: string): LoadedChantConfig {
   if (typeof raw !== "object" || raw === null) {
     return DEFAULT_CHANT_CONFIG;
   }
@@ -826,8 +931,20 @@ function normalizeConfig(raw: Record<string, unknown>, source?: string): ChantCo
     const issue = result.error.issues[0];
     const path = issue.path.length > 0 ? issue.path.join(".") : undefined;
     const loc = source ? ` in ${source}` : "";
-    throw new Error(`Invalid chant config${loc}: ${path ? `${path}: ` : ""}${issue.message}`);
+    throw new InvalidChantConfigError(
+      `Invalid chant config${loc}: ${path ? `${path}: ` : ""}${issue.message}`,
+      issue.path.length > 0 ? String(issue.path[0]) : undefined,
+    );
+  }
+
+  // chant #2520 — a lexicon declared by module path. The path is recorded for
+  // the loaders and the entry becomes its name, so every reader of `lexicons`
+  // keeps seeing names. A config of plain names is returned untouched.
+  const lexicons = (raw as ChantConfig).lexicons;
+  const recorded = registerLexiconDeclarations(lexicons, source !== undefined ? dirname(source) : process.cwd());
+  if (Object.keys(recorded).length > 0) {
+    return { ...(raw as ChantConfig), lexicons: lexiconNames(lexicons), lexiconModules: recorded };
   }
 
-  return raw as ChantConfig;
+  return raw as LoadedChantConfig;
 }

package/src/discovery/convergence.test.ts

@@ -0,0 +1,231 @@
+import { describe, test, expect, vi, beforeEach, afterEach } from "vitest";
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync, realpathSync } from "node:fs";
+import { execFileSync } from "node:child_process";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { findInfraFiles } from "./files";
+import { resetDiscoveryWarnings, warnDiscoveryChanges } from "./convergence";
+import { discoverComponents } from "../components/discover";
+
+/**
+ * chant#2527's warning release. Each category of file whose discovery status
+ * changes when the walkers converge gets a warning naming the file, the
+ * change and the glob that keeps today's behaviour, while the walker's
+ * returned list stays exactly what it is today. Lint's category lives in
+ * `../cli/commands/lint.test.ts`, the Op category in
+ * `../op/discover-convergence.test.ts`.
+ */
+
+let dir: string;
+let stderr: string[];
+
+function write(rel: string, body = "export const x = 1;\n"): void {
+  mkdirSync(dirname(join(dir, rel)), { recursive: true });
+  writeFileSync(join(dir, rel), body);
+}
+
+/** A project config (it declares a project-level key, so it is no lint-only fragment). */
+function project(rel = ".", extra: Record<string, unknown> = {}): void {
+  write(join(rel, "chant.config.json"), JSON.stringify({ lexicons: ["aws"], ...extra }));
+}
+
+const rel = (files: string[]): string[] => files.map((f) => f.slice(dir.length + 1)).sort();
+
+beforeEach(() => {
+  dir = realpathSync(mkdtempSync(join(tmpdir(), "chant-2527-")));
+  resetDiscoveryWarnings();
+  stderr = [];
+  vi.spyOn(console, "error").mockImplementation((...args: unknown[]) => void stderr.push(args.join(" ")));
+  vi.spyOn(process, "cwd").mockReturnValue(dir);
+});
+
+afterEach(() => {
+  vi.restoreAllMocks();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+describe("source discovery warns about files the converged walker reads differently (#2527)", () => {
+  test("a .ts file in a dot-directory", async () => {
+    project();
+    write("src/app.ts");
+    write(".cache/gen.ts");
+
+    expect(rel(await findInfraFiles(dir))).toEqual([".cache/gen.ts", "src/app.ts"]);
+    expect(stderr).toEqual([
+      "warning: Source discovery (build, graph, list, explain) under the current directory changes in the next release (chant#2527):\n" +
+        "  .cache/gen.ts: .cache/ is a dot-directory. Every walker skips dot-directories from the next release. " +
+        'To keep reading it after the change, add ".cache" to include in chant.config.json, which the next release honours.',
+    ]);
+  });
+
+  test("a git-ignored .ts file, ignored by a nested .gitignore", async () => {
+    execFileSync("git", ["init", "-q"], { cwd: dir });
+    project();
+    write("src/app.ts");
+    write("src/vendored.ts");
+    write("src/.gitignore", "vendored.ts\n");
+
+    expect(rel(await findInfraFiles(dir))).toEqual(["src/app.ts", "src/vendored.ts"]);
+    expect(stderr).toHaveLength(1);
+    expect(stderr[0]).toContain(
+      "  src/vendored.ts is git-ignored. Every walker skips git-ignored files from the next release. " +
+        'To keep reading it after the change, add "src/vendored.ts" to include in chant.config.json, which the next release honours.',
+    );
+  });
+
+  test("an ignored directory is named once, by its outermost ignored path", async () => {
+    execFileSync("git", ["init", "-q"], { cwd: dir });
+    project();
+    write("src/app.ts");
+    write(".gitignore", "vendor/\n");
+    for (const name of ["a", "b", "c", "d"]) write(`vendor/lib/${name}.ts`);
+
+    expect(rel(await findInfraFiles(dir))).toHaveLength(5);
+    expect(stderr[0]).toContain("  4 files under vendor/: vendor/ is git-ignored.");
+    expect(stderr[0]).toContain('add "vendor" to include in chant.config.json, which the next release honours.');
+  });
+
+  test("a .ts file under dist", async () => {
+    project();
+    write("src/app.ts");
+    write("dist/out.ts");
+
+    expect(rel(await findInfraFiles(dir))).toEqual(["dist/out.ts", "src/app.ts"]);
+    expect(stderr[0]).toContain(
+      "  dist/out.ts: dist/ is a dist directory. Every walker skips dist from the next release. " +
+        'To keep reading it after the change, add "dist" to include in chant.config.json, which the next release honours.',
+    );
+  });
+
+  test("a nested child project whose chant.config.json declares a project", async () => {
+    project();
+    write("src/app.ts");
+    project("stacks/east");
+    write("stacks/east/east.ts");
+    // A lint-only fragment is not a boundary: no warning for this one.
+    write("stacks/west/chant.config.json", JSON.stringify({ rules: { COR004: "off" } }));
+    write("stacks/west/west.ts");
+
+    expect(rel(await findInfraFiles(dir))).toEqual(["src/app.ts", "stacks/east/east.ts", "stacks/west/west.ts"]);
+    expect(stderr).toHaveLength(1);
+    expect(stderr[0]).toContain(
+      "  stacks/east/east.ts: stacks/east/ is a child project with its own chant.config. " +
+        "Discovery stops at child projects from the next release. " +
+        'To keep reading it after the change, add "stacks/east" to include in chant.config.json, which the next release honours.',
+    );
+    expect(stderr[0]).not.toContain("west");
+  });
+
+  test("the first child project met is no longer read as the project's own source (the quirk)", async () => {
+    project();
+    write("app/chant.config.ts", "export default {};\n");
+    write("app/main.ts");
+
+    expect(rel(await findInfraFiles(dir))).toEqual(["app/chant.config.ts", "app/main.ts"]);
+    expect(stderr[0]).toContain(
+      "  app/chant.config.ts, app/main.ts: app/ is a child project with its own chant.config, " +
+        "read today as this project's own source because the walk met it first. " +
+        "From the next release discovery stops there, as at any child project. " +
+        'To keep reading them after the change, add "app" to include in chant.config.json, which the next release honours.',
+    );
+  });
+
+  test("the glob in the config is the fix: with it, nothing is printed and the list is the same", async () => {
+    project(".", { include: [".cache", "dist"] });
+    write("src/app.ts");
+    write(".cache/gen.ts");
+    write("dist/out.ts");
+
+    expect(rel(await findInfraFiles(dir))).toEqual([".cache/gen.ts", "dist/out.ts", "src/app.ts"]);
+    expect(stderr).toEqual([]);
+  });
+
+  test("a clean project prints nothing", async () => {
+    execFileSync("git", ["init", "-q"], { cwd: dir });
+    project();
+    write(".gitignore", "node_modules/\n.chant/types/\n");
+    write("src/app.ts");
+    write("src/lib/util.ts");
+    write("node_modules/dep/index.ts");
+    // chant init's generated type stubs: read today, never worth a warning.
+    write(".chant/types/core/index.d.ts");
+
+    expect(rel(await findInfraFiles(dir))).toEqual([".chant/types/core/index.d.ts", "src/app.ts", "src/lib/util.ts"]);
+    expect(stderr).toEqual([]);
+  });
+
+  test("a project-local lexicon named by module path (#2520) prints nothing", async () => {
+    execFileSync("git", ["init", "-q"], { cwd: dir });
+    write("chant.config.json", JSON.stringify({ lexicons: [{ name: "local", module: "./lexicon/index.ts" }] }));
+    write("lexicon/index.ts");
+    write("lexicon/rules/r1.ts");
+    write("src/app.ts");
+
+    expect(rel(await findInfraFiles(dir))).toEqual(["lexicon/index.ts", "lexicon/rules/r1.ts", "src/app.ts"]);
+    expect(stderr).toEqual([]);
+  });
+
+  test("a project-local lexicon's config still yields the glob when something else warns", async () => {
+    write("chant.config.json", JSON.stringify({ lexicons: [{ name: "local", module: "./lexicon/index.ts" }], include: [".cache"] }));
+    write("lexicon/index.ts");
+    write(".cache/gen.ts");
+    write("dist/out.ts");
+
+    expect(rel(await findInfraFiles(dir))).toEqual([".cache/gen.ts", "dist/out.ts", "lexicon/index.ts"]);
+    expect(stderr).toHaveLength(1);
+    expect(stderr[0]).not.toContain(".cache");
+    expect(stderr[0]).toContain('add "dist" to include in chant.config.json');
+  });
+
+  test("a warning prints once per process, however often the tree is walked", async () => {
+    project();
+    write(".cache/gen.ts");
+    await findInfraFiles(dir);
+    await findInfraFiles(dir);
+    expect(stderr).toHaveLength(1);
+  });
+});
+
+describe("component discovery outside a project reads every child project next release (#2527)", () => {
+  test("the second child project's component is named, with the exclude glob", async () => {
+    // No config and no .git/package.json at or above `dir`: outside a project.
+    write("one/chant.config.ts", "export default {};\n");
+    write("one/one.component.ts");
+    write("two/chant.config.ts", "export default {};\n");
+    write("two/two.component.ts");
+
+    const { sourceFiles } = await discoverComponents(dir);
+    // Today: whichever child the walk met first, and only that one.
+    expect(sourceFiles).toHaveLength(1);
+    const [read] = rel(sourceFiles);
+    const skipped = read.startsWith("one/") ? "two" : "one";
+
+    expect(stderr).toHaveLength(1);
+    expect(stderr[0]).toContain(
+      `warning: Component discovery under the current directory changes in the next release (chant#2527):\n` +
+        `  ${skipped}/${skipped}.component.ts: ${skipped}/ is a child project, skipped today ` +
+        "because the walk met another child project first. From the next release every child project is read, " +
+        "since the current directory is not inside a chant project. " +
+        `To keep skipping it, add "${skipped}" to exclude in chant.config.ts (a new file).`,
+    );
+  });
+});
+
+describe("the audit walk (#2527)", () => {
+  test("warns about a git-ignored candidate and leaves Terraform state to TF023", async () => {
+    execFileSync("git", ["init", "-q"], { cwd: dir });
+    write(".gitignore", ".env\n");
+    write(".env", "TOKEN=x\n");
+    write("k8s/deploy.yaml", "kind: Deployment\n");
+
+    const lines = await warnDiscoveryChanges({
+      walker: "audit",
+      root: dir,
+      files: [join(dir, ".env"), join(dir, "k8s/deploy.yaml"), join(dir, ".github/workflows/ci.yml")],
+    });
+    expect(lines).toEqual([
+      "  .env is git-ignored. Every walker skips git-ignored files from the next release. " +
+        'To keep reading it after the change, add ".env" to include in chant.config.ts (a new file), which the next release honours.',
+    ]);
+  });
+});