@intentius/chant 0.79.0 → 0.80.0

package/src/discovery/files.ts

@@ -1,6 +1,10 @@
 import { readdir, stat, open } from "node:fs/promises";
 import { existsSync } from "node:fs";
-import { join, relative } from "node:path";
+import { join, relative, sep, isAbsolute } from "node:path";
+// @ts-ignore — picomatch has no types declaration
+import picomatch from "picomatch";
+import { resolveDiscoveryGlobs, type DiscoveryGlobs } from "../config";
+import { warnDiscoveryChanges } from "./convergence";
 
 /**
  * Marker chant writes at the top of files it generates (a hosting lexicon's
@@ -12,13 +16,38 @@ import { join, relative } from "node:path";
  */
 export const GENERATED_MARKER = "Generated by chant — do not edit directly";
 
-/** True when a file's header carries the {@link GENERATED_MARKER}. Reads only the head. */
-async function isGeneratedFile(file: string): Promise<boolean> {
+/**
+ * Comment a hand-written file carries to keep discovery out of it (#2519): a
+ * runnable script with top-level side effects, say, that lives next to the
+ * project's declarations. It must be a comment of its own (`// chant-discovery-skip`
+ * or inside a block comment) within the first {@link SKIP_MARKER_HEAD_BYTES}
+ * bytes of the file. See {@link hasSkipMarker}.
+ */
+export const SKIP_MARKER = "chant-discovery-skip";
+
+/** How much of a file's head {@link hasSkipMarker} reads: room for a shebang and a license header. */
+const SKIP_MARKER_HEAD_BYTES = 1024;
+
+/** {@link SKIP_MARKER} opening a `//`, `/*` or ` * ` comment line, as a whole word. */
+const SKIP_MARKER_PATTERN = /^[ \t]*(?:\/\/|\/\*+|\*)[ \t]*chant-discovery-skip(?![\w-])/m;
+
+/** True when `head`, the start of a file, carries the {@link SKIP_MARKER} comment. */
+export function hasSkipMarker(head: string): boolean {
+  return SKIP_MARKER_PATTERN.test(head.slice(0, SKIP_MARKER_HEAD_BYTES));
+}
+
+/**
+ * Whether a file's head carries the {@link GENERATED_MARKER} or the
+ * {@link SKIP_MARKER}. Reads only the head. The generated marker is looked for
+ * in the first 256 bytes, as it always was.
+ */
+export async function hasDiscoveryMarker(file: string): Promise<boolean> {
   let fh;
   try {
     fh = await open(file, "r");
-    const { buffer, bytesRead } = await fh.read({ buffer: Buffer.alloc(256), position: 0 });
-    return buffer.toString("utf8", 0, bytesRead).includes(GENERATED_MARKER);
+    const { buffer, bytesRead } = await fh.read({ buffer: Buffer.alloc(SKIP_MARKER_HEAD_BYTES), position: 0 });
+    if (buffer.toString("utf8", 0, Math.min(bytesRead, 256)).includes(GENERATED_MARKER)) return true;
+    return hasSkipMarker(buffer.toString("utf8", 0, bytesRead));
   } catch {
     return false;
   } finally {
@@ -26,14 +55,55 @@ async function isGeneratedFile(file: string): Promise<boolean> {
   }
 }
 
+/**
+ * Compile a project's `exclude`/`include` globs (#2519) into a predicate that
+ * is true for a file discovery must skip, or `undefined` when there is nothing
+ * to skip.
+ *
+ * Each pattern is matched, with picomatch and dot files included, against the
+ * file's path relative to `globs.root` (forward slashes) and against the path
+ * of every directory above it, so a pattern naming a directory covers
+ * everything under it. A file is skipped when some `exclude` pattern matches
+ * and no `include` pattern does: `include` always wins. Files outside
+ * `globs.root` are never skipped.
+ */
+export function compileDiscoveryFilter(globs: DiscoveryGlobs | undefined): ((file: string) => boolean) | undefined {
+  if (!globs || globs.exclude.length === 0) return undefined;
+  const excluded = picomatch(globs.exclude, { dot: true });
+  const included = globs.include.length > 0 ? picomatch(globs.include, { dot: true }) : () => false;
+  return (file: string): boolean => {
+    const rel = relative(globs.root, file);
+    if (rel === "" || rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) return false;
+    const parts = rel.split(sep);
+    const candidates = parts.map((_, i) => parts.slice(0, i + 1).join("/"));
+    return candidates.some((c) => excluded(c)) && !candidates.some((c) => included(c));
+  };
+}
+
+/** Options for {@link findInfraFiles}. */
+export interface FindInfraFilesOptions {
+  /**
+   * The project's discovery globs. Omitted, they are read from the project
+   * config `path` belongs to ({@link resolveDiscoveryGlobs}); `null` applies none.
+   */
+  globs?: DiscoveryGlobs | null;
+}
+
 /**
  * Recursively find all TypeScript infrastructure files in a directory
  * @param path - The directory path to search
- * @returns Array of file paths to .ts files (excluding test files and chant-generated files)
+ * @returns Array of file paths to .ts files (excluding test files, chant-generated
+ *   files, files carrying the skip marker, and files the project's `exclude` globs name)
  */
-export async function findInfraFiles(path: string): Promise<string[]> {
+export async function findInfraFiles(path: string, options?: FindInfraFilesOptions): Promise<string[]> {
   const files: string[] = [];
+  const skip = compileDiscoveryFilter(
+    options?.globs === null ? undefined : (options?.globs ?? (await resolveDiscoveryGlobs(path))),
+  );
   let sourceRoot: string | null = null;
+  // Child projects skipped because the source root was already set: the
+  // next release reads them when `path` is outside a project (#2527).
+  const skippedChildren: string[] = [];
 
   async function scanDirectory(dir: string): Promise<void> {
     let entries;
@@ -63,17 +133,20 @@ export async function findInfraFiles(path: string): Promise<string[]> {
           if (sourceRoot === null) {
             sourceRoot = fullPath;
           } else {
+            skippedChildren.push(fullPath);
             continue;
           }
         }
         await scanDirectory(fullPath);
       } else if (entry.isFile()) {
-        // Include only .ts files, exclude test files and chant-generated files
+        // Include only .ts files, exclude test files, files the project's
+        // globs exclude, and chant-generated or skip-marked files
         if (
           entry.name.endsWith(".ts") &&
           !entry.name.endsWith(".test.ts") &&
           !entry.name.endsWith(".spec.ts") &&
-          !(await isGeneratedFile(fullPath))
+          !skip?.(fullPath) &&
+          !(await hasDiscoveryMarker(fullPath))
         ) {
           files.push(fullPath);
         }
@@ -82,5 +155,20 @@ export async function findInfraFiles(path: string): Promise<string[]> {
   }
 
   await scanDirectory(path);
+  // The warning release for #2527: say which files the converged walker will
+  // read differently. The list returned is today's, unchanged.
+  await warnDiscoveryChanges({
+    walker: "source",
+    root: path,
+    files,
+    sourceRoot,
+    skippedChildren,
+    fileOk: async (name, full) =>
+      name.endsWith(".ts") &&
+      !name.endsWith(".test.ts") &&
+      !name.endsWith(".spec.ts") &&
+      !skip?.(full) &&
+      !(await hasDiscoveryMarker(full)),
+  });
   return files;
 }

package/src/discovery/fold-import.ts

@@ -27,6 +27,7 @@ import {
   type FoldedCompositeStepCall,
 } from "../fold/fold";
 import { isChantOwnedSpecifier, isFoldableHelperName } from "../fold/foldable-helpers";
+import { lexiconModulePath } from "../lexicon-module";
 import {
   briefNodeText,
   callExpressionMessage,
@@ -247,6 +248,21 @@ export interface FoldSession {
    * established one.
    */
   readonly lexiconPackages: ReadonlySet<string>;
+  /**
+   * chant#2577 — the absolute module paths of this build's lexicons that
+   * `chant.config.ts` declares by path (#2520), each resolved to the file an
+   * import of it reaches. Such a lexicon has no package specifier, so a
+   * project imports it by a relative or absolute path. A binding whose path
+   * resolves to one of these is treated exactly like a binding of a
+   * {@link lexiconPackages} entry: its data exports resolve as values, and
+   * under `--sandbox` it is on {@link isTrustedExecutableBinding}'s allowlist,
+   * because `loadPlugins` has already imported and run it in this process.
+   *
+   * Only the declared module itself is in the set. A package has a root that
+   * bounds its subpaths, and a module path has none, so other files beside it
+   * stay project files.
+   */
+  readonly lexiconModules: ReadonlySet<string>;
   /**
    * chant #1093 — this build asked for the #1045 sandbox
    * (`DiscoveryOptions.sandbox`, `chant build --sandbox`), so fold must not
@@ -327,6 +343,15 @@ export function createFoldSession(
   /** chant#2455 — `ι = executing`. Mutually exclusive with `sandbox`. */
   executing = false,
 ): FoldSession {
+  // chant#2577 — a lexicon declared by path is that module and no package, so
+  // its name contributes the module path and not `@intentius/chant-lexicon-<name>`.
+  const packages = new Set(lexiconPackages);
+  const modules = new Set<string>();
+  for (const name of lexicons) {
+    const modulePath = lexiconModulePath(name);
+    if (modulePath === undefined) packages.add(lexiconPackageName(name));
+    else modules.add(resolveModulePath(modulePath, modulePath));
+  }
   return {
     intrinsics,
     cache: new Map(),
@@ -334,7 +359,8 @@ export function createFoldSession(
     importCache: new Map(),
     resolvePathCache: new Map(),
     buildParams,
-    lexiconPackages: new Set([...lexicons.map(lexiconPackageName), ...lexiconPackages]),
+    lexiconPackages: packages,
+    lexiconModules: modules,
     sandbox,
     executing,
     factoryModules: new Map(),
@@ -1104,6 +1130,29 @@ function activeLexiconPackage(specifier: string, lexiconPackages: ReadonlySet<st
   return lexiconPackages.has(specifier) ? specifier : undefined;
 }
 
+/**
+ * chant#2577 — the path-declared counterpart of {@link activeLexiconPackage}:
+ * the resolved module path when a relative or absolute `specifier` reaches
+ * one of this build's {@link FoldSession.lexiconModules}, `undefined`
+ * otherwise. A bare specifier is never resolved here, and a build with no
+ * path-declared lexicon resolves nothing.
+ */
+function activeLexiconModule(
+  specifier: string,
+  fromFile: string,
+  lexiconModules: ReadonlySet<string>,
+  resolvePathCache: Map<string, string>,
+): string | undefined {
+  if (lexiconModules.size === 0 || !isProjectFileSpecifier(specifier)) return undefined;
+  let modulePath: string;
+  try {
+    modulePath = resolveModulePathMemoized(specifier, fromFile, resolvePathCache);
+  } catch {
+    return undefined;
+  }
+  return lexiconModules.has(modulePath) ? modulePath : undefined;
+}
+
 /**
  * chant #1995 — the package-ROOT portion of a bare subpath specifier
  * (`@intentius/chant-lexicon-azure/generated/index` ->
@@ -1168,13 +1217,17 @@ async function resolveActiveLexiconExport(
   fromFile: string,
   session: FoldSession,
 ): Promise<{ value: unknown } | undefined> {
-  if (!activeLexiconPackage(binding.specifier, session.lexiconPackages)) return undefined;
-
-  let modulePath: string;
-  try {
-    modulePath = resolveModulePathMemoized(binding.specifier, fromFile, session.resolvePathCache);
-  } catch {
-    return undefined;
+  let modulePath: string | undefined;
+  if (activeLexiconPackage(binding.specifier, session.lexiconPackages)) {
+    try {
+      modulePath = resolveModulePathMemoized(binding.specifier, fromFile, session.resolvePathCache);
+    } catch {
+      return undefined;
+    }
+  } else {
+    // chant#2577 — a lexicon declared by path, imported by its path.
+    modulePath = activeLexiconModule(binding.specifier, fromFile, session.lexiconModules, session.resolvePathCache);
+    if (modulePath === undefined) return undefined;
   }
 
   let mod: Record<string, unknown>;
@@ -1278,6 +1331,8 @@ interface ResolveCtx {
    * site that imports and executes a module.
    */
   lexiconPackages: ReadonlySet<string>;
+  /** chant#2577 — see {@link FoldSession.lexiconModules}. */
+  lexiconModules: ReadonlySet<string>;
   /** chant #1093 — see {@link FoldSession.sandbox}. */
   sandbox: boolean;
   /** chant#2455 — see {@link FoldSession.executing}. */
@@ -2053,7 +2108,10 @@ function findCompositeDefinition(
       // chant build the set holds only `@intentius/chant-lexicon-*`, and no
       // lexicon exports a `Composite`, so nothing about a chant project moves.
       const chantOwned = isChantOwnedHelperBinding(compositeBinding, { ...ctx, file: scope.file });
-      const hostOwned = activeLexiconPackage(compositeBinding.specifier, ctx.lexiconPackages) !== undefined;
+      const hostOwned =
+        activeLexiconPackage(compositeBinding.specifier, ctx.lexiconPackages) !== undefined ||
+        activeLexiconModule(compositeBinding.specifier, scope.file, ctx.lexiconModules, ctx.resolvePathCache) !==
+          undefined;
       if (!chantOwned && !hostOwned) return undefined;
 
       const [fnArg, nameArg] = init.arguments;
@@ -2425,6 +2483,7 @@ async function interpretCompositeFactory(
     importCache: ctx.importCache,
     resolvePathCache: ctx.resolvePathCache,
     lexiconPackages: ctx.lexiconPackages,
+    lexiconModules: ctx.lexiconModules,
     sandbox: ctx.sandbox,
     executing: ctx.executing,
     session: ctx.session,
@@ -3048,6 +3107,9 @@ function isTrustedExecutableBinding(binding: ImportBinding, ctx: ResolveCtx): bo
   } catch {
     return false;
   }
+  // chant#2577 — arm 1 for a lexicon declared by path: the module
+  // `loadPlugins` already imported, reached by the path the project imports it by.
+  if (ctx.lexiconModules.has(targetPath)) return true;
   const root = chantCoreRoot();
   return targetPath === root || targetPath.startsWith(root + sep);
 }
@@ -3607,6 +3669,14 @@ async function buildExternals(
     } catch {
       continue;
     }
+    // chant#2577 — a path-declared lexicon is a lexicon, not a project file:
+    // resolve its data export the way the bare-specifier branch above does,
+    // with no `liveSources` edge for the same reason.
+    if (session.lexiconModules.has(targetPath)) {
+      const lexiconExport = await resolveActiveLexiconExport(binding, file, session);
+      if (lexiconExport) externals.set(localName, lexiconExport.value);
+      continue;
+    }
     const result = await foldFileMemoized(targetPath, session);
     if (!result.ok) {
       failures.set(localName, locatedMessage(binding.specifierNode, result.reason));
@@ -3632,6 +3702,9 @@ async function buildExternals(
     } catch {
       continue;
     }
+    // chant#2577 — a namespace import of a lexicon is left alone, as the
+    // bare-specifier check above leaves a package's alone.
+    if (session.lexiconModules.has(targetPath)) continue;
     const result = await foldFileMemoized(targetPath, session);
     if (!result.ok) {
       failures.set(localName, locatedMessage(binding.specifierNode, result.reason));
@@ -3738,6 +3811,7 @@ async function tryFoldFileCore(file: string, session: FoldSession): Promise<Fold
       importCache: session.importCache,
       resolvePathCache: session.resolvePathCache,
       lexiconPackages: session.lexiconPackages,
+      lexiconModules: session.lexiconModules,
       sandbox: session.sandbox,
       executing: session.executing,
       session,

package/src/discovery/sandbox/path-lexicon-parity.test.ts

@@ -0,0 +1,162 @@
+/**
+ * chant#2577 — a lexicon declared by path in `chant.config.ts` (#2520) gets the
+ * same fold treatment as the same lexicon installed as a package, with and
+ * without `--sandbox`.
+ *
+ * One lexicon source is written twice: once as the package
+ * `@intentius/chant-lexicon-demo` under the fixture's own `node_modules`, and
+ * once as `./lexicon/index.ts` named by path. The two projects differ only in
+ * how `chant.config.ts` names the lexicon and in the specifier `main.ts`
+ * imports it by. Each project's config is loaded, which is what records a
+ * path-declared lexicon, and each is built with the lexicon list a CLI build
+ * passes (the loaded plugins' names). The outputs and fold verdicts are compared.
+ */
+import { describe, test, expect, beforeAll, afterAll, afterEach } from "vitest";
+import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { tmpdir } from "node:os";
+import { build } from "../../build";
+import { loadChantConfig } from "../../config";
+import { resetLexiconModules } from "../../lexicon-module";
+import type { LexiconPlugin } from "../../lexicon";
+
+const runtimePath = resolve(import.meta.dirname, "../../runtime");
+const compositePath = resolve(import.meta.dirname, "../../composite");
+
+const LEXICON_SOURCE = `
+import { createResource } from ${JSON.stringify(runtimePath)};
+import { Composite } from ${JSON.stringify(compositePath)};
+
+export const Bucket = createResource("Demo::Bucket", "demo", { arn: "Arn" });
+export const Role = createResource("Demo::Role", "demo", {});
+export const Regions = { primary: "us-east-1" };
+
+export const WebApp = Composite((props) => {
+  const bucket = new Bucket({ bucketName: props.name });
+  const role = new Role({ resource: bucket.arn });
+  return { bucket, role };
+}, "WebApp");
+
+function plain(value) {
+  if (value && typeof value === "object" && typeof value.getLogicalName === "function") {
+    return { ref: value.getLogicalName(), attr: value.attribute };
+  }
+  if (Array.isArray(value)) return value.map(plain);
+  if (value && typeof value === "object") {
+    return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, plain(v)]));
+  }
+  return value;
+}
+
+export const demoPlugin = {
+  name: "demo",
+  serializer: {
+    name: "demo",
+    rulePrefix: "DEMO",
+    serialize: (entities) =>
+      JSON.stringify(
+        Object.fromEntries(
+          [...entities.keys()].sort().map((name) => {
+            const entity = entities.get(name);
+            return [name, { type: entity.entityType, props: plain(entity.props) }];
+          }),
+        ),
+      ),
+  },
+  generate: async () => {},
+  validate: async () => {},
+  coverage: async () => {},
+  package: async () => {},
+};
+`;
+
+function mainSource(specifier: string): string {
+  return `
+import { Bucket, Regions, WebApp } from ${JSON.stringify(specifier)};
+
+export const logs = new Bucket({ bucketName: Regions.primary });
+export const web = WebApp({ name: "data" });
+`;
+}
+
+let root: string;
+
+beforeAll(async () => {
+  root = await realpath(await mkdtemp(join(tmpdir(), "chant-2577-")));
+
+  const pkg = join(root, "pkg");
+  const pkgDir = join(pkg, "node_modules", "@intentius", "chant-lexicon-demo");
+  await mkdir(pkgDir, { recursive: true });
+  await writeFile(
+    join(pkgDir, "package.json"),
+    JSON.stringify({ name: "@intentius/chant-lexicon-demo", type: "module", main: "index.ts" }),
+  );
+  await writeFile(join(pkgDir, "index.ts"), LEXICON_SOURCE);
+  await mkdir(join(pkg, "src"));
+  await writeFile(join(pkg, "src", "main.ts"), mainSource("@intentius/chant-lexicon-demo"));
+  await writeFile(join(pkg, "chant.config.ts"), `export default { lexicons: ["demo"] };\n`);
+
+  const path = join(root, "path");
+  await mkdir(join(path, "lexicon"), { recursive: true });
+  await writeFile(join(path, "lexicon", "index.ts"), LEXICON_SOURCE);
+  await mkdir(join(path, "src"));
+  await writeFile(join(path, "src", "main.ts"), mainSource("../lexicon"));
+  await writeFile(
+    join(path, "chant.config.ts"),
+    `export default { lexicons: [{ name: "demo", module: "./lexicon/index.ts" }] };\n`,
+  );
+});
+
+afterAll(async () => {
+  await rm(root, { recursive: true, force: true });
+});
+
+afterEach(() => {
+  resetLexiconModules();
+});
+
+async function buildProject(project: "pkg" | "path", modes: { fold: boolean; sandbox: boolean }) {
+  const dir = join(root, project);
+  const lexiconFile =
+    project === "pkg"
+      ? join(dir, "node_modules", "@intentius", "chant-lexicon-demo", "index.ts")
+      : join(dir, "lexicon", "index.ts");
+  await loadChantConfig(dir);
+  const plugin = (await import(lexiconFile)).demoPlugin as LexiconPlugin;
+  const result = await build(join(dir, "src"), [plugin.serializer], undefined, {
+    ...modes,
+    lexicons: [plugin.name],
+  });
+  return {
+    errors: result.errors,
+    output: result.outputs.get("demo") as string,
+    decisions: result.foldDecisions.map((d) => ({ file: d.file.slice(dir.length), mode: d.mode, reason: d.reason })),
+  };
+}
+
+describe("a lexicon declared by path folds like the same lexicon installed as a package (chant#2577)", () => {
+  for (const modes of [
+    { fold: true, sandbox: true },
+    { fold: true, sandbox: false },
+  ]) {
+    test(
+      `--fold${modes.sandbox ? " --sandbox" : ""}: same output, same fold verdicts`,
+      async () => {
+        const asPackage = await buildProject("pkg", modes);
+        const byPath = await buildProject("path", modes);
+
+        expect(asPackage.errors).toEqual([]);
+        expect(byPath.errors).toEqual([]);
+        expect(byPath.output).toBe(asPackage.output);
+        expect(JSON.parse(byPath.output)).toEqual({
+          logs: { type: "Demo::Bucket", props: { bucketName: "us-east-1" } },
+          webBucket: { type: "Demo::Bucket", props: { bucketName: "data" } },
+          webRole: { type: "Demo::Role", props: { resource: { ref: "webBucket", attr: "Arn" } } },
+        });
+        expect(byPath.decisions).toEqual(asPackage.decisions);
+        expect(byPath.decisions).toEqual([{ file: "/src/main.ts", mode: "fold", reason: undefined }]);
+      },
+      60_000,
+    );
+  }
+});

package/src/graph-ir.ts

@@ -290,8 +290,22 @@ export interface IRPipeline {
   edges: IRPipelineEdge[];
 }
 
+/**
+ * The version `chant graph --format ir` writes on its output (#2529, #2524 D8).
+ * The format changes only additively: a new field or edge kind never bumps
+ * it, and a consumer ignores the fields and edge kinds it does not know. The
+ * number moves only for a change a consumer could not ignore.
+ */
+export const GRAPH_IR_VERSION = 1;
+
 /** The full graph IR for a project at the default (declarable) detail level. */
 export interface GraphIR {
+  /**
+   * {@link GRAPH_IR_VERSION}, stamped by `chant graph --format ir` when it
+   * writes the IR (#2529). Absent on an IR built in memory, and on output
+   * from a chant older than the field, which a reader treats as version 1.
+   */
+  version?: number;
   nodes: IRNode[];
   edges: IREdge[];
   groups: IRGroups;