@intentius/chant 0.80.0 → 0.81.0

package/src/discovery/files.ts

@@ -1,10 +1,9 @@
-import { readdir, stat, open } from "node:fs/promises";
-import { existsSync } from "node:fs";
-import { join, relative, sep, isAbsolute } from "node:path";
+import { open } from "node:fs/promises";
+import { relative, sep, isAbsolute } from "node:path";
 // @ts-ignore — picomatch has no types declaration
 import picomatch from "picomatch";
 import { resolveDiscoveryGlobs, type DiscoveryGlobs } from "../config";
-import { warnDiscoveryChanges } from "./convergence";
+import { readHead, walkDiscovery, workspaceMemberDirs } from "./walk";
 
 /**
  * Marker chant writes at the top of files it generates (a hosting lexicon's
@@ -55,6 +54,12 @@ export async function hasDiscoveryMarker(file: string): Promise<boolean> {
   }
 }
 
+/** Whether a file's head carries the {@link GENERATED_MARKER} or the {@link SKIP_MARKER}. The synchronous form of {@link hasDiscoveryMarker}. */
+export function hasDiscoveryMarkerSync(file: string): boolean {
+  const head = readHead(file, SKIP_MARKER_HEAD_BYTES);
+  return head.slice(0, 256).includes(GENERATED_MARKER) || hasSkipMarker(head);
+}
+
 /**
  * Compile a project's `exclude`/`include` globs (#2519) into a predicate that
  * is true for a file discovery must skip, or `undefined` when there is nothing
@@ -65,7 +70,8 @@ export async function hasDiscoveryMarker(file: string): Promise<boolean> {
  * of every directory above it, so a pattern naming a directory covers
  * everything under it. A file is skipped when some `exclude` pattern matches
  * and no `include` pattern does: `include` always wins. Files outside
- * `globs.root` are never skipped.
+ * `globs.root` are never skipped. The discovery walk (`./walk.ts`) applies the
+ * same rule, and also lets `include` re-admit what its default rules skip.
  */
 export function compileDiscoveryFilter(globs: DiscoveryGlobs | undefined): ((file: string) => boolean) | undefined {
   if (!globs || globs.exclude.length === 0) return undefined;
@@ -89,86 +95,24 @@ export interface FindInfraFilesOptions {
   globs?: DiscoveryGlobs | null;
 }
 
+/** A TypeScript source file by name: `.ts`, not a test or spec. */
+export function isSourceFileName(name: string): boolean {
+  return name.endsWith(".ts") && !name.endsWith(".test.ts") && !name.endsWith(".spec.ts");
+}
+
 /**
- * Recursively find all TypeScript infrastructure files in a directory
- * @param path - The directory path to search
- * @returns Array of file paths to .ts files (excluding test files, chant-generated
- *   files, files carrying the skip marker, and files the project's `exclude` globs name)
+ * Find every TypeScript infrastructure file under `path`, through the one
+ * discovery walk (`./walk.ts`, #2527): test and spec files, chant-generated
+ * files, skip-marked files and whatever the walk's rules and the project's
+ * globs skip are left out.
  */
 export async function findInfraFiles(path: string, options?: FindInfraFilesOptions): Promise<string[]> {
-  const files: string[] = [];
-  const skip = compileDiscoveryFilter(
-    options?.globs === null ? undefined : (options?.globs ?? (await resolveDiscoveryGlobs(path))),
-  );
-  let sourceRoot: string | null = null;
-  // Child projects skipped because the source root was already set: the
-  // next release reads them when `path` is outside a project (#2527).
-  const skippedChildren: string[] = [];
-
-  async function scanDirectory(dir: string): Promise<void> {
-    let entries;
-
-    try {
-      entries = await readdir(dir, { withFileTypes: true });
-    } catch (error) {
-      // Skip directories we can't read
-      return;
-    }
-
-    for (const entry of entries) {
-      const fullPath = join(dir, entry.name);
-
-      // Skip node_modules
-      if (entry.isDirectory() && entry.name === "node_modules") {
-        continue;
-      }
-
-      if (entry.isDirectory()) {
-        // Child project boundary — a directory with its own chant.config.ts
-        // is a separate scope, but only if we've already found the project's
-        // own source root. The first config directory is the source root, not
-        // a child project.
-        const configPath = join(fullPath, "chant.config.ts");
-        if (existsSync(configPath)) {
-          if (sourceRoot === null) {
-            sourceRoot = fullPath;
-          } else {
-            skippedChildren.push(fullPath);
-            continue;
-          }
-        }
-        await scanDirectory(fullPath);
-      } else if (entry.isFile()) {
-        // Include only .ts files, exclude test files, files the project's
-        // globs exclude, and chant-generated or skip-marked files
-        if (
-          entry.name.endsWith(".ts") &&
-          !entry.name.endsWith(".test.ts") &&
-          !entry.name.endsWith(".spec.ts") &&
-          !skip?.(fullPath) &&
-          !(await hasDiscoveryMarker(fullPath))
-        ) {
-          files.push(fullPath);
-        }
-      }
-    }
-  }
-
-  await scanDirectory(path);
-  // The warning release for #2527: say which files the converged walker will
-  // read differently. The list returned is today's, unchanged.
-  await warnDiscoveryChanges({
+  const globs = options?.globs === null ? undefined : (options?.globs ?? (await resolveDiscoveryGlobs(path)));
+  return walkDiscovery({
     walker: "source",
     root: path,
-    files,
-    sourceRoot,
-    skippedChildren,
-    fileOk: async (name, full) =>
-      name.endsWith(".ts") &&
-      !name.endsWith(".test.ts") &&
-      !name.endsWith(".spec.ts") &&
-      !skip?.(full) &&
-      !(await hasDiscoveryMarker(full)),
+    globs,
+    excludeDirs: await workspaceMemberDirs(path),
+    accept: (name, full) => isSourceFileName(name) && !hasDiscoveryMarkerSync(full),
   });
-  return files;
 }

package/src/discovery/fold-import.ts

@@ -27,7 +27,7 @@ import {
   type FoldedCompositeStepCall,
 } from "../fold/fold";
 import { isChantOwnedSpecifier, isFoldableHelperName } from "../fold/foldable-helpers";
-import { lexiconModulePath } from "../lexicon-module";
+import { lexiconModulePath, lexiconModuleRoot } from "../lexicon-module";
 import {
   briefNodeText,
   callExpressionMessage,
@@ -258,11 +258,22 @@ export interface FoldSession {
    * under `--sandbox` it is on {@link isTrustedExecutableBinding}'s allowlist,
    * because `loadPlugins` has already imported and run it in this process.
    *
-   * Only the declared module itself is in the set. A package has a root that
-   * bounds its subpaths, and a module path has none, so other files beside it
-   * stay project files.
+   * Only the declared module itself is in the set. The other files of a
+   * multi-file path lexicon are covered by {@link lexiconRoots}.
    */
   readonly lexiconModules: ReadonlySet<string>;
+  /**
+   * chant#2590 — the directory trusted for each path-declared lexicon that has
+   * one (`../lexicon-module.ts`'s `pathLexiconRoot`: the declared `root`, or
+   * the module's own directory, inside the project and not containing its
+   * source directory). A file under one of these gets what a subpath of an
+   * active lexicon package gets: a place on the `--sandbox` trust list, and
+   * no fold as a project file. Its data exports are not read as values,
+   * which is also how a package subpath is treated. A directory that holds
+   * any of the build's own source files is dropped, so the build's source is
+   * never trusted this way.
+   */
+  readonly lexiconRoots: readonly string[];
   /**
    * chant #1093 — this build asked for the #1045 sandbox
    * (`DiscoveryOptions.sandbox`, `chant build --sandbox`), so fold must not
@@ -342,15 +353,23 @@ export function createFoldSession(
   lexiconPackages: readonly string[] = [],
   /** chant#2455 — `ι = executing`. Mutually exclusive with `sandbox`. */
   executing = false,
+  /**
+   * chant#2590 — the build's own source files. A path lexicon's trusted
+   * directory that holds any of them is dropped (see {@link FoldSession.lexiconRoots}).
+   */
+  sourceFiles: readonly string[] = [],
 ): FoldSession {
   // chant#2577 — a lexicon declared by path is that module and no package, so
   // its name contributes the module path and not `@intentius/chant-lexicon-<name>`.
   const packages = new Set(lexiconPackages);
   const modules = new Set<string>();
+  const roots: string[] = [];
   for (const name of lexicons) {
     const modulePath = lexiconModulePath(name);
     if (modulePath === undefined) packages.add(lexiconPackageName(name));
     else modules.add(resolveModulePath(modulePath, modulePath));
+    const root = lexiconModuleRoot(name);
+    if (root !== undefined && !sourceFiles.some((file) => isUnderDirectory(resolvePath(file), root))) roots.push(root);
   }
   return {
     intrinsics,
@@ -361,6 +380,7 @@ export function createFoldSession(
     buildParams,
     lexiconPackages: packages,
     lexiconModules: modules,
+    lexiconRoots: roots,
     sandbox,
     executing,
     factoryModules: new Map(),
@@ -1153,6 +1173,16 @@ function activeLexiconModule(
   return lexiconModules.has(modulePath) ? modulePath : undefined;
 }
 
+/** chant#2590 — true when absolute `file` lies under absolute directory `dir`. */
+function isUnderDirectory(file: string, dir: string): boolean {
+  return file.startsWith(dir.endsWith(sep) ? dir : dir + sep);
+}
+
+/** chant#2590 — true when `path` lies under one of the build's trusted path-lexicon directories. */
+function inPathLexiconRoot(path: string, roots: readonly string[]): boolean {
+  return roots.some((root) => isUnderDirectory(path, root));
+}
+
 /**
  * chant #1995 — the package-ROOT portion of a bare subpath specifier
  * (`@intentius/chant-lexicon-azure/generated/index` ->
@@ -1333,6 +1363,8 @@ interface ResolveCtx {
   lexiconPackages: ReadonlySet<string>;
   /** chant#2577 — see {@link FoldSession.lexiconModules}. */
   lexiconModules: ReadonlySet<string>;
+  /** chant#2590 — see {@link FoldSession.lexiconRoots}. */
+  lexiconRoots: readonly string[];
   /** chant #1093 — see {@link FoldSession.sandbox}. */
   sandbox: boolean;
   /** chant#2455 — see {@link FoldSession.executing}. */
@@ -2484,6 +2516,7 @@ async function interpretCompositeFactory(
     resolvePathCache: ctx.resolvePathCache,
     lexiconPackages: ctx.lexiconPackages,
     lexiconModules: ctx.lexiconModules,
+    lexiconRoots: ctx.lexiconRoots,
     sandbox: ctx.sandbox,
     executing: ctx.executing,
     session: ctx.session,
@@ -3110,6 +3143,9 @@ function isTrustedExecutableBinding(binding: ImportBinding, ctx: ResolveCtx): bo
   // chant#2577 — arm 1 for a lexicon declared by path: the module
   // `loadPlugins` already imported, reached by the path the project imports it by.
   if (ctx.lexiconModules.has(targetPath)) return true;
+  // chant#2590 — the subpath case of the same arm: another file of a
+  // multi-file path lexicon, under its trusted directory.
+  if (inPathLexiconRoot(targetPath, ctx.lexiconRoots)) return true;
   const root = chantCoreRoot();
   return targetPath === root || targetPath.startsWith(root + sep);
 }
@@ -3677,6 +3713,12 @@ async function buildExternals(
       if (lexiconExport) externals.set(localName, lexiconExport.value);
       continue;
     }
+    // chant#2590 — another file of a multi-file path lexicon is a lexicon
+    // subpath, not a project file: it is not folded, and like a package
+    // subpath (see the bare-specifier branch above) its data exports are not
+    // read as values. A constructor or composite from it resolves through the
+    // import path, where `isTrustedExecutableBinding` admits it.
+    if (inPathLexiconRoot(targetPath, session.lexiconRoots)) continue;
     const result = await foldFileMemoized(targetPath, session);
     if (!result.ok) {
       failures.set(localName, locatedMessage(binding.specifierNode, result.reason));
@@ -3704,7 +3746,7 @@ async function buildExternals(
     }
     // chant#2577 — a namespace import of a lexicon is left alone, as the
     // bare-specifier check above leaves a package's alone.
-    if (session.lexiconModules.has(targetPath)) continue;
+    if (session.lexiconModules.has(targetPath) || inPathLexiconRoot(targetPath, session.lexiconRoots)) continue;
     const result = await foldFileMemoized(targetPath, session);
     if (!result.ok) {
       failures.set(localName, locatedMessage(binding.specifierNode, result.reason));
@@ -3812,6 +3854,7 @@ async function tryFoldFileCore(file: string, session: FoldSession): Promise<Fold
       resolvePathCache: session.resolvePathCache,
       lexiconPackages: session.lexiconPackages,
       lexiconModules: session.lexiconModules,
+      lexiconRoots: session.lexiconRoots,
       sandbox: session.sandbox,
       executing: session.executing,
       session,
@@ -4341,6 +4384,7 @@ export async function foldProject(
     options.sandbox ?? false,
     options.lexiconPackages ?? [],
     options.executing ?? false,
+    files,
   );
   const attempts = new Map<string, FoldFileResult>();
   for (const file of files) attempts.set(file, await tryFoldFile(file, intrinsics, session));

package/src/discovery/index.ts

@@ -291,6 +291,7 @@ export async function discover(path: string, options?: DiscoveryOptions): Promis
         options.sandbox === true,
         [],
         options.executing === true,
+        files,
       )
     : undefined;
   if (options?.fold) {

package/src/discovery/sandbox/driver.ts

@@ -70,6 +70,49 @@ function lit(value: unknown): string {
   return JSON.stringify(value);
 }
 
+/**
+ * chant#2461 — the `send` every driver uses, as generated source lines.
+ *
+ * `process.send` is asynchronous: it queues the message on the IPC channel and
+ * returns. Node's contract is that the optional callback runs once the message
+ * has been written, or with an error if it could not be. Without the callback a
+ * failed send is not reported to the caller at all, so the driver could return
+ * from `main()` believing it had answered.
+ *
+ * So `send` returns a promise that settles only when the write has completed,
+ * and every call site awaits it before `main()` returns. A send that fails, or
+ * throws synchronously (a payload that cannot be serialized), writes the reason
+ * to stderr with a synchronous `writeSync` and exits 1. That way a result that
+ * never left the child cannot end as exit 0 with empty stderr, which is the
+ * signature that looks like success went missing.
+ *
+ * `before` is spliced in at the top of the function body (the run driver marks
+ * itself `reported` there).
+ */
+export function sendFunctionSource(before: readonly string[] = []): string[] {
+  return [
+    `function send(payload) {`,
+    ...before.map((line) => `  ${line}`),
+    `  if (typeof process.send !== "function") {`,
+    `    console.log(JSON.stringify(payload));`,
+    `    return Promise.resolve();`,
+    `  }`,
+    `  return new Promise((resolve) => {`,
+    `    const lost = (err) => {`,
+    `      writeSync(2, "chant: the sandboxed child could not hand its result to the parent: " +`,
+    `        (err && err.message ? err.message : String(err)) + " (chant#2461)\\n");`,
+    `      process.exit(1);`,
+    `    };`,
+    `    try {`,
+    `      process.send(payload, (err) => (err ? lost(err) : resolve()));`,
+    `    } catch (err) {`,
+    `      lost(err);`,
+    `    }`,
+    `  });`,
+    `}`,
+  ];
+}
+
 /**
  * Generate the driver module's full TypeScript source. Written to a tmp file
  * and bundled (see `./bundle.ts`) before being handed to a sandboxed child —
@@ -117,11 +160,7 @@ export function generateDriverSource(options: GenerateDriverOptions): string {
     `    "Nothing is thrown, so nothing else can name it (chant#2461).\\n");`,
     `});`,
     ``,
-    `function send(payload) {`,
-    `  reported = true;`,
-    `  if (typeof process.send === "function") process.send(payload);`,
-    `  else console.log(JSON.stringify(payload));`,
-    `}`,
+    ...sendFunctionSource([`reported = true;`]),
     ``,
     // collectEntities (bundled, real DiscoveryError instances) already names
     // the exact offending file on a same-directory duplicate — reuse that
@@ -156,7 +195,7 @@ export function generateDriverSource(options: GenerateDriverOptions): string {
     `    entities = collectEntities(modules, BUILD_ROOT);`,
     `  } catch (err) {`,
     `    errors.push(classifyChildError(errFile(err), err, "resolution").toJSON());`,
-    `    send({ entitySet: { entities: [] }, errors, provenanceByName: {} });`,
+    `    await send({ entitySet: { entities: [] }, errors, provenanceByName: {} });`,
     `    return;`,
     `  }`,
     ``,
@@ -178,18 +217,20 @@ export function generateDriverSource(options: GenerateDriverOptions): string {
     `    errors.push(classifyChildError("", err, "resolution").toJSON());`,
     `  }`,
     ``,
+    `  let entitySet;`,
     `  try {`,
-    `    const entitySet = encodeEntitySet(entities);`,
-    `    send({ entitySet, errors, provenanceByName });`,
+    `    entitySet = encodeEntitySet(entities);`,
     `  } catch (err) {`,
     `    errors.push(classifyChildError("", err, "resolution").toJSON());`,
-    `    send({ entitySet: { entities: [] }, errors, provenanceByName });`,
+    `    await send({ entitySet: { entities: [] }, errors, provenanceByName });`,
+    `    return;`,
     `  }`,
+    `  await send({ entitySet, errors, provenanceByName });`,
     `}`,
     ``,
-    `main().catch((err) => {`,
-    `  send({ entitySet: { entities: [] }, errors: [classifyChildError("", err).toJSON()], provenanceByName: {}, fatal: true });`,
-    `});`,
+    `main().catch((err) =>`,
+    `  send({ entitySet: { entities: [] }, errors: [classifyChildError("", err).toJSON()], provenanceByName: {}, fatal: true }),`,
+    `);`,
   );
 
   return lines.join("\n");
@@ -218,18 +259,16 @@ export function generateConfigDriverSource(configPath: string): string {
   return [
     `import { classifyChildError } from ${lit(CHILD_ERRORS_MODULE)};`,
     `import { scanConfigWireSafety } from ${lit(CONFIG_WIRE_MODULE)};`,
+    `import { writeSync } from "node:fs";`,
     ``,
-    `function send(payload) {`,
-    `  if (typeof process.send === "function") process.send(payload);`,
-    `  else console.log(JSON.stringify(payload));`,
-    `}`,
+    ...sendFunctionSource(),
     ``,
     `async function main() {`,
     `  let namespace;`,
     `  try {`,
     `    namespace = await import(${lit(configPath)});`,
     `  } catch (err) {`,
-    `    send({ kind: "chant-config", ok: false, error: classifyChildError(${lit(configPath)}, err).toJSON() });`,
+    `    await send({ kind: "chant-config", ok: false, error: classifyChildError(${lit(configPath)}, err).toJSON() });`,
     `    return;`,
     `  }`,
     ``,
@@ -240,24 +279,26 @@ export function generateConfigDriverSource(configPath: string): string {
     ``,
     `  const offenders = scanConfigWireSafety(selected);`,
     `  if (offenders.length > 0) {`,
-    `    send({ kind: "chant-config", ok: false, offenders });`,
+    `    await send({ kind: "chant-config", ok: false, offenders });`,
     `    return;`,
     `  }`,
     ``,
+    `  let config;`,
     `  try {`,
     // Round-trip here, not just at the IPC boundary: this is what proves the
     // payload really is JSON before it leaves the child, and turns anything
     // the scan somehow missed into a named error instead of a quiet drop.
-    `    const config = JSON.parse(JSON.stringify(selected ?? {}));`,
-    `    send({ kind: "chant-config", ok: true, config });`,
+    `    config = JSON.parse(JSON.stringify(selected ?? {}));`,
     `  } catch (err) {`,
-    `    send({ kind: "chant-config", ok: false, error: classifyChildError(${lit(configPath)}, err, "resolution").toJSON() });`,
+    `    await send({ kind: "chant-config", ok: false, error: classifyChildError(${lit(configPath)}, err, "resolution").toJSON() });`,
+    `    return;`,
     `  }`,
+    `  await send({ kind: "chant-config", ok: true, config });`,
     `}`,
     ``,
-    `main().catch((err) => {`,
-    `  send({ kind: "chant-config", ok: false, error: classifyChildError(${lit(configPath)}, err).toJSON() });`,
-    `});`,
+    `main().catch((err) =>`,
+    `  send({ kind: "chant-config", ok: false, error: classifyChildError(${lit(configPath)}, err).toJSON() }),`,
+    `);`,
   ].join("\n");
 }
 
@@ -294,18 +335,15 @@ export function generatePolicyDriverSource(policyPaths: readonly string[]): stri
     `import { classifyChildError } from ${lit(CHILD_ERRORS_MODULE)};`,
     `import { decodePolicyBuildResult, scanPolicyDiagnostics, PolicyWireError } from ${lit(POLICY_WIRE_MODULE)};`,
     `import { runPostSynthChecks, isPostSynthCheck } from ${lit(POST_SYNTH_MODULE)};`,
+    `import { writeSync } from "node:fs";`,
     ``,
-    `function send(payload) {`,
-    `  if (typeof process.send === "function") process.send(payload);`,
-    `  else console.log(JSON.stringify(payload));`,
-    `}`,
+    ...sendFunctionSource(),
     ``,
     `function fail(file, err, type) {`,
     `  if (err instanceof PolicyWireError) {`,
-    `    send({ kind: "chant-policy", ok: false, offenders: err.offenders });`,
-    `    return;`,
+    `    return send({ kind: "chant-policy", ok: false, offenders: err.offenders });`,
     `  }`,
-    `  send({ kind: "chant-policy", ok: false, error: classifyChildError(file, err, type).toJSON() });`,
+    `  return send({ kind: "chant-policy", ok: false, error: classifyChildError(file, err, type).toJSON() });`,
     `}`,
     ``,
     // The wrapper described in the doc above: same id/description so any
@@ -329,7 +367,7 @@ export function generatePolicyDriverSource(policyPaths: readonly string[]): stri
     `  try {`,
     `    buildResult = decodePolicyBuildResult(request.buildResult);`,
     `  } catch (err) {`,
-    `    fail("", err, "resolution");`,
+    `    await fail("", err, "resolution");`,
     `    return;`,
     `  }`,
     ``,
@@ -347,7 +385,7 @@ export function generatePolicyDriverSource(policyPaths: readonly string[]): stri
       `      if (isPostSynthCheck(value)) checks.push(guard(value, ${lit(policyPath)}));`,
       `    }`,
       `  } catch (err) {`,
-      `    fail(${lit(policyPath)}, err, "import");`,
+      `    await fail(${lit(policyPath)}, err, "import");`,
       `    return;`,
       `  }`,
     );
@@ -359,11 +397,11 @@ export function generatePolicyDriverSource(policyPaths: readonly string[]): stri
     `  try {`,
     `    diagnostics = runPostSynthChecks(checks, buildResult, request.env ?? undefined);`,
     `  } catch (err) {`,
-    `    fail("", err, "resolution");`,
+    `    await fail("", err, "resolution");`,
     `    return;`,
     `  }`,
     ``,
-    `  send({ kind: "chant-policy", ok: true, diagnostics });`,
+    `  await send({ kind: "chant-policy", ok: true, diagnostics });`,
     `}`,
     ``,
     // Registered synchronously at module top level — see the doc above on why