@intentius/chant 0.124.12 → 0.124.14

package/dist/discovery/index.js

@@ -0,0 +1,333 @@
+import { relative } from "node:path";
+import { findInfraFiles } from "./files.js";
+import { importModule } from "./import.js";
+import { collectEntities } from "./collect.js";
+import { resolveAttrRefs } from "./resolve.js";
+import { buildDependencyGraph } from "./graph.js";
+import { tryFoldFile, planFoldTaintWithEdges, createFoldSession } from "./fold-import.js";
+import { getProvenance } from "../provenance.js";
+import { buildParamValues } from "../build-params.js";
+import { setBuildParams } from "../params.js";
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, join, parse } from "node:path";
+/**
+ * Warn when resolved build parameters cannot reach project source (#1421).
+ *
+ * chant's core is ESM. When the project is CommonJS — `"type": "commonjs"`, or
+ * no `type` field — tsx loads project source through the CommonJS transform, so
+ * the project's `require` of `params.ts` and core's `import` of it produce two
+ * separate module records. {@link setBuildParams} mutates one object in place;
+ * project source reads the other, and sees `{}`.
+ *
+ * The failure is silence. chant prints `[param] tier = "prod" (cli)` and then
+ * emits the graph for the default branch. `chant graph` is always affected
+ * because it always takes the run path; `chant build --no-fold` likewise. Plain
+ * `chant build` usually escapes because folding substitutes parameters
+ * statically — but a file that falls back to run inside a folded build is wrong
+ * the same way, which is why this warns regardless of `fold`.
+ *
+ * Only fires when parameters were actually resolved, so a CJS project that uses
+ * none is never nagged. `chant doctor`'s `package-type-module` check is the
+ * ambient version of the same advice.
+ *
+ * Best-effort and never throws: a project whose `package.json` cannot be found
+ * or parsed gets no warning rather than a failed build.
+ */
+function warnIfParamsCannotReachProject(path, values) {
+    if (Object.keys(values).length === 0)
+        return;
+    try {
+        const pkgPath = findPackageJsonUpward(path);
+        if (!pkgPath)
+            return;
+        const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
+        if (pkg.type === "module")
+            return;
+        const found = pkg.type ? `"type": "${pkg.type}"` : "no `type` field";
+        const names = Object.keys(values).sort().join(", ");
+        console.error(`warning: ${pkgPath} has ${found}, but chant is ESM — build parameters (${names}) ` +
+            `will read as empty in project source on the run path, so declarations conditioned ` +
+            `on them take their default branch. Set "type": "module". (chant #1421)`);
+    }
+    catch {
+        // Unreadable or unparseable package.json — say nothing rather than fail.
+    }
+}
+/** Nearest `package.json` at or above `startDir`, or undefined. */
+function findPackageJsonUpward(startDir) {
+    let dir = startDir;
+    const { root } = parse(dir);
+    for (;;) {
+        const candidate = join(dir, "package.json");
+        if (existsSync(candidate))
+            return candidate;
+        if (dir === root)
+            return undefined;
+        const parent = dirname(dir);
+        if (parent === dir)
+            return undefined;
+        dir = parent;
+    }
+}
+/**
+ * Discovers all declarable entities in a directory by scanning files,
+ * importing modules, collecting entities, resolving references, and building
+ * a dependency graph.
+ *
+ * @param path - The directory path to discover entities in
+ * @param options - Optional discovery behavior, e.g. {@link DiscoveryOptions.fold}
+ * @returns DiscoveryResult with entities, dependencies, sourceFiles, and errors
+ */
+export async function discover(path, options) {
+    const errors = [];
+    const sourceFiles = [];
+    const foldDecisions = [];
+    // chant #1064 — populate the shared build-time-parameters object BEFORE any
+    // project file is imported or folded, so both paths observe the identical
+    // values: the fold path substitutes them directly (see
+    // fold-import.ts's `buildExternals`), and a run(-fallback) file that
+    // imports "@intentius/chant/params" for real sees them too, since
+    // `setBuildParams` mutates the shared object in place rather than
+    // rebinding it. Unconditional (not just when `buildParams` is set) so a
+    // stale value from a PRIOR `discover()` call in the same process (tests,
+    // `--watch`) never leaks into a build that supplied none.
+    const buildParamValuesMap = buildParamValues(options?.buildParams ?? []);
+    setBuildParams(buildParamValuesMap);
+    warnIfParamsCannotReachProject(path, buildParamValuesMap);
+    // Step 1: Scan for TypeScript files
+    const files = await findInfraFiles(path);
+    sourceFiles.push(...files);
+    // Step 2: Import all modules
+    const modules = [];
+    // chant #1045 Phase 2 — files that would reach the in-process importModule()
+    // call below are instead queued here when options.sandbox is set, and run
+    // together afterward in one isolated child (see the merge step after
+    // collectEntities).
+    const sandboxFiles = [];
+    // Fold path (#1022/#1023): try the static folder on EVERY file first,
+    // independently, before committing any decision. A file that folds
+    // successfully in isolation must still be forced back to run if some
+    // OTHER file that itself falls back to run imports it (directly or
+    // transitively) — otherwise the run-fallback file's real re-import
+    // produces a SECOND, non-identical copy of the same entities this file
+    // already folded, and cross-file AttrRefs between them can never resolve
+    // (see {@link planFoldTaint}'s doc for the full story). This is only
+    // knowable after every file's fold has been attempted, hence the two
+    // passes instead of committing per-file as they're visited.
+    const foldAttempts = new Map();
+    // chant #1020 — ONE session, shared by every top-level `tryFoldFile` call
+    // below AND by any cross-file reference one file's fold attempt makes into
+    // another (see fold-import.ts's `FoldSession` doc): this is what guarantees
+    // a project file imported by several others is folded exactly once, so
+    // every referrer shares the identical constructed Declarable/
+    // CompositeInstance objects rather than each building its own copy.
+    // chant #1093 — `sandbox` is threaded into the fold session, not just used
+    // for the run-fallback set below: fold itself resolves a composite factory /
+    // resource constructor / intrinsic tag by IMPORTING the module that defines
+    // it and invoking it, which for a project-owned one means executing project
+    // code in this process even though the file is reported as "folded". Under
+    // `sandbox` the session refuses those imports and the file demotes to the
+    // run path — which, under `sandbox`, is the isolated child. See
+    // fold-import.ts's `sandboxedExecutionRefusal`.
+    if (options?.sandbox === true && options?.executing === true) {
+        throw new Error("discover: `sandbox` and `executing` are mutually exclusive — the first refuses to import " +
+            "project code and the second exists to invoke it. Pass one.");
+    }
+    const foldSession = options?.fold
+        ? createFoldSession(options.intrinsics, buildParamValuesMap, options.lexicons, options.sandbox === true, [], options.executing === true, files)
+        : undefined;
+    if (options?.fold) {
+        for (const file of files) {
+            foldAttempts.set(file, await tryFoldFile(file, options.intrinsics, foldSession));
+        }
+    }
+    const taintPlan = options?.fold
+        ? await planFoldTaintWithEdges(files, new Map(files.map((file) => [file, foldAttempts.get(file)?.ok === true])), 
+        // chant #1044 — which files' OBJECTS each successful fold captured,
+        // so a file forced back to run also invalidates the folds that
+        // already hold its instances (see planFoldTaint's doc).
+        new Map(files.flatMap((file) => {
+            const attempt = foldAttempts.get(file);
+            return attempt?.ok === true ? [[file, attempt.liveSources]] : [];
+        })))
+        : { tainted: new Set(), reachedBy: new Map() };
+    const taintedFiles = taintPlan.tainted;
+    for (const file of files) {
+        if (options?.fold) {
+            const folded = foldAttempts.get(file);
+            if (folded.ok && !taintedFiles.has(file)) {
+                // chant #1112 — hand `collectEntities` the file's WHOLE folded export
+                // namespace, exactly as the run path hands it the real module
+                // namespace object `importModule` returns. It used to get only the
+                // `Declarable`/`CompositeInstance` subset fold had already picked
+                // out, which quietly made fold-import a SECOND owner of the "which
+                // export becomes an entity" decision — and it had one fewer case than
+                // the real owner (`./collect.ts`'s `enumerateEntries`): a
+                // `LexiconOutput` (`export const oArn = output(bucket.Arn, "oArn")`)
+                // was resolved, dropped, and the template lost its whole `Outputs`
+                // section with no warning and exit 0. Fold now decides nothing here;
+                // `collectEntities` filters both paths' exports the same way, so a
+                // shape it learns about (today: outputs and arrays of declarables)
+                // cannot reach one path and not the other.
+                modules.push({ file, exports: Object.fromEntries(folded.exportedValues) });
+                foldDecisions.push({ file, mode: "fold", resourceCount: folded.entities.length });
+                continue;
+            }
+            // chant#2406 — name the edge that actually fired. A reverse-tainted file
+            // is not imported by anything that falls back; saying so sends a reader
+            // looking for an importer that does not exist.
+            const edge = taintPlan.reachedBy.get(file);
+            const reason = !folded.ok
+                ? folded.reason
+                : edge?.kind === "capture"
+                    ? `would fold in isolation, but it captured objects from ${relative(process.cwd(), edge.from)}, which falls back to run — folding independently would hold an instance the build never collects`
+                    : `would fold in isolation, but a file that imports it (directly or transitively) falls back to run — folding independently would create a duplicate, non-identical instance`;
+            foldDecisions.push({ file, mode: "run", reason, reverseTainted: folded.ok });
+        }
+        if (options?.sandbox) {
+            sandboxFiles.push(file);
+            continue;
+        }
+        try {
+            const exports = await importModule(file);
+            modules.push({ file, exports });
+        }
+        catch (error) {
+            // Collect import errors but continue processing other files
+            if (error instanceof Error && error.name === "DiscoveryError") {
+                errors.push(error);
+            }
+            else {
+                // Convert unexpected errors to DiscoveryError
+                const { DiscoveryError: DiscoveryErrorClass } = await import("../errors.js");
+                errors.push(new DiscoveryErrorClass(file, error instanceof Error ? error.message : String(error), "import"));
+            }
+        }
+    }
+    // Step 3: Collect entities from imported modules
+    let entities = new Map();
+    try {
+        entities = collectEntities(modules, path);
+    }
+    catch (error) {
+        // Collect resolution errors
+        if (error instanceof Error && error.name === "DiscoveryError") {
+            errors.push(error);
+        }
+        else {
+            const { DiscoveryError: DiscoveryErrorClass } = await import("../errors.js");
+            errors.push(new DiscoveryErrorClass("", error instanceof Error ? error.message : String(error), "resolution"));
+        }
+        // If collection fails, return early with empty results
+        return {
+            entities: new Map(),
+            dependencies: new Map(),
+            sourceFiles,
+            errors,
+            foldDecisions,
+        };
+    }
+    // chant #1045 Phase 2 — run the queued sandbox files together, isolated, in
+    // one child process, and merge its already-named, already-ref-resolved
+    // entities in. The child performs its OWN collectEntities/resolveAttrRefs
+    // over just this subset (see ./sandbox/run.ts) rather than sharing live
+    // objects with the fold-only `entities` map above — a real object can't
+    // cross a process boundary, and `planFoldTaint` already guarantees no
+    // folded file is ever referenced by a run-fallback one (or vice versa), so
+    // the two subsets never share cross-references. A name collision between
+    // them is therefore always a genuine duplicate (never the same object
+    // legitimately re-exported), reported exactly like collectEntities' own
+    // same-key check above rather than silently overwritten. (One narrower gap
+    // than the single unified collectEntities call this replaces: cross-
+    // directory stack-prefix disambiguation, #932, is computed separately for
+    // each subset, so a bare-name collision that spans a folded directory and a
+    // run-fallback directory won't be disambiguated the same way a single
+    // combined call would. Not hit by any corpus entry today.)
+    if (options?.sandbox && sandboxFiles.length > 0) {
+        try {
+            // Dynamic, not static — `./sandbox/run` imports `esbuild`, a large CJS
+            // package with its own module-scope filesystem access (the same class
+            // of thing `entity-wire.ts`'s split from `entity-wire-codec.ts` avoids
+            // for `typescript`). `discover()` is re-exported from the package root
+            // (`@intentius/chant`), and project source commonly imports that root
+            // — a STATIC top-level import here would make `esbuild` transitively
+            // reachable, and therefore BUNDLED AND EAGERLY EVALUATED, by every
+            // project file the sandboxed child imports that happens to import
+            // chant itself. A dynamic import is only ever actually reached here,
+            // at runtime, when a caller opts into `sandbox: true` — never from
+            // inside a bundled project file (project code never calls `discover()`
+            // itself), so esbuild only ever bundles it as inert, un-evaluated code
+            // when it's pulled in transitively.
+            const { runFallbackFilesSandboxed } = await import("./sandbox/run.js");
+            const sandboxResult = await runFallbackFilesSandboxed(sandboxFiles, path);
+            errors.push(...sandboxResult.errors);
+            for (const [name, entity] of sandboxResult.entities) {
+                if (entities.has(name)) {
+                    const { DiscoveryError: DiscoveryErrorClass } = await import("../errors.js");
+                    const file = sandboxResult.provenanceByName[name] ?? path;
+                    errors.push(new DiscoveryErrorClass(file, `Duplicate export name "${name}" found`, "resolution"));
+                    continue;
+                }
+                entities.set(name, entity);
+            }
+            // Re-order the merged map to match original file discovery order.
+            // Without this, every fold entity (inserted above via the parent's own
+            // collectEntities call) sorts before every sandboxed entity (appended
+            // by the loop just above) — a fold-block-then-run-block order, not the
+            // per-file INTERLEAVED order a single unified collectEntities call
+            // over `files` would produce. Several downstream consumers iterate the
+            // entities map directly and are order-sensitive (e.g. a serializer's
+            // auto-detected cross-lexicon `Parameters`), so this isn't cosmetic —
+            // uncorrected, it's real, if harmless (still a valid template),
+            // byte-level drift. Ordered by each entity's OWN provenance
+            // (`../provenance.ts`) rather than by which loop produced it: a fold
+            // entity's provenance is read directly (a live object, still in this
+            // process); a sandboxed entity's provenance didn't survive the wire —
+            // `encodeEntitySet` deliberately drops build metadata, not declared
+            // configuration — so `sandboxResult.provenanceByName` (a side channel
+            // the child computes for exactly this, see ./sandbox/run.ts) is used
+            // instead. `Array.prototype.sort` is stable (guaranteed since ES2019),
+            // so entities from the same file (already correctly ordered by
+            // whichever collectEntities call produced them) keep their relative
+            // order — only which FILE's block comes first changes.
+            const fileOrder = new Map(files.map((file, i) => [file, i]));
+            const withIndex = [...entities.entries()].map(([name, entity]) => {
+                const sourceFile = getProvenance(entity)?.sourceFile ?? sandboxResult.provenanceByName[name];
+                const index = sourceFile !== undefined ? fileOrder.get(sourceFile) : undefined;
+                return { name, entity, index: index ?? Number.MAX_SAFE_INTEGER };
+            });
+            withIndex.sort((a, b) => a.index - b.index);
+            entities = new Map(withIndex.map(({ name, entity }) => [name, entity]));
+        }
+        catch (error) {
+            const { DiscoveryError: DiscoveryErrorClass } = await import("../errors.js");
+            errors.push(new DiscoveryErrorClass(path, error instanceof Error ? error.message : String(error), "resolution"));
+        }
+    }
+    // Step 4: Resolve AttrRefs
+    try {
+        resolveAttrRefs(entities);
+    }
+    catch (error) {
+        // Collect resolution errors but continue
+        const { DiscoveryError: DiscoveryErrorClass } = await import("../errors.js");
+        errors.push(new DiscoveryErrorClass("", error instanceof Error ? error.message : String(error), "resolution"));
+    }
+    // Step 5: Build dependency graph
+    let dependencies = new Map();
+    try {
+        dependencies = buildDependencyGraph(entities);
+    }
+    catch (error) {
+        // Collect graph building errors
+        const { DiscoveryError: DiscoveryErrorClass } = await import("../errors.js");
+        errors.push(new DiscoveryErrorClass("", error instanceof Error ? error.message : String(error), "resolution"));
+    }
+    return {
+        entities,
+        dependencies,
+        sourceFiles,
+        errors,
+        foldDecisions,
+    };
+}