@goodbones/core 0.1.0-beta.2 → 0.1.0-beta.4

package/src/core/graph.ts

@@ -215,6 +215,18 @@ const stronglyConnected = (
   return components;
 };
 
+// Every cycle in the graph, each as its sorted member set. What `infer` asks
+// before it writes a no-cycles rule: a rule the tree fails on day one is a
+// baseline entry, not a description of the tree.
+export const cyclesIn = (graph: Graph): ReadonlyArray<ReadonlyArray<string>> =>
+  stronglyConnected(graph.files, graph).filter((component) => {
+    const [first] = component;
+    return (
+      first !== undefined &&
+      (component.length > 1 || (graph.edges.get(first) ?? []).includes(first))
+    );
+  });
+
 const evaluateCycles = (rule: CompiledGraphCycleRule, graph: Graph): ReadonlyArray<Violation> => {
   const nodes = graph.files.filter((file) => inScope(rule, file));
   const violations: Array<Violation> = [];

package/src/domain/manifest-location.ts

@@ -11,9 +11,31 @@ export type ManifestPosition = {
   // 1-based, as editors count.
   readonly line: number;
   readonly column: number;
+  // The file the position is in, when it is not the manifest itself: a
+  // manifest assembled from several files through `include` answers with the
+  // included file's path, relative to the manifest's own directory. Absent
+  // for a position in the manifest file.
+  readonly file?: string | undefined;
 };
 
 // Answers with the position of the value at `path`, or the nearest ancestor
 // that exists when the path names something the file does not contain (a
 // missing key), or `null` when the source has no positions to give.
 export type ManifestLocator = (path: ManifestPath) => ManifestPosition | null;
+
+const isIdentifier = (key: string): boolean => /^[A-Za-z_$][\w$]*$/.test(key);
+
+// `tree["~/core/"].members[0].subject` — dotted where a key reads as a name,
+// bracketed where it does not, so a node key that is a path pattern stays
+// legible.
+export const renderManifestPath = (path: ManifestPath): string =>
+  path.length === 0
+    ? "(root)"
+    : path
+        .map((segment, index) => {
+          if (typeof segment === "number") return `[${String(segment)}]`;
+          const key = String(segment);
+          if (isIdentifier(key)) return index === 0 ? key : `.${key}`;
+          return `[${JSON.stringify(key)}]`;
+        })
+        .join("");

package/src/index.ts

@@ -37,6 +37,7 @@ export {
 export {
   type CompiledGraph,
   compileGraphRules,
+  cyclesIn,
   EMPTY_GRAPH_RULES,
   evaluateGraph,
   type Graph,
@@ -112,6 +113,7 @@ export {
   type ManifestLocator,
   type ManifestPath,
   type ManifestPosition,
+  renderManifestPath,
 } from "./domain/manifest-location.js";
 export {
   fingerprintOf,
@@ -127,7 +129,19 @@ export {
   type ManifestFile,
   readManifestFile,
 } from "./infrastructure/manifest-file.js";
-export { listSourceFiles, type WalkedLanguage } from "./infrastructure/walk.js";
+export {
+  expandIncludes,
+  type IncludedManifest,
+  type IncludeReader,
+  type SourceDocument,
+} from "./infrastructure/manifest-include.js";
+export {
+  listPackageRoots,
+  listSourceFiles,
+  type PackagedLanguage,
+  type PackageRoot,
+  type WalkedLanguage,
+} from "./infrastructure/walk.js";
 export { type LoadedPolicy, loadPolicy, type LoadPolicyInput } from "./load/policy.js";
 export { type LoweredRules, lowerManifest, type ProbeLanguage } from "./manifest/compile.js";
 export {
@@ -138,7 +152,24 @@ export {
   originOf,
   type Substitution,
 } from "./manifest/expand.js";
-export { MANIFEST_SCHEMA_ID, manifestJsonSchema } from "./manifest/json-schema.js";
+export {
+  type Candidate,
+  candidatesOf,
+  type Generalization,
+  type InferInput,
+  inferManifest,
+  type InferOptions,
+  type InferPackage,
+  type Inferred,
+  type InferredTarget,
+  singularOf,
+} from "./manifest/infer.js";
+export {
+  MANIFEST_NODE_SCHEMA_ID,
+  MANIFEST_SCHEMA_ID,
+  manifestJsonSchema,
+  manifestNodeJsonSchema,
+} from "./manifest/json-schema.js";
 export {
   type DecodedManifest,
   decodeManifest,

package/src/infrastructure/manifest-file.ts

@@ -21,14 +21,17 @@ import type {
   ManifestPath,
   ManifestPosition,
 } from "../domain/manifest-location.js";
+import { expandIncludes, type SourceDocument } from "./manifest-include.js";
 
-// The manifest, as the file on disk states it, before any decoding. A data
+// The manifest, as the files on disk state it, before any decoding. A data
 // file — YAML, or JSON, which YAML 1.2 contains — is what any host in any
 // language can read, and is the form the docs are written in. A JavaScript
 // module is the escape hatch a Node host alone honours: for a manifest
 // generated from other data, and for what the ecosystem expects of a config
-// file. Whichever it is, what comes out is one `unknown` value for the decoder,
-// and, from the data formats, a way to turn a path in it back into a line.
+// file. Either may `include` further data files, so a monorepo's policy can
+// live beside the packages it governs. Whichever it is, what comes out is one
+// `unknown` value for the decoder, and, from the data formats, a way to turn
+// a path in it back into a file and a line.
 
 // In discovery order. A repository with none of these is told all four; one
 // with more than one is refused, so nobody edits the wrong file for a week.
@@ -42,8 +45,12 @@ export const MANIFEST_FILENAMES = [
 export type ManifestFile = {
   readonly configPath: string;
   readonly manifest: unknown;
-  // Absent for a JavaScript module, which has no positions to give.
+  // Absent for a JavaScript module that includes nothing, which has no
+  // positions to give.
   readonly locate: ManifestLocator | undefined;
+  // Every file the manifest was read from, the root first: what a host that
+  // caches or watches the policy has to look at.
+  readonly files: ReadonlyArray<string>;
 };
 
 export const findManifestFile = (repoRoot: string): string => {
@@ -76,24 +83,38 @@ const isDataManifest = (configPath: string): boolean =>
 const isModuleManifest = (configPath: string): boolean =>
   [".mjs", ".js", ".cjs"].includes(extensionOf(configPath));
 
+// Whichever form the root takes, an `include` in it names a data file, read
+// by the same parser; the locator that comes back answers across every file.
 export const readManifestFile = async (configPath: string): Promise<ManifestFile> => {
-  if (isDataManifest(configPath)) return readDataManifest(configPath);
-  if (isModuleManifest(configPath)) return readModuleManifest(configPath);
-  throw new ConfigInvalid({
+  const root = isDataManifest(configPath)
+    ? parseDataFile(configPath)
+    : isModuleManifest(configPath)
+      ? await readModule(configPath)
+      : undefined;
+  if (root === undefined) {
+    throw new ConfigInvalid({
+      configPath,
+      detail:
+        "a manifest is a .yaml, .yml or .json file, or a .mjs/.js module — " +
+        `not ${JSON.stringify(path.basename(configPath))}.`,
+    });
+  }
+  const assembled = expandIncludes(configPath, root, { exists: existsSync, read: parseDataFile });
+  return {
     configPath,
-    detail:
-      "a manifest is a .yaml, .yml or .json file, or a .mjs/.js module — " +
-      `not ${JSON.stringify(path.basename(configPath))}.`,
-  });
+    manifest: assembled.value,
+    locate: assembled.locate,
+    files: assembled.files,
+  };
 };
 
-const readModuleManifest = async (configPath: string): Promise<ManifestFile> => {
+const readModule = async (configPath: string): Promise<SourceDocument> => {
   const module: unknown = await import(pathToFileURL(configPath).href).catch((cause: unknown) => {
     throw new ConfigInvalid({ configPath, detail: String(cause) });
   });
-  const manifest =
+  const value =
     typeof module === "object" && module !== null && "default" in module ? module.default : module;
-  return { configPath, manifest, locate: undefined };
+  return { value, locate: undefined };
 };
 
 // YAML 1.2 core schema, which is what a reader without a YAML background
@@ -102,7 +123,7 @@ const readModuleManifest = async (configPath: string): Promise<ManifestFile> =>
 // duplicate keys are refused rather than last-one-wins. A tag the parser does
 // not know is refused too — a manifest is data, and a `!!js/function` in it
 // would be a manifest only one runtime could read.
-const readDataManifest = (configPath: string): ManifestFile => {
+const parseDataFile = (configPath: string): SourceDocument => {
   let text: string;
   try {
     text = readFileSync(configPath, "utf8");
@@ -140,8 +161,7 @@ const readDataManifest = (configPath: string): ManifestFile => {
   }
 
   return {
-    configPath,
-    manifest: document.toJS({ mapAsMap: false }) as unknown,
+    value: document.toJS({ mapAsMap: false }) as unknown,
     locate: makeLocator(document, lines),
   };
 };

package/src/infrastructure/manifest-include.ts

@@ -0,0 +1,318 @@
+import * as path from "node:path";
+
+import { ConfigInvalid } from "../domain/architecture-error.js";
+import {
+  type ManifestLocator,
+  type ManifestPath,
+  renderManifestPath,
+} from "../domain/manifest-location.js";
+
+// A manifest split across files. `{ include: "<path>" }` standing anywhere in
+// a manifest — a node under `tree`, a whole section, one entry of a list — is
+// replaced by the value of the file it names, read by the same parser and
+// resolved relative to the file that wrote the reference. This runs on the raw
+// value before anything else looks at it: the `defs`/`use` expansion sees one
+// document, the decoder sees one document, and the schema never learns that a
+// reference existed.
+//
+// What is deliberately not here: no merging (a value is replaced, full stop),
+// no parameters, no glob of files. An included file is YAML or JSON only — a
+// module would make a data manifest readable by one runtime — and a file may
+// carry a top-level `defs` of its own, which joins the manifest's under one
+// namespace, and a `$schema` for its editor, which is dropped.
+
+// One file, as the reader parsed it.
+export type SourceDocument = {
+  readonly value: unknown;
+  readonly locate: ManifestLocator | undefined;
+};
+
+export type IncludeReader = {
+  readonly exists: (file: string) => boolean;
+  // Throws `ConfigInvalid` naming the file when it does not parse.
+  readonly read: (file: string) => SourceDocument;
+};
+
+export type IncludedManifest = {
+  readonly value: unknown;
+  // Answers for every file: a position inside an included file carries that
+  // file's path, relative to the root manifest's directory.
+  readonly locate: ManifestLocator | undefined;
+  // Every file that took part, root first, as absolute paths.
+  readonly files: ReadonlyArray<string>;
+};
+
+const INCLUDABLE = [".yaml", ".yml", ".json"];
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+  typeof value === "object" && value !== null && !Array.isArray(value);
+
+type IncludeReference = Record<string, unknown> & { readonly include: unknown };
+
+const isInclude = (value: unknown): value is IncludeReference =>
+  isRecord(value) && "include" in value;
+
+const isPrefix = (prefix: ManifestPath, whole: ManifestPath): boolean =>
+  prefix.length <= whole.length && prefix.every((segment, index) => whole[index] === segment);
+
+// Which file a region of the assembled document came from: everything under
+// `at` was written in the file whose locator this is, starting at `origin`
+// there. The root file mounts at the root; each include mounts where it
+// landed; a fragment hoisted out of an included file mounts under `defs`; and
+// a list item that moved — spliced in from another file, or shifted by a
+// splice before it — mounts on its own, since its index is not the one it
+// was written at.
+type Mount = {
+  readonly at: ManifestPath;
+  readonly origin: ManifestPath;
+  readonly label: string | undefined;
+  readonly locate: ManifestLocator | undefined;
+};
+
+// The file being walked.
+type Source = {
+  readonly file: string;
+  // How the file is named in a message: its path relative to the root
+  // manifest's directory. The root file itself has none, and is named by
+  // whoever reports the error.
+  readonly label: string | undefined;
+  readonly locate: ManifestLocator | undefined;
+  // Every file on the way here, for the cycle check.
+  readonly stack: ReadonlyArray<string>;
+};
+
+type Opened = {
+  readonly document: SourceDocument;
+  readonly source: Source;
+};
+
+const makeLocator = (mounts: ReadonlyArray<Mount>): ManifestLocator | undefined => {
+  if (!mounts.some((mount) => mount.locate !== undefined)) return undefined;
+  return (manifestPath) => {
+    const innermost = mounts
+      .filter((mount) => isPrefix(mount.at, manifestPath))
+      .sort((a, b) => a.at.length - b.at.length)
+      .at(-1);
+    if (innermost === undefined) return null;
+    const found =
+      innermost.locate?.([...innermost.origin, ...manifestPath.slice(innermost.at.length)]) ?? null;
+    if (found === null) return null;
+    return innermost.label === undefined ? found : { ...found, file: innermost.label };
+  };
+};
+
+export const expandIncludes = (
+  rootPath: string,
+  root: SourceDocument,
+  reader: IncludeReader,
+): IncludedManifest => {
+  const rootDirectory = path.dirname(rootPath);
+  const labelOf = (file: string): string =>
+    path.relative(rootDirectory, file).split(path.sep).join("/");
+  const nameOf = (source: Source): string => source.label ?? path.basename(rootPath);
+
+  const files: Array<string> = [rootPath];
+  const mounts: Array<Mount> = [{ at: [], origin: [], label: undefined, locate: root.locate }];
+  // Fragments hoisted out of included files, and the file each name came from.
+  const hoisted: Record<string, unknown> = {};
+  const definedIn = new Map<string, string>();
+
+  const refuse = (source: Source, origin: ManifestPath, detail: string): never => {
+    const position = source.locate?.(origin) ?? null;
+    const at =
+      position === null
+        ? ""
+        : `${nameOf(source)}:${String(position.line)}:${String(position.column)}  `;
+    throw new ConfigInvalid({
+      configPath: rootPath,
+      detail: `the manifest does not include:\n  ${at}${renderManifestPath(origin)}: ${detail}`,
+    });
+  };
+
+  // Checks a reference and reads the file it names. Nothing is placed yet:
+  // where the value lands depends on whether it is a list spliced into a list.
+  const open = (reference: IncludeReference, origin: ManifestPath, source: Source): Opened => {
+    const { include: specifier, ...rest } = reference;
+    if (typeof specifier !== "string") {
+      return refuse(source, origin, "`include` names a file, as a string.");
+    }
+    const shown = `\`include: ${JSON.stringify(specifier)}\``;
+    const beside = Object.keys(rest);
+    if (beside.length > 0) {
+      return refuse(
+        source,
+        origin,
+        `${shown} stands alone: an included file is replaced whole, so there is nothing for ` +
+          `${beside.map((key) => `\`${key}\``).join(", ")} to override. ` +
+          `To override a fragment, put it under \`defs\` and \`use\` it.`,
+      );
+    }
+    const target = path.resolve(path.dirname(source.file), specifier);
+    if (!INCLUDABLE.includes(path.extname(target).toLowerCase())) {
+      return refuse(
+        source,
+        origin,
+        `${shown} names a file that is not YAML or JSON. Only a data file can be included: ` +
+          `a module would make the manifest readable by one runtime only.`,
+      );
+    }
+    if (source.stack.includes(target)) {
+      return refuse(
+        source,
+        origin,
+        `${shown} includes a file that is already being included: ` +
+          `${[...source.stack, target].map(labelOf).join(" → ")}.`,
+      );
+    }
+    if (!reader.exists(target)) {
+      return refuse(
+        source,
+        origin,
+        `${shown} names a file that does not exist (looked for ${labelOf(target)}, ` +
+          `relative to ${nameOf(source)}).`,
+      );
+    }
+    const document = reader.read(target);
+    files.push(target);
+    return {
+      document,
+      source: {
+        file: target,
+        label: labelOf(target),
+        locate: document.locate,
+        stack: [...source.stack, target],
+      },
+    };
+  };
+
+  // Places an opened file at `at`, whole, and walks it so no reference is
+  // left inside. A top-level `defs` in the file joins the root's, and its
+  // `$schema` is for the editor — unless the file was included somewhere
+  // under `defs`, where it is a fragment or the map itself, and a key by
+  // either name is the author's.
+  const place = (opened: Opened, at: ManifestPath): unknown => {
+    const { document, source } = opened;
+    mounts.push({ at, origin: [], label: source.label, locate: source.locate });
+    if (!isRecord(document.value) || at[0] === "defs") {
+      return walk(document.value, at, [], source);
+    }
+    const { $schema: _schema, defs, ...body } = document.value;
+    if (defs !== undefined) {
+      if (!isRecord(defs)) {
+        return refuse(source, ["defs"], "`defs` must be a map of named fragments.");
+      }
+      for (const [name, fragment] of Object.entries(defs)) {
+        const already = definedIn.get(name);
+        if (already !== undefined) {
+          return refuse(
+            source,
+            ["defs", name],
+            `\`defs.${name}\` is already defined in ${already}. Every file's \`defs\` share ` +
+              `one namespace; rename one of them.`,
+          );
+        }
+        definedIn.set(name, nameOf(source));
+        mounts.push({
+          at: ["defs", name],
+          origin: ["defs", name],
+          label: source.label,
+          locate: source.locate,
+        });
+        hoisted[name] = walk(fragment, ["defs", name], ["defs", name], source);
+      }
+    }
+    return walk(body, at, [], source);
+  };
+
+  // Appends each item of `list` to `items`. An include standing as an item
+  // whose file holds a list is spliced in, so a list can be split across
+  // files; an item that does not sit at the index it was written at is
+  // mounted on its own.
+  const append = (
+    items: Array<unknown>,
+    list: ReadonlyArray<unknown>,
+    at: ManifestPath,
+    origin: ManifestPath,
+    source: Source,
+    spliced: boolean,
+  ): void => {
+    for (const [index, item] of list.entries()) {
+      const itemOrigin = [...origin, index];
+      const slot = [...at, items.length];
+      if (isInclude(item)) {
+        const opened = open(item, itemOrigin, source);
+        if (Array.isArray(opened.document.value)) {
+          append(items, opened.document.value, at, [], opened.source, true);
+        } else {
+          items.push(place(opened, slot));
+        }
+        continue;
+      }
+      if (spliced || items.length !== index) {
+        mounts.push({ at: slot, origin: itemOrigin, label: source.label, locate: source.locate });
+      }
+      items.push(walk(item, slot, itemOrigin, source));
+    }
+  };
+
+  const walk = (
+    value: unknown,
+    at: ManifestPath,
+    origin: ManifestPath,
+    source: Source,
+  ): unknown => {
+    if (isInclude(value)) return place(open(value, origin, source), at);
+
+    if (Array.isArray(value)) {
+      const items: Array<unknown> = [];
+      append(items, value, at, origin, source, false);
+      return items;
+    }
+
+    if (isRecord(value)) {
+      const entries: Record<string, unknown> = {};
+      for (const [key, item] of Object.entries(value)) {
+        entries[key] = walk(item, [...at, key], [...origin, key], source);
+      }
+      return entries;
+    }
+
+    return value;
+  };
+
+  const rootSource: Source = {
+    file: rootPath,
+    label: undefined,
+    locate: root.locate,
+    stack: [rootPath],
+  };
+  const value = walk(root.value, [], [], rootSource);
+
+  // Nothing was included: the value and the locator are the reader's own.
+  if (files.length === 1) return { value, locate: root.locate, files };
+
+  const names = Object.keys(hoisted);
+  if (names.length === 0 || !isRecord(value)) {
+    return { value, locate: makeLocator(mounts), files };
+  }
+
+  const own = value.defs;
+  if (own !== undefined && !isRecord(own)) {
+    return refuse(rootSource, ["defs"], "`defs` must be a map of named fragments.");
+  }
+  for (const name of names) {
+    if (own !== undefined && name in own) {
+      return refuse(
+        rootSource,
+        ["defs", name],
+        `\`defs.${name}\` is also defined in ${definedIn.get(name) ?? "an included file"}. ` +
+          `Every file's \`defs\` share one namespace; rename one of them.`,
+      );
+    }
+  }
+  return {
+    value: { ...value, defs: { ...(own ?? {}), ...hoisted } },
+    locate: makeLocator(mounts),
+    files,
+  };
+};

package/src/infrastructure/walk.ts

@@ -1,4 +1,4 @@
-import { readdirSync, statSync } from "node:fs";
+import { existsSync, readdirSync, statSync } from "node:fs";
 import * as path from "node:path";
 
 import type { Language } from "../ports/language.js";
@@ -52,3 +52,72 @@ export const listSourceFiles = (
   }
   return found.sort();
 };
+
+// What `infer` needs to know about a language beyond the walker's two: what
+// marks a package of it, and where a package keeps its source.
+export type PackagedLanguage = Pick<Language, "packageMarkers" | "sourceRoots">;
+
+export type PackageRoot = {
+  // Repo-relative folder with forward slashes; `""` is the repository itself.
+  readonly root: string;
+  // The package's source folder, repo-relative, when one of the language's
+  // `sourceRoots` exists under it; otherwise `null`, and source sits at the root.
+  readonly source: string | null;
+};
+
+// The packages under (and above) `roots`: every folder holding one of the
+// languages' marker files, the walk roots' own ancestors included, so a walk
+// rooted at `src/` still finds the `package.json` beside it. Sorted by path.
+export const listPackageRoots = (
+  repoRoot: string,
+  roots: ReadonlyArray<string>,
+  languages: ReadonlyArray<PackagedLanguage>,
+): ReadonlyArray<PackageRoot> => {
+  const markers = languages.flatMap((language) => language.packageMarkers);
+  const sourceRoots = languages.flatMap((language) => language.sourceRoots);
+  if (markers.length === 0) return [];
+
+  const isDirectory = (absolute: string): boolean => {
+    try {
+      return statSync(absolute).isDirectory();
+    } catch {
+      return false;
+    }
+  };
+  const relative = (absolute: string): string =>
+    path.relative(repoRoot, absolute).replaceAll(path.sep, "/");
+
+  const found = new Map<string, PackageRoot>();
+  const consider = (absolute: string): void => {
+    const key = relative(absolute);
+    if (found.has(key)) return;
+    if (!markers.some((marker) => existsSync(path.join(absolute, marker)))) return;
+    const source = sourceRoots.find((name) => isDirectory(path.join(absolute, name)));
+    found.set(key, {
+      root: key,
+      source: source === undefined ? null : relative(path.join(absolute, source)),
+    });
+  };
+  const walk = (absolute: string): void => {
+    consider(absolute);
+    for (const entry of readdirSync(absolute)) {
+      if (SKIPPED.has(entry)) continue;
+      const child = path.join(absolute, entry);
+      if (isDirectory(child)) walk(child);
+    }
+  };
+
+  const top = path.resolve(repoRoot);
+  for (const root of roots) {
+    let absolute = path.resolve(repoRoot, root);
+    if (!isDirectory(absolute)) absolute = path.dirname(absolute);
+    walk(absolute);
+    // The ancestors, up to the repository: a package is as often above the
+    // walk root as inside it.
+    for (let above = absolute; above.startsWith(top); above = path.dirname(above)) {
+      consider(above);
+      if (above === top) break;
+    }
+  }
+  return [...found.values()].sort((a, b) => a.root.localeCompare(b.root));
+};