@goodbones/core 0.1.0-beta.1 → 0.1.0-beta.10

package/src/infrastructure/manifest-include.ts

@@ -0,0 +1,318 @@
+import * as path from "node:path";
+
+import { ConfigInvalid } from "../domain/architecture-error.js";
+import {
+  type ManifestLocator,
+  type ManifestPath,
+  renderManifestPath,
+} from "../domain/manifest-location.js";
+
+// A manifest split across files. `{ include: "<path>" }` standing anywhere in
+// a manifest — a node under `tree`, a whole section, one entry of a list — is
+// replaced by the value of the file it names, read by the same parser and
+// resolved relative to the file that wrote the reference. This runs on the raw
+// value before anything else looks at it: the `defs`/`use` expansion sees one
+// document, the decoder sees one document, and the schema never learns that a
+// reference existed.
+//
+// What is deliberately not here: no merging (a value is replaced, full stop),
+// no parameters, no glob of files. An included file is YAML or JSON only — a
+// module would make a data manifest readable by one runtime — and a file may
+// carry a top-level `defs` of its own, which joins the manifest's under one
+// namespace, and a `$schema` for its editor, which is dropped.
+
+// One file, as the reader parsed it.
+export type SourceDocument = {
+  readonly value: unknown;
+  readonly locate: ManifestLocator | undefined;
+};
+
+export type IncludeReader = {
+  readonly exists: (file: string) => boolean;
+  // Throws `ConfigInvalid` naming the file when it does not parse.
+  readonly read: (file: string) => SourceDocument;
+};
+
+export type IncludedManifest = {
+  readonly value: unknown;
+  // Answers for every file: a position inside an included file carries that
+  // file's path, relative to the root manifest's directory.
+  readonly locate: ManifestLocator | undefined;
+  // Every file that took part, root first, as absolute paths.
+  readonly files: ReadonlyArray<string>;
+};
+
+const INCLUDABLE = [".yaml", ".yml", ".json"];
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+  typeof value === "object" && value !== null && !Array.isArray(value);
+
+type IncludeReference = Record<string, unknown> & { readonly include: unknown };
+
+const isInclude = (value: unknown): value is IncludeReference =>
+  isRecord(value) && "include" in value;
+
+const isPrefix = (prefix: ManifestPath, whole: ManifestPath): boolean =>
+  prefix.length <= whole.length && prefix.every((segment, index) => whole[index] === segment);
+
+// Which file a region of the assembled document came from: everything under
+// `at` was written in the file whose locator this is, starting at `origin`
+// there. The root file mounts at the root; each include mounts where it
+// landed; a fragment hoisted out of an included file mounts under `defs`; and
+// a list item that moved — spliced in from another file, or shifted by a
+// splice before it — mounts on its own, since its index is not the one it
+// was written at.
+type Mount = {
+  readonly at: ManifestPath;
+  readonly origin: ManifestPath;
+  readonly label: string | undefined;
+  readonly locate: ManifestLocator | undefined;
+};
+
+// The file being walked.
+type Source = {
+  readonly file: string;
+  // How the file is named in a message: its path relative to the root
+  // manifest's directory. The root file itself has none, and is named by
+  // whoever reports the error.
+  readonly label: string | undefined;
+  readonly locate: ManifestLocator | undefined;
+  // Every file on the way here, for the cycle check.
+  readonly stack: ReadonlyArray<string>;
+};
+
+type Opened = {
+  readonly document: SourceDocument;
+  readonly source: Source;
+};
+
+const makeLocator = (mounts: ReadonlyArray<Mount>): ManifestLocator | undefined => {
+  if (!mounts.some((mount) => mount.locate !== undefined)) return undefined;
+  return (manifestPath) => {
+    const innermost = mounts
+      .filter((mount) => isPrefix(mount.at, manifestPath))
+      .sort((a, b) => a.at.length - b.at.length)
+      .at(-1);
+    if (innermost === undefined) return null;
+    const found =
+      innermost.locate?.([...innermost.origin, ...manifestPath.slice(innermost.at.length)]) ?? null;
+    if (found === null) return null;
+    return innermost.label === undefined ? found : { ...found, file: innermost.label };
+  };
+};
+
+export const expandIncludes = (
+  rootPath: string,
+  root: SourceDocument,
+  reader: IncludeReader,
+): IncludedManifest => {
+  const rootDirectory = path.dirname(rootPath);
+  const labelOf = (file: string): string =>
+    path.relative(rootDirectory, file).split(path.sep).join("/");
+  const nameOf = (source: Source): string => source.label ?? path.basename(rootPath);
+
+  const files: Array<string> = [rootPath];
+  const mounts: Array<Mount> = [{ at: [], origin: [], label: undefined, locate: root.locate }];
+  // Fragments hoisted out of included files, and the file each name came from.
+  const hoisted: Record<string, unknown> = {};
+  const definedIn = new Map<string, string>();
+
+  const refuse = (source: Source, origin: ManifestPath, detail: string): never => {
+    const position = source.locate?.(origin) ?? null;
+    const at =
+      position === null
+        ? ""
+        : `${nameOf(source)}:${String(position.line)}:${String(position.column)}  `;
+    throw new ConfigInvalid({
+      configPath: rootPath,
+      detail: `the manifest does not include:\n  ${at}${renderManifestPath(origin)}: ${detail}`,
+    });
+  };
+
+  // Checks a reference and reads the file it names. Nothing is placed yet:
+  // where the value lands depends on whether it is a list spliced into a list.
+  const open = (reference: IncludeReference, origin: ManifestPath, source: Source): Opened => {
+    const { include: specifier, ...rest } = reference;
+    if (typeof specifier !== "string") {
+      return refuse(source, origin, "`include` names a file, as a string.");
+    }
+    const shown = `\`include: ${JSON.stringify(specifier)}\``;
+    const beside = Object.keys(rest);
+    if (beside.length > 0) {
+      return refuse(
+        source,
+        origin,
+        `${shown} stands alone: an included file is replaced whole, so there is nothing for ` +
+          `${beside.map((key) => `\`${key}\``).join(", ")} to override. ` +
+          `To override a fragment, put it under \`defs\` and \`use\` it.`,
+      );
+    }
+    const target = path.resolve(path.dirname(source.file), specifier);
+    if (!INCLUDABLE.includes(path.extname(target).toLowerCase())) {
+      return refuse(
+        source,
+        origin,
+        `${shown} names a file that is not YAML or JSON. Only a data file can be included: ` +
+          `a module would make the manifest readable by one runtime only.`,
+      );
+    }
+    if (source.stack.includes(target)) {
+      return refuse(
+        source,
+        origin,
+        `${shown} includes a file that is already being included: ` +
+          `${[...source.stack, target].map(labelOf).join(" → ")}.`,
+      );
+    }
+    if (!reader.exists(target)) {
+      return refuse(
+        source,
+        origin,
+        `${shown} names a file that does not exist (looked for ${labelOf(target)}, ` +
+          `relative to ${nameOf(source)}).`,
+      );
+    }
+    const document = reader.read(target);
+    files.push(target);
+    return {
+      document,
+      source: {
+        file: target,
+        label: labelOf(target),
+        locate: document.locate,
+        stack: [...source.stack, target],
+      },
+    };
+  };
+
+  // Places an opened file at `at`, whole, and walks it so no reference is
+  // left inside. A top-level `defs` in the file joins the root's, and its
+  // `$schema` is for the editor — unless the file was included somewhere
+  // under `defs`, where it is a fragment or the map itself, and a key by
+  // either name is the author's.
+  const place = (opened: Opened, at: ManifestPath): unknown => {
+    const { document, source } = opened;
+    mounts.push({ at, origin: [], label: source.label, locate: source.locate });
+    if (!isRecord(document.value) || at[0] === "defs") {
+      return walk(document.value, at, [], source);
+    }
+    const { $schema: _schema, defs, ...body } = document.value;
+    if (defs !== undefined) {
+      if (!isRecord(defs)) {
+        return refuse(source, ["defs"], "`defs` must be a map of named fragments.");
+      }
+      for (const [name, fragment] of Object.entries(defs)) {
+        const already = definedIn.get(name);
+        if (already !== undefined) {
+          return refuse(
+            source,
+            ["defs", name],
+            `\`defs.${name}\` is already defined in ${already}. Every file's \`defs\` share ` +
+              `one namespace; rename one of them.`,
+          );
+        }
+        definedIn.set(name, nameOf(source));
+        mounts.push({
+          at: ["defs", name],
+          origin: ["defs", name],
+          label: source.label,
+          locate: source.locate,
+        });
+        hoisted[name] = walk(fragment, ["defs", name], ["defs", name], source);
+      }
+    }
+    return walk(body, at, [], source);
+  };
+
+  // Appends each item of `list` to `items`. An include standing as an item
+  // whose file holds a list is spliced in, so a list can be split across
+  // files; an item that does not sit at the index it was written at is
+  // mounted on its own.
+  const append = (
+    items: Array<unknown>,
+    list: ReadonlyArray<unknown>,
+    at: ManifestPath,
+    origin: ManifestPath,
+    source: Source,
+    spliced: boolean,
+  ): void => {
+    for (const [index, item] of list.entries()) {
+      const itemOrigin = [...origin, index];
+      const slot = [...at, items.length];
+      if (isInclude(item)) {
+        const opened = open(item, itemOrigin, source);
+        if (Array.isArray(opened.document.value)) {
+          append(items, opened.document.value, at, [], opened.source, true);
+        } else {
+          items.push(place(opened, slot));
+        }
+        continue;
+      }
+      if (spliced || items.length !== index) {
+        mounts.push({ at: slot, origin: itemOrigin, label: source.label, locate: source.locate });
+      }
+      items.push(walk(item, slot, itemOrigin, source));
+    }
+  };
+
+  const walk = (
+    value: unknown,
+    at: ManifestPath,
+    origin: ManifestPath,
+    source: Source,
+  ): unknown => {
+    if (isInclude(value)) return place(open(value, origin, source), at);
+
+    if (Array.isArray(value)) {
+      const items: Array<unknown> = [];
+      append(items, value, at, origin, source, false);
+      return items;
+    }
+
+    if (isRecord(value)) {
+      const entries: Record<string, unknown> = {};
+      for (const [key, item] of Object.entries(value)) {
+        entries[key] = walk(item, [...at, key], [...origin, key], source);
+      }
+      return entries;
+    }
+
+    return value;
+  };
+
+  const rootSource: Source = {
+    file: rootPath,
+    label: undefined,
+    locate: root.locate,
+    stack: [rootPath],
+  };
+  const value = walk(root.value, [], [], rootSource);
+
+  // Nothing was included: the value and the locator are the reader's own.
+  if (files.length === 1) return { value, locate: root.locate, files };
+
+  const names = Object.keys(hoisted);
+  if (names.length === 0 || !isRecord(value)) {
+    return { value, locate: makeLocator(mounts), files };
+  }
+
+  const own = value.defs;
+  if (own !== undefined && !isRecord(own)) {
+    return refuse(rootSource, ["defs"], "`defs` must be a map of named fragments.");
+  }
+  for (const name of names) {
+    if (own !== undefined && name in own) {
+      return refuse(
+        rootSource,
+        ["defs", name],
+        `\`defs.${name}\` is also defined in ${definedIn.get(name) ?? "an included file"}. ` +
+          `Every file's \`defs\` share one namespace; rename one of them.`,
+      );
+    }
+  }
+  return {
+    value: { ...value, defs: { ...(own ?? {}), ...hoisted } },
+    locate: makeLocator(mounts),
+    files,
+  };
+};

package/src/infrastructure/report-source-fake.ts

@@ -0,0 +1,10 @@
+import type { Diagnostic } from "../domain/report.js";
+import type { ReportSource } from "../ports/report-source.js";
+
+// Keyed by file: a test states which diagnostics another tool reported on
+// which file, and never runs the tool. Every spec is answered alike.
+export const makeReportSourceFake = (
+  staged: Readonly<Record<string, ReadonlyArray<Omit<Diagnostic, "file">>>>,
+): ReportSource => ({
+  diagnosticsOf: (_spec, file) => (staged[file] ?? []).map((one) => ({ ...one, file })),
+});

package/src/infrastructure/report-source-live.ts

@@ -0,0 +1,192 @@
+import { spawn, spawnSync } from "node:child_process";
+import { readFileSync } from "node:fs";
+import { availableParallelism } from "node:os";
+import * as path from "node:path";
+
+import { ReportUnavailable } from "../domain/architecture-error.js";
+import { type Diagnostic, indexByFile, parseReport, uniqueDiagnostics } from "../domain/report.js";
+import { type ReportSource, reportSourcesOf, type ReportSpec } from "../ports/report-source.js";
+
+// The report source on this host: runs a `report` term's commands from the
+// repository root, or reads the files it names, and parses the output once
+// — a report is about the whole repository, a campaign asks per file, and
+// the plugin asks for one file at a time across a whole lint run. Cached
+// per spec for the life of the process: the CLI is one process per `check`,
+// and oxlint's language server is one process per editor session, which
+// sees the report as of when it loaded the plugin. A report the build
+// writes to a file is the predictable form for the editor, and for CI.
+//
+// A term may name several commands — a tool that checks one project at a
+// time, run once per project — and their outputs are one report: parsed
+// each, joined, and a diagnostic two of them print kept once. `read` runs
+// them concurrently, as many at a time as the machine has cores, which is
+// why both hosts call it before any file asks; `diagnosticsOf` with no
+// answer kept runs them one after another, synchronously, and keeps that.
+//
+// A non-zero exit is not a failure — `tsc` exits 2 when there are errors,
+// which is the case the term exists for. A command that cannot be spawned,
+// or a file that is not there, is — and that failure is cached as the report
+// would have been, so the second file to ask gets the same answer without a
+// second spawn. The command is forked from the asking process; under the
+// oxlint plugin that is the linter, which Linux refuses to fork once its AST
+// buffers have merged into one mapping larger than RAM and swap — so the
+// plugin asks at load, and retrying per file would only repeat the refusal.
+
+const MAX_BUFFER = 256 * 1024 * 1024;
+
+const fileTextOf = (repoRoot: string, file: string): string => {
+  const at = path.resolve(repoRoot, file);
+  try {
+    return readFileSync(at, "utf8");
+  } catch (cause) {
+    throw new ReportUnavailable({ kind: "file", source: file, detail: String(cause) });
+  }
+};
+
+const commandTextSync = (repoRoot: string, command: string): string => {
+  const run = spawnSync(command, {
+    cwd: repoRoot,
+    shell: true,
+    encoding: "utf8",
+    maxBuffer: MAX_BUFFER,
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+  if (run.error !== undefined) {
+    throw new ReportUnavailable({ kind: "command", source: command, detail: run.error.message });
+  }
+  return run.stdout;
+};
+
+const commandText = (repoRoot: string, command: string): Promise<string> =>
+  new Promise((resolve, reject) => {
+    const child = spawn(command, {
+      cwd: repoRoot,
+      shell: true,
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    const chunks: Array<Buffer> = [];
+    let size = 0;
+    child.stdout.on("data", (chunk: Buffer) => {
+      size += chunk.length;
+      if (size > MAX_BUFFER) {
+        child.kill();
+        reject(
+          new ReportUnavailable({
+            kind: "command",
+            source: command,
+            detail: `stdout exceeded ${String(MAX_BUFFER)} bytes`,
+          }),
+        );
+        return;
+      }
+      chunks.push(chunk);
+    });
+    child.stderr.resume();
+    child.on("error", (cause) => {
+      reject(new ReportUnavailable({ kind: "command", source: command, detail: cause.message }));
+    });
+    child.on("close", () => {
+      resolve(Buffer.concat(chunks).toString("utf8"));
+    });
+  });
+
+// `work` over `items`, at most `lanes` at a time, answers in order.
+const inLanes = async <A, B>(
+  items: ReadonlyArray<A>,
+  lanes: number,
+  work: (item: A) => Promise<B>,
+): Promise<ReadonlyArray<B>> => {
+  const answers = new Array<B>(items.length);
+  let next = 0;
+  const lane = async (): Promise<void> => {
+    while (next < items.length) {
+      const index = next;
+      next += 1;
+      answers[index] = await work(items[index] as A);
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(lanes, items.length) }, lane));
+  return answers;
+};
+
+// One command named as a string and the same one as a list of one are the
+// same report.
+const keyOf = (spec: ReportSpec): string =>
+  JSON.stringify([
+    spec.file === undefined ? "command" : "file",
+    reportSourcesOf(spec),
+    spec.format,
+    spec.pattern ?? null,
+  ]);
+
+type Indexed = ReadonlyMap<string, ReadonlyArray<Diagnostic>>;
+
+export const makeReportSourceLive = (repoRoot: string): ReportSource => {
+  const cache = new Map<string, Indexed | ReportUnavailable>();
+  const reading = new Map<string, Promise<void>>();
+
+  // The outputs, in the order named, as one report.
+  const indexOf = (spec: ReportSpec, texts: ReadonlyArray<string>): Indexed =>
+    indexByFile(
+      uniqueDiagnostics(
+        texts.flatMap((text) =>
+          parseReport(spec.format, text, { repoRoot, pattern: spec.pattern }),
+        ),
+      ),
+    );
+
+  const answerSync = (spec: ReportSpec): Indexed | ReportUnavailable => {
+    try {
+      const texts = reportSourcesOf(spec).map((source) =>
+        spec.file === undefined ? commandTextSync(repoRoot, source) : fileTextOf(repoRoot, source),
+      );
+      return indexOf(spec, texts);
+    } catch (cause) {
+      if (cause instanceof ReportUnavailable) return cause;
+      throw cause;
+    }
+  };
+
+  const answer = async (spec: ReportSpec): Promise<Indexed | ReportUnavailable> => {
+    try {
+      const sources = reportSourcesOf(spec);
+      const texts =
+        spec.file === undefined
+          ? await inLanes(sources, availableParallelism(), (command) =>
+              commandText(repoRoot, command),
+            )
+          : sources.map((file) => fileTextOf(repoRoot, file));
+      return indexOf(spec, texts);
+    } catch (cause) {
+      if (cause instanceof ReportUnavailable) return cause;
+      throw cause;
+    }
+  };
+
+  const read = (spec: ReportSpec): Promise<void> => {
+    const key = keyOf(spec);
+    if (cache.has(key)) return Promise.resolve();
+    const pending = reading.get(key);
+    if (pending !== undefined) return pending;
+    const started = answer(spec).then((found) => {
+      cache.set(key, found);
+      reading.delete(key);
+    });
+    reading.set(key, started);
+    return started;
+  };
+
+  return {
+    read,
+    diagnosticsOf: (spec, file) => {
+      const key = keyOf(spec);
+      let found = cache.get(key);
+      if (found === undefined) {
+        found = answerSync(spec);
+        cache.set(key, found);
+      }
+      if (found instanceof ReportUnavailable) throw found;
+      return found.get(file) ?? [];
+    },
+  };
+};

package/src/infrastructure/syntax-matcher-fake.ts

@@ -0,0 +1,51 @@
+import type { SyntaxMatch, SyntaxMatcher } from "../ports/syntax-matcher.js";
+
+// One match as a test states it: the text, its captures as a plain object,
+// the anchor, and optionally a line — the rest of the range is filled in.
+// `rule` narrows the match to one rule (compared by its JSON), so a test
+// about two `syntax` terms can answer each differently; omit it to answer
+// every rule the same.
+export type StagedMatch = {
+  readonly text: string;
+  readonly captures?: Readonly<Record<string, string>>;
+  readonly anchor?: string | null;
+  readonly line?: number;
+  readonly rule?: unknown;
+};
+
+const ZERO = { line: 0, column: 0 };
+
+// Keyed by the source text, as the extractor fake is: a core test states
+// what a snippet's syntax yields and never the parse, which has its own
+// tests in the matcher package. A text not staged parses to no matches; a
+// file whose extension is not among `extensions` does not parse at all,
+// which is the shape of a language with no matcher.
+export const makeSyntaxMatcherFake = (
+  staged: Readonly<Record<string, ReadonlyArray<StagedMatch>>>,
+  extensions: ReadonlyArray<string> | null = null,
+): SyntaxMatcher => ({
+  parse: (file, text) => {
+    if (extensions !== null && !extensions.some((extension) => file.endsWith(extension))) {
+      return null;
+    }
+    const matches = staged[text] ?? [];
+    return {
+      // A staged match's line is what a position is anchored by.
+      anchorAt: (position) => matches.find((one) => one.line === position.line)?.anchor ?? null,
+      findAll: (rule) => {
+        const asked = JSON.stringify(rule);
+        return matches
+          .filter((one) => one.rule === undefined || JSON.stringify(one.rule) === asked)
+          .map((one): SyntaxMatch => ({
+            text: one.text,
+            captures: new Map(Object.entries(one.captures ?? {})),
+            range: {
+              start: one.line === undefined ? ZERO : { line: one.line, column: 0 },
+              end: one.line === undefined ? ZERO : { line: one.line, column: one.text.length },
+            },
+            anchor: one.anchor ?? null,
+          }));
+      },
+    };
+  },
+});

package/src/infrastructure/walk.ts

@@ -1,4 +1,4 @@
-import { readdirSync, statSync } from "node:fs";
+import { existsSync, readdirSync, statSync } from "node:fs";
 import * as path from "node:path";
 
 import type { Language } from "../ports/language.js";
@@ -52,3 +52,72 @@ export const listSourceFiles = (
   }
   return found.sort();
 };
+
+// What `infer` needs to know about a language beyond the walker's two: what
+// marks a package of it, and where a package keeps its source.
+export type PackagedLanguage = Pick<Language, "packageMarkers" | "sourceRoots">;
+
+export type PackageRoot = {
+  // Repo-relative folder with forward slashes; `""` is the repository itself.
+  readonly root: string;
+  // The package's source folder, repo-relative, when one of the language's
+  // `sourceRoots` exists under it; otherwise `null`, and source sits at the root.
+  readonly source: string | null;
+};
+
+// The packages under (and above) `roots`: every folder holding one of the
+// languages' marker files, the walk roots' own ancestors included, so a walk
+// rooted at `src/` still finds the `package.json` beside it. Sorted by path.
+export const listPackageRoots = (
+  repoRoot: string,
+  roots: ReadonlyArray<string>,
+  languages: ReadonlyArray<PackagedLanguage>,
+): ReadonlyArray<PackageRoot> => {
+  const markers = languages.flatMap((language) => language.packageMarkers);
+  const sourceRoots = languages.flatMap((language) => language.sourceRoots);
+  if (markers.length === 0) return [];
+
+  const isDirectory = (absolute: string): boolean => {
+    try {
+      return statSync(absolute).isDirectory();
+    } catch {
+      return false;
+    }
+  };
+  const relative = (absolute: string): string =>
+    path.relative(repoRoot, absolute).replaceAll(path.sep, "/");
+
+  const found = new Map<string, PackageRoot>();
+  const consider = (absolute: string): void => {
+    const key = relative(absolute);
+    if (found.has(key)) return;
+    if (!markers.some((marker) => existsSync(path.join(absolute, marker)))) return;
+    const source = sourceRoots.find((name) => isDirectory(path.join(absolute, name)));
+    found.set(key, {
+      root: key,
+      source: source === undefined ? null : relative(path.join(absolute, source)),
+    });
+  };
+  const walk = (absolute: string): void => {
+    consider(absolute);
+    for (const entry of readdirSync(absolute)) {
+      if (SKIPPED.has(entry)) continue;
+      const child = path.join(absolute, entry);
+      if (isDirectory(child)) walk(child);
+    }
+  };
+
+  const top = path.resolve(repoRoot);
+  for (const root of roots) {
+    let absolute = path.resolve(repoRoot, root);
+    if (!isDirectory(absolute)) absolute = path.dirname(absolute);
+    walk(absolute);
+    // The ancestors, up to the repository: a package is as often above the
+    // walk root as inside it.
+    for (let above = absolute; above.startsWith(top); above = path.dirname(above)) {
+      consider(above);
+      if (above === top) break;
+    }
+  }
+  return [...found.values()].sort((a, b) => a.root.localeCompare(b.root));
+};