@timiagama/ecr 0.0.1 → 0.1.0

package/src/cli/corpus-loader.ts

@@ -0,0 +1,482 @@
+/**
+ * Corpus Loading
+ *
+ * File discovery is the host's responsibility, not the linter's. This module
+ * is the CLI's implementation of that responsibility: it walks a directory,
+ * excludes meta-documents, and produces the inputs the linter consumes.
+ */
+
+import { lstatSync, readFileSync, readdirSync, statSync } from 'node:fs';
+import type { Stats } from 'node:fs';
+import { isAbsolute, join, relative } from 'node:path';
+
+import { MetaDocumentFilter } from '../meta-documents.js';
+import type { CorpusDocumentInput } from '../ecr.js';
+
+/**
+ * A path the walk reached but could not read.
+ */
+export interface UnreadablePath {
+  /** Corpus-relative path, with forward slashes. */
+  readonly path: string;
+  /** Why it could not be read: the file system's error code, or its message. */
+  readonly reason: string;
+}
+
+/**
+ * The outcome of walking a corpus directory.
+ */
+export interface LoadedCorpus {
+  /** Documents to validate, in path order. */
+  readonly documents: readonly CorpusDocumentInput[];
+  /**
+   * Corpus-relative paths excluded as meta-documents or by ignore pattern. A
+   * directory excluded whole by a pattern is listed once, with a trailing
+   * slash, and is not walked.
+   */
+  readonly excludedPaths: readonly string[];
+  /**
+   * Links beneath the corpus root that were not followed: every link to a
+   * directory, every link named like a document, and every link whose target
+   * is missing.
+   */
+  readonly notFollowedPaths: readonly string[];
+  /**
+   * Paths that could not be read. A corpus with any of these was not fully
+   * read, so its validation result cannot stand for the whole corpus.
+   */
+  readonly unreadablePaths: readonly UnreadablePath[];
+  /**
+   * Whether the project's `.ecrignore` excludes the corpus root itself, in
+   * which case nothing beneath it was walked.
+   */
+  readonly rootExcluded: boolean;
+}
+
+/**
+ * A project's own ignore patterns, from its `.ecrignore`, and the directory
+ * they are relative to.
+ */
+export interface ProjectIgnore {
+  /** The project root: the directory the command runs from. */
+  readonly workingDirectory: string;
+  /** Patterns relative to the working directory. */
+  readonly patterns: readonly string[];
+}
+
+/**
+ * Directory-walking characters that mark a path as hidden on every platform
+ * this runs on. Hidden directories are skipped so that `.git`, `.obsidian` and
+ * similar never reach the linter.
+ */
+const HIDDEN_PREFIX: string = '.';
+
+/**
+ * The directory package managers install dependencies into. It is skipped
+ * wherever the walk meets it: an installed copy of this package carries the
+ * specification, whose DocIDs would collide with a project's own, and any
+ * other package's Markdown is not the project's documentation. A corpus root
+ * named explicitly is still walked, even inside one.
+ */
+const DEPENDENCY_DIRECTORY: string = 'node_modules';
+
+/**
+ * Extension identifying a Markdown document.
+ */
+const MARKDOWN_EXTENSION: string = '.md';
+
+/**
+ * Discovers and reads the Markdown documents of a corpus.
+ *
+ * Paths are reported relative to the corpus root and normalised to forward
+ * slashes, so that ignore patterns and diagnostics read the same on every
+ * platform.
+ *
+ * Symbolic links and junctions beneath the root are not followed, because the
+ * published recursive searches (`rg`, `grep -r`) do not follow them: a
+ * document reached only through a link would be validated but could not be
+ * found (1#9.11). A root that is itself a link is followed, as both search
+ * engines follow a link named on their command line.
+ */
+export class CorpusLoader {
+  /** Absolute or relative path to the corpus root directory. */
+  private readonly corpusRoot: string;
+
+  /** Decides which discovered documents are excluded from validation. */
+  private readonly metaDocumentFilter: MetaDocumentFilter;
+
+  /**
+   * Applies the project's `.ecrignore` patterns, which are relative to the
+   * working directory rather than the corpus root; absent when there are none.
+   */
+  private readonly projectFilter: MetaDocumentFilter | undefined;
+
+  /** The directory the project's patterns are relative to. */
+  private readonly workingDirectory: string | undefined;
+
+  /**
+   * Creates a loader for one corpus directory.
+   *
+   * @param corpusRoot - Path to the directory to walk
+   * @param ignorePatterns - Glob patterns relative to the corpus root, from `--ignore`
+   * @param projectIgnore - The project's `.ecrignore` patterns, relative to the working directory
+   */
+  public constructor(
+    corpusRoot: string,
+    ignorePatterns: readonly string[] = [],
+    projectIgnore?: ProjectIgnore,
+  ) {
+    this.corpusRoot = corpusRoot;
+    this.metaDocumentFilter = new MetaDocumentFilter(ignorePatterns);
+    // No meta-document names: those are already applied by the corpus filter.
+    this.projectFilter =
+      projectIgnore === undefined ? undefined : new MetaDocumentFilter(projectIgnore.patterns, []);
+    this.workingDirectory = projectIgnore?.workingDirectory;
+  }
+
+  /**
+   * Walks the corpus root and reads every Markdown document that is not
+   * excluded.
+   *
+   * A path that cannot be read is recorded and the walk continues, so that
+   * every such path is reported at once.
+   *
+   * @returns The documents to validate, and the paths excluded, not followed, or not readable
+   */
+  public load(): LoadedCorpus {
+    const walk: CorpusWalk = new CorpusWalk();
+    const rootExcluded: boolean = this.tellProjectExcludesRoot();
+
+    if (!rootExcluded) {
+      this.walkDirectory(this.corpusRoot, walk);
+    }
+
+    // Sorted by corpus-relative path, so the order is the same on every
+    // platform whatever the path separator.
+    walk.documents.sort((left: CorpusDocumentInput, right: CorpusDocumentInput): number =>
+      CorpusLoader.comparePaths(left.uri, right.uri),
+    );
+    walk.excludedPaths.sort((left: string, right: string): number =>
+      CorpusLoader.comparePaths(left, right),
+    );
+    walk.notFollowedPaths.sort((left: string, right: string): number =>
+      CorpusLoader.comparePaths(left, right),
+    );
+    walk.unreadablePaths.sort((left: UnreadablePath, right: UnreadablePath): number =>
+      CorpusLoader.comparePaths(left.path, right.path),
+    );
+
+    return {
+      documents: walk.documents,
+      excludedPaths: walk.excludedPaths,
+      notFollowedPaths: walk.notFollowedPaths,
+      unreadablePaths: walk.unreadablePaths,
+      rootExcluded,
+    };
+  }
+
+  /**
+   * Orders two corpus-relative paths by their UTF-16 code units, as
+   * `Array.prototype.sort` does by default.
+   *
+   * @param left - A path
+   * @param right - Another path
+   * @returns Negative, zero or positive, as for a sort comparator
+   */
+  private static comparePaths(left: string, right: string): number {
+    if (left === right) {
+      return 0;
+    }
+
+    return left < right ? -1 : 1;
+  }
+
+  /**
+   * Walks one directory, in name order.
+   *
+   * @param directory - Directory to walk
+   * @param walk - What the walk has found so far
+   */
+  private walkDirectory(directory: string, walk: CorpusWalk): void {
+    let entryNames: string[];
+
+    try {
+      entryNames = readdirSync(directory).sort((left: string, right: string): number =>
+        CorpusLoader.comparePaths(left, right),
+      );
+    } catch (error: unknown) {
+      walk.recordUnreadable(this.toCorpusRelativePath(directory), error);
+      return;
+    }
+
+    for (const entryName of entryNames) {
+      if (entryName.startsWith(HIDDEN_PREFIX)) {
+        continue;
+      }
+
+      this.walkEntry(join(directory, entryName), entryName, walk);
+    }
+  }
+
+  /**
+   * Walks one directory entry: a link is recorded and not followed, a
+   * directory is descended into unless an ignore pattern excludes it whole,
+   * and a Markdown file is read unless it is excluded.
+   *
+   * @param entryPath - Path of the entry
+   * @param entryName - The entry's own name
+   * @param walk - What the walk has found so far
+   */
+  private walkEntry(entryPath: string, entryName: string, walk: CorpusWalk): void {
+    const relativePath: string = this.toCorpusRelativePath(entryPath);
+    let entryStats: Stats;
+
+    try {
+      entryStats = lstatSync(entryPath);
+    } catch (error: unknown) {
+      if (this.tellExcludedWhateverItIs(relativePath, entryPath)) {
+        walk.excludedPaths.push(relativePath);
+        return;
+      }
+
+      walk.recordUnreadable(relativePath, error);
+      return;
+    }
+
+    if (entryStats.isSymbolicLink()) {
+      this.recordLink(entryPath, entryName, relativePath, walk);
+      return;
+    }
+
+    if (entryStats.isDirectory()) {
+      if (
+        entryName === DEPENDENCY_DIRECTORY ||
+        this.metaDocumentFilter.excludesDirectory(relativePath) ||
+        this.tellProjectExcludesDirectory(entryPath)
+      ) {
+        walk.excludedPaths.push(`${relativePath}/`);
+        return;
+      }
+
+      this.walkDirectory(entryPath, walk);
+      return;
+    }
+
+    if (!CorpusLoader.tellMarkdownName(entryName)) {
+      return;
+    }
+
+    if (
+      this.metaDocumentFilter.shouldExclude(relativePath) ||
+      this.tellProjectExcludesFile(entryPath)
+    ) {
+      walk.excludedPaths.push(relativePath);
+      return;
+    }
+
+    try {
+      walk.documents.push({
+        uri: relativePath,
+        markdownText: readFileSync(entryPath, 'utf8'),
+      });
+    } catch (error: unknown) {
+      walk.recordUnreadable(relativePath, error);
+    }
+  }
+
+  /**
+   * Records a link without following it. An ignore pattern covering it
+   * excludes it instead; a link to a file that is not Markdown is of no
+   * interest, as it could never be a document.
+   *
+   * @param entryPath - Path of the link
+   * @param entryName - The link's own name
+   * @param relativePath - The link's corpus-relative path
+   * @param walk - What the walk has found so far
+   */
+  private recordLink(
+    entryPath: string,
+    entryName: string,
+    relativePath: string,
+    walk: CorpusWalk,
+  ): void {
+    if (this.tellExcludedWhateverItIs(relativePath, entryPath)) {
+      walk.excludedPaths.push(relativePath);
+      return;
+    }
+
+    if (!CorpusLoader.tellMarkdownName(entryName) && CorpusLoader.tellLinksToFile(entryPath)) {
+      return;
+    }
+
+    walk.notFollowedPaths.push(relativePath);
+  }
+
+  /**
+   * Determines whether an ignore pattern of either kind covers a path whose
+   * kind cannot or need not be known, such as a link or a vanished entry.
+   *
+   * @param relativePath - Corpus-relative path
+   * @param entryPath - Path of the entry, for the project's patterns
+   * @returns `true` when a file pattern or a whole-directory pattern covers it
+   */
+  private tellExcludedWhateverItIs(relativePath: string, entryPath: string): boolean {
+    return (
+      this.metaDocumentFilter.excludesDirectory(relativePath) ||
+      this.metaDocumentFilter.matchesIgnorePattern(relativePath) ||
+      this.tellProjectExcludesDirectory(entryPath) ||
+      this.tellProjectExcludesFile(entryPath)
+    );
+  }
+
+  /**
+   * Determines whether a `.ecrignore` pattern excludes the corpus root, either
+   * itself or through a directory above it: with `ecr/**` ignored, a root of
+   * `ecr/sub` is excluded as surely as `ecr`.
+   *
+   * @returns `true` when the root, or any directory between it and the project root, is excluded whole
+   */
+  private tellProjectExcludesRoot(): boolean {
+    const projectPath: string | undefined = this.toProjectRelativePath(this.corpusRoot);
+
+    if (projectPath === undefined || projectPath === '') {
+      return false;
+    }
+
+    const segments: readonly string[] = projectPath.split('/');
+
+    return segments.some((_segment: string, index: number): boolean =>
+      this.projectFilter?.excludesDirectory(segments.slice(0, index + 1).join('/')) === true,
+    );
+  }
+
+  /**
+   * Determines whether a `.ecrignore` pattern excludes a whole directory.
+   *
+   * @param directoryPath - Path of the directory
+   * @returns `true` when the project's patterns cover everything beneath it
+   */
+  private tellProjectExcludesDirectory(directoryPath: string): boolean {
+    const projectPath: string | undefined = this.toProjectRelativePath(directoryPath);
+
+    return projectPath !== undefined && projectPath !== '' &&
+      this.projectFilter?.excludesDirectory(projectPath) === true;
+  }
+
+  /**
+   * Determines whether a `.ecrignore` pattern excludes a file.
+   *
+   * @param filePath - Path of the file
+   * @returns `true` when one of the project's patterns matches it
+   */
+  private tellProjectExcludesFile(filePath: string): boolean {
+    const projectPath: string | undefined = this.toProjectRelativePath(filePath);
+
+    return projectPath !== undefined && this.projectFilter?.matchesIgnorePattern(projectPath) === true;
+  }
+
+  /**
+   * Converts a path into one relative to the working directory, with forward
+   * slashes, which is what `.ecrignore` patterns are matched against.
+   *
+   * @param path - A path
+   * @returns The path relative to the working directory, or `undefined` when there are no project patterns or the path lies outside the project
+   */
+  private toProjectRelativePath(path: string): string | undefined {
+    if (this.workingDirectory === undefined) {
+      return undefined;
+    }
+
+    const projectPath: string = relative(this.workingDirectory, path);
+
+    // The project's patterns say nothing about paths outside the project.
+    if (isAbsolute(projectPath) || /^\.\.(?:[\\/]|$)/.test(projectPath)) {
+      return undefined;
+    }
+
+    return projectPath.replaceAll('\\', '/');
+  }
+
+  /**
+   * Determines whether a name is a Markdown document's.
+   *
+   * @param entryName - A file or link name
+   * @returns `true` for a `.md` name, in any case
+   */
+  private static tellMarkdownName(entryName: string): boolean {
+    return entryName.toLowerCase().endsWith(MARKDOWN_EXTENSION);
+  }
+
+  /**
+   * Determines whether a link resolves to something other than a directory.
+   *
+   * @param linkPath - Path of the link
+   * @returns `true` when the target exists and is not a directory
+   */
+  private static tellLinksToFile(linkPath: string): boolean {
+    try {
+      return !statSync(linkPath).isDirectory();
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Converts an absolute path into a corpus-relative path with forward slashes.
+   *
+   * @param absolutePath - A path beneath the corpus root
+   * @returns The path relative to the corpus root
+   */
+  private toCorpusRelativePath(absolutePath: string): string {
+    const withoutRoot: string = absolutePath.startsWith(this.corpusRoot)
+      ? absolutePath.slice(this.corpusRoot.length)
+      : absolutePath;
+
+    return withoutRoot
+      .split('\\')
+      .join('/')
+      .replace(/^\/+/, '');
+  }
+}
+
+/**
+ * What one walk of a corpus has found so far.
+ */
+class CorpusWalk {
+  /** Documents read, in the order reached. */
+  public readonly documents: CorpusDocumentInput[] = [];
+
+  /** Paths excluded as meta-documents or by ignore pattern. */
+  public readonly excludedPaths: string[] = [];
+
+  /** Links recorded and not followed. */
+  public readonly notFollowedPaths: string[] = [];
+
+  /** Paths reached but not readable. */
+  public readonly unreadablePaths: UnreadablePath[] = [];
+
+  /**
+   * Records a path that could not be read.
+   *
+   * @param path - The corpus-relative path; empty for the corpus root itself
+   * @param error - What reading it threw
+   */
+  public recordUnreadable(path: string, error: unknown): void {
+    this.unreadablePaths.push({ path: path === '' ? '.' : path, reason: CorpusWalk.showReason(error) });
+  }
+
+  /**
+   * Describes a file-system error briefly.
+   *
+   * @param error - What was thrown
+   * @returns The error's code, such as `ENOENT`, or its message
+   */
+  private static showReason(error: unknown): string {
+    if (error instanceof Error) {
+      const code: unknown = (error as NodeJS.ErrnoException).code;
+      return typeof code === 'string' ? code : error.message;
+    }
+
+    return String(error);
+  }
+}

package/src/cli/corpus-statistics.ts

@@ -0,0 +1,196 @@
+/**
+ * Corpus Statistics
+ *
+ * Counts the structure ECR makes explicit, so that a reader can measure their
+ * own corpus rather than take anyone's figures on trust. Every number here is
+ * derived from validated artefacts, not from a regular expression over the
+ * source, so the same corpus always yields the same counts.
+ */
+
+import { UNPARSABLE_DOCUMENT_RULE_ID } from '../per-document-visitor.js';
+import type { CorpusResult, ReferenceDirection, ExtractedDocument } from '../types.js';
+
+/**
+ * Counts of References-section entries by their declared direction.
+ */
+export interface DirectionCounts {
+  /** Entries declaring the cited document as an authority. */
+  readonly authority: number;
+  /** Entries declaring the cited document as a constraint. */
+  readonly constraint: number;
+  /** Entries declaring the cited document as a contract. */
+  readonly contract: number;
+  /** Entries declaring the cited document as a dependency. */
+  readonly dependency: number;
+}
+
+/**
+ * A measured summary of an ECR corpus.
+ */
+export interface CorpusStatisticsReport {
+  /** Documents validated, excluding meta-documents. */
+  readonly documents: number;
+  /** Documents that carry at least one References entry. */
+  readonly documentsWithReferences: number;
+  /** Sections extracted across the corpus, excluding the root H1 of each document. */
+  readonly sections: number;
+  /** Unique DocIDs found in the corpus. */
+  readonly docIds: number;
+  /** Unique SectionIDs found in the corpus. */
+  readonly sectionIds: number;
+  /** References-section entries, the document-level edges. */
+  readonly referenceEntries: number;
+  /** References-section entries broken down by direction. */
+  readonly referencesByDirection: DirectionCounts;
+  /** Inline `see`/`per` references. */
+  readonly inlineReferences: number;
+  /** Inline references naming a precise section rather than a whole document. */
+  readonly sectionPreciseInlineReferences: number;
+  /** Every edge in the corpus: References entries plus inline references. */
+  readonly totalEdges: number;
+  /**
+   * Whether every input document was parsed successfully. This does not imply
+   * ECR conformance or complete extraction.
+   *
+   * A document that was parsed but has no recoverable DocID contributes
+   * nothing to any count, and one with structural errors contributes whatever
+   * did pass validation; both leave this `true`, because both were parsed.
+   * Only a document the parser could not read at all makes it `false`, and
+   * {@link unparsable} names those documents.
+   */
+  readonly complete: boolean;
+  /**
+   * Documents the parser could not read at all, which contribute nothing.
+   * A document with ordinary validation errors is not listed here: it still
+   * contributes whatever passed validation.
+   */
+  readonly unparsable: readonly string[];
+}
+
+/**
+ * Derives a statistical summary from a corpus validation result.
+ *
+ * Only documents that produced extracted artefacts contribute, which means
+ * every document with a valid H1 DocID. A document with other structural
+ * errors still counts, and contributes whatever was extracted from it: a
+ * heading or References entry that could not be read at all adds nothing,
+ * while one reported for something other than its structure -- a heading
+ * whose source form breaks the navigation guarantee, say -- is extracted and
+ * counted like any other. A document with no recoverable DocID contributes
+ * nothing at all.
+ */
+export class CorpusStatistics {
+  /** The validation result to summarise. */
+  private readonly corpusResult: CorpusResult;
+
+  /**
+   * Creates a summariser for one validation result.
+   *
+   * @param corpusResult - The result produced by validating the corpus
+   */
+  public constructor(corpusResult: CorpusResult) {
+    this.corpusResult = corpusResult;
+  }
+
+  /**
+   * Computes the summary.
+   *
+   * @returns Counts describing the corpus's explicit structure
+   */
+  public summarise(): CorpusStatisticsReport {
+    const extractedDocuments: readonly ExtractedDocument[] = this.collectExtractedDocuments();
+    const unparsable: readonly string[] = this.collectUnparsableDocuments();
+
+    let documentsWithReferences: number = 0;
+    let sections: number = 0;
+    let referenceEntries: number = 0;
+    let inlineReferences: number = 0;
+    let sectionPreciseInlineReferences: number = 0;
+
+    const referencesByDirection: Record<ReferenceDirection, number> = {
+      authority: 0,
+      constraint: 0,
+      contract: 0,
+      dependency: 0,
+    };
+
+    for (const document of extractedDocuments) {
+      if (document.references.length > 0) {
+        documentsWithReferences += 1;
+      }
+
+      // The root H1 is a node but not a section, so it is not counted here.
+      sections += Math.max(document.sections.length - 1, 0);
+      referenceEntries += document.references.length;
+      inlineReferences += document.inlineReferences.length;
+
+      for (const reference of document.references) {
+        referencesByDirection[reference.direction] += 1;
+      }
+
+      for (const inlineReference of document.inlineReferences) {
+        if (inlineReference.toId.includes('#')) {
+          sectionPreciseInlineReferences += 1;
+        }
+      }
+    }
+
+    return {
+      documents: extractedDocuments.length,
+      documentsWithReferences,
+      sections,
+      docIds: this.countIndexEntries(this.corpusResult.index?.docIds),
+      sectionIds: this.countIndexEntries(this.corpusResult.index?.sectionIds),
+      referenceEntries,
+      referencesByDirection: { ...referencesByDirection },
+      inlineReferences,
+      sectionPreciseInlineReferences,
+      totalEdges: referenceEntries + inlineReferences,
+      complete: unparsable.length === 0,
+      unparsable,
+    };
+  }
+
+  /**
+   * Gathers the extracted artefacts of every document that produced them.
+   *
+   * @returns Extracted documents, in corpus order
+   */
+  private collectExtractedDocuments(): readonly ExtractedDocument[] {
+    const extracted: ExtractedDocument[] = [];
+
+    for (const entry of this.corpusResult.documents) {
+      if (entry.result.extracted !== undefined) {
+        extracted.push(entry.result.extracted);
+      }
+    }
+
+    return extracted;
+  }
+
+  /**
+   * Names the documents the parser could not read, so that a reader knows
+   * what these counts leave out rather than taking them for the whole corpus.
+   *
+   * @returns Their URIs, in corpus order
+   */
+  private collectUnparsableDocuments(): readonly string[] {
+    return this.corpusResult.documents
+      .filter((entry): boolean =>
+        entry.result.diagnostics.some(
+          (diagnostic): boolean => diagnostic.ruleId === UNPARSABLE_DOCUMENT_RULE_ID,
+        ),
+      )
+      .map((entry): string => entry.uri);
+  }
+
+  /**
+   * Counts the entries of a corpus index map.
+   *
+   * @param index - A DocID or SectionID index, absent when indexing did not succeed
+   * @returns The number of entries, or 0 when the index is absent
+   */
+  private countIndexEntries(index: Readonly<Record<string, string>> | undefined): number {
+    return index === undefined ? 0 : Object.keys(index).length;
+  }
+}