@ng-doc/builder 22.0.0-beta.2 → 22.1.0

package/generator/content/html-pipeline.d.ts

@@ -0,0 +1,179 @@
+import type { ContentAnchor, KeywordExport, SearchRecord, ShikiLanguage } from '../contracts.js';
+import type { HighlightBlock } from './highlight-cache.js';
+/**
+ * The back half of content compilation: the HTML pipeline of `@ng-doc/utils` (process, post-process,
+ * keyword replacement, indexable content and search records). It is a pure function of strings and
+ * JSON values: it reads no file, no program and no user value, and records no dependency, so the
+ * same code runs in the main thread and in a render thread (`./html-worker`, `./html-pool`), and
+ * both give the same result for the same input. Only the code-block highlight cache is state, and a
+ * cache hit gives the same HTML as a miss (`./highlight-cache`).
+ *
+ * Everything that reaches this module from a thread crosses as `encode`d JSON text
+ * (`worker/protocol`); the thread files import nothing but this module, `@ng-doc/utils`, the
+ * search parser, the digest and the protocol (the `source-boundaries` check).
+ */
+/** The `@ng-doc/utils` pipeline, as the generator calls it. */
+interface HtmlUtilities {
+    processHtml(html: string, config: {
+        headings?: string[];
+        route?: string;
+        lightTheme?: string;
+        darkTheme?: string;
+        langs?: readonly ShikiLanguage[];
+        grammars?: 'all' | 'used';
+        highlight?: PipelineHighlight;
+    }): Promise<{
+        content: string;
+        anchors: ContentAnchor[];
+        error?: unknown;
+    }>;
+    postProcessHtml(html: string): Promise<{
+        content: string;
+        usedKeywords: string[];
+        error?: unknown;
+    }>;
+    replaceKeywords(html: string, config: {
+        getKeyword(key: string): unknown;
+    }): Promise<string>;
+    removeNotIndexableContent(html: string): Promise<string>;
+}
+export declare const htmlUtilities: () => Promise<HtmlUtilities>;
+/** The highlight cache of one `processHtml` call (`HighlightCall` of `./highlight-cache`). */
+export interface PipelineHighlight {
+    key(block: HighlightBlock): string;
+    get(key: string): string | undefined;
+    set(key: string, value: string): void;
+    loads(themes: HighlightBlock['themes']): boolean;
+    readonly verify: boolean;
+    mismatch(key: string): void;
+    readonly mismatches: readonly string[];
+}
+/** One HTML document of a render task: one `processHtml` and one `postProcessHtml` call. */
+export interface RenderDocument {
+    readonly html: string;
+    readonly route?: string;
+    /** The configured anchor headings; absent for a demo asset, which takes none. */
+    readonly headings?: string[];
+}
+/** The back half of one IR: its documents, processed in order until the first one fails. */
+export interface RenderTask {
+    readonly documents: readonly RenderDocument[];
+    readonly themes: {
+        readonly light: string;
+        readonly dark: string;
+    };
+    /** The configured Shiki languages (`shiki.langs`); absent without any. */
+    readonly langs?: readonly ShikiLanguage[];
+    /**
+     * `all` when highlighting loads every bundled Shiki grammar (the switch is off); absent when it
+     * loads only the grammars code blocks reach. The HTML is the same.
+     */
+    readonly grammars?: 'all';
+}
+/**
+ * What processing one document gave. `mismatches` counts the cached code blocks that `verify`
+ * found to differ (reported before any failure, as the content compiler always did).
+ */
+export type RenderedDocument = ({
+    readonly html: string;
+    readonly anchors: ContentAnchor[];
+    readonly usedKeywords: string[];
+    readonly mismatches: number;
+} | {
+    readonly failed: 'process' | 'post-process';
+    readonly message: string;
+    readonly mismatches: number;
+} | {
+    readonly aborted: true;
+    readonly mismatches: number;
+}) & ParallelCheck;
+/**
+ * `NGDOC_PARALLEL_RENDER=verify`: where a task's thread result differed from the main thread's,
+ * which is used (on the first document of a render task).
+ */
+export interface ParallelCheck {
+    readonly differs?: string;
+}
+/** The back half of linking one IR. */
+export interface LinkTask {
+    readonly html: string;
+    readonly title: string;
+    readonly absoluteRoute: string;
+    readonly breadcrumbs: string[];
+    readonly pageType: 'guide' | 'api';
+}
+/** What linking one IR gave; `consulted` lists every key it looked up, in lookup order. */
+export type LinkedDocument = ({
+    readonly html: string;
+    readonly searchRecords: SearchRecord[];
+} | {
+    readonly failed: string;
+} | {
+    readonly aborted: true;
+}) & {
+    readonly consulted: string[];
+} & ParallelCheck;
+/**
+ * Processes a task's documents in order and stops after the first that does not succeed, as the
+ * content compiler always did: a demo asset that fails ends its IR. `highlight` gives each document
+ * its own highlight cache call; `aborted` is checked between the steps (only the main thread has
+ * the signal: a thread's result is checked once it arrives).
+ */
+export declare function renderDocuments(task: RenderTask, highlight: () => PipelineHighlight | undefined, aborted?: () => boolean): Promise<RenderedDocument[]>;
+/**
+ * Links one IR: replaces its keywords, removes what is not indexable and builds its search records.
+ * Every key the keyword plugin looks up goes to `consulted`, before any branch on its binding.
+ */
+export declare function linkDocument(task: LinkTask, getKeyword: (key: string) => KeywordExport | undefined, aborted?: () => boolean): Promise<LinkedDocument>;
+/** The bindings of a keyword set by key; a later duplicate key wins, as in `new Map(...)`. */
+export declare function bindingsOf(keywords: readonly KeywordExport[]): Map<string, KeywordExport>;
+/**
+ * What one document's highlighting met, for the main thread to apply in plan order
+ * (`HighlightSession.merge`): the keys it hit or highlighted, what it highlighted, and the keys
+ * whose cached value `verify` found to differ.
+ */
+export interface HighlightRecord {
+    used: string[];
+    fresh: Array<[string, string]>;
+    mismatched: string[];
+}
+/** Where a recording highlight cache reads and keeps entries. */
+export interface HighlightStore {
+    get(key: string): string | undefined;
+    set(key: string, value: string): void;
+    /** Whether the context's themes are known to load. */
+    proven(): boolean;
+    prove(): void;
+}
+/**
+ * A highlight cache that records, per document, what it hit, highlighted and found to differ,
+ * instead of deciding anything shared: a render thread (and the main thread when it renders out
+ * of plan order) uses it, and the main thread merges the records in plan order, so what the
+ * shared cache holds and which document reports a `verify` difference are those of a sequential
+ * render, whatever ran where and when. `next` starts the record of the next document.
+ */
+export declare function recordingHighlight(settings: {
+    context: string;
+    verify: boolean;
+}, themes: RenderTask['themes'], store: HighlightStore): {
+    next(): PipelineHighlight;
+    records: HighlightRecord[];
+};
+/** A port a render thread receives jobs on and replies through (`parentPort`, or a test channel). */
+export interface ThreadPort {
+    on(event: 'message', listener: (value: unknown) => void): unknown;
+    postMessage(value: string): void;
+}
+/**
+ * Serves render, link and warm-up jobs on `port`: the body of a render thread. Every message is
+ * `encode`d JSON text both ways. A job replies `{ type: 'done', id, result }`, or
+ * `{ type: 'failed', id, message }` when it could not run (the main thread then runs it itself).
+ *
+ * The thread keeps a copy of the main thread's highlighted blocks, which the main thread sends as
+ * `highlight` messages (all of them after a reset, then the new ones), and reports what each render
+ * hit (`used`) and highlighted (`fresh`), which the main thread merges in plan order. It keeps the
+ * generation's keyword set, sent once as a `keywords` message and named by its number in each
+ * link job.
+ */
+export declare function serveHtmlThread(port: ThreadPort): void;
+export {};

package/generator/content/html-pool.d.ts

@@ -0,0 +1,245 @@
+import { Worker } from 'node:worker_threads';
+import type { CompilationRequest, KeywordExport, ShikiLanguage } from '../contracts.js';
+import type { ContentBack } from './content-compiler.js';
+export { PARALLEL_RENDER_MISMATCH } from './content-compiler.js';
+import { type HighlightSession } from './highlight-cache.js';
+import { type LinkedDocument, type LinkTask, type RenderedDocument, type RenderTask } from './html-pipeline.js';
+/**
+ * Parallel rendering: the HTML pipeline of a large generation's content (`./html-pipeline`) runs on
+ * worker threads of the compiler runtime, while the main thread runs the fronts (files, templates,
+ * semantic queries) strictly in plan order and settles every result in plan order
+ * (`compiler/render.ts`, `compiler/link.ts`).
+ *
+ * The output cannot depend on where a task ran: a task is a pure function of its JSON input, and
+ * every message to and from a thread is `encode`d JSON text. A thread that cannot start, crashes,
+ * exits, replies with something malformed or fails a job gets the job run again in the main thread
+ * through the same pipeline, and the rest of the generation runs there; no diagnostic is reported,
+ * since a cold build would not have one (`renderPoolCounters` counts it).
+ *
+ * Threads are used only where they pay: never on the targeted path or the reference path
+ * (`incrementalReuse: false`), and in a generation only after its first
+ * {@link PARALLEL_RENDER_THRESHOLD} tasks, which run in the main thread while the threads start.
+ * A production build or a start without a previous snapshot starts them before the semantic phase
+ * (`prestartRenderThreads`), so their start-up hides behind it.
+ *
+ * The pool belongs to the runtime: a long-lived compiler runtime keeps it warm across generations
+ * and terminates it after {@link RENDER_POOL_IDLE_MS} without work; a one-shot compile terminates
+ * it when its service is disposed (`disposeHtmlPool`). Threads never keep the process alive, add
+ * no process listeners and change no environment, and they die with the runtime's process.
+ *
+ * `NGDOC_PARALLEL_RENDER=0` (`CompilationOptions.parallelRender: false`) runs everything in the main
+ * thread, as before. `verify` also runs every thread task in the main thread, uses that result and
+ * reports `CONTENT_PARALLEL_MISMATCH` when the two differ.
+ */
+/** The default number of render threads at most. */
+export declare const RENDER_THREAD_LIMIT = 4;
+/** A generation's first tasks run in the main thread: a small generation never waits for a thread. */
+export declare const PARALLEL_RENDER_THRESHOLD = 32;
+/**
+ * The entries a likely large generation has at least before its threads start ahead of time: every
+ * entry renders a header and at least one tab, so fewer stay below the threshold anyway.
+ */
+export declare const PRESTART_ENTRIES: number;
+/** A pool without work for this long terminates its threads, to give their memory back. */
+export declare const RENDER_POOL_IDLE_MS = 30000;
+/** Tasks in flight per live thread, at most (`compiler/render.ts` bounds its units by it too). */
+export declare const RENDER_WINDOW_PER_THREAD = 4;
+/** The default thread count: up to four, leaving room for the main thread and Prettier's. */
+export declare function defaultRenderThreads(): number;
+/** The switch: `off`, `on` or `verify`. */
+export declare function parallelRenderSwitch(options: {
+    parallelRender?: boolean | 'verify';
+}): 'off' | 'on' | 'verify';
+/** Why a thread could not give a task's result; the task then runs in the main thread. */
+export declare class ThreadFailure extends Error {
+}
+/**
+ * Counters of this runtime, for tests: where tasks ran (`rendered`: the render tasks among the
+ * thread tasks) and how often a thread failed.
+ */
+declare const counters: {
+    thread: number;
+    rendered: number;
+    main: number;
+    fallbacks: number;
+    mismatches: number;
+};
+/** Tests: a copy of the counters. */
+export declare function renderPoolCounters(): typeof counters;
+/** Tests: the counters back to zero. */
+export declare function resetRenderPoolCounters(): void;
+/**
+ * Tests only: run the threads from `entry` (a bundled `html-worker.ts`; the sources cannot run in
+ * a thread), with another threshold or idle time. `undefined` restores the default.
+ */
+export declare function configureRenderPool(value: {
+    entry?: URL;
+    threshold?: number;
+    idleMs?: number;
+}): void;
+type Reply = Record<string, unknown>;
+/** One render thread and the jobs it has not answered. */
+declare class HtmlThread {
+    private readonly changed;
+    readonly worker: Worker;
+    readonly pending: Map<number, {
+        resolve(reply: Reply): void;
+        reject(error: Error): void;
+    }>;
+    failure?: Error;
+    /** How far this thread replayed the highlight journal (`highlightJournal`). */
+    highlight?: {
+        epoch: number;
+        cursor: number;
+    };
+    /** The keyword set this thread holds. */
+    keywords?: readonly KeywordExport[];
+    private readonly exited;
+    constructor(entry: URL, changed: () => void);
+    run(id: number, messages: readonly string[]): Promise<Reply>;
+    post(message: string): void;
+    private receive;
+    /** Ends the thread: every job it has not answered fails, and it is not used again. */
+    fail(error: Error): void;
+    terminate(): Promise<void>;
+}
+/**
+ * A warm-up: the themes and languages to set the highlighter up for, and whether the cached
+ * plugin is used.
+ */
+export interface RenderWarmUp {
+    themes: {
+        light: string;
+        dark: string;
+    };
+    langs?: readonly ShikiLanguage[];
+    /** `all` when highlighting loads every bundled grammar. */
+    grammars?: 'all';
+    cache: boolean;
+}
+/** Up to `size` render threads, started on demand. */
+export declare class HtmlPool {
+    readonly size: number;
+    private readonly entry;
+    private readonly idleMs;
+    private readonly threads;
+    private nextId;
+    private idle;
+    private warm;
+    constructor(size: number, entry: URL, idleMs: number);
+    /** The threads that have not failed. */
+    get live(): number;
+    /** Starts the missing threads (failed ones are replaced), each warmed up when `warm` is given. */
+    start(warm?: RenderWarmUp | undefined): void;
+    /**
+     * Runs a job on the live thread with the fewest jobs in flight (the first on a tie). `messages`
+     * builds what to post to that thread for job `id`: any messages that bring the thread up to date,
+     * then the job. A thread that fails rejects the job with a {@link ThreadFailure}.
+     */
+    run(messages: (thread: HtmlThread, id: number) => string[]): Promise<{
+        thread: HtmlThread;
+        reply: Reply;
+    }>;
+    /** Terminates every thread and resolves when all of them have stopped. */
+    terminate(): Promise<void>;
+    /**
+     * Every change (a start, a job, a reply, a release) starts the idle time again: with no job in
+     * flight and no compile holding the pool (`holdHtmlPool`), the threads end once it has passed
+     * without another change (a timer that keeps nothing alive).
+     */
+    changed(): void;
+}
+/**
+ * Holds the pool for a compile: threads started before its semantic phase are still there for its
+ * render, however long that phase takes. The returned release (idempotent) starts the idle time.
+ */
+export declare function holdHtmlPool(): () => void;
+/** This runtime's pool of `size` threads, or undefined when there is no thread entry or no size. */
+export declare function htmlPool(size: number): HtmlPool | undefined;
+/** The live threads of this runtime's pool (tests: nothing may outlive a one-shot compile). */
+export declare function renderThreadsAlive(): number;
+/** Terminates this runtime's pool and resolves when every thread has stopped. */
+export declare function disposeHtmlPool(): Promise<void>;
+/**
+ * Whether disposing a compilation service leaves the pool running: a long-lived compiler runtime
+ * (`worker/entry.ts`) creates a service per generation and keeps its pool warm across them.
+ */
+export declare function keepHtmlPool(keep: boolean): void;
+/** Whether a disposed compilation service leaves the pool to its runtime (`keepHtmlPool`). */
+export declare function htmlPoolKept(): boolean;
+/** The options parallel rendering reads (`CompilationOptions`). */
+export interface ParallelRenderOptions {
+    incrementalReuse?: boolean;
+    parallelRender?: boolean | 'verify';
+    renderThreads?: number;
+}
+/**
+ * Starts the threads of a generation that is likely large, before its semantic phase: a production
+ * build or a start without a previous snapshot, of at least {@link PRESTART_ENTRIES} entries. Other
+ * generations start them only once they have more than {@link PARALLEL_RENDER_THRESHOLD} tasks, so a
+ * small site never starts one.
+ */
+export declare function prestartRenderThreads(options: ParallelRenderOptions, request: Pick<CompilationRequest, 'mode' | 'previous'>, entries: number, warm: RenderWarmUp): void;
+/**
+ * The back of one generation's content (`ContentBack`), or undefined when everything runs in the
+ * main thread: with the switch off, on the reference path and without threads.
+ */
+export declare function createRenderBack(options: ParallelRenderOptions, highlight: HighlightSession | undefined, warm: RenderWarmUp): RenderBack | undefined;
+/**
+ * One generation's back. Its tasks run in the main thread until the generation is admitted (its
+ * full path) and has passed the threshold, then on the pool. Results of thread render tasks feed
+ * the highlight cache in the order the tasks were submitted, which is plan order.
+ */
+export declare class RenderBack implements ContentBack {
+    readonly mode: 'on' | 'verify';
+    readonly threads: number;
+    private readonly highlight;
+    private readonly warm;
+    private admitted;
+    /** A thread failed in this generation: its remaining tasks run in the main thread. */
+    private off;
+    private tasks;
+    private submitted;
+    private merging;
+    private readonly held;
+    private readonly keywordSets;
+    private nextKeywordSet;
+    constructor(mode: 'on' | 'verify', threads: number, highlight: HighlightSession | undefined, warm: RenderWarmUp);
+    /** The generation's full path may use threads (the targeted path never does). */
+    admit(): void;
+    /** Whether tasks may go to threads: the units in flight are worth bounding. */
+    get parallel(): boolean;
+    /** How many units may wait for their back half before the next front starts. */
+    window(): number;
+    render(task: RenderTask, signal: AbortSignal): Promise<RenderedDocument[]>;
+    /** A render task's documents and highlight records, from a thread or from this thread. */
+    private renderAnywhere;
+    link(task: LinkTask, keywords: readonly KeywordExport[], signal: AbortSignal): Promise<LinkedDocument>;
+    /** The pool this task may use, or undefined when it runs in the main thread. */
+    private pool;
+    /**
+     * Runs a job on a thread: its validated result, `aborted` when the generation was aborted first
+     * (the thread's reply is then ignored), or undefined after a failure, when the task must run in
+     * the main thread (and so must the rest of the generation's).
+     */
+    private onThread;
+    /**
+     * Renders in this thread. Its highlighting is recorded rather than applied, like a thread's, so
+     * that tasks rendered here out of plan order (ahead of earlier thread tasks, or concurrently with
+     * each other) still feed the cache in plan order.
+     */
+    private renderHere;
+    private linkHere;
+    private mismatch;
+    /**
+     * The highlight entries a thread lacks before a render job: all of them after it started or the
+     * cache was cleared, then those written since its last job.
+     */
+    private highlightSync;
+    /**
+     * Resolves once every render task submitted before `order` has had its turn, after running
+     * `apply` (this task's highlight merge): turns are taken in submission order, which is plan order.
+     */
+    private inTurn;
+    private keywordSet;
+}