@plebeiantech/lat.md 0.12.3-fork.13 → 0.12.3-fork.15

package/dist/src/external-documents.js

@@ -0,0 +1,404 @@
+import { posix } from 'node:path';
+import { performance } from 'node:perf_hooks';
+import { flattenSections } from './lattice-model.js';
+import { documentFormat } from './document-formats.js';
+import { loadMarkdownAnalyzer } from './markdown-analyzer-loader.js';
+import { PARSER_CACHE_VERSION, hashParserContent, parsedCachePath, readParsedCache, writeParsedCache, } from './parser-cache.js';
+function unique(values) {
+    const seen = new Set();
+    const result = [];
+    for (const value of values) {
+        const trimmed = value?.trim();
+        if (!trimmed || seen.has(trimmed.toLowerCase()))
+            continue;
+        seen.add(trimmed.toLowerCase());
+        result.push(trimmed);
+    }
+    return result;
+}
+function closeSections(sections, finalLine) {
+    return sections.map((section, index) => {
+        const next = sections
+            .slice(index + 1)
+            .find((candidate) => candidate.depth <= section.depth);
+        return {
+            ...section,
+            endLine: next
+                ? Math.max(section.startLine, next.startLine - 1)
+                : finalLine,
+        };
+    });
+}
+async function markdownAnalysis(path, content, onParserImport, detail = path) {
+    const analyzeMarkdownFile = await loadMarkdownAnalyzer(onParserImport, detail);
+    const virtual = `/external/${path}`;
+    const analysis = analyzeMarkdownFile(virtual, content, '/', '/');
+    const parents = [];
+    const sections = flattenSections(analysis.sections).map((section, index) => {
+        const title = analysis.headingTitles[index] ?? section.heading;
+        while (parents.length > 0 &&
+            parents[parents.length - 1].depth >= section.depth) {
+            parents.pop();
+        }
+        const hierarchy = [...parents.map((parent) => parent.title), title];
+        const legacyHierarchy = section.id.split('#').slice(1);
+        parents.push({ depth: section.depth, title });
+        return {
+            title,
+            depth: section.depth,
+            anchor: section.githubSlug ?? '',
+            aliases: unique([
+                hierarchy.join('#'),
+                legacyHierarchy.join('#'),
+                title,
+                section.heading,
+                section.githubSlug,
+            ]),
+            hierarchy,
+            startLine: section.startLine,
+        };
+    });
+    return {
+        format: 'markdown',
+        title: analysis.headingTitles[0] ?? posix.basename(path, '.md'),
+        sections: closeSections(sections, content.split('\n').length),
+    };
+}
+async function restructuredTextAnalysis(path, content, onParserImport, detail = path) {
+    const importStarted = performance.now();
+    const { RstSection, RstToHtmlCompiler } = await import('rst-compiler');
+    onParserImport?.({
+        parser: 'reStructuredText parser',
+        imported: true,
+        durationMs: performance.now() - importStarted,
+        detail,
+    });
+    const compiler = new RstToHtmlCompiler();
+    const parsed = compiler.parse(content, {
+        disableErrors: true,
+        disableWarnings: true,
+    });
+    const nodes = parsed.root
+        .findAllChildren('Section')
+        .filter((node) => Boolean(node instanceof RstSection));
+    const aliasesByNode = new Map();
+    for (const [alias, node] of parsed.simpleNameResolver
+        .nodesLinkableFromOutside) {
+        const aliases = aliasesByNode.get(node) ?? [];
+        aliases.push(alias);
+        aliasesByNode.set(node, aliases);
+    }
+    const hierarchy = [];
+    const sections = nodes.map((node) => {
+        hierarchy.length = node.level - 1;
+        hierarchy.push(node.textContent);
+        const anchor = parsed.htmlAttrResolver.getNodeHtmlId(node) ?? '';
+        return {
+            title: node.textContent,
+            depth: node.level,
+            anchor,
+            aliases: unique([
+                hierarchy.join('#'),
+                node.textContent,
+                anchor,
+                ...(aliasesByNode.get(node) ?? []),
+            ]),
+            hierarchy: [...hierarchy],
+            startLine: node.source.startLineIdx + 1,
+        };
+    });
+    return {
+        format: 'restructuredtext',
+        title: sections[0]?.title ?? posix.basename(path, '.rst'),
+        sections: closeSections(sections, content.split('\n').length),
+    };
+}
+/** Accept legacy AsciiDoc source listings whose delimiter lengths do not match. */
+export function asciidocCompatibleContent(content) {
+    const lines = content.split('\n');
+    for (let index = 0; index + 1 < lines.length; index++) {
+        if (!/^\[(?:source|listing)(?:,|\])/i.test(lines[index].trim()))
+            continue;
+        if (!/^-{5,}\r?$/.test(lines[index + 1]))
+            continue;
+        for (let closing = index + 2; closing < lines.length; closing++) {
+            if (!/^-{4,}\r?$/.test(lines[closing]))
+                continue;
+            const openingCr = lines[index + 1].endsWith('\r') ? '\r' : '';
+            const closingCr = lines[closing].endsWith('\r') ? '\r' : '';
+            lines[index + 1] = `----${openingCr}`;
+            lines[closing] = `----${closingCr}`;
+            index = closing;
+            break;
+        }
+    }
+    return lines.join('\n');
+}
+async function asciidocAnalysis(path, content, onParserImport, detail = path) {
+    const importStarted = performance.now();
+    const { load } = await import('@asciidoctor/core');
+    onParserImport?.({
+        parser: 'AsciiDoc parser',
+        imported: true,
+        durationMs: performance.now() - importStarted,
+        detail,
+    });
+    const document = await load(asciidocCompatibleContent(content), {
+        safe: 'secure',
+        sourcemap: true,
+        attributes: { showtitle: true },
+    });
+    const sections = [];
+    const documentTitle = document.getTitle();
+    const documentId = document.getId() ?? '';
+    const rootHierarchy = documentTitle ? [documentTitle] : [];
+    if (documentTitle) {
+        sections.push({
+            title: documentTitle,
+            depth: 1,
+            anchor: documentId,
+            aliases: unique([documentTitle, documentId]),
+            hierarchy: rootHierarchy,
+            startLine: document.getLineNumber() ?? 1,
+        });
+    }
+    const visit = (nodes, parents) => {
+        for (const node of nodes) {
+            const title = node.getTitle() ?? '';
+            const hierarchy = [...parents, title];
+            const anchor = node.getId() ?? '';
+            const level = node.getLevel();
+            sections.push({
+                title,
+                depth: level == null ? hierarchy.length : level + (documentTitle ? 1 : 0),
+                anchor,
+                aliases: unique([hierarchy.join('#'), title, anchor]),
+                hierarchy,
+                startLine: node.getLineNumber() ?? 1,
+            });
+            visit(node.getSections(), hierarchy);
+        }
+    };
+    visit(document.getSections(), rootHierarchy);
+    const extension = path.toLowerCase().endsWith('.asciidoc')
+        ? '.asciidoc'
+        : '.adoc';
+    return {
+        format: 'asciidoc',
+        title: documentTitle ?? sections[0]?.title ?? posix.basename(path, extension),
+        sections: closeSections(sections, content.split('\n').length),
+    };
+}
+export async function analyzeExternalDocument(path, content, onParserImport, importDetail = path) {
+    switch (documentFormat(path)) {
+        case 'markdown':
+            return markdownAnalysis(path, content, onParserImport, importDetail);
+        case 'restructuredtext':
+            return restructuredTextAnalysis(path, content, onParserImport, importDetail);
+        case 'asciidoc':
+            return asciidocAnalysis(path, content, onParserImport, importDetail);
+        default:
+            throw new Error(`unsupported external document format for "${path}"`);
+    }
+}
+function parserImportEvent(format, imported, durationMs, detail) {
+    return {
+        parser: format === 'markdown'
+            ? 'Markdown analyzer'
+            : format === 'restructuredtext'
+                ? 'reStructuredText parser'
+                : 'AsciiDoc parser',
+        imported,
+        durationMs,
+        detail,
+    };
+}
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function isStringArray(value) {
+    return (Array.isArray(value) && value.every((item) => typeof item === 'string'));
+}
+function isExternalDocumentSection(value) {
+    if (!isRecord(value))
+        return false;
+    return (typeof value.title === 'string' &&
+        Number.isInteger(value.depth) &&
+        value.depth > 0 &&
+        typeof value.anchor === 'string' &&
+        isStringArray(value.aliases) &&
+        isStringArray(value.hierarchy) &&
+        Number.isInteger(value.startLine) &&
+        value.startLine > 0 &&
+        Number.isInteger(value.endLine) &&
+        value.endLine >= value.startLine);
+}
+function cachedExternalDocumentAnalysis(entry, contentHash, identity, timings) {
+    if (!entry ||
+        entry.version !== PARSER_CACHE_VERSION ||
+        entry.contentHash !== contentHash ||
+        !isRecord(entry.value) ||
+        entry.value.path !== identity ||
+        !isRecord(entry.value.document) ||
+        !['markdown', 'restructuredtext', 'asciidoc'].includes(String(entry.value.document.format)) ||
+        typeof entry.value.document.title !== 'string' ||
+        !Array.isArray(entry.value.document.sections) ||
+        !entry.value.document.sections.every(isExternalDocumentSection)) {
+        return null;
+    }
+    return {
+        path: identity,
+        document: entry.value.document,
+        timings: {
+            ...timings,
+            cacheStatus: 'hit',
+            cacheWriteMs: 0,
+            parseMs: 0,
+        },
+    };
+}
+/** Request-scoped owner for external document parser work. */
+export class ExternalDocumentParserRuntime {
+    analyses = new Map();
+    clear() {
+        this.analyses.clear();
+    }
+}
+const defaultExternalDocumentParserRuntime = new ExternalDocumentParserRuntime();
+/** Return the shared parsed-cache path for one external document identity. */
+export function externalDocumentAnalysisCachePath(latDir, identity) {
+    return parsedCachePath(latDir, identity);
+}
+/** Analyze an external document through the shared persistent parser cache. */
+export async function analyzeExternalDocumentCached(path, content, latDir, options = {}) {
+    const format = documentFormat(path);
+    const identity = (options.identity ?? path)
+        .replaceAll('\\', '/')
+        .normalize('NFC');
+    const hashStarted = performance.now();
+    const contentHash = hashParserContent(content);
+    const hashMs = performance.now() - hashStarted;
+    const cache = options.cache !== false;
+    const cachePath = externalDocumentAnalysisCachePath(latDir, identity);
+    const promiseKey = `${cache ? 'cache' : 'direct'}\0${cachePath}\0${contentHash}`;
+    const runtime = options.runtime ?? defaultExternalDocumentParserRuntime;
+    let analysis = runtime.analyses.get(promiseKey);
+    const created = !analysis;
+    if (!analysis) {
+        analysis = (async () => {
+            const cacheStarted = performance.now();
+            const entry = cache ? await readParsedCache(cachePath) : null;
+            const cacheReadMs = cache ? performance.now() - cacheStarted : 0;
+            const timings = {
+                hashMs,
+                cacheReadMs,
+                cacheStatus: cache ? 'miss' : 'disabled',
+            };
+            const cached = cache
+                ? cachedExternalDocumentAnalysis(entry, contentHash, identity, timings)
+                : null;
+            if (cached) {
+                if (format) {
+                    options.onParserImport?.(parserImportEvent(format, false, 0, identity));
+                }
+                return cached;
+            }
+            const parseStarted = performance.now();
+            const document = await analyzeExternalDocument(path, content, options.onParserImport, identity);
+            const result = {
+                path: identity,
+                document,
+                timings: {
+                    ...timings,
+                    cacheWriteMs: 0,
+                    parseMs: performance.now() - parseStarted,
+                },
+            };
+            if (!cache)
+                return result;
+            const writeStarted = performance.now();
+            try {
+                await writeParsedCache(cachePath, contentHash, result);
+            }
+            catch {
+                // Parsed caches are disposable; document resolution must work read-only.
+            }
+            result.timings.cacheWriteMs = performance.now() - writeStarted;
+            return result;
+        })();
+        runtime.analyses.set(promiseKey, analysis);
+    }
+    const result = await analysis;
+    if (created)
+        options.onFileAnalyzed?.(result);
+    return result;
+}
+export function findExternalDocumentSection(analysis, fragment) {
+    const wanted = fragment.toLowerCase();
+    return analysis.sections.find((section) => section.aliases.some((alias) => alias.toLowerCase() === wanted));
+}
+/** Project format-neutral document sections into the existing browser TOC model. */
+export function externalDocumentSections(path, analysis) {
+    const roots = [];
+    const parents = [];
+    for (const source of analysis.sections) {
+        const section = {
+            id: `${path}#${source.hierarchy.join('#')}`,
+            heading: source.title,
+            depth: source.depth,
+            file: path,
+            filePath: path,
+            children: [],
+            startLine: source.startLine,
+            endLine: source.endLine,
+            firstParagraph: '',
+            githubSlug: source.anchor,
+        };
+        while (parents.length > 0 &&
+            parents[parents.length - 1].depth >= section.depth) {
+            parents.pop();
+        }
+        const parent = parents[parents.length - 1];
+        if (parent)
+            parent.children.push(section);
+        else
+            roots.push(section);
+        parents.push(section);
+    }
+    return roots;
+}
+/** Add zero-size alias anchors to the normalized external document tree. */
+export function addExternalDocumentAliasAnchors(source, analysis) {
+    const tree = structuredClone(source);
+    const canonical = new Set(analysis.sections.map((section) => section.anchor).filter(Boolean));
+    for (const section of analysis.sections) {
+        if (!section.anchor)
+            continue;
+        const aliases = section.aliases.filter((alias) => alias !== section.anchor && !canonical.has(alias));
+        if (aliases.length === 0)
+            continue;
+        const insert = (children) => {
+            for (let index = 0; index < children.length; index++) {
+                const node = children[index];
+                if (node.type !== 'element')
+                    continue;
+                if (/^h[1-6]$/.test(node.tagName) &&
+                    node.properties.id === section.anchor) {
+                    const anchors = aliases.map((alias) => ({
+                        type: 'element',
+                        tagName: 'span',
+                        properties: { id: alias, ariaHidden: 'true' },
+                        children: [],
+                    }));
+                    children.splice(index, 0, ...anchors);
+                    return true;
+                }
+                if (insert(node.children))
+                    return true;
+            }
+            return false;
+        };
+        insert(tree.children);
+    }
+    return tree;
+}

package/dist/src/external-sources.d.ts

@@ -0,0 +1,136 @@
+import { type ExternalDocumentAnalysis, type ExternalDocumentFileAnalysis } from './external-documents.js';
+import type { ParserImportObserver } from './parser-import.js';
+/** Version of the external cache metadata and on-disk provider layouts. */
+export declare const EXTERNAL_SOURCES_SCHEMA_VER = 1;
+export type ExternalStrategy = 'fetch' | 'checkout';
+export type EffectiveExternalStrategy = ExternalStrategy | 'local';
+export type CanonicalExternalSource = {
+    handle: string;
+    repo: string;
+    source: string;
+    commit: string;
+    prefix: string;
+    defaultFileExtension?: string;
+    strategy: ExternalStrategy;
+    fetchUrl?: string;
+};
+export type LocalExternalOverride = {
+    localPath?: string;
+    commit?: string;
+};
+export type EffectiveExternalSource = CanonicalExternalSource & {
+    canonicalCommit: string;
+    effectiveStrategy: EffectiveExternalStrategy;
+    localPath?: string;
+    localError?: string;
+};
+export type ExternalSourcesSnapshot = {
+    canonicalPath: string;
+    localPath: string;
+    sources: Map<string, EffectiveExternalSource>;
+    errors: ExternalConfigError[];
+    validCanonical: boolean;
+};
+export type ExternalConfigError = {
+    file: string;
+    message: string;
+};
+export type ExternalTarget = {
+    target: string;
+    handle: string;
+    authoredPath: string;
+    resolvedPath: string;
+    repositoryPath: string;
+    fragment: string;
+    identity: string;
+};
+type ResolvedExternalContentBase = {
+    target: ExternalTarget;
+    source: EffectiveExternalSource;
+    provider: EffectiveExternalStrategy;
+    content: string;
+    fullContent: string;
+    startLine: number;
+    endLine: number;
+};
+export type ResolvedExternalContent = ResolvedExternalContentBase & ({
+    kind: 'document';
+    document: ExternalDocumentAnalysis;
+} | {
+    kind: 'source';
+    signature?: string;
+});
+export type ExternalCacheMetadata = {
+    ver: number;
+    source: string;
+    commit: string;
+    strategy: EffectiveExternalStrategy;
+};
+export declare function normalizeExternalDefaultFileExtension(value: string): string;
+export declare function normalizeExternalRepoUrl(value: string): string;
+export declare function inferExternalFetchUrl(repo: string): string | null;
+export declare function loadExternalSources(latDir: string, projectRoot?: string, options?: {
+    ignoreLocal?: boolean;
+}): Promise<ExternalSourcesSnapshot>;
+export declare function parseExternalTarget(value: string, snapshot: Pick<ExternalSourcesSnapshot, 'sources'>): ExternalTarget | null;
+export declare function externalCachePaths(latDir: string, handle: string): {
+    root: string;
+    directory: string;
+    metadata: string;
+};
+export declare function readExternalCacheMetadata(latDir: string, handle: string): ExternalCacheMetadata | null;
+export declare class ExternalResolver {
+    readonly latDir: string;
+    readonly projectRoot: string;
+    readonly snapshot: ExternalSourcesSnapshot;
+    private readonly ca?;
+    private readonly ignoreLocal;
+    private readonly onDocumentAnalyzed?;
+    private readonly onParserImport?;
+    private reconciliationPromise?;
+    private readonly externalFilePromises;
+    private readonly externalDocumentParserRuntime;
+    private readonly sourceParserRuntime;
+    constructor(latDir: string, projectRoot: string, snapshot: ExternalSourcesSnapshot, ca?: (string | Buffer) | undefined, ignoreLocal?: boolean, onDocumentAnalyzed?: ((analysis: ExternalDocumentFileAnalysis) => void) | undefined, onParserImport?: ParserImportObserver | undefined);
+    parse(target: string): ExternalTarget | null;
+    unknownTargetMessage(target: string): string | null;
+    reconcile(): Promise<void>;
+    private readFile;
+    resolve(targetValue: string): Promise<ResolvedExternalContent>;
+}
+export declare function createExternalResolver(latDir: string, projectRoot?: string, options?: {
+    ignoreLocal?: boolean;
+    ca?: string | Buffer;
+    onDocumentAnalyzed?: (analysis: ExternalDocumentFileAnalysis) => void;
+    onParserImport?: ParserImportObserver;
+}): Promise<ExternalResolver>;
+export type ExternalSourceDescription = {
+    handle: string;
+    repo: string;
+    canonicalCommit: string;
+    effectiveCommit: string;
+    prefix: string;
+    defaultFileExtension?: string;
+    strategy: ExternalStrategy;
+    effectiveStrategy: EffectiveExternalStrategy;
+    fetchUrl?: string;
+    localPath?: string;
+    localError?: string;
+    cache: ExternalCacheMetadata | null;
+    checkout: {
+        command: string;
+        args: string[];
+    }[];
+};
+export declare function describeExternalSources(latDir: string, snapshot: ExternalSourcesSnapshot): ExternalSourceDescription[];
+export declare function resolveExternalCommit(repo: string, ref: string): Promise<string>;
+export declare function addCanonicalExternalSource(latDir: string, input: {
+    handle: string;
+    repo: string;
+    commit: string;
+    prefix?: string;
+    defaultFileExtension?: string;
+    strategy: ExternalStrategy;
+    fetchUrl?: string;
+}): Promise<CanonicalExternalSource>;
+export {};