@wrtnlabs/evidence 0.0.0

package/lib/adapters/markdown/MarkdownScanner.js

@@ -0,0 +1,471 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MarkdownScanner = void 0;
+const EvidenceDocumentation_1 = require("../../parsers/EvidenceDocumentation");
+const EvidenceTagParser_1 = require("../../parsers/EvidenceTagParser");
+const MarkdownSyntax_1 = require("./MarkdownSyntax");
+const SourceText_1 = require("../../internal/SourceText");
+/**
+ * Materializes one Markdown source as a file unit and its heading units.
+ *
+ * The scanner also maps HTML-comment annotations while excluding rendered and
+ * example content that must remain ordinary Markdown text.
+ */
+class MarkdownScanner {
+    inventory;
+    source;
+    /**
+     * Maps UTF-16 offsets in the original source content to Evidence ranges.
+     *
+     * All sites, content ranges, and diagnostics use this source-preserving mapper.
+     */
+    text;
+    lineStarts = [0];
+    lineEnds = [];
+    fenced = [];
+    rendered = [];
+    commentOnly = [];
+    comments = [];
+    owners = [];
+    hostUnits = [];
+    hostSites = [];
+    hostProblems = [];
+    sites = new Map();
+    unitSites = new Map();
+    /**
+     * Binds the destination inventory and one physical Markdown source.
+     *
+     * Scan mutates only the supplied inventory with units, hosts, and diagnostics from this source.
+     */
+    constructor(inventory, source) {
+        this.inventory = inventory;
+        this.source = source;
+        this.text = new SourceText_1.SourceText(source.content);
+    }
+    /**
+     * Scans Markdown structure, materializes units, and parses attached annotations.
+     *
+     * The phases run in source order because unit and host ownership depend on line classification.
+     */
+    scan() {
+        this.splitLines();
+        this.markExamples();
+        this.findComments();
+        this.materializeUnits();
+        this.assignContent();
+        this.materializeComments();
+        this.reportRenderedAnnotations();
+    }
+    /**
+     * Indexes physical source lines and initializes their classification state.
+     *
+     * Carriage returns are excluded from line content while offsets remain tied to the original source.
+     */
+    splitLines() {
+        for (let index = 0; index < this.source.content.length; ++index)
+            if (this.source.content[index] === "\n")
+                this.lineStarts.push(index + 1);
+        for (let index = 0; index < this.lineStarts.length; ++index) {
+            const next = this.lineStarts[index + 1];
+            const rawEnd = next === undefined ? this.source.content.length : next - 1;
+            this.lineEnds.push(this.source.content[rawEnd - 1] === "\r" ? rawEnd - 1 : rawEnd);
+            this.fenced.push(false);
+            this.rendered.push(false);
+            this.commentOnly.push(false);
+        }
+    }
+    /**
+     * Marks fenced, indented, rendered, and comment-only lines before heading parsing.
+     *
+     * These regions cannot introduce Markdown units or active Evidence annotation hosts.
+     */
+    markExamples() {
+        let fenceMarker;
+        let fenceLength = 0;
+        let rendered = false;
+        let comment = false;
+        for (let index = 0; index < this.lineStarts.length; ++index) {
+            const line = this.line(index);
+            if (fenceMarker !== undefined) {
+                this.fenced[index] = true;
+                const delimiter = MarkdownSyntax_1.MarkdownSyntax.fence(line);
+                if (delimiter !== undefined &&
+                    delimiter.marker === fenceMarker &&
+                    delimiter.length >= fenceLength &&
+                    delimiter.remainder.trim() === "") {
+                    fenceMarker = undefined;
+                    fenceLength = 0;
+                }
+                continue;
+            }
+            if (rendered) {
+                this.rendered[index] = true;
+                const edge = MarkdownSyntax_1.MarkdownSyntax.renderedEdge(line);
+                if (edge === "close" || edge === "both")
+                    rendered = false;
+                continue;
+            }
+            if (comment) {
+                this.commentOnly[index] = true;
+                if (line.includes("-->"))
+                    comment = false;
+                continue;
+            }
+            const delimiter = MarkdownSyntax_1.MarkdownSyntax.fence(line);
+            if (delimiter !== undefined) {
+                this.fenced[index] = true;
+                fenceMarker = delimiter.marker;
+                fenceLength = delimiter.length;
+                continue;
+            }
+            if (MarkdownSyntax_1.MarkdownSyntax.indentedCode(line))
+                continue;
+            const opening = line.indexOf("<!--");
+            if (opening >= 0 && line.slice(0, opening).trim() === "") {
+                this.commentOnly[index] = true;
+                if (!line.slice(opening + 4).includes("-->"))
+                    comment = true;
+                continue;
+            }
+            let outside = "";
+            let cursor = 0;
+            while (cursor < line.length) {
+                const commentStart = line.indexOf("<!--", cursor);
+                if (commentStart < 0) {
+                    outside += line.slice(cursor);
+                    cursor = line.length;
+                    continue;
+                }
+                outside += line.slice(cursor, commentStart);
+                const commentEnd = line.indexOf("-->", commentStart + 4);
+                if (commentEnd < 0) {
+                    comment = true;
+                    cursor = line.length;
+                    continue;
+                }
+                cursor = commentEnd + 3;
+            }
+            const edge = MarkdownSyntax_1.MarkdownSyntax.renderedEdge(outside);
+            if (edge !== "none") {
+                this.rendered[index] = true;
+                rendered = edge === "open";
+            }
+        }
+    }
+    /**
+     * Finds HTML comments that occur outside examples and inline code.
+     *
+     * The resulting carriers retain source spans for later host and documentation construction.
+     */
+    findComments() {
+        let cursor = 0;
+        while (cursor < this.source.content.length) {
+            const start = this.source.content.indexOf("<!--", cursor);
+            if (start < 0)
+                break;
+            const closing = this.source.content.indexOf("-->", start + 4);
+            if (closing < 0)
+                break;
+            const end = closing + 3;
+            const startLine = this.lineAt(start);
+            const endLine = this.lineAt(Math.max(start, end - 1));
+            const example = this.fenced[startLine] === true ||
+                this.rendered[startLine] === true ||
+                MarkdownSyntax_1.MarkdownSyntax.indentedCode(this.line(startLine)) ||
+                MarkdownSyntax_1.MarkdownSyntax.inlineCode(this.line(startLine), start - (this.lineStarts[startLine] ?? 0));
+            if (!example) {
+                this.comments.push({ start, end, startLine, endLine });
+                const prefixStart = this.lineStarts[startLine] ?? 0;
+                if (this.source.content.slice(prefixStart, start).trim() === "")
+                    this.commentOnly[startLine] = true;
+                for (let line = startLine + 1; line <= endLine; ++line)
+                    this.commentOnly[line] = true;
+            }
+            cursor = end > start ? end : start + 4;
+        }
+    }
+    /**
+     * Materializes file and supported heading units for every targetable address.
+     *
+     * Heading ownership is maintained by structural depth so content and annotations attach consistently.
+     */
+    materializeUnits() {
+        const targetable = this.source.addresses.filter((address) => !MarkdownSyntax_1.MarkdownSyntax.hasWhitespace(address.relative));
+        for (const address of this.source.addresses)
+            if (MarkdownSyntax_1.MarkdownSyntax.hasWhitespace(address.relative))
+                this.inventory.diagnostics.push({
+                    code: "markdown-path",
+                    severity: "error",
+                    message: `Markdown path '${address.relative}' cannot form one evidence target token.`,
+                    repair: "Rename the file so its population-relative path contains no whitespace.",
+                    location: { file: this.source.physicalPath },
+                });
+        const first = targetable[0];
+        let fileId;
+        let fileSiteId;
+        if (first !== undefined) {
+            fileId = `markdown:${this.source.id}:file`;
+            fileSiteId = fileId + ":site";
+            const site = {
+                id: fileSiteId,
+                file: this.source.physicalPath,
+                range: this.text.range(0, this.source.content.length),
+                content: [],
+            };
+            const unit = {
+                id: fileId,
+                type: "markdown",
+                symbol: "file",
+                identity: [],
+                name: first.relative,
+                sites: [site],
+                withdrawals: [],
+            };
+            this.inventory.units.push(unit);
+            this.sites.set(fileSiteId, site);
+            this.unitSites.set(fileId, fileSiteId);
+            for (const address of targetable)
+                this.inventory.addresses.push({
+                    unitId: fileId,
+                    file: address.absolute,
+                    segments: [],
+                });
+        }
+        const structural = [
+            undefined,
+            undefined,
+            undefined,
+            undefined,
+            undefined,
+        ];
+        const openSites = [
+            undefined,
+            undefined,
+            undefined,
+            undefined,
+            undefined,
+        ];
+        let digestOwner = fileId;
+        let hostUnit = fileId;
+        let hostSite = fileSiteId;
+        let hostProblem = fileId === undefined
+            ? "Rename the Markdown file so its relative path can form an evidence target."
+            : undefined;
+        for (let index = 0; index < this.lineStarts.length; ++index) {
+            const heading = this.fenced[index] === true ||
+                this.rendered[index] === true ||
+                this.commentOnly[index] === true
+                ? undefined
+                : MarkdownSyntax_1.MarkdownSyntax.heading(this.line(index));
+            if (heading !== undefined) {
+                hostUnit = undefined;
+                hostSite = undefined;
+                hostProblem =
+                    heading.level > 4
+                        ? `Move the annotation under a Markdown file, H1, H2, H3, or H4 host; H${heading.level} is not an Evidence unit.`
+                        : "Give this heading a resolvable title or explicit {#anchor}, then attach the annotation to it.";
+                if (heading.level <= 4) {
+                    for (let level = heading.level; level <= 4; ++level) {
+                        const open = openSites[level];
+                        if (open !== undefined) {
+                            const site = this.sites.get(open);
+                            if (site !== undefined)
+                                site.range = this.text.range(site.range.start.offset, this.lineStarts[index] ?? this.source.content.length);
+                        }
+                        openSites[level] = undefined;
+                        structural[level] = undefined;
+                    }
+                }
+                const parent = this.parent(structural, heading.level, fileId);
+                digestOwner = parent;
+                if (heading.level <= 4 &&
+                    heading.anchor !== "" &&
+                    fileId !== undefined) {
+                    const start = this.lineStarts[index] ?? 0;
+                    const id = `markdown:${this.source.id}:h${heading.level}:${start}`;
+                    const siteId = id + ":site";
+                    const site = {
+                        id: siteId,
+                        file: this.source.physicalPath,
+                        range: this.text.range(start, this.source.content.length),
+                        content: [],
+                    };
+                    const unit = {
+                        id,
+                        type: "markdown",
+                        symbol: MarkdownSyntax_1.MarkdownSyntax.symbol(heading.level),
+                        identity: [heading.anchor],
+                        name: heading.title,
+                        sites: [site],
+                        withdrawals: [],
+                        ...(parent === undefined ? {} : { parentId: parent }),
+                    };
+                    this.inventory.units.push(unit);
+                    this.sites.set(siteId, site);
+                    this.unitSites.set(id, siteId);
+                    structural[heading.level] = id;
+                    openSites[heading.level] = siteId;
+                    digestOwner = id;
+                    hostUnit = id;
+                    hostSite = siteId;
+                    hostProblem = undefined;
+                    for (const address of targetable)
+                        this.inventory.addresses.push({
+                            unitId: id,
+                            file: address.absolute,
+                            segments: [heading.anchor],
+                        });
+                }
+                else if (heading.level <= 4 &&
+                    heading.anchor === "" &&
+                    fileId !== undefined)
+                    this.inventory.diagnostics.push({
+                        code: "markdown-heading",
+                        severity: "error",
+                        message: `Markdown H${heading.level} has no resolvable anchor.`,
+                        repair: "Add a nonempty heading title or a valid explicit {#anchor} suffix.",
+                        location: {
+                            file: this.source.physicalPath,
+                            range: this.text.range(this.lineStarts[index] ?? 0, this.lineEnds[index] ?? this.source.content.length),
+                        },
+                    });
+            }
+            this.owners[index] = digestOwner;
+            this.hostUnits[index] = hostUnit;
+            this.hostSites[index] = hostSite;
+            this.hostProblems[index] = hostProblem;
+        }
+        for (const siteId of openSites) {
+            if (siteId === undefined)
+                continue;
+            const site = this.sites.get(siteId);
+            if (site !== undefined)
+                site.range = this.text.range(site.range.start.offset, this.source.content.length);
+        }
+    }
+    /**
+     * Assigns non-comment source lines to their current semantic unit sites.
+     *
+     * Annotation-only lines do not contribute to fingerprints of their documentation host.
+     */
+    assignContent() {
+        for (let index = 0; index < this.lineStarts.length; ++index) {
+            const owner = this.owners[index];
+            if (owner === undefined || this.commentOnly[index] === true)
+                continue;
+            const siteId = this.unitSites.get(owner);
+            const site = siteId === undefined ? undefined : this.sites.get(siteId);
+            if (site === undefined)
+                continue;
+            site.content.push(this.text.range(this.lineStarts[index] ?? 0, this.lineEnds[index] ?? this.source.content.length));
+        }
+    }
+    /**
+     * Creates annotation hosts and parses each recognized HTML comment.
+     *
+     * Unsupported attachment positions remain hosts so diagnostics retain their original location.
+     */
+    materializeComments() {
+        const origins = this.source.addresses.map((address) => address.absolute);
+        for (const comment of this.comments) {
+            this.inventory.annotationRanges.push({
+                file: this.source.physicalPath,
+                range: this.text.range(comment.start, comment.end),
+            });
+            const unitId = this.hostUnits[comment.startLine];
+            const siteId = this.hostSites[comment.startLine];
+            const problem = this.hostProblems[comment.startLine];
+            const attached = unitId !== undefined && siteId !== undefined;
+            const host = {
+                id: `markdown:${this.source.id}:comment:${comment.start}`,
+                file: this.source.physicalPath,
+                range: this.text.range(comment.start, comment.end),
+                origins,
+                unitIds: attached ? [unitId] : [],
+                attachment: attached ? "attached" : "unsupported",
+                ...(attached ? { siteId } : {}),
+                ...(!attached && problem !== undefined ? { problem } : {}),
+            };
+            this.inventory.hosts.push(host);
+            const documentation = EvidenceDocumentation_1.EvidenceDocumentation.read(this.source.content, host.id, host.range, {
+                opening: "<!--",
+                closing: "-->",
+                tagBoundaries: false,
+                allowWithdrawal: false,
+            });
+            const parsed = EvidenceTagParser_1.EvidenceTagParser.parse(this.source.content, host, documentation);
+            this.inventory.declarations.push(...parsed.declarations);
+            this.inventory.reviews.push(...parsed.reviews);
+            this.inventory.diagnostics.push(...parsed.diagnostics);
+        }
+    }
+    /**
+     * Diagnoses annotation markers rendered as ordinary Markdown prose.
+     *
+     * A visible marker is not an Evidence annotation until wrapped in an HTML comment.
+     */
+    reportRenderedAnnotations() {
+        for (let index = 0; index < this.lineStarts.length; ++index) {
+            const line = this.line(index);
+            if (this.fenced[index] === true ||
+                this.rendered[index] === true ||
+                this.commentOnly[index] === true ||
+                MarkdownSyntax_1.MarkdownSyntax.indentedCode(line))
+                continue;
+            const marker = MarkdownSyntax_1.MarkdownSyntax.annotation(line);
+            if (marker === undefined)
+                continue;
+            const at = line.indexOf(marker);
+            const start = (this.lineStarts[index] ?? 0) + Math.max(0, at);
+            this.inventory.diagnostics.push({
+                code: "unsupported-annotation-host",
+                severity: "error",
+                message: `${marker} is rendered as Markdown prose, so it is not an Evidence annotation.`,
+                repair: `Wrap it in an HTML comment such as '<!-- ${marker} <target> ${marker.endsWith("Review") ? "<what you checked>" : "<reason>"} -->'.`,
+                location: {
+                    file: this.source.physicalPath,
+                    range: this.text.range(start, this.lineEnds[index] ?? start),
+                },
+            });
+        }
+    }
+    /**
+     * Finds the nearest structural heading ancestor for a newly materialized unit.
+     *
+     * The file unit is the fallback owner when no prior heading level remains open.
+     */
+    parent(structural, level, fileId) {
+        for (let ancestor = Math.min(level - 1, 4); ancestor >= 1; --ancestor) {
+            const id = structural[ancestor];
+            if (id !== undefined)
+                return id;
+        }
+        return fileId;
+    }
+    /**
+     * Returns one indexed line without its line-ending characters.
+     *
+     * Classification helpers use this normalized view while source ranges retain original offsets.
+     */
+    line(index) {
+        return this.source.content.slice(this.lineStarts[index] ?? 0, this.lineEnds[index] ?? this.source.content.length);
+    }
+    /**
+     * Locates the source line containing a UTF-16 offset.
+     *
+     * Binary search keeps comment lookup proportional to the logarithm of the file length.
+     */
+    lineAt(offset) {
+        let left = 0;
+        let right = this.lineStarts.length;
+        while (left + 1 < right) {
+            const middle = Math.floor((left + right) / 2);
+            if ((this.lineStarts[middle] ?? 0) <= offset)
+                left = middle;
+            else
+                right = middle;
+        }
+        return left;
+    }
+}
+exports.MarkdownScanner = MarkdownScanner;

package/lib/adapters/markdown/MarkdownSyntax.d.ts

@@ -0,0 +1,34 @@
+import type { EvidenceMarkdownSymbol } from "../../typings/EvidenceMarkdownSymbol";
+import type { EvidenceMarkdownRenderedEdge } from "./EvidenceMarkdownRenderedEdge";
+import type { IMarkdownFence } from "./IMarkdownFence";
+import type { IMarkdownHeading } from "./IMarkdownHeading";
+/**
+ * Markdown syntax shared by unit materialization and unreadable-tag diagnostics.
+ *
+ * The Markdown adapter uses these lexical helpers before it creates units or
+ * reports annotations that appear in non-readable source regions.
+ */
+export declare namespace MarkdownSyntax {
+    function fence(line: string): IMarkdownFence | undefined;
+    function heading(line: string): IMarkdownHeading | undefined;
+    function slug(title: string): string;
+    function annotation(line: string): string | undefined;
+    function lineContent(line: string): string;
+    function renderedEdge(line: string): EvidenceMarkdownRenderedEdge;
+    function indentedCode(line: string): boolean;
+    /**
+     * Reports whether a column lies inside a closed backtick code span.
+     *
+     * Annotation scanning uses this result to reject tags in inline examples while
+     * ignoring unmatched delimiters that do not establish a Markdown code region.
+     */
+    function inlineCode(line: string, column: number): boolean;
+    function hasWhitespace(value: string): boolean;
+    /**
+     * Maps one supported ATX depth to its public Markdown selector.
+     *
+     * Only headings that Evidence materializes as units have selectors; other
+     * depths fail here instead of silently receiving a public symbol.
+     */
+    function symbol(level: number): EvidenceMarkdownSymbol;
+}