@aia-matrix/llms-txt-validator 0.0.0-stage → 1.0.0

package/dist/action.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/action.js

@@ -0,0 +1,35 @@
+import { appendFile } from "node:fs/promises";
+import { validate } from "./validator.js";
+import { human, exitCode } from "./report.js";
+async function main() {
+    const url = process.env.INPUT_URL;
+    if (!url)
+        throw Error("url input is required");
+    const failOn = process.env["INPUT_FAIL-ON"] ?? "error";
+    if (failOn !== "warn" && failOn !== "error")
+        throw Error("fail-on must be warn or error");
+    const numeric = (name, fallback, minimum) => {
+        const raw = process.env[`INPUT_${name.toUpperCase()}`] ?? fallback;
+        const value = Number(raw);
+        if (!/^\d+$/.test(raw) || !Number.isSafeInteger(value) || value < minimum)
+            throw Error(`${name} must be ${minimum === 0 ? "a nonnegative" : "a positive"} integer`);
+        return value;
+    };
+    const maxLinks = numeric("max-links", "50", 0);
+    const timeout = numeric("timeout", "10000", 1);
+    const report = await validate(url, {
+        failOn,
+        maxLinks,
+        timeout,
+    });
+    console.log(human(report));
+    if (process.env.GITHUB_STEP_SUMMARY) {
+        const escape = (s) => s.replace(/[&<>]/g, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;" })[c]);
+        await appendFile(process.env.GITHUB_STEP_SUMMARY, `## llms.txt validation\n\n${report.summary.pass} pass · ${report.summary.info} info · ${report.summary.warn} warn · ${report.summary.fail} fail\n\n<pre>${escape(human(report))}</pre>\n`);
+    }
+    process.exitCode = exitCode(report, failOn);
+}
+main().catch((e) => {
+    console.error(`Tool error: ${e instanceof Error ? e.message : String(e)}`);
+    process.exitCode = 2;
+});

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli.js

@@ -0,0 +1,52 @@
+#!/usr/bin/env node
+import { parseArgs } from "node:util";
+import { validate } from "./validator.js";
+import { human, exitCode } from "./report.js";
+import { VERSION } from "./spec.js";
+let json = process.argv.slice(2).includes("--json");
+try {
+    const { values, positionals } = parseArgs({
+        allowPositionals: true,
+        options: {
+            json: { type: "boolean" },
+            "max-links": { type: "string" },
+            timeout: { type: "string" },
+            "fail-on": { type: "string" },
+            "no-network": { type: "boolean" },
+            help: { type: "boolean", short: "h" },
+            version: { type: "boolean", short: "v" },
+        },
+    });
+    json = values.json ?? false;
+    if (values.help) {
+        console.log("Usage: llms-txt-validator <url|file> [--json] [--max-links 50] [--timeout 10000] [--fail-on warn|error] [--no-network]");
+    }
+    else if (values.version) {
+        console.log(VERSION);
+    }
+    else {
+        if (positionals.length !== 1)
+            throw Error("Provide exactly one website URL or local llms.txt file");
+        const failOn = values["fail-on"] ?? "error";
+        if (failOn !== "warn" && failOn !== "error")
+            throw Error("--fail-on must be warn or error");
+        const report = await validate(positionals[0], {
+            maxLinks: values["max-links"] === undefined ? 50 : Number(values["max-links"]),
+            timeout: values.timeout === undefined ? 10000 : Number(values.timeout),
+            failOn,
+            network: !values["no-network"],
+        });
+        console.log(json ? JSON.stringify(report, null, 2) : human(report));
+        process.exitCode = exitCode(report, failOn);
+    }
+}
+catch (e) {
+    if (json)
+        console.log(JSON.stringify({
+            version: "1.0",
+            error: { message: e instanceof Error ? e.message : String(e) },
+        }));
+    else
+        console.error(`Tool error: ${e instanceof Error ? e.message : String(e)}`);
+    process.exitCode = 2;
+}

package/dist/discovery.d.ts

@@ -0,0 +1,7 @@
+export interface Relation {
+    href: string;
+    rel: string;
+    type?: string;
+}
+export declare function relations(body: string, header: string | null, base: string): Relation[];
+export declare const hasRel: (r: Relation, name: string) => boolean;

package/dist/discovery.js

@@ -0,0 +1,55 @@
+import { parse } from "parse5";
+export function relations(body, header, base) {
+    const out = [];
+    const add = (href, rel, type) => {
+        try {
+            out.push({ href: new URL(href, base).href, rel, type });
+        }
+        catch {
+            /* Invalid relation is not usable. */
+        }
+    };
+    for (const match of (header ?? "").matchAll(/<([^>]+)>((?:\s*;\s*[\w-]+\s*=\s*(?:"[^"]*"|[^;,]+))*)/g)) {
+        const attrs = new Map();
+        for (const a of match[2].matchAll(/;\s*([\w-]+)\s*=\s*(?:"([^"]*)"|([^;,]+))/g))
+            attrs.set(a[1].toLowerCase(), (a[2] ?? a[3]).trim());
+        add(match[1], attrs.get("rel") ?? "", attrs.get("type"));
+    }
+    const document = parse(body);
+    let effective = base;
+    const walk = (node, visit) => {
+        const n = node;
+        if (n.tagName)
+            visit(n.tagName, new Map((n.attrs ?? []).map((a) => [a.name, a.value])));
+        for (const child of n.childNodes ?? [])
+            walk(child, visit);
+    };
+    let foundBase = false;
+    walk(document, (tag, a) => {
+        if (tag === "base" && a.has("href") && !foundBase) {
+            try {
+                effective = new URL(a.get("href"), base).href;
+                foundBase = true;
+            }
+            catch {
+                /* Ignore invalid base. */
+            }
+        }
+    });
+    walk(document, (tag, a) => {
+        if (tag === "link" && a.has("href")) {
+            try {
+                out.push({
+                    href: new URL(a.get("href"), effective).href,
+                    rel: a.get("rel") ?? "",
+                    type: a.get("type"),
+                });
+            }
+            catch {
+                /* Ignore invalid link. */
+            }
+        }
+    });
+    return out;
+}
+export const hasRel = (r, name) => r.rel.toLowerCase().split(/\s+/).includes(name);

package/dist/format.d.ts

@@ -0,0 +1,6 @@
+import type { Result } from "./types.js";
+export interface Parsed {
+    results: Result[];
+    links: string[];
+}
+export declare function checkFormat(input: string, url: string): Parsed;

package/dist/format.js

@@ -0,0 +1,117 @@
+import MarkdownIt from "markdown-it";
+import { SPEC } from "./spec.js";
+const parser = new MarkdownIt();
+export function checkFormat(input, url) {
+    const text = input.replace(/^\uFEFF/, "");
+    const results = [];
+    const links = [];
+    const add = (id, ok, message) => results.push({
+        id,
+        status: ok ? "pass" : "fail",
+        message,
+        specRef: SPEC.format,
+        url,
+    });
+    const tokens = parser.parse(text, {});
+    const heads = tokens.filter((t) => t.type === "heading_open");
+    add("format.h1", heads.filter((t) => t.tag === "h1").length === 1 &&
+        tokens[0]?.type === "heading_open" &&
+        tokens[0]?.tag === "h1" &&
+        !!tokens[1]?.content.trim(), "File must begin with exactly one nonempty H1.");
+    add("format.heading-levels", heads.every((t) => t.tag === "h1" || t.tag === "h2"), "Only H1 and H2 headings are allowed.");
+    let section = false;
+    let count = 0;
+    let sectionNumber = 0;
+    let sectionBad = false;
+    let preambleStarted = false;
+    let summarySeen = false;
+    const warn = (id, message) => results.push({ id, status: "warn", message, specRef: SPEC.format, url });
+    const close = () => {
+        if (section)
+            add(`format.section.${sectionNumber}`, count > 0 && !sectionBad, "Each H2 section must contain nonempty link-list items.");
+    };
+    for (let i = 0; i < tokens.length; i++) {
+        const t = tokens[i];
+        if (t.level !== 0)
+            continue;
+        if (t.type === "heading_open") {
+            if (t.tag === "h2") {
+                close();
+                section = true;
+                count = 0;
+                sectionBad = !tokens[i + 1]?.content.trim();
+                sectionNumber++;
+            }
+            i += 2;
+            continue;
+        }
+        if (!section) {
+            if (t.type === "blockquote_open") {
+                if (preambleStarted || summarySeen)
+                    warn("format.summary-order", "An optional summary should precede the non-heading preamble.");
+                else
+                    add("format.summary-order", true, "Summary precedes the non-heading preamble.");
+                summarySeen = true;
+            }
+            else if (t.type.endsWith("_open") ||
+                t.type === "fence" ||
+                t.type === "html_block")
+                preambleStarted = true;
+            continue;
+        }
+        if (t.type === "bullet_list_open" || t.type === "ordered_list_open") {
+            const end = tokens.findIndex((x, j) => j > i &&
+                x.type === t.type.replace("_open", "_close") &&
+                x.level === 0);
+            for (let j = i + 1; j < end; j++) {
+                if (tokens[j].type !== "list_item_open")
+                    continue;
+                count++;
+                const level = tokens[j].level;
+                const itemEnd = tokens.findIndex((x, k) => k > j && x.type === "list_item_close" && x.level === level);
+                const inlines = tokens
+                    .slice(j + 1, itemEnd)
+                    .filter((x) => x.type === "inline");
+                const children = inlines.flatMap((x) => x.children ?? []);
+                const firstLink = children.find((x) => x.type === "link_open");
+                // Preserve links even when the item's arrangement needs a warning.
+                for (const child of children.filter((x) => x.type === "link_open")) {
+                    const href = child.attrGet("href");
+                    if (href)
+                        links.push(href);
+                }
+                if (!firstLink)
+                    sectionBad = true;
+                else if (children[0]?.type !== "link_open")
+                    warn("format.item-prefix", "A file-list item should start with its link.");
+                else {
+                    const linkEnd = children.findIndex((x) => x.type === "link_close");
+                    const suffix = children
+                        .slice(linkEnd + 1)
+                        .map((x) => x.content)
+                        .join("")
+                        .trim();
+                    if (suffix && !suffix.startsWith(":"))
+                        warn("format.item-notes", "Notes after a file-list link should start with a colon.");
+                }
+            }
+            i = end;
+            continue;
+        }
+        if (!t.type.endsWith("_close"))
+            warn("format.section-prose", "H2 sections should contain file lists; additional prose is present.");
+    }
+    close();
+    const bytes = Buffer.byteLength(input);
+    const size = bytes >= 1024 ? `${(bytes / 1024).toFixed(1)} KiB` : `${bytes} B`;
+    results.push({
+        id: "file.size",
+        status: bytes > 50 * 1024 ? "warn" : "info",
+        message: bytes > 50 * 1024
+            ? `llms.txt is ${size}, above the 50 KiB guidance; agents may truncate or skip it (not a spec requirement).`
+            : `llms.txt is ${size} (guidance: 50 KiB or less; not a spec requirement).`,
+        specRef: SPEC.proposal,
+        url,
+    });
+    return { results, links };
+}

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export { validate } from "./validator.js";
+export { checkFormat } from "./format.js";
+export { exitCode } from "./report.js";
+export type { Options, Report, Result, Status } from "./types.js";

package/dist/index.js

@@ -0,0 +1,3 @@
+export { validate } from "./validator.js";
+export { checkFormat } from "./format.js";
+export { exitCode } from "./report.js";

package/dist/markdown-versions.d.ts

@@ -0,0 +1,2 @@
+export declare function markdownCandidates(page: string): string[];
+export declare function pageCandidates(markdown: string): string[];

package/dist/markdown-versions.js

@@ -0,0 +1,32 @@
+export function markdownCandidates(page) {
+    const u = new URL(page);
+    const path = u.pathname;
+    const paths = path.endsWith("/")
+        ? [path + "index.md", path + "index.html.md"]
+        : [path + ".md", path.replace(/\.[^/.]+$/, ".md")];
+    return [
+        ...new Set(paths.map((p) => {
+            const v = new URL(u);
+            v.pathname = p;
+            return v.href;
+        })),
+    ];
+}
+export function pageCandidates(markdown) {
+    const u = new URL(markdown);
+    if (!u.pathname.endsWith(".md"))
+        return [];
+    const path = u.pathname.slice(0, -3);
+    const paths = path.endsWith("/index")
+        ? [path.slice(0, -5), path + ".html"]
+        : path.endsWith("/index.html")
+            ? [path.slice(0, -10), path]
+            : [path, ...(!/\.[^/]+$/.test(path) ? [path + ".html"] : [])];
+    return [
+        ...new Set(paths.map((p) => {
+            const v = new URL(u);
+            v.pathname = p;
+            return v.href;
+        })),
+    ];
+}

package/dist/network.d.ts

@@ -0,0 +1,11 @@
+import type { Resource } from "./types.js";
+export declare function networkMessage(error: unknown, timeout?: number): string;
+export declare class Client {
+    private timeout;
+    private cache;
+    constructor(timeout: number);
+    request(url: string, method?: string): Promise<Resource>;
+    private fetch;
+    resolve(url: string): Promise<Resource>;
+}
+export declare function pooled<T>(items: T[], work: (item: T) => Promise<void>): Promise<void>;

package/dist/network.js

@@ -0,0 +1,107 @@
+import { VERSION } from "./spec.js";
+export function networkMessage(error, timeout) {
+    const e = error;
+    if (e?.name === "AbortError" || e?.name === "TimeoutError")
+        return timeout ? `timed out after ${timeout} ms` : "timed out";
+    const message = e?.message ?? String(error);
+    const cause = [e?.cause?.code, e?.cause?.message].filter(Boolean).join(" ");
+    return cause ? `${message}: ${cause}` : message;
+}
+export class Client {
+    timeout;
+    cache = new Map();
+    constructor(timeout) {
+        this.timeout = timeout;
+    }
+    async request(url, method = "GET") {
+        const key = method + " " + url;
+        const existing = this.cache.get(key);
+        if (existing)
+            return existing;
+        const pending = this.fetch(url, method);
+        this.cache.set(key, pending);
+        return pending;
+    }
+    async fetch(url, method) {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), this.timeout);
+        try {
+            let current = url;
+            const seen = new Set();
+            for (let n = 0; n <= 10; n++) {
+                if (seen.has(current))
+                    throw Error("Redirect loop");
+                seen.add(current);
+                const parsed = new URL(current);
+                if (!["http:", "https:"].includes(parsed.protocol))
+                    throw Error("Only HTTP(S) URLs are supported");
+                const response = await fetch(current, {
+                    method,
+                    redirect: "manual",
+                    signal: controller.signal,
+                    headers: {
+                        "User-Agent": `aiamatrix-llms-txt-validator/${VERSION} (+https://github.com/aiamatrix/llms-txt-validator)`,
+                    },
+                });
+                if ([301, 302, 303, 307, 308].includes(response.status)) {
+                    const location = response.headers.get("location");
+                    await response.body?.cancel();
+                    if (!location)
+                        throw Error("Redirect missing Location");
+                    current = new URL(location, current).href;
+                    continue;
+                }
+                const reader = response.body?.getReader();
+                const chunks = [];
+                let bytes = 0;
+                if (reader)
+                    while (true) {
+                        const { done, value } = await reader.read();
+                        if (done)
+                            break;
+                        bytes += value.length;
+                        if (bytes > 2 * 1024 * 1024) {
+                            await reader.cancel();
+                            throw Error("Response exceeds 2 MiB safety limit");
+                        }
+                        chunks.push(value);
+                    }
+                return {
+                    url: current,
+                    status: response.status,
+                    headers: response.headers,
+                    body: Buffer.concat(chunks).toString("utf8"),
+                };
+            }
+            throw Error("More than 10 redirects");
+        }
+        catch (e) {
+            throw new Error(controller.signal.aborted
+                ? `timed out after ${this.timeout} ms`
+                : networkMessage(e, this.timeout));
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    async resolve(url) {
+        try {
+            const head = await this.request(url, "HEAD");
+            if (head.status === 200)
+                return head;
+        }
+        catch {
+            /* Retry with GET. */
+        }
+        return this.request(url);
+    }
+}
+export async function pooled(items, work) {
+    let next = 0;
+    await Promise.all(Array.from({ length: Math.min(5, items.length) }, async () => {
+        while (next < items.length) {
+            const item = items[next++];
+            await work(item);
+        }
+    }));
+}

package/dist/report.d.ts

@@ -0,0 +1,3 @@
+import type { Report } from "./types.js";
+export declare function exitCode(report: Report, failOn?: "warn" | "error"): 0 | 1;
+export declare function human(report: Report): string;

package/dist/report.js

@@ -0,0 +1,17 @@
+export function exitCode(report, failOn = "error") {
+    return report.summary.fail > 0 ||
+        (failOn === "warn" && report.summary.warn > 0)
+        ? 1
+        : 0;
+}
+export function human(report) {
+    const groups = new Map();
+    for (const r of report.results) {
+        const key = r.id.split(".")[0];
+        const group = groups.get(key) ?? [];
+        group.push(`  ${r.status.toUpperCase()} ${r.id}: ${r.message}\n    ${r.url}\n    ${r.specRef}`);
+        groups.set(key, group);
+    }
+    return ([...groups].map(([k, v]) => `${k}\n${v.join("\n")}`).join("\n\n") +
+        `\n\nSummary: ${report.summary.pass} pass, ${report.summary.info} info, ${report.summary.warn} warn, ${report.summary.fail} fail\nGet a full AI-readiness report: https://aiamatrix.com`);
+}

package/dist/scope.d.ts

@@ -0,0 +1,2 @@
+export declare function scopedCandidates(page: string): string[];
+export declare function applicable(page: string, indexes: string[]): string | undefined;

package/dist/scope.js

@@ -0,0 +1,20 @@
+export function scopedCandidates(page) {
+    const u = new URL(page);
+    const dirs = u.pathname
+        .slice(0, u.pathname.lastIndexOf("/") + 1)
+        .split("/")
+        .filter(Boolean);
+    const result = [];
+    for (let i = dirs.length; i >= 0; i--)
+        result.push(new URL("/" + dirs.slice(0, i).join("/") + (i ? "/" : "") + "llms.txt", u.origin).href);
+    return result;
+}
+export function applicable(page, indexes) {
+    const p = new URL(page);
+    return indexes
+        .filter((s) => {
+        const u = new URL(s);
+        return (u.origin === p.origin && p.pathname.startsWith(u.pathname.slice(0, -8)));
+    })
+        .sort((a, b) => b.length - a.length)[0];
+}

package/dist/spec.d.ts

@@ -0,0 +1,6 @@
+export { VERSION } from "./version.js";
+export declare const SPEC: {
+    format: string;
+    proposal: string;
+    optional: string;
+};

package/dist/spec.js

@@ -0,0 +1,6 @@
+export { VERSION } from "./version.js";
+export const SPEC = {
+    format: "https://llmstxt.org/#format",
+    proposal: "https://llmstxt.org/#proposal",
+    optional: "https://llmstxt.org/changes.html#v2-august-2026",
+};

package/dist/types.d.ts

@@ -0,0 +1,27 @@
+export type Status = "pass" | "info" | "warn" | "fail";
+export interface Result {
+    id: string;
+    status: Status;
+    message: string;
+    specRef: string;
+    url: string;
+}
+export interface Report {
+    version: "1.0";
+    url: string;
+    checkedAt: string;
+    summary: Record<Status, number>;
+    results: Result[];
+}
+export interface Options {
+    maxLinks?: number;
+    timeout?: number;
+    failOn?: "warn" | "error";
+    network?: boolean;
+}
+export interface Resource {
+    url: string;
+    status: number;
+    headers: Headers;
+    body: string;
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/dist/validator.d.ts

@@ -0,0 +1,2 @@
+import type { Options, Report } from "./types.js";
+export declare function validate(target: string, options?: Options): Promise<Report>;