@queryrev/ai-readiness 0.0.0-stage → 1.0.0

package/dist/facts/reader.d.ts

@@ -0,0 +1,2 @@
+/** The product token this reader uses when it matches robots.txt rules (its user agent's product). */
+export declare const READER_TOKEN = "QueryRev-AIReadiness";

package/dist/facts/schema-org-organization-types.d.ts

@@ -0,0 +1,6 @@
+/** The schema.org release the list below was read from. */
+export declare const SCHEMA_ORG_VERSION = "30.1";
+/** SHA-256 of that release's vocabulary file (schemaorg-current-https.jsonld). */
+export declare const SCHEMA_ORG_SHA256 = "0a4b8c4c910fc831ec8695196510973390906a5ee34dc6b862dfec4d21a12f74";
+/** `Organization` and every schema.org type that is a subtype of it, sorted. */
+export declare const ORGANIZATION_TYPES: readonly string[];

package/dist/facts/schema-org-types.d.ts

@@ -0,0 +1,2 @@
+/** Every type of schema.org 30.1 (classes and data types), sorted. */
+export declare const SCHEMA_ORG_TYPES: readonly string[];

package/dist/facts/sitemap.d.ts

@@ -0,0 +1,24 @@
+import type { SitemapFacts } from './types.js';
+/** A sitemap is read to this many bytes after decompression (the fetch cap for a sitemap). */
+export declare const SITEMAP_MAX_BYTES: number;
+/** `<loc>` entries or text lines counted at most. */
+export declare const MAX_LOCS_COUNTED = 100000;
+export interface ReadSitemapOptions {
+    /** The bytes were already cut at the fetch cap. */
+    readonly truncated?: boolean;
+    readonly registrableDomain?: (host: string) => string;
+}
+/**
+ * The sitemap rules on text that is already decoded and inflated. `truncated` says the text was
+ * cut at the cap: it is then read as far as it goes.
+ */
+export declare function readSitemapText(text: string, contentType: string | null, siteUrl: string, options?: ReadSitemapOptions & {
+    readonly gzip?: boolean;
+}): SitemapFacts;
+/**
+ * Reads a sitemap file from its bytes as served. A gzip file (the `.gz` sitemap, found by its
+ * two first bytes) is inflated to at most `SITEMAP_MAX_BYTES`; a file that goes past the cap is
+ * cut there and reported as `truncated`, and valid as far as it was read. `siteUrl` is the
+ * site's own address: only `<loc>` entries on its registrable domain count.
+ */
+export declare function readSitemap(bytes: Uint8Array, contentType: string | null, siteUrl: string, options?: ReadSitemapOptions): Promise<SitemapFacts>;

package/dist/facts/types.d.ts

@@ -0,0 +1,214 @@
+import type { TextFetcher } from '../net/fetch-text.js';
+/**
+ * The result types of the readers and of `checkSite`. They are plain data (no classes, no
+ * functions), so a result can be written to JSON and read back: the study runner, the check
+ * pages and the command line all build on these shapes.
+ *
+ * Nothing here is a score. A check returns findings with evidence; counts and shares are for
+ * whoever collects many of them.
+ */
+/** One result of a check, stable enough to be matched by `id` across versions. */
+export type FindingLevel = 'pass' | 'warn' | 'fail' | 'info';
+export interface Finding {
+    /** Stable name of the check (`organization-schema`, `sitemap-valid`). Never reused. */
+    readonly id: string;
+    readonly level: FindingLevel;
+    /** One plain sentence about this site. */
+    readonly message: string;
+    /** Where the check is explained. */
+    readonly docUrl: string;
+    /** The evidence: the line, value or text the finding rests on, cut short. */
+    readonly snippet?: string;
+}
+/** How much of the home page's text is in the HTML as served. */
+export type HomeContent = 'text_in_html' | 'little_text' | 'app_shell';
+/** Where a profile in `sameAs` points, by host. */
+export type ProfileKind = 'wikidata' | 'wikipedia' | 'linkedin' | 'x' | 'facebook' | 'instagram' | 'youtube' | 'tiktok' | 'github' | 'crunchbase' | 'pinterest' | 'other';
+/** Where the organization was found. JSON-LD wins over microdata. */
+export type OrganizationSource = 'json_ld' | 'microdata' | 'none';
+/** A limit of the JSON-LD reader that was reached; the rest of the data was not read. */
+export type JsonLdBound = 'blocks' | 'block_size' | 'total_size' | 'depth' | 'nodes';
+/** What the page says about its primary Organization node. */
+export interface OrganizationFacts {
+    /** The node's `@type` that names Organization or one of its subtypes. */
+    readonly type: string;
+    /** The node's `name`, as written, cut to 200 characters; null without a usable one. */
+    readonly name: string | null;
+    /** The node's `url` when it is an absolute http(s) URL, else null. */
+    readonly url: string | null;
+    /** A non-empty `name` and an absolute http(s) `url`. */
+    readonly nameAndUrl: boolean;
+    /** `logo` is a URL, or an ImageObject with a `url`. */
+    readonly logo: boolean;
+    /**
+     * The distinct `sameAs` URLs (absolute http(s), not on the site's own registrable domain),
+     * at most 100, in the order written.
+     */
+    readonly sameAs: readonly string[];
+    /** The kinds of `sameAs`, each once, in a fixed order. */
+    readonly sameAsKinds: readonly ProfileKind[];
+}
+/** What the `<script type="application/ld+json">` blocks of the page held. */
+export interface JsonLdFacts {
+    /** ld+json scripts read (at most 50). */
+    readonly scripts: number;
+    /** Scripts that parse as JSON. */
+    readonly parsed: number;
+    /** Scripts that parse and whose top-level `@context` names schema.org. */
+    readonly schemaOrgBlocks: number;
+    /** Scripts that do not parse. */
+    readonly errors: number;
+    /** The limits that were reached, each once. Empty when the whole page was read. */
+    readonly bounds: readonly JsonLdBound[];
+}
+/** Everything `readHomePage` reads from one page. */
+export interface HomePageFacts {
+    /** The address the page was read at (the final one, after redirects). */
+    readonly url: string;
+    readonly host: string;
+    /** Size of the HTML in bytes (UTF-8). */
+    readonly bytes: number;
+    /** The scan stopped at its limit, so the end of a very large page was not read. */
+    readonly scanTruncated: boolean;
+    /** Words of main text; a lower bound when `wordsCapped`. */
+    readonly mainWords: number;
+    /** Counting stopped at its word limit or its character limit, so `mainWords` is a lower bound. */
+    readonly wordsCapped: boolean;
+    /** Whether the words are those of `<main>` / `[role=main]` or of the body less its chrome. */
+    readonly mainSource: 'main' | 'body';
+    readonly content: HomeContent;
+    /** An application root element is present (`#root`, `#__next`, `app-root`, and the rest). */
+    readonly appRoot: boolean;
+    /** The page asks the reader to enable JavaScript. */
+    readonly jsNotice: boolean;
+    /** A small page whose title or visible text is that of a bot check (a 202 is decided apart). */
+    readonly botCheck: boolean;
+    readonly noindex: boolean;
+    readonly noindexSource: 'meta' | 'header' | 'both' | null;
+    readonly jsonLd: JsonLdFacts;
+    /** schema.org type names found in JSON-LD and microdata, each once, at most 200. */
+    readonly types: readonly string[];
+    /** The `itemtype` values of the microdata, as type names, each once. */
+    readonly microdataTypes: readonly string[];
+    readonly organization: OrganizationSource;
+    /** Type of the primary node, or the first Organization type seen in microdata. */
+    readonly organizationType: string | null;
+    /** The primary Organization node of the JSON-LD; null without one. */
+    readonly primary: OrganizationFacts | null;
+    /** A `WebSite` node. */
+    readonly website: boolean;
+    /** A `WebSite` node with a `potentialAction` of type `SearchAction`. */
+    readonly searchAction: boolean;
+    /** A `FAQPage` node (JSON-LD or microdata). */
+    readonly faqPage: boolean;
+}
+export type SitemapOutcome = 'valid' | 'not_a_sitemap' | 'malformed';
+/** Why a file is not a valid sitemap. */
+export type SitemapReason = 'html' | 'empty' | 'doctype' | 'bad_root' | 'no_loc' | 'off_site' | 'not_closed' | 'text_lines' | 'gzip_failed';
+/** What `readSitemap` found in one file. */
+export interface SitemapFacts {
+    readonly outcome: SitemapOutcome;
+    /** `urlset` and `index` are XML (sitemaps.org); `text` is one URL per line. */
+    readonly kind: 'urlset' | 'index' | 'text' | null;
+    readonly reason: SitemapReason | null;
+    /** `<loc>` entries (or text lines) counted, up to 100,000. */
+    readonly locs: number;
+    /** The file was gzip and was inflated. */
+    readonly gzip: boolean;
+    /** The file was cut at the size cap: it is valid as far as it was read. */
+    readonly truncated: boolean;
+    /** Bytes read after decompression. */
+    readonly bytes: number;
+}
+/** Why the home page counts as read or not. */
+export type HomeOutcome = 'read'
+/** robots.txt keeps this reader from `/`. */
+ | 'not_allowed' | 'forbidden' | 'challenge' | 'placeholder' | 'not_html' | 'error_status' | 'off_site' | 'redirect_error' | 'network';
+/** The robots.txt of a site as a check read it. */
+export type RobotsOutcome = 
+/** A file, parsed. */
+'file'
+/** A 4xx (not 401, 403, 429): there is no file, and nothing is closed. */
+ | 'missing'
+/** Not read: 401, 403, 429, 5xx, a bot check, or no answer. Nothing more is asked. */
+ | 'not_read';
+export interface RobotsRead {
+    readonly outcome: RobotsOutcome;
+    readonly url: string;
+    readonly status: number | null;
+    readonly bytes: number;
+    readonly truncated: boolean;
+    /** `Sitemap:` lines, as written. */
+    readonly sitemaps: readonly string[];
+    /** Whether this reader's own token may fetch `/`. True without a file. */
+    readonly allowsThisReader: boolean;
+}
+export interface HomeRead {
+    readonly outcome: HomeOutcome;
+    readonly requestedUrl: string;
+    /** The last URL requested; the page was read there. */
+    readonly finalUrl: string;
+    readonly status: number | null;
+    readonly redirects: number;
+    readonly truncated: boolean;
+    /** Present when the outcome is `read`. */
+    readonly facts: HomePageFacts | null;
+}
+export interface LlmsRead {
+    /** False when robots.txt keeps this reader from the file. */
+    readonly asked: boolean;
+    readonly present: boolean;
+    /** The verdict of `validateLlmsTxt`; null when not asked. */
+    readonly verdict: string | null;
+    readonly bytes: number;
+    readonly truncated: boolean;
+    readonly h1: string | null;
+    readonly sections: number;
+    readonly links: number;
+}
+export type SitemapSource = 'robots' | 'well_known';
+export interface SitemapRead {
+    /** False when robots.txt keeps this reader from the file, or the robots.txt was not read. */
+    readonly asked: boolean;
+    readonly source: SitemapSource | null;
+    readonly url: string | null;
+    readonly status: number | null;
+    /** Null when the file was not read or the answer was not 2xx. */
+    readonly facts: SitemapFacts | null;
+}
+/** Access of one covered crawler to `/`, from the site's robots.txt. */
+export interface CrawlerRootAccess {
+    readonly token: string;
+    readonly purpose: 'training' | 'search' | 'user';
+    readonly access: 'allowed' | 'partial' | 'blocked';
+    /** The crawler may fetch `/`. */
+    readonly root: boolean;
+}
+/** The result of `checkSite`: facts gathered from files, and the findings drawn from them. */
+export interface SiteCheck {
+    readonly schemaVersion: 1;
+    /** The site as asked: scheme and host. */
+    readonly origin: string;
+    readonly host: string;
+    /** The crawlers-covered line every report carries. */
+    readonly crawlersCovered: string;
+    readonly robots: RobotsRead;
+    /** The covered crawlers' access; empty when robots.txt was not read. */
+    readonly crawlerAccess: readonly CrawlerRootAccess[];
+    /** Null when no request was made (robots.txt not read, or `/` not allowed). */
+    readonly home: HomeRead | null;
+    readonly llms: LlmsRead;
+    readonly sitemap: SitemapRead;
+    readonly findings: readonly Finding[];
+}
+export interface CheckOptions {
+    /** Reads files over the guarded network layer. By default one is made and closed per check. */
+    readonly fetcher?: TextFetcher;
+    /** The caller's overall budget; by default 30 seconds. */
+    readonly signal?: AbortSignal;
+    /**
+     * Maps a host name to its registrable domain. The built-in rule knows the common two-label
+     * suffixes only; pass a full public-suffix lookup to be exact.
+     */
+    readonly registrableDomain?: (host: string) => string;
+}

package/dist/facts/words.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Word counting by the Unicode word-break rules (`Intl.Segmenter`, word-like segments), so
+ * a page in Chinese or Japanese is counted too, where splitting on spaces would find one word.
+ */
+/** Counting stops here: no rule of the library looks past it, and a page cannot make it slow. */
+export declare const MAX_WORDS_COUNTED = 100000;
+/** Characters given to the word-break rules at most (text that is not plain ASCII). */
+export declare const MAX_SEGMENTED_CHARS: number;
+export interface WordCount {
+    readonly words: number;
+    /** The count is a lower bound: counting stopped at the word limit or the character limit. */
+    readonly capped: boolean;
+}
+/** Words in `text`, counted up to `limit`. Linear in the length of the text. */
+export declare function countWords(text: string, limit?: number): WordCount;

package/dist/generate/generate-llms.d.ts

@@ -0,0 +1,51 @@
+import { type TextFetcher } from '../net/fetch-text.js';
+import { type LlmsPage, type LlmsSite } from './index.js';
+/** Pages listed when the caller does not say. */
+export declare const GENERATE_DEFAULT_PAGES = 30;
+/** The most pages one run lists. */
+export declare const GENERATE_MAX_PAGES = 100;
+/** Sitemap files read in one run. */
+export declare const GENERATE_MAX_SITEMAPS = 5;
+/** Bytes of each page that are read: the part `pageMeta` looks at. */
+export declare const GENERATE_PAGE_BYTES: number;
+export type SkippedReason = 'robots' | 'noindex' | 'not_html' | 'status' | 'off_site' | 'unreachable' | 'duplicate';
+export interface SkippedPage {
+    readonly url: string;
+    readonly reason: SkippedReason;
+}
+export type GenerateOutcome = 
+/** A file was built. */
+'generated'
+/** The address cannot be checked (not a web site name). */
+ | 'refused'
+/** robots.txt could not be read, so nothing else was requested. */
+ | 'robots_unreadable'
+/** No page could be read: robots.txt keeps this reader out, or nothing answered. */
+ | 'nothing_found';
+export interface GeneratedLlms {
+    readonly schemaVersion: 1;
+    readonly outcome: GenerateOutcome;
+    readonly origin: string;
+    readonly host: string;
+    /** The file; null unless the outcome is `generated`. */
+    readonly text: string | null;
+    readonly site: LlmsSite | null;
+    readonly pages: readonly LlmsPage[];
+    /** Where the page addresses came from. */
+    readonly source: 'sitemap' | 'home_links' | 'none';
+    readonly skipped: readonly SkippedPage[];
+    /** The run ran out of time and listed what it had. */
+    readonly budgetReached: boolean;
+    /** One plain sentence about anything the reader should know. */
+    readonly notes: readonly string[];
+}
+export interface GenerateOptions {
+    /** Pages to list: 30 by default, at most 100. */
+    readonly maxPages?: number;
+    readonly fetcher?: TextFetcher;
+    readonly signal?: AbortSignal;
+    readonly registrableDomain?: (host: string) => string;
+}
+/** The addresses of the `<a href>` links of a page, resolved against `base`; at most `max`. */
+export declare function homeLinks(html: string, base: string, max?: number): string[];
+export declare function generateLlmsTxt(input: string, options?: GenerateOptions): Promise<GeneratedLlms>;

package/dist/generate/index.d.ts

@@ -0,0 +1,70 @@
+/**
+ * llms.txt generation, as pure functions of text: what a page says about itself (its title,
+ * first H1 and meta description, as written), the addresses a sitemap lists, and the file
+ * built from them. Nothing here fetches; a caller reads the files with `fetchText` and passes
+ * the text in.
+ */
+/** What a page says about itself. */
+export interface PageMeta {
+    /** The `<title>`, as written (entities decoded, whitespace collapsed); null without one. */
+    readonly title: string | null;
+    /** The first `<h1>`'s text; null without one. */
+    readonly h1: string | null;
+    /** `<meta name="description">`; null without one. */
+    readonly description: string | null;
+    /** A `<meta name="robots">` that says `noindex`. */
+    readonly noindex: boolean;
+}
+export declare function decodeEntities(text: string): string;
+/** Text from a site made safe to show: control and reordering characters removed. */
+export declare function stripUnsafeCharacters(text: string): string;
+/** Characters of a page read for its title, H1 and meta tags: its head and the start of its body. */
+export declare const PAGE_SCAN_MAX_CHARS: number;
+/** Characters of a sitemap scanned: its whole size cap (5 MiB), in one linear pass. */
+export declare const SITEMAP_SCAN_MAX_CHARS: number;
+/**
+ * `<loc>` tags looked at in one sitemap, empty or unusable ones included, before the scan
+ * stops: enough for any list the tools take addresses from, and a bound on the work a file of
+ * millions of empty tags can cause.
+ */
+export declare const MAX_LOC_TAGS_SCANNED = 10000;
+/** A sitemap address longer than this is left out. */
+export declare const MAX_LOCATION_LENGTH = 2048;
+export declare function isSpace(char: string | undefined): boolean;
+/** The attributes of one tag's text (between its name and its `>`), in one pass. */
+export declare function attributes(tag: string): Map<string, string>;
+/** Reads a page's title, first H1, meta description and robots meta from its HTML. */
+export declare function pageMeta(input: string): PageMeta;
+/** The addresses a sitemap file lists. */
+export interface SitemapLocations {
+    /** `urlset` lists pages; `index` lists other sitemaps; `unknown` is neither. */
+    readonly kind: 'urlset' | 'index' | 'unknown';
+    /** The `<loc>` values in order, entities decoded, at most `max`; none over 2,048 characters. */
+    readonly locations: readonly string[];
+}
+/** Reads the `<loc>` addresses of a sitemap (sitemaps.org), up to `max` of them. */
+export declare function sitemapLocations(input: string, max: number): SitemapLocations;
+/** The site an llms.txt is about. */
+export interface LlmsSite {
+    /** Its name: the file's H1. */
+    readonly name: string;
+    /** One or two sentences about it: the summary blockquote. Null leaves it out. */
+    readonly summary: string | null;
+}
+/** One page to list. */
+export interface LlmsPage {
+    /** Its absolute address. */
+    readonly url: string;
+    readonly title: string;
+    readonly description: string | null;
+}
+/** The section of pages that sit at the top of the site (no first path segment). */
+export declare const MAIN_SECTION = "Main pages";
+/**
+ * The llms.txt for a site and its pages (llmstxt.org): the site's name as the H1, its summary
+ * as a blockquote, then one H2 section per folder (first path segment with pages under it), in
+ * the order the pages come, the other pages first under "Main pages". Each page is a list item
+ * `- [title](url): description`, the description left out when there is none. Titles and
+ * descriptions are used as written, on one line.
+ */
+export declare function renderLlmsTxt(site: LlmsSite, pages: readonly LlmsPage[]): string;

package/dist/index.d.ts

@@ -0,0 +1,10 @@
+export { crawlDelayFor, isAllowedByRobots, matchRobots, parseRobots, ROBOTS_MAX_AGE_MS, ROBOTS_MAX_BYTES, ROBOTS_MAX_REDIRECTS, type RobotsGroup, type RobotsRule, type RobotsTxt, type RuleMatch, } from './robots/index.js';
+export { agentVerdict, AI_AGENTS, CRAWLER_PURPOSES, CRAWLERS, crawlerAccess, crawlersCoveredLine, STUDY_AGENTS, verdictsFor, type AgentAccess, type AgentVerdict, type AiAgent, type CrawlerAccess, type CrawlerInfo, type CrawlerPurpose, type StudyAgent, } from './crawlers/index.js';
+export { llmsStructure, looksLikeHtml, looksLikeLlmsTxt, looksLikeRobotsTxt, LLMS_H1_MAX, validateLlmsTxt, type LlmsReport, type LlmsStructure, type LlmsVerdict, } from './llms/index.js';
+export { llmsFindings, LLMS_FINDING_IDS } from './llms/findings.js';
+export { MAIN_SECTION, MAX_LOC_TAGS_SCANNED, pageMeta, renderLlmsTxt, sitemapLocations, stripUnsafeCharacters, type LlmsPage, type LlmsSite, type PageMeta, type SitemapLocations, } from './generate/index.js';
+export { decodeHtml, detectCharset, type DecodedText } from './net/charset.js';
+export { classifyAddress, type AddressClass, type AddressRefusal } from './net/address.js';
+export { CRAWL_PORTS, createNetworkGuard, DNS_TIMEOUT_MS, guardedConnector, NetworkGuardError, systemResolver, type GuardErrorCode, type HostResolver, type NetworkGuard, } from './net/guard.js';
+export { AI_READINESS_USER_AGENT, CHECK_BUDGET_MS, CONNECT_TIMEOUT_MS, createTextFetcher, FETCH_LIMITS, GENERATE_BUDGET_MS, HOST_INTERVAL_MS, KEPT_HEADERS, MAX_CRAWL_DELAY_SECONDS, MAX_HOST_LENGTH, MAX_REDIRECTS, REQUEST_TIMEOUT_MS, type FetchKind, type FetchTextOptions, type KeptHeader, type RedirectHop, type TextFetcher, type TextFetcherOptions, type TextFetchOutcome, type TextFetchResult, type UrlRefusal, } from './net/fetch-text.js';
+export * from './facts/index.js';