explorbot 0.4.0 → 0.4.2

package/dist/src/utils/overlay.js

@@ -1,62 +1,34 @@
 import { detectFocusArea } from './aria.js';
-import { pathToXPath } from './html-diff.js';
-import { extractHeadings } from './html.js';
+import { HTML_SELECTORS } from './html.js';
 import { createDebug } from './logger.js';
+import { Region, addedSubtrees } from './region.js';
 const debugLog = createDebug('explorbot:overlay');
-export class Overlay {
-    type;
-    name;
-    root;
-    html;
-    xpath;
-    parent;
+export class Overlay extends Region {
     constructor(data = {}) {
-        this.type = data.type ?? null;
-        this.name = data.name ?? null;
-        this.root = data.root ?? null;
-        this.html = data.html ?? null;
-        this.xpath = data.xpath ?? null;
-        this.parent = data.parent ?? null;
+        super({ ...data, type: 'overlay' });
     }
-    get detected() {
-        return this.type !== null && this.type !== 'region';
-    }
-    get present() {
-        return this.type !== null;
-    }
-    describe() {
-        if (!this.present)
-            return '';
-        let text = `${this.type} "${this.name || 'unnamed'}" opened`;
-        if (this.root)
-            text += `, scope: ${this.root}`;
-        return text;
+    get isModal() {
+        return true;
     }
     withGeometry(geometry) {
         return new Overlay({
-            type: this.type,
             name: this.name || geometry.name,
             root: geometry.root,
             html: geometry.html,
             xpath: geometry.xpath,
         });
     }
-    withParent(parent) {
-        return new Overlay({
-            type: this.type,
-            name: this.name,
-            root: this.root,
-            html: this.html,
-            xpath: this.xpath,
-            parent: { type: parent.type, name: parent.name, root: parent.root, xpath: parent.xpath },
-        });
-    }
     static fromAria(snapshot) {
-        return new Overlay(detectFocusArea(snapshot));
+        const focus = detectFocusArea(snapshot);
+        if (!focus.type)
+            return new Region();
+        return new Overlay({ name: focus.name });
     }
     static resolve(data) {
-        if (data.overlay)
+        if (data.overlay?.type === 'overlay')
             return new Overlay(data.overlay);
+        if (data.overlay)
+            return new Region(data.overlay);
         return Overlay.fromAria(data.ariaSnapshot ?? null);
     }
 }
@@ -68,146 +40,120 @@ export class OverlayPage {
     async detectRegion(diff) {
         if (!diff.sameUrl && diff.similarity < SOFT_NAVIGATION_SIMILARITY)
             return null;
-        for (const subRoot of this.appearedSubRoots(diff)) {
-            const probe = await this.probe(subRoot.elementXPath);
-            if (probe?.found && probe.onScreen && !probe.centerBelongs) {
-                debugLog('Appeared region is hidden or covered, trying the next candidate');
+        for (const added of addedSubtrees(diff)) {
+            const layout = await this.measure(added.xpath);
+            if (layout?.rendered && layout.inViewport && !layout.centerBelongs) {
+                debugLog('Added content is hidden or covered, trying the next candidate');
+                continue;
+            }
+            const overlaying = !!layout?.rendered && layout.outOfFlow && layout.inViewport;
+            if (overlaying && modalScore(layout) < MIN_MODAL_SCORE) {
+                debugLog('Out-of-flow content is a popover, not an overlay; ignoring it');
                 continue;
             }
-            const overlay = this.toOverlay(subRoot, probe, diff.previousHtml);
-            if (!overlay.name && !overlay.root) {
-                debugLog('Appeared region has no name and no semantic root, ignoring it');
+            if (!overlaying && !added.sizable) {
+                debugLog('In-flow content is under the region floor, ignoring it');
                 continue;
             }
-            debugLog(`Region detected: ${overlay.describe()}`);
-            return overlay;
+            const data = { name: added.name, root: added.root, html: added.html, xpath: added.xpath };
+            let region = new Region({ ...data, type: 'region' });
+            if (overlaying)
+                region = new Overlay(data);
+            debugLog(`Region detected: ${region.describe()}`);
+            return region;
         }
         return null;
     }
-    async isStillOpen(overlay) {
-        if (!overlay.xpath)
+    async isStillOpen(region) {
+        if (!region.xpath)
             return true;
-        const probe = await this.probe(overlay.xpath);
+        const probe = await this.measure(region.xpath);
         if (!probe)
             return true;
-        if (!probe.found)
+        if (!probe.rendered)
             return false;
-        if (probe.onScreen && !probe.centerBelongs && !probe.coveredByFloating)
+        if (probe.inViewport && !probe.centerBelongs && !probe.coveredOutOfFlow)
             return false;
         return true;
     }
-    appearedSubRoots(diff) {
-        const candidates = [];
-        for (const part of diff.parts) {
-            const appeared = part.added.find((line) => line.startsWith('ELEMENT:'));
-            if (!appeared)
-                continue;
-            if (part.subtree.length < SUBROOT_MIN_HTML)
-                continue;
-            if (diff.pageSize > 0 && part.rawSize > REGION_MAX_RATIO * diff.pageSize) {
-                debugLog(`Appeared subtree spans ${Math.round((part.rawSize / diff.pageSize) * 100)}% of the page — a new state, not a region`);
-                continue;
-            }
-            candidates.push({
-                container: part.container,
-                elementXPath: pathToXPath(appeared.slice('ELEMENT:'.length)),
-                subtree: part.subtree,
-                size: part.subtree.length,
-                rawSize: part.rawSize,
-                appearedSelector: part.appearedSelector,
-                fresh: !!this.freshHeading(part.subtree, diff.previousHtml),
-            });
-        }
-        if (candidates.length === 0)
-            return [];
-        const totalRawSize = candidates.reduce((sum, c) => sum + c.rawSize, 0);
-        const largest = Math.max(...candidates.map((c) => c.rawSize));
-        if (largest < REGION_DOMINANCE * totalRawSize) {
-            debugLog('Changes are scattered across the page, no dominant region');
-            return [];
-        }
-        candidates.sort((a, b) => Number(b.fresh) - Number(a.fresh) || b.size - a.size);
-        return candidates;
-    }
-    async probe(xpath) {
+    async measure(xpath) {
         if (!this.page)
             return null;
-        return this.page
-            .evaluate(({ probeSource, config }) => {
-            const probe = new Function(`return ${probeSource}`)();
-            return probe(config);
-        }, { probeSource: inspectRegion.toString(), config: { xpath } })
-            .catch((err) => {
-            debugLog('Region probe failed:', err.message);
+        return this.page.evaluate(measureLayout, { config: { xpath, controlsSelector: HTML_SELECTORS.interactiveControl } }).catch((err) => {
+            debugLog('Layout measurement failed:', err.message);
             return null;
         });
     }
-    toOverlay(subRoot, probe, previousHtml) {
-        let type = 'region';
-        if (probe?.centerBelongs && probe.floating)
-            type = 'modal';
-        let root = null;
-        if (subRoot.container !== 'body' && !subRoot.container.startsWith('//'))
-            root = subRoot.container;
-        if (!root && subRoot.appearedSelector)
-            root = subRoot.appearedSelector;
-        return new Overlay({ type, name: this.nameFrom(subRoot.subtree, previousHtml), root, html: subRoot.subtree, xpath: subRoot.elementXPath });
-    }
-    nameFrom(html, previousHtml) {
-        const fresh = this.freshHeading(html, previousHtml);
-        if (fresh)
-            return fresh;
-        const candidates = this.headingsOf(html);
-        if (candidates.length === 0)
-            return null;
-        return candidates.join(' ');
-    }
-    freshHeading(html, previousHtml) {
-        const fresh = this.headingsOf(html).filter((heading) => !previousHtml.includes(heading));
-        return fresh[0] ?? null;
-    }
-    headingsOf(html) {
-        const headings = extractHeadings(html);
-        return [headings.h1, headings.h2, headings.h3, headings.h4].filter(Boolean);
-    }
 }
-const SUBROOT_MIN_HTML = 5_000;
 const SOFT_NAVIGATION_SIMILARITY = 50;
-const REGION_MAX_RATIO = 0.6;
-const REGION_DOMINANCE = 0.7;
-// Serialized via toString() into page.evaluate — must stay a plain function with no outer-scope references.
-function inspectRegion(config) {
-    const probe = { found: false, onScreen: false, floating: false, centerBelongs: false, coveredByFloating: false };
-    const isFloating = (start) => {
+const TYPICAL_DIALOG_CONTROLS = 3;
+const MIN_MODAL_SCORE = 0.25;
+const modalScore = (layout) => {
+    if (!layout)
+        return 0;
+    let blocked = layout.viewportCoverage;
+    if (layout.holdsViewportCenter)
+        blocked = 1;
+    return blocked * Math.min(1, layout.controls / TYPICAL_DIALOG_CONTROLS);
+};
+// Passed to page.evaluate as a function — must stay self-contained with no outer-scope references.
+function measureLayout({ config }) {
+    const layout = { rendered: false, inViewport: false, outOfFlow: false, centerBelongs: false, coveredOutOfFlow: false, holdsViewportCenter: false, viewportCoverage: 0, controls: 0 };
+    const isOutOfFlow = (start) => {
         let node = start;
         while (node && node !== document.body) {
-            const style = window.getComputedStyle(node);
-            if (style.position === 'fixed' || style.position === 'absolute' || (Number.parseInt(style.zIndex || '0', 10) || 0) > 0)
+            const position = window.getComputedStyle(node).position;
+            if (position === 'fixed' || position === 'absolute')
                 return true;
             node = node.parentElement;
         }
         return false;
     };
+    const clip = (rect) => {
+        const left = Math.max(rect.left, 0);
+        const top = Math.max(rect.top, 0);
+        const right = Math.min(rect.right, window.innerWidth);
+        const bottom = Math.min(rect.bottom, window.innerHeight);
+        return { left, top, right, bottom, area: Math.max(0, right - left) * Math.max(0, bottom - top) };
+    };
     const result = document.evaluate(config.xpath, document, null, 9, null);
     const node = result.singleNodeValue;
     if (!node || node.nodeType !== 1)
-        return probe;
+        return layout;
     const element = node;
-    const rect = element.getBoundingClientRect();
+    const belongs = (hit) => !!hit && (hit === element || element.contains(hit));
+    let box = element;
+    let outOfFlow = isOutOfFlow(element);
+    let boxArea = 0;
+    if (outOfFlow)
+        boxArea = clip(element.getBoundingClientRect()).area;
+    const MAX_DESCENDANTS_SCANNED = 200;
+    const descendants = Array.from(element.querySelectorAll('*')).slice(0, MAX_DESCENDANTS_SCANNED);
+    for (const descendant of descendants) {
+        if (!isOutOfFlow(descendant))
+            continue;
+        const area = clip(descendant.getBoundingClientRect()).area;
+        if (area <= boxArea)
+            continue;
+        box = descendant;
+        boxArea = area;
+        outOfFlow = true;
+    }
+    const rect = box.getBoundingClientRect();
     if (rect.width === 0 && rect.height === 0)
-        return probe;
-    probe.found = true;
-    probe.floating = isFloating(element);
-    const left = Math.max(rect.left, 0);
-    const top = Math.max(rect.top, 0);
-    const right = Math.min(rect.right, window.innerWidth);
-    const bottom = Math.min(rect.bottom, window.innerHeight);
+        return layout;
+    layout.rendered = true;
+    layout.outOfFlow = outOfFlow;
+    layout.controls = Array.from(element.querySelectorAll(config.controlsSelector)).filter((control) => control.getClientRects().length > 0).length;
+    const { left, top, right, bottom, area } = clip(rect);
     if (right <= left || bottom <= top)
-        return probe;
-    probe.onScreen = true;
+        return layout;
+    layout.inViewport = true;
+    layout.viewportCoverage = area / (window.innerWidth * window.innerHeight);
     const hit = document.elementFromPoint((left + right) / 2, (top + bottom) / 2);
-    probe.centerBelongs = !!hit && (hit === element || element.contains(hit));
-    if (hit && !probe.centerBelongs)
-        probe.coveredByFloating = isFloating(hit);
-    return probe;
+    layout.centerBelongs = belongs(hit);
+    if (hit && !layout.centerBelongs)
+        layout.coveredOutOfFlow = isOutOfFlow(hit);
+    layout.holdsViewportCenter = belongs(document.elementFromPoint(window.innerWidth / 2, window.innerHeight / 2));
+    return layout;
 }

package/dist/src/utils/region.d.ts

@@ -0,0 +1,41 @@
+import type { HtmlDiffPart } from './html-diff.js';
+export declare class Region {
+    readonly type: RegionType | null;
+    readonly name: string | null;
+    readonly root: string | null;
+    readonly html: string | null;
+    readonly xpath: string | null;
+    readonly parent: RegionData | null;
+    constructor(data?: RegionData);
+    get isOpen(): boolean;
+    get isModal(): boolean;
+    describe(): string;
+    withParent(parent: Region): Region;
+}
+export declare function addedSubtrees(diff: RegionDiff): AddedSubtree[];
+export type RegionType = 'overlay' | 'region';
+export type RegionData = {
+    type?: RegionType | null;
+    name?: string | null;
+    root?: string | null;
+    html?: string | null;
+    xpath?: string | null;
+    parent?: RegionData | null;
+};
+export interface RegionDiff {
+    parts: HtmlDiffPart[];
+    pageSize: number;
+    similarity: number;
+    sameUrl: boolean;
+    previousHtml: string;
+}
+export interface AddedSubtree {
+    xpath: string;
+    html: string;
+    size: number;
+    rawSize: number;
+    sizable: boolean;
+    name: string | null;
+    root: string | null;
+    hasNewHeading: boolean;
+}

package/dist/src/utils/region.js

@@ -0,0 +1,99 @@
+import { pathToXPath } from './html-diff.js';
+import { extractHeadings } from './html.js';
+import { createDebug } from './logger.js';
+const debugLog = createDebug('explorbot:region');
+export class Region {
+    type;
+    name;
+    root;
+    html;
+    xpath;
+    parent;
+    constructor(data = {}) {
+        this.type = data.type ?? null;
+        this.name = data.name ?? null;
+        this.root = data.root ?? null;
+        this.html = data.html ?? null;
+        this.xpath = data.xpath ?? null;
+        this.parent = data.parent ?? null;
+    }
+    get isOpen() {
+        return this.type !== null;
+    }
+    get isModal() {
+        return false;
+    }
+    describe() {
+        if (!this.isOpen)
+            return '';
+        let text = `${this.type} "${this.name || 'unnamed'}" opened`;
+        if (this.root)
+            text += `, scope: ${this.root}`;
+        return text;
+    }
+    withParent(parent) {
+        const Ctor = this.constructor;
+        return new Ctor({
+            type: this.type,
+            name: this.name,
+            root: this.root,
+            html: this.html,
+            xpath: this.xpath,
+            parent: { type: parent.type, name: parent.name, root: parent.root, xpath: parent.xpath },
+        });
+    }
+}
+export function addedSubtrees(diff) {
+    const candidates = [];
+    for (const part of diff.parts) {
+        const appeared = part.added.find((line) => line.startsWith('ELEMENT:'));
+        if (!appeared)
+            continue;
+        if (diff.pageSize > 0 && part.rawSize > REGION_MAX_RATIO * diff.pageSize) {
+            debugLog(`Added subtree spans ${Math.round((part.rawSize / diff.pageSize) * 100)}% of the page — a new state, not a region`);
+            continue;
+        }
+        const headings = headingsOf(part.subtree);
+        const newHeading = headings.find((heading) => !diff.previousHtml.includes(heading)) ?? null;
+        let name = newHeading;
+        if (!name && headings.length > 0)
+            name = headings.join(' ');
+        const namedContainer = part.container !== 'body' && !part.container.startsWith('//');
+        let root = null;
+        if (namedContainer && part.containerWasEmpty)
+            root = part.container;
+        if (!root)
+            root = part.appearedSelector ?? null;
+        if (!root && namedContainer)
+            root = part.container;
+        candidates.push({
+            xpath: pathToXPath(appeared.slice('ELEMENT:'.length)),
+            html: part.subtree,
+            size: part.subtree.length,
+            rawSize: part.rawSize,
+            sizable: part.subtree.length >= REGION_MIN_HTML,
+            name,
+            root,
+            hasNewHeading: !!newHeading,
+        });
+    }
+    const sizable = candidates.filter((candidate) => candidate.sizable);
+    const totalRawSize = sizable.reduce((sum, candidate) => sum + candidate.rawSize, 0);
+    const largest = Math.max(0, ...sizable.map((candidate) => candidate.rawSize));
+    if (largest < REGION_DOMINANCE * totalRawSize) {
+        debugLog('Changes are scattered across the page, no dominant region');
+        return [];
+    }
+    const identified = candidates.filter((candidate) => candidate.name || candidate.root);
+    if (identified.length < candidates.length)
+        debugLog('Dropped added content with no name and no scoping root');
+    identified.sort((a, b) => Number(b.hasNewHeading) - Number(a.hasNewHeading) || b.size - a.size);
+    return identified;
+}
+const headingsOf = (html) => {
+    const headings = extractHeadings(html);
+    return [headings.h1, headings.h2, headings.h3, headings.h4].filter(Boolean);
+};
+const REGION_MIN_HTML = 5_000;
+const REGION_MAX_RATIO = 0.6;
+const REGION_DOMINANCE = 0.7;

package/dist/src/utils/similarity.d.ts

@@ -0,0 +1 @@
+export declare function diceSimilarity(a: Set<string>, b: Set<string>): number;

package/dist/src/utils/similarity.js

@@ -0,0 +1,11 @@
+export function diceSimilarity(a, b) {
+    let intersection = 0;
+    for (const item of a) {
+        if (b.has(item))
+            intersection++;
+    }
+    const total = a.size + b.size;
+    if (total === 0)
+        return 100;
+    return Math.round(((2 * intersection) / total) * 100);
+}

package/dist/src/utils/url-matcher.d.ts

@@ -2,7 +2,7 @@ export declare function isDynamicSegment(segment: string): boolean;
 export declare function hasDynamicUrlSegment(url: string): boolean;
 export declare function isSamePageFamily(urlA: string, urlB: string): boolean;
 export declare function generalizeSegment(segment: string): string;
-export declare function generalizeUrl(url: string): string;
+export declare function generalizeUrl(url: string, replaceSegment?: (segment: string) => string): string;
 export declare function matchesUrl(pattern: string, path: string): boolean;
 export declare function extractStatePath(url: string): string;
 export declare function matchesNavigationUrl(expected: string, current: string): boolean;

package/dist/src/utils/url-matcher.js

@@ -3,8 +3,8 @@ import { ConfigParser } from '../config.js';
 export function isDynamicSegment(segment) {
     try {
         const configRegex = ConfigParser.getInstance().getConfig().dynamicPageRegex;
-        if (configRegex)
-            return new RegExp(configRegex, 'i').test(segment);
+        if (configRegex && new RegExp(configRegex, 'i').test(segment))
+            return true;
     }
     catch {
         /* config not loaded yet */
@@ -62,10 +62,10 @@ export function generalizeSegment(segment) {
         return '[a-f0-9]+';
     return '[^/]+';
 }
-export function generalizeUrl(url) {
+export function generalizeUrl(url, replaceSegment = generalizeSegment) {
     return url
         .split('/')
-        .map((seg) => (seg.length > 0 && isDynamicSegment(seg) ? generalizeSegment(seg) : seg))
+        .map((seg) => (seg.length > 0 && isDynamicSegment(seg) ? replaceSegment(seg) : seg))
         .join('/');
 }
 export function matchesUrl(pattern, path) {

package/docs/api-testing/basics.md

@@ -56,6 +56,21 @@ api: {
 
 A matching `teardown` hook runs after all tests finish — use it to clean up data.
 
+### Without a config file
+
+Chief and Curler need three things: where the API is, what its spec says, and how to authenticate. Pass all three on the command line and no config file is needed:
+
+```bash
+npx explorbot api plan /users \
+  --endpoint https://api.example.com/v1 \
+  --spec ./openapi.yaml \
+  --knowledge 'Send X-Api-Key: ${env.API_KEY} on every request'
+```
+
+`--endpoint` and `--spec` each have an environment twin — `EXPLORBOT_URL` and `EXPLORBOT_API_SPEC` — and the flag wins when both are set. `--knowledge` adds to the facts `EXPLORBOT_KNOWLEDGE` and `EXPLORBOT_KNOWLEDGE_FILE` bring in rather than replacing them. Configure your models once with `npx explorbot init --global` and every run stores its plans and requests per host under `~/.explorbot/sites/<host>/`, so a later `api test` against the same API picks up where the last one left off. Knowledge given on the command line lasts for the run; `api know` is what writes it down.
+
+`--endpoint` keeps its path prefix: given `https://api.example.com/v1`, steps stay relative (`/users`) and Curler sends them to `https://api.example.com/v1/users`. `api test`, which takes a plan file rather than an endpoint, reads it from the flag or the variable.
+
 ### A dedicated API project
 
 If you don't have a web `explorbot.config.js`, run `npx explorbot api init`. It asks for your base endpoint, spec, and a one-line description of the API, then writes a standalone `apibot.config.ts` (with an `ai` and `api` section) plus `output/` and `knowledge/` directories. When both files exist, `apibot.config.*` takes precedence over `explorbot.config.*`.

package/docs/api-testing/planning.md

@@ -21,7 +21,16 @@ endpoint: "/users"
 CRUD for users. Admin role required for writes. IDs are UUIDs.
 ```
 
-Chief loads knowledge matching the endpoint it's planning. Running `know` again on the same endpoint appends to the file. See [knowledge](../workflow/knowledge.md) for how matching and files work.
+Chief loads knowledge matching the endpoint it's planning, and Curler loads it again for the endpoint it's testing, so auth headers and payload rules reach the requests themselves. Running `know` again on the same endpoint appends to the file. See [knowledge](../workflow/knowledge.md) for how matching and files work.
+
+For a fact that should not be stored — a token, a one-off fixture — pass `--knowledge` instead. It applies to the run only:
+
+```bash
+npx explorbot api explore /users --knowledge '---
+endpoint: /users/*
+---
+Send X-Api-Key: ${env.API_KEY} on every request'
+```
 
 ## Choose a planning style
 

package/docs/doc-collection/basics.md

@@ -105,22 +105,28 @@ Title: User Management
 
 Lists all user accounts and provides entry points for managing them.
 
-## Screenshots
-
-![Page screenshot](../screenshots/admin_users_page.png)
-
 ## User Can
 
 - user can search users by name -> list of items
   Proof: Search input above the users table.
+  ![user can search users by name](../screenshots/admin_users_can_1.png)
 
 ## User Might
 
 - user might export the user list -> all items
   Signal: Toolbar shows an unlabeled download icon.
+
+## Screenshots
+
+![Page screenshot](../screenshots/admin_users_page.png)
+
+## Navigation
+
+- Create user: /admin/users/new
+- Audit log: /admin/audit
 ```
 
-`User Can` lists capabilities proven by visible UI, each with its evidence. `User Might` lists capabilities the UI suggests but research could not fully confirm. Pages where research found almost nothing are skipped rather than padded with guesses.
+`User Can` lists capabilities proven by visible UI, each with its evidence. `User Might` lists capabilities the UI suggests but research could not fully confirm. Links to other pages are collected in their own `Navigation` section and never counted as capabilities. Pages where research found almost nothing are skipped rather than padded with guesses.
 
 ## Next steps
 

package/docs/doc-collection/crawling.md

@@ -4,7 +4,7 @@ With defaults, a run crawls everything it can reach on the same origin, up to 10
 
 ## How the queue is built
 
-After documenting a page, the crawler queues new targets from three sources: links found on the page, navigation entries identified by research (this is how hash-navigated pages such as OpenAPI reference docs get crawled), and URLs discovered through clicks when [interactive mode](./interactive-mode.md) is on. Every target must pass the filters on this page before it is queued. Pages already visited in this session are not revisited, and the crawl stops early if it detects a dead loop.
+After documenting a page, the crawler queues new targets from three sources: links found on the page, navigation entries identified by research (this is how hash-navigated pages such as OpenAPI reference docs get crawled), and URLs discovered through clicks when [interactive mode](./interactive-mode.md) is on. Every target must pass the filters on this page before it is queued. Pages already visited in this session are not revisited.
 
 ## maxPages — the budget
 
@@ -65,3 +65,23 @@ docs: {
 ```
 
 Set it to `false` when such pages genuinely differ and you want each URL documented separately. Expect the page budget to fill up faster.
+
+## collapseTemplatePages — skip pages with a documented layout
+
+URL collapsing cannot see slugs: `/blog/my-post` and `/blog/other-post` look like different pages, yet they share one layout and differ only in their text. By default, after a page is loaded the crawler compares the page structure — the tree of ARIA roles without any text — against the pages it has already documented. A page whose layout matches one of them is skipped and listed under Skipped in the index with the reason `same layout as <url> (only content differs)`. Skipped pages do not consume the page budget.
+
+The first page of each layout is always documented in full; only its clones are skipped.
+
+A page counts as a clone when its structure is at least 90% similar to a documented page. Identical templates score close to 100; pages that merely look alike (same kind of form or list, different purpose) score around 80 and are still documented. If your site leans either way, move the bar with `docs.templateSimilarity` or per-run with `--template-similarity <percent>`; invalid values fall back to the default.
+
+```ts
+docs: {
+  collapseTemplatePages: false,
+}
+```
+
+For a one-off run over every page, use the CLI flag instead:
+
+```bash
+explorbot docs collect /blog --no-collapse-template-pages
+```