geopilot 0.0.0-stage → 0.0.1

package/dist/cli/io.js

@@ -0,0 +1,28 @@
+import { createInterface } from 'node:readline/promises';
+import { colorEnabled, palette } from "../core/tty.js";
+export function isCi(env) {
+    return env.CI !== undefined && env.CI !== '' && env.CI !== '0' && env.CI !== 'false';
+}
+export function processIo(opts, env = process.env) {
+    const interactive = !opts.nonInteractive && !isCi(env) && Boolean(process.stdin.isTTY && process.stdout.isTTY);
+    const ask = async (question) => {
+        const rl = createInterface({ input: process.stdin, output: process.stderr });
+        try {
+            return (await rl.question(question)).trim();
+        }
+        finally {
+            rl.close();
+        }
+    };
+    return {
+        out: (t) => process.stdout.write(t),
+        err: (t) => process.stderr.write(t),
+        interactive,
+        color: palette(!opts.noColor && colorEnabled(env, Boolean(process.stdout.isTTY)), env),
+        errColor: palette(!opts.noColor && colorEnabled(env, Boolean(process.stderr.isTTY)), env),
+        confirm: async (q) => interactive && /^y(es)?$/i.test(await ask(`${q} [y/N] `)),
+        ask: async (q) => (interactive ? ask(q) : ''),
+        env,
+    };
+}
+//# sourceMappingURL=io.js.map

package/dist/cli/io.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"io.js","sourceRoot":"","sources":["../../src/cli/io.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,OAAO,EAAgB,MAAM,gBAAgB,CAAC;AAerE,MAAM,UAAU,IAAI,CAAC,GAAsB;IACzC,OAAO,GAAG,CAAC,EAAE,KAAK,SAAS,IAAI,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,GAAG,CAAC,EAAE,KAAK,OAAO,CAAC;AACvF,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAmD,EAAE,MAAyB,OAAO,CAAC,GAAG;IACjH,MAAM,WAAW,GAAG,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC/G,MAAM,GAAG,GAAG,KAAK,EAAE,QAAgB,EAAmB,EAAE;QACtD,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7E,IAAI,CAAC;YACH,OAAO,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC9C,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,KAAK,EAAE,CAAC;QACb,CAAC;IACH,CAAC,CAAC;IACF,OAAO;QACL,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACnC,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACnC,WAAW;QACX,KAAK,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,IAAI,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;QACtF,QAAQ,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,IAAI,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;QACzF,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,WAAW,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC/E,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,GAAG;KACJ,CAAC;AACJ,CAAC"}

package/dist/cli/main.d.ts

@@ -0,0 +1,15 @@
+#!/usr/bin/env node
+import { type Io } from './io.ts';
+import type { Deps } from './context.ts';
+import { COMMANDS } from './commands-list.ts';
+export { COMMANDS };
+export declare function helpText(): string;
+export declare function commandHelp(name: string): string | null;
+export interface RunDeps {
+    readonly io?: Io;
+    readonly cwd?: string;
+    readonly env?: NodeJS.ProcessEnv;
+    readonly fetcher?: Deps['fetcher'];
+    readonly now?: () => Date;
+}
+export declare function run(argv: readonly string[], d?: RunDeps): Promise<number>;

package/dist/cli/main.js

@@ -0,0 +1,118 @@
+#!/usr/bin/env node
+import { realpathSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { EXIT, isGeoError } from "../core/errors.js";
+import { DISCLAIMER, TOOL_VERSION } from "../core/version.js";
+import { processIo } from "./io.js";
+import { scanCommand } from "./commands/scan.js";
+import { explainCommand, rulesCommand } from "./commands/rules.js";
+import { fixCommand } from "./commands/fix.js";
+import { initCommand } from "./commands/init.js";
+import { agentCommand } from "./commands/agent.js";
+import { insightsCommand, reportCommand } from "./commands/insights.js";
+import { COMMANDS } from "./commands-list.js";
+export { COMMANDS };
+export function helpText() {
+    const lines = [
+        `geopilot ${TOOL_VERSION} — local-first technical readiness auditor`,
+        '',
+        'Usage: geopilot <command> [options]',
+        '',
+        'Commands:',
+        ...COMMANDS.map((c) => `  ${c.name.padEnd(10)} ${c.summary}${c.phase ? ` (${c.phase}; not in this release)` : ''}`),
+        '',
+        'Global options:',
+        '  --config <file>  --cwd <dir>  --format pretty|json|markdown  --output <file>',
+        '  --non-interactive  --yes  --quiet  --verbose  --no-color  --no-cache  --print-config',
+        '',
+        'Exit codes: 0 ok · 1 gate failed / invalid insights · 2 usage or missing approval · 3 invalid config',
+        '            4 incomplete (with --fail-on-incomplete) · 5 fix rolled back · 6 applied but unverified · 70 internal error',
+        '',
+        `Docs: https://moelzanaty3.github.io/geopilot/docs/`,
+        '',
+        DISCLAIMER,
+        '',
+    ];
+    return lines.join('\n');
+}
+export function commandHelp(name) {
+    const c = COMMANDS.find((x) => x.name === name);
+    return c ? `geopilot ${c.usage}\n\n${c.summary}.${c.phase ? ` ${c.phase}: not available in this release.` : ''}\n` : null;
+}
+export async function run(argv, d = {}) {
+    const env = d.env ?? process.env;
+    const io = d.io ?? processIo({ nonInteractive: argv.includes('--non-interactive'), noColor: argv.includes('--no-color') }, env);
+    const deps = { io, cwd: d.cwd ?? process.cwd(), env, ...(d.fetcher ? { fetcher: d.fetcher } : {}), ...(d.now ? { now: d.now } : {}) };
+    const [command, ...rest] = argv;
+    try {
+        if (command === undefined || command === '--help' || command === '-h' || command === 'help') {
+            const topic = command === 'help' ? rest[0] : undefined;
+            io.out(topic ? commandHelp(topic) ?? helpText() : helpText());
+            return EXIT.ok;
+        }
+        if (command === '--version' || command === '-v' || command === 'version') {
+            io.out(`${TOOL_VERSION}\n`);
+            return EXIT.ok;
+        }
+        if (rest.includes('--help') || rest.includes('-h')) {
+            const h = commandHelp(command);
+            if (h) {
+                io.out(h);
+                return EXIT.ok;
+            }
+        }
+        switch (command) {
+            case 'scan':
+                return await scanCommand(rest, deps);
+            case 'explain':
+                return await explainCommand(rest, deps);
+            case 'rules':
+                return await rulesCommand(rest, deps);
+            case 'fix':
+                return await fixCommand(rest, deps);
+            case 'init':
+                return await initCommand(rest, deps);
+            case 'agent':
+                return await agentCommand(rest, deps);
+            case 'insights':
+                return await insightsCommand(rest, deps);
+            case 'report':
+                return await reportCommand(rest, deps);
+            case 'review':
+            case 'baseline':
+            case 'hook':
+                io.err(`${command} is a Phase 3 command and is not available in this release.\n`);
+                return EXIT.usage;
+            default:
+                io.err(`unknown command ${command}\n${helpText()}`);
+                return EXIT.usage;
+        }
+    }
+    catch (e) {
+        if (isGeoError(e)) {
+            io.err(`${io.errColor.red('error')}  ${e.message}\n`);
+            if (e.hint)
+                io.err(`       ${io.errColor.dim(e.hint)}\n`);
+            return e.exitCode;
+        }
+        io.err(`${io.errColor.red('internal error')}  ${e.stack ?? String(e)}\nPlease report it: https://github.com/moelzanaty3/geopilot/issues\n`);
+        return EXIT.internal;
+    }
+}
+function isMain() {
+    try {
+        return process.argv[1] !== undefined && realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url));
+    }
+    catch {
+        return false;
+    }
+}
+if (isMain()) {
+    run(process.argv.slice(2)).then((code) => {
+        process.exitCode = code;
+    }, (e) => {
+        process.stderr.write(`internal error ${String(e)}\n`);
+        process.exitCode = EXIT.internal;
+    });
+}
+//# sourceMappingURL=main.js.map

package/dist/cli/main.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"main.js","sourceRoot":"","sources":["../../src/cli/main.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,SAAS,EAAW,MAAM,SAAS,CAAC;AAE7C,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnE,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAExE,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAE9C,OAAO,EAAE,QAAQ,EAAE,CAAC;AAEpB,MAAM,UAAU,QAAQ;IACtB,MAAM,KAAK,GAAG;QACZ,YAAY,YAAY,4CAA4C;QACpE,EAAE;QACF,qCAAqC;QACrC,EAAE;QACF,WAAW;QACX,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,wBAAwB,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACnH,EAAE;QACF,iBAAiB;QACjB,gFAAgF;QAChF,wFAAwF;QACxF,EAAE;QACF,sGAAsG;QACtG,yHAAyH;QACzH,EAAE;QACF,oDAAoD;QACpD,EAAE;QACF,UAAU;QACV,EAAE;KACH,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;IAChD,OAAO,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,kCAAkC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;AAC5H,CAAC;AAUD,MAAM,CAAC,KAAK,UAAU,GAAG,CAAC,IAAuB,EAAE,IAAa,EAAE;IAChE,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACjC,MAAM,EAAE,GAAG,CAAC,CAAC,EAAE,IAAI,SAAS,CAAC,EAAE,cAAc,EAAE,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAChI,MAAM,IAAI,GAAS,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IAC5I,MAAM,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;IAChC,IAAI,CAAC;QACH,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;YAC5F,MAAM,KAAK,GAAG,OAAO,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACvD,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,QAAQ,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC9D,OAAO,IAAI,CAAC,EAAE,CAAC;QACjB,CAAC;QACD,IAAI,OAAO,KAAK,WAAW,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YACzE,EAAE,CAAC,GAAG,CAAC,GAAG,YAAY,IAAI,CAAC,CAAC;YAC5B,OAAO,IAAI,CAAC,EAAE,CAAC;QACjB,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACnD,MAAM,CAAC,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;YAC/B,IAAI,CAAC,EAAE,CAAC;gBACN,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACV,OAAO,IAAI,CAAC,EAAE,CAAC;YACjB,CAAC;QACH,CAAC;QACD,QAAQ,OAAO,EAAE,CAAC;YAChB,KAAK,MAAM;gBACT,OAAO,MAAM,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACvC,KAAK,SAAS;gBACZ,OAAO,MAAM,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC1C,KAAK,OAAO;gBACV,OAAO,MAAM,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACxC,KAAK,KAAK;gBACR,OAAO,MAAM,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACtC,KAAK,MAAM;gBACT,OAAO,MAAM,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACvC,KAAK,OAAO;gBACV,OAAO,MAAM,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACxC,KAAK,UAAU;gBACb,OAAO,MAAM,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC3C,KAAK,QAAQ;gBACX,OAAO,MAAM,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACzC,KAAK,QAAQ,CAAC;YACd,KAAK,UAAU,CAAC;YAChB,KAAK,MAAM;gBACT,EAAE,CAAC,GAAG,CAAC,GAAG,OAAO,+DAA+D,CAAC,CAAC;gBAClF,OAAO,IAAI,CAAC,KAAK,CAAC;YACpB;gBACE,EAAE,CAAC,GAAG,CAAC,mBAAmB,OAAO,KAAK,QAAQ,EAAE,EAAE,CAAC,CAAC;gBACpD,OAAO,IAAI,CAAC,KAAK,CAAC;QACtB,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;YAClB,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC;YACtD,IAAI,CAAC,CAAC,IAAI;gBAAE,EAAE,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1D,OAAO,CAAC,CAAC,QAAQ,CAAC;QACpB,CAAC;QACD,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,gBAAgB,CAAC,KAAM,CAAW,CAAC,KAAK,IAAI,MAAM,CAAC,CAAC,CAAC,sEAAsE,CAAC,CAAC;QACvJ,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,MAAM;IACb,IAAI,CAAC;QACH,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,YAAY,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACzH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,IAAI,MAAM,EAAE,EAAE,CAAC;IACb,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAC7B,CAAC,IAAI,EAAE,EAAE;QACP,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAC1B,CAAC,EACD,CAAC,CAAU,EAAE,EAAE;QACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACtD,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;IACnC,CAAC,CACF,CAAC;AACJ,CAAC"}

package/dist/collect/crawl.d.ts

@@ -0,0 +1,51 @@
+import { type BodyKind, type FetchResult, type NetPolicy } from '../net/fetch.ts';
+import { type Document, type HeadFacts } from '../html/parse.ts';
+import { type ParsedRobots } from '../robots/robots.ts';
+import { type ParsedSitemap } from '../sitemap/sitemap.ts';
+export type Fetcher = (url: string, kind: BodyKind) => Promise<FetchResult>;
+export interface PageObservation {
+    readonly url: string;
+    readonly result: FetchResult;
+    readonly via: 'target' | 'sitemap' | 'link';
+    readonly depth: number;
+    readonly doc: Document | null;
+    readonly facts: HeadFacts | null;
+    readonly isHtml: boolean;
+}
+export interface UrlEvidence {
+    readonly origin: string;
+    readonly targets: readonly string[];
+    readonly robots: {
+        readonly url: string;
+        readonly result: FetchResult;
+        readonly parsed: ParsedRobots | null;
+    };
+    readonly sitemaps: readonly {
+        readonly url: string;
+        readonly result: FetchResult;
+        readonly parsed: ParsedSitemap | null;
+        readonly via: 'robots' | 'default' | 'index';
+    }[];
+    readonly sitemapUrls: readonly string[];
+    readonly pages: ReadonlyMap<string, PageObservation>;
+    /** Normalised target URL → pages whose server HTML links to it. */
+    readonly inbound: ReadonlyMap<string, ReadonlySet<string>>;
+    readonly extras: ReadonlyMap<string, FetchResult>;
+    readonly skippedByRobots: readonly string[];
+    readonly ignoreRobots: boolean;
+    readonly notes: readonly string[];
+}
+export interface CrawlOptions {
+    readonly targets: readonly string[];
+    readonly maxPages: number;
+    readonly maxDepth: number;
+    readonly followExternal: boolean;
+    readonly ignoreRobots: boolean;
+    readonly include: (path: string) => boolean;
+    readonly extraPaths: readonly string[];
+    readonly fetcher: Fetcher;
+}
+export declare const GEO_UA_TOKEN = "geopilot";
+export declare function normaliseUrl(input: string, base?: string): string | null;
+export declare function defaultFetcher(policy: NetPolicy): Fetcher;
+export declare function crawl(opts: CrawlOptions): Promise<UrlEvidence>;

package/dist/collect/crawl.js

@@ -0,0 +1,187 @@
+import { guardedFetch, HostScheduler } from "../net/fetch.js";
+import { isLoopbackUrl } from "../net/address.js";
+import { headFacts, parseHtml } from "../html/parse.js";
+import { evaluate, parseRobots } from "../robots/robots.js";
+import { parseSitemap } from "../sitemap/sitemap.js";
+export const GEO_UA_TOKEN = 'geopilot';
+export function normaliseUrl(input, base) {
+    try {
+        const u = base ? new URL(input, base) : new URL(input);
+        if (u.protocol !== 'http:' && u.protocol !== 'https:')
+            return null;
+        u.hash = '';
+        return u.href;
+    }
+    catch {
+        return null;
+    }
+}
+export function defaultFetcher(policy) {
+    const scheduler = new HostScheduler();
+    return (url, kind) => {
+        let host = url;
+        let local = false;
+        try {
+            const u = new URL(url);
+            host = u.host;
+            local = isLoopbackUrl(u);
+        }
+        catch {
+            // guardedFetch reports the malformed URL itself
+        }
+        return scheduler.run(host, local, () => guardedFetch(url, kind, policy));
+    };
+}
+function allowedHost(url, origin, followExternal) {
+    if (url.origin === origin.origin)
+        return true;
+    if (!followExternal)
+        return false;
+    const base = origin.hostname.replace(/^www\./, '');
+    return url.hostname === base || url.hostname.endsWith(`.${base}`);
+}
+function isHtmlResponse(r) {
+    const ct = (r.headers['content-type'] ?? '').toLowerCase();
+    return ct === '' ? /^\s*(<!doctype html|<html)/i.test(r.body) : ct.includes('html');
+}
+export async function crawl(opts) {
+    const first = new URL(opts.targets[0]);
+    const origin = new URL(first.origin);
+    const notes = [];
+    const f = opts.fetcher;
+    const robotsUrl = new URL('/robots.txt', origin).href;
+    const robotsResult = await f(robotsUrl, 'robots');
+    const robotsParsed = robotsResult.status !== null && robotsResult.status >= 200 && robotsResult.status < 300 ? parseRobots(robotsResult.body) : null;
+    if (robotsParsed) {
+        const delay = robotsParsed.groups.find((g) => g.agents.includes(GEO_UA_TOKEN) || g.agents.includes('*'))?.crawlDelay;
+        if (delay)
+            notes.push(`robots.txt asks for a crawl-delay of ${delay}s; GEO honours up to 2s`);
+    }
+    const robotsServerError = robotsResult.status !== null && robotsResult.status >= 500;
+    const robotsUnreachable = robotsResult.error !== null;
+    const mayFetch = (url) => {
+        if (opts.ignoreRobots)
+            return true;
+        // RFC 9309 2.3.1.4: an unreachable or 5xx robots.txt means assume complete disallow.
+        if (robotsServerError || robotsUnreachable)
+            return false;
+        if (!robotsParsed)
+            return true;
+        const u = new URL(url);
+        return evaluate(robotsParsed, GEO_UA_TOKEN, `${u.pathname}${u.search}`).allowed;
+    };
+    const sitemaps = [];
+    const sitemapUrls = [];
+    const sitemapQueue = [];
+    for (const s of robotsParsed?.sitemaps ?? []) {
+        const n = normaliseUrl(s.url);
+        if (n && allowedHost(new URL(n), origin, opts.followExternal))
+            sitemapQueue.push({ url: n, via: 'robots' });
+    }
+    if (sitemapQueue.length === 0)
+        sitemapQueue.push({ url: new URL('/sitemap.xml', origin).href, via: 'default' });
+    const seenSitemaps = new Set();
+    while (sitemapQueue.length > 0 && seenSitemaps.size < 6) {
+        const next = sitemapQueue.shift();
+        if (seenSitemaps.has(next.url))
+            continue;
+        seenSitemaps.add(next.url);
+        if (!mayFetch(next.url))
+            continue;
+        const result = await f(next.url, 'sitemap');
+        const ok = result.status !== null && result.status >= 200 && result.status < 300;
+        const parsed = ok ? parseSitemap(result.body) : null;
+        sitemaps.push({ url: next.url, result, parsed, via: next.via });
+        if (parsed?.kind === 'sitemapindex') {
+            for (const loc of parsed.locs.slice(0, 5))
+                sitemapQueue.push({ url: loc, via: 'index' });
+        }
+        else if (parsed?.kind === 'urlset') {
+            for (const loc of parsed.locs)
+                sitemapUrls.push(loc);
+        }
+    }
+    const pages = new Map();
+    const inbound = new Map();
+    const skippedByRobots = [];
+    const queue = [];
+    const queued = new Set();
+    const enqueue = (url, via, depth) => {
+        const n = normaliseUrl(url);
+        if (!n || queued.has(n))
+            return;
+        const u = new URL(n);
+        if (!allowedHost(u, origin, opts.followExternal) || !opts.include(u.pathname))
+            return;
+        queued.add(n);
+        queue.push({ url: n, via, depth });
+    };
+    for (const t of opts.targets)
+        enqueue(t, 'target', 0);
+    for (const s of sitemapUrls)
+        enqueue(s, 'sitemap', 1);
+    while (queue.length > 0 && pages.size < opts.maxPages) {
+        const batch = queue.splice(0, Math.min(4, opts.maxPages - pages.size));
+        await Promise.all(batch.map(async (item) => {
+            if (!mayFetch(item.url)) {
+                skippedByRobots.push(item.url);
+                return;
+            }
+            const result = await f(item.url, 'html');
+            const html = result.error === null && isHtmlResponse(result);
+            const doc = html ? parseHtml(result.body) : null;
+            const facts = doc ? headFacts(doc) : null;
+            pages.set(item.url, { url: item.url, result, via: item.via, depth: item.depth, doc, facts, isHtml: html });
+            if (!facts)
+                return;
+            for (const href of facts.links) {
+                const target = normaliseUrl(href, result.finalUrl);
+                if (!target)
+                    continue;
+                if (!inbound.has(target))
+                    inbound.set(target, new Set());
+                inbound.get(target).add(item.url);
+                if (item.depth < opts.maxDepth)
+                    enqueue(target, 'link', item.depth + 1);
+            }
+        }));
+    }
+    if (queue.length > 0)
+        notes.push(`page budget of ${opts.maxPages} reached; ${queue.length} discovered URLs were not fetched`);
+    if (skippedByRobots.length > 0)
+        notes.push(`${skippedByRobots.length} URLs were not fetched because robots.txt disallows ${GEO_UA_TOKEN}`);
+    const extras = new Map();
+    for (const path of opts.extraPaths) {
+        const url = new URL(path, origin).href;
+        if (!mayFetch(url))
+            continue;
+        extras.set(url, await f(url, path.endsWith('.json') || path.includes('apple-app-site') ? 'json' : 'robots'));
+    }
+    const canonicalTargets = new Set();
+    for (const page of pages.values()) {
+        for (const c of page.facts?.canonical ?? []) {
+            const n = normaliseUrl(c, page.result.finalUrl);
+            if (n && !pages.has(n) && allowedHost(new URL(n), origin, opts.followExternal))
+                canonicalTargets.add(n);
+        }
+    }
+    for (const target of [...canonicalTargets].slice(0, 25)) {
+        if (!mayFetch(target))
+            continue;
+        extras.set(target, await f(target, 'html'));
+    }
+    return {
+        origin: origin.origin,
+        targets: opts.targets,
+        robots: { url: robotsUrl, result: robotsResult, parsed: robotsParsed },
+        sitemaps,
+        sitemapUrls,
+        pages,
+        inbound,
+        extras,
+        skippedByRobots,
+        ignoreRobots: opts.ignoreRobots,
+        notes,
+    };
+}
+//# sourceMappingURL=crawl.js.map

package/dist/collect/crawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/collect/crawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAmD,MAAM,iBAAiB,CAAC;AAC/G,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,SAAS,EAAiC,MAAM,kBAAkB,CAAC;AACvF,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAqB,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAAE,YAAY,EAAsB,MAAM,uBAAuB,CAAC;AAwCzE,MAAM,CAAC,MAAM,YAAY,GAAG,UAAU,CAAC;AAEvC,MAAM,UAAU,YAAY,CAAC,KAAa,EAAE,IAAa;IACvD,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QACvD,IAAI,CAAC,CAAC,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC,QAAQ,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QACnE,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC;QACZ,OAAO,CAAC,CAAC,IAAI,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAAiB;IAC9C,MAAM,SAAS,GAAG,IAAI,aAAa,EAAE,CAAC;IACtC,OAAO,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QACnB,IAAI,IAAI,GAAG,GAAG,CAAC;QACf,IAAI,KAAK,GAAG,KAAK,CAAC;QAClB,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YACvB,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;YACd,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QAC3B,CAAC;QAAC,MAAM,CAAC;YACP,gDAAgD;QAClD,CAAC;QACD,OAAO,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAC3E,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,GAAQ,EAAE,MAAW,EAAE,cAAuB;IACjE,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAC9C,IAAI,CAAC,cAAc;QAAE,OAAO,KAAK,CAAC;IAClC,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACnD,OAAO,GAAG,CAAC,QAAQ,KAAK,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;AACpE,CAAC;AAED,SAAS,cAAc,CAAC,CAAc;IACpC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAC3D,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,6BAA6B,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AACtF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,IAAkB;IAC5C,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC;IAEvB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;IACtD,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAClD,MAAM,YAAY,GAAG,YAAY,CAAC,MAAM,KAAK,IAAI,IAAI,YAAY,CAAC,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACrJ,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC;QACrH,IAAI,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,wCAAwC,KAAK,yBAAyB,CAAC,CAAC;IAChG,CAAC;IACD,MAAM,iBAAiB,GAAG,YAAY,CAAC,MAAM,KAAK,IAAI,IAAI,YAAY,CAAC,MAAM,IAAI,GAAG,CAAC;IACrF,MAAM,iBAAiB,GAAG,YAAY,CAAC,KAAK,KAAK,IAAI,CAAC;IAEtD,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAW,EAAE;QACxC,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO,IAAI,CAAC;QACnC,qFAAqF;QACrF,IAAI,iBAAiB,IAAI,iBAAiB;YAAE,OAAO,KAAK,CAAC;QACzD,IAAI,CAAC,YAAY;YAAE,OAAO,IAAI,CAAC;QAC/B,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,OAAO,QAAQ,CAAC,YAAY,EAAE,YAAY,EAAE,GAAG,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC;IAClF,CAAC,CAAC;IAEF,MAAM,QAAQ,GAA8G,EAAE,CAAC;IAC/H,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,MAAM,YAAY,GAA2D,EAAE,CAAC;IAChF,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,QAAQ,IAAI,EAAE,EAAE,CAAC;QAC7C,MAAM,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI,WAAW,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,cAAc,CAAC;YAAE,YAAY,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC9G,CAAC;IACD,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,YAAY,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC;IAChH,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACvC,OAAO,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACxD,MAAM,IAAI,GAAG,YAAY,CAAC,KAAK,EAAG,CAAC;QACnC,IAAI,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAS;QACzC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAS;QAClC,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAC5C,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC;QACjF,MAAM,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACrD,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAChE,IAAI,MAAM,EAAE,IAAI,KAAK,cAAc,EAAE,CAAC;YACpC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBAAE,YAAY,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;QAC3F,CAAC;aAAM,IAAI,MAAM,EAAE,IAAI,KAAK,QAAQ,EAAE,CAAC;YACrC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI;gBAAE,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,GAAG,EAA2B,CAAC;IACjD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC/C,MAAM,eAAe,GAAa,EAAE,CAAC;IACrC,MAAM,KAAK,GAAkE,EAAE,CAAC;IAChF,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,MAAM,OAAO,GAAG,CAAC,GAAW,EAAE,GAA2B,EAAE,KAAa,EAAQ,EAAE;QAChF,MAAM,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,CAAC,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,OAAO;QAChC,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;QACrB,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;YAAE,OAAO;QACtF,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IACrC,CAAC,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO;QAAE,OAAO,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;IACtD,KAAK,MAAM,CAAC,IAAI,WAAW;QAAE,OAAO,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;IAEtD,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QACvE,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxB,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC/B,OAAO;YACT,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YACzC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,KAAK,IAAI,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;YAC7D,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACjD,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAC1C,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YAC3G,IAAI,CAAC,KAAK;gBAAE,OAAO;YACnB,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC/B,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;gBACnD,IAAI,CAAC,MAAM;oBAAE,SAAS;gBACtB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;oBAAE,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;gBACzD,OAAO,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACnC,IAAI,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,QAAQ;oBAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,QAAQ,aAAa,KAAK,CAAC,MAAM,mCAAmC,CAAC,CAAC;IAC9H,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,MAAM,uDAAuD,YAAY,EAAE,CAAC,CAAC;IAE3I,MAAM,MAAM,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC9C,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QACnC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;QACvC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,SAAS;QAC7B,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/G,CAAC;IACD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC3C,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;QAClC,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,SAAS,IAAI,EAAE,EAAE,CAAC;YAC5C,MAAM,CAAC,GAAG,YAAY,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAChD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,WAAW,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,cAAc,CAAC;gBAAE,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC1G,CAAC;IACH,CAAC;IACD,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,gBAAgB,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;QACxD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,SAAS;QAChC,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,MAAM,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE;QACtE,QAAQ;QACR,WAAW;QACX,KAAK;QACL,OAAO;QACP,MAAM;QACN,eAAe;QACf,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,KAAK;KACN,CAAC;AACJ,CAAC"}

package/dist/config/config.d.ts

@@ -0,0 +1,86 @@
+import type { RuleId, Severity } from '../types.ts';
+export type RenderMode = 'auto' | 'always' | 'never';
+export type CrawlerPolicy = 'allow' | 'block' | 'undecided';
+export type FrameworkSetting = 'auto' | 'nextjs-app' | 'static' | 'generic';
+export interface Suppression {
+    readonly rule: RuleId;
+    readonly routes: readonly string[];
+    readonly reason: string;
+    readonly expires: string | null;
+}
+export interface GeoConfig {
+    readonly version: 1;
+    readonly framework: FrameworkSetting;
+    readonly profile: 'web' | 'app';
+    readonly site: {
+        readonly origin: string | null;
+        readonly language: string | null;
+    };
+    readonly routes: {
+        readonly include: readonly string[];
+        readonly exclude: readonly string[];
+    };
+    readonly crawl: {
+        readonly maxPages: number | null;
+        readonly maxDepth: number;
+        readonly render: RenderMode;
+        readonly requestTimeoutMs: number;
+    };
+    readonly policy: {
+        readonly aiSearchCrawlers: CrawlerPolicy;
+        readonly aiTrainingCrawlers: CrawlerPolicy;
+        readonly userTriggeredFetchers: CrawlerPolicy;
+    };
+    readonly appLinks: {
+        readonly ios: {
+            readonly appIds: readonly string[];
+        } | null;
+        readonly android: {
+            readonly packageName: string;
+            readonly sha256CertFingerprints: readonly string[];
+        } | null;
+    };
+    readonly rules: Readonly<Partial<Record<RuleId, 'off' | {
+        readonly severity: Severity;
+    }>>>;
+    readonly suppress: readonly Suppression[];
+    readonly fix: {
+        readonly verifyCommands: readonly (readonly string[])[];
+        readonly runtimeTimeoutSeconds: number;
+        readonly serve: {
+            readonly command: readonly string[] | null;
+            readonly url: string | null;
+            readonly readyTimeoutSeconds: number;
+        };
+    };
+    readonly thresholds: {
+        readonly failOn: Severity | null;
+        readonly minReadiness: number | null;
+        readonly minCoverage: number;
+    };
+    readonly hook: {
+        readonly failOn: Severity | null;
+        readonly timeoutSeconds: number;
+    };
+    readonly experimental: boolean;
+}
+export declare const DEFAULT_CONFIG: GeoConfig;
+export declare const CONFIG_FILE = "geo.config.json";
+/**
+ * Validate a parsed geo.config.json and merge it over `base`. Unknown keys are
+ * errors (spec 6): a typo in a policy key must not silently fall back to a
+ * default the owner did not choose.
+ */
+export declare function validateConfig(raw: unknown, base?: GeoConfig, source?: string): GeoConfig;
+export declare function readConfigFile(path: string, base: GeoConfig): GeoConfig;
+/** Overrides applied after files: documented GEO_* variables. */
+export declare function envOverrides(env: NodeJS.ProcessEnv, c: GeoConfig): GeoConfig;
+export interface LoadedConfig {
+    readonly config: GeoConfig;
+    readonly files: readonly string[];
+    readonly hash: string;
+}
+/** Root config first, then the workspace config in `cwd`, then environment. */
+export declare function loadConfig(cwd: string, repoRoot: string | null, explicit: string | null, env: NodeJS.ProcessEnv): LoadedConfig;
+/** The effective configuration for --print-config. It holds no secrets by construction. */
+export declare function printableConfig(c: GeoConfig): string;