@casoon/astro-crawler-policy 0.1.0

package/dist/compile.js

@@ -0,0 +1,109 @@
+import { presetDefaults } from './presets.js';
+import { defaultRegistry } from './registry.js';
+function mergeOptions(base, override) {
+    return {
+        ...base,
+        ...override,
+        contentSignals: {
+            ...base.contentSignals,
+            ...override?.contentSignals
+        },
+        bots: {
+            ...base.bots,
+            ...override?.bots
+        },
+        groups: {
+            ...base.groups,
+            ...override?.groups
+        },
+        output: {
+            ...base.output,
+            ...override?.output
+        },
+        audit: {
+            ...base.audit,
+            ...override?.audit
+        },
+        rules: override?.rules ?? base.rules,
+        sitemaps: override?.sitemaps ?? base.sitemaps
+    };
+}
+function classifyDefaultAction(bot, options) {
+    if (bot.verified === false && options.groups?.unknownAi) {
+        return options.groups.unknownAi;
+    }
+    if (bot.categories.includes('search') && options.groups?.searchEngines) {
+        return options.groups.searchEngines;
+    }
+    if (bot.verified &&
+        bot.categories.some((category) => ['ai-search', 'ai-input', 'ai-training'].includes(category)) &&
+        options.groups?.verifiedAi) {
+        return options.groups.verifiedAi;
+    }
+    if (bot.categories.includes('unknown-ai') && options.groups?.unknownAi) {
+        return options.groups.unknownAi;
+    }
+    return bot.defaultAction ?? 'inherit';
+}
+function normalizeRules(rules) {
+    return (rules ?? []).map((rule) => ({
+        ...rule,
+        userAgent: Array.isArray(rule.userAgent) ? rule.userAgent : [rule.userAgent],
+        allow: rule.allow ?? [],
+        disallow: rule.disallow ?? []
+    }));
+}
+function normalizeSitemaps(site, sitemaps) {
+    const values = sitemaps ?? [];
+    const deduped = new Set();
+    for (const sitemap of values) {
+        if (/^https?:\/\//.test(sitemap)) {
+            deduped.add(sitemap);
+            continue;
+        }
+        if (site) {
+            deduped.add(new URL(sitemap, site).toString());
+            continue;
+        }
+        deduped.add(sitemap);
+    }
+    return [...deduped];
+}
+export function compilePolicy(input) {
+    const extraBots = input.options.extraBots ?? [];
+    const registry = [...(input.registry ?? defaultRegistry), ...extraBots];
+    const environment = input.environment;
+    const baseOptions = input.options;
+    if (baseOptions.preset !== undefined && !Object.prototype.hasOwnProperty.call(presetDefaults, baseOptions.preset)) {
+        throw new Error(`astro-crawler-policy: Unknown preset "${baseOptions.preset}". Valid presets are: ${Object.keys(presetDefaults).join(', ')}.`);
+    }
+    const basePreset = baseOptions.preset ?? 'citationFriendly';
+    const presetOptions = presetDefaults[basePreset] ?? {};
+    const withPreset = mergeOptions(presetOptions, baseOptions);
+    const withEnv = mergeOptions(withPreset, environment ? withPreset.env?.[environment] : undefined);
+    const preset = withEnv.preset ?? basePreset;
+    const rules = normalizeRules(withEnv.rules);
+    const botRules = registry.flatMap((bot) => {
+        const resolvedAction = withEnv.bots?.[bot.id] ?? classifyDefaultAction(bot, withEnv);
+        if (resolvedAction === 'inherit') {
+            return [];
+        }
+        return bot.userAgents.map((userAgent) => ({
+            id: bot.id,
+            userAgent,
+            provider: bot.provider,
+            action: resolvedAction,
+            categories: bot.categories,
+            verified: bot.verified ?? false
+        }));
+    });
+    return {
+        preset,
+        mergeStrategy: withEnv.mergeStrategy ?? 'prepend',
+        contentSignals: withEnv.contentSignals ?? {},
+        rules,
+        botRules,
+        sitemaps: normalizeSitemaps(input.site, withEnv.sitemaps),
+        host: withEnv.host
+    };
+}

package/dist/generate.d.ts

@@ -0,0 +1,11 @@
+import { compilePolicy } from './compile.js';
+import type { AstroCrawlerPolicyOptions } from './types.js';
+export declare function generateRobotsTxt(input: {
+    options: AstroCrawlerPolicyOptions;
+    existingRobotsTxt?: string | null;
+    environment?: string;
+    site?: string;
+}): {
+    content: string;
+    policy: ReturnType<typeof compilePolicy>;
+};

package/dist/generate.js

@@ -0,0 +1,13 @@
+import { compilePolicy } from './compile.js';
+import { mergeRobotsTxt } from './merge.js';
+import { renderRobotsTxt } from './render.js';
+export function generateRobotsTxt(input) {
+    const policy = compilePolicy({
+        options: input.options,
+        environment: input.environment,
+        site: input.site
+    });
+    const generated = renderRobotsTxt(policy);
+    const content = mergeRobotsTxt(generated, input.existingRobotsTxt ?? null, policy.mergeStrategy);
+    return { content, policy };
+}

package/dist/index.d.ts

@@ -0,0 +1,9 @@
+export { auditPolicy } from './audit.js';
+export { compilePolicy } from './compile.js';
+export { generateRobotsTxt } from './generate.js';
+export { defaultRegistry, REGISTRY_VERSION } from './registry.js';
+export { mergeRobotsTxt } from './merge.js';
+export { presetDefaults } from './presets.js';
+export { renderLlmsTxt, renderRobotsTxt } from './render.js';
+export type { AstroCrawlerPolicyOptions, AuditIssue, BotAction, BotCategory, ContentSignals, MergeStrategy, Preset, RegistryBot, ResolvedPolicy, RobotsRule } from './types.js';
+export { default } from './integration.js';

package/dist/index.js

@@ -0,0 +1,8 @@
+export { auditPolicy } from './audit.js';
+export { compilePolicy } from './compile.js';
+export { generateRobotsTxt } from './generate.js';
+export { defaultRegistry, REGISTRY_VERSION } from './registry.js';
+export { mergeRobotsTxt } from './merge.js';
+export { presetDefaults } from './presets.js';
+export { renderLlmsTxt, renderRobotsTxt } from './render.js';
+export { default } from './integration.js';

package/dist/integration.d.ts

@@ -0,0 +1,11 @@
+import type { AstroBuildDoneContextLike, AstroConfigLike, AstroCrawlerPolicyOptions, AstroLoggerLike } from './types.js';
+export default function crawlerPolicy(options?: AstroCrawlerPolicyOptions): {
+    name: string;
+    hooks: {
+        'astro:config:setup': ({ config: astroConfig, logger }: {
+            config: AstroConfigLike;
+            logger?: AstroLoggerLike;
+        }) => void;
+        'astro:build:done': ({ dir, logger }: AstroBuildDoneContextLike) => Promise<void>;
+    };
+};

package/dist/integration.js

@@ -0,0 +1,96 @@
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { auditPolicy } from './audit.js';
+import { generateRobotsTxt } from './generate.js';
+import { defaultRegistry, REGISTRY_VERSION } from './registry.js';
+import { renderLlmsTxt } from './render.js';
+function logIssue(logger, issue) {
+    if (issue.level === 'error') {
+        logger?.error?.(issue.message);
+        return;
+    }
+    if (issue.level === 'warn') {
+        logger?.warn?.(issue.message);
+        return;
+    }
+    logger?.info?.(issue.message);
+}
+async function readOptionalFile(pathname) {
+    try {
+        return await readFile(pathname, 'utf8');
+    }
+    catch {
+        return null;
+    }
+}
+export default function crawlerPolicy(options = {}) {
+    let config;
+    return {
+        name: '@casoon/astro-crawler-policy',
+        hooks: {
+            'astro:config:setup': ({ config: astroConfig, logger }) => {
+                config = astroConfig;
+                logger?.info?.('Initializing astro-crawler-policy');
+            },
+            'astro:build:done': async ({ dir, logger }) => {
+                if (options.output?.robotsTxt === false) {
+                    logger?.info?.('Skipping robots.txt generation because output.robotsTxt is false');
+                    return;
+                }
+                const publicDir = config?.publicDir ? fileURLToPath(config.publicDir) : join(process.cwd(), 'public');
+                const site = config?.site ? String(config.site) : undefined;
+                const environment = process.env.CONTEXT ??
+                    process.env.DEPLOYMENT_ENVIRONMENT ??
+                    process.env.NODE_ENV ??
+                    'production';
+                const existingRobotsTxt = await readOptionalFile(join(publicDir, 'robots.txt'));
+                const { content, policy } = generateRobotsTxt({
+                    options,
+                    existingRobotsTxt,
+                    environment,
+                    site
+                });
+                const outputPath = join(fileURLToPath(dir), 'robots.txt');
+                await mkdir(dirname(outputPath), { recursive: true });
+                await writeFile(outputPath, content, 'utf8');
+                if (options.debug) {
+                    logger?.info?.(`[debug] registry version: ${REGISTRY_VERSION}${options.extraBots?.length ? ` + ${options.extraBots.length} extra bot(s)` : ''}`);
+                    logger?.info?.(`[debug] environment: ${environment}`);
+                    logger?.info?.(`[debug] preset: ${policy.preset}`);
+                    const cs = policy.contentSignals;
+                    const signals = Object.entries(cs)
+                        .filter(([, v]) => v !== undefined)
+                        .map(([k, v]) => `${k}=${v ? 'yes' : 'no'}`)
+                        .join(', ');
+                    if (signals)
+                        logger?.info?.(`[debug] content signals: ${signals}`);
+                    for (const rule of policy.botRules) {
+                        logger?.info?.(`[debug] bot: ${rule.userAgent} → ${rule.action}`);
+                    }
+                    for (const sitemap of policy.sitemaps) {
+                        logger?.info?.(`[debug] sitemap: ${sitemap}`);
+                    }
+                }
+                const registry = [...defaultRegistry, ...(options.extraBots ?? [])];
+                const issues = auditPolicy(policy, {
+                    site,
+                    environment,
+                    registry,
+                    rawOptions: options,
+                    warnOnMissingSitemap: options.audit?.warnOnMissingSitemap,
+                    warnOnConflicts: options.audit?.warnOnConflicts
+                });
+                for (const issue of issues) {
+                    logIssue(logger, issue);
+                }
+                logger?.info?.(`Generated robots.txt at ${outputPath}`);
+                if (options.output?.llmsTxt === true) {
+                    const llmsPath = join(fileURLToPath(dir), 'llms.txt');
+                    await writeFile(llmsPath, renderLlmsTxt(policy, site), 'utf8');
+                    logger?.info?.(`Generated llms.txt at ${llmsPath}`);
+                }
+            }
+        }
+    };
+}

package/dist/merge.d.ts

@@ -0,0 +1,2 @@
+import type { MergeStrategy } from './types.js';
+export declare function mergeRobotsTxt(generated: string, existing: string | null, strategy: MergeStrategy): string;

package/dist/merge.js

@@ -0,0 +1,13 @@
+export function mergeRobotsTxt(generated, existing, strategy) {
+    if (!existing?.trim()) {
+        return generated.endsWith('\n') ? generated : `${generated}\n`;
+    }
+    switch (strategy) {
+        case 'replace':
+            return generated.endsWith('\n') ? generated : `${generated}\n`;
+        case 'prepend':
+            return `${generated.trim()}\n\n${existing.trim()}\n`;
+        case 'append':
+            return `${existing.trim()}\n\n${generated.trim()}\n`;
+    }
+}

package/dist/presets.d.ts

@@ -0,0 +1,2 @@
+import type { AstroCrawlerPolicyOptions, Preset } from './types.js';
+export declare const presetDefaults: Record<Preset, Partial<AstroCrawlerPolicyOptions>>;

package/dist/presets.js

@@ -0,0 +1,73 @@
+export const presetDefaults = {
+    seoOnly: {
+        contentSignals: {
+            search: true,
+            aiInput: false,
+            aiTrain: false
+        },
+        groups: {
+            searchEngines: 'allow',
+            verifiedAi: 'disallow',
+            unknownAi: 'disallow'
+        }
+    },
+    citationFriendly: {
+        contentSignals: {
+            search: true,
+            aiInput: true,
+            aiTrain: false
+        },
+        groups: {
+            searchEngines: 'allow',
+            verifiedAi: 'allow',
+            unknownAi: 'disallow'
+        }
+    },
+    openToAi: {
+        contentSignals: {
+            search: true,
+            aiInput: true,
+            aiTrain: true
+        },
+        groups: {
+            searchEngines: 'allow',
+            verifiedAi: 'allow',
+            unknownAi: 'allow'
+        }
+    },
+    blockTraining: {
+        contentSignals: {
+            search: true,
+            aiInput: true,
+            aiTrain: false
+        },
+        groups: {
+            searchEngines: 'allow',
+            verifiedAi: 'allow',
+            unknownAi: 'disallow'
+        },
+        bots: {
+            GPTBot: 'disallow',
+            'Google-Extended': 'disallow',
+            CCBot: 'disallow'
+        }
+    },
+    lockdown: {
+        contentSignals: {
+            search: false,
+            aiInput: false,
+            aiTrain: false
+        },
+        groups: {
+            searchEngines: 'disallow',
+            verifiedAi: 'disallow',
+            unknownAi: 'disallow'
+        },
+        rules: [
+            {
+                userAgent: '*',
+                disallow: ['/']
+            }
+        ]
+    }
+};

package/dist/registry.d.ts

@@ -0,0 +1,4 @@
+import type { RegistryBot } from './types.js';
+/** ISO date of the last registry update. Shown in debug output. */
+export declare const REGISTRY_VERSION = "2026-04-09";
+export declare const defaultRegistry: RegistryBot[];

package/dist/registry.js

@@ -0,0 +1,95 @@
+/** ISO date of the last registry update. Shown in debug output. */
+export const REGISTRY_VERSION = '2026-04-09';
+export const defaultRegistry = [
+    {
+        id: 'GPTBot',
+        provider: 'OpenAI',
+        userAgents: ['GPTBot'],
+        categories: ['ai-training'],
+        verified: true
+    },
+    {
+        id: 'OAI-SearchBot',
+        provider: 'OpenAI',
+        userAgents: ['OAI-SearchBot'],
+        categories: ['ai-search', 'ai-input'],
+        verified: true
+    },
+    {
+        id: 'ClaudeBot',
+        provider: 'Anthropic',
+        userAgents: ['ClaudeBot'],
+        categories: ['ai-input', 'ai-training'],
+        verified: true
+    },
+    {
+        id: 'claude-web',
+        provider: 'Anthropic',
+        userAgents: ['claude-web'],
+        categories: ['ai-input'],
+        verified: true
+    },
+    {
+        id: 'Google-Extended',
+        provider: 'Google',
+        userAgents: ['Google-Extended'],
+        categories: ['ai-training'],
+        verified: true
+    },
+    {
+        id: 'CCBot',
+        provider: 'Common Crawl',
+        userAgents: ['CCBot'],
+        categories: ['ai-training'],
+        verified: true
+    },
+    {
+        id: 'PerplexityBot',
+        provider: 'Perplexity',
+        userAgents: ['PerplexityBot'],
+        categories: ['ai-search', 'ai-input'],
+        verified: true
+    },
+    {
+        id: 'Bytespider',
+        provider: 'ByteDance',
+        userAgents: ['Bytespider'],
+        categories: ['ai-training'],
+        verified: true
+    },
+    {
+        id: 'meta-externalagent',
+        provider: 'Meta',
+        userAgents: ['meta-externalagent'],
+        categories: ['ai-input', 'ai-training'],
+        verified: true
+    },
+    {
+        id: 'Amazonbot',
+        provider: 'Amazon',
+        userAgents: ['Amazonbot'],
+        categories: ['ai-search', 'ai-input'],
+        verified: true
+    },
+    {
+        id: 'Applebot-Extended',
+        provider: 'Apple',
+        userAgents: ['Applebot-Extended'],
+        categories: ['ai-training'],
+        verified: true
+    },
+    {
+        id: 'Googlebot',
+        provider: 'Google',
+        userAgents: ['Googlebot'],
+        categories: ['search'],
+        verified: true
+    },
+    {
+        id: 'Bingbot',
+        provider: 'Microsoft',
+        userAgents: ['Bingbot'],
+        categories: ['search'],
+        verified: true
+    }
+];

package/dist/render.d.ts

@@ -0,0 +1,3 @@
+import type { ResolvedPolicy } from './types.js';
+export declare function renderLlmsTxt(policy: ResolvedPolicy, site?: string): string;
+export declare function renderRobotsTxt(policy: ResolvedPolicy): string;

package/dist/render.js

@@ -0,0 +1,134 @@
+export function renderLlmsTxt(policy, site) {
+    let domain = 'this site';
+    if (site) {
+        try {
+            domain = new URL(site).hostname;
+        }
+        catch {
+            domain = site;
+        }
+    }
+    const lines = [];
+    lines.push(`# ${domain}`);
+    lines.push('');
+    lines.push(`> AI content access policy for ${domain}.`);
+    lines.push(`> Generated by astro-crawler-policy (preset: ${policy.preset}).`);
+    lines.push('');
+    lines.push('## Content Policy');
+    lines.push('');
+    const { search, aiInput, aiTrain } = policy.contentSignals;
+    if (search !== undefined) {
+        lines.push(`- Search indexing: ${search ? 'allowed' : 'not allowed'}`);
+    }
+    if (aiInput !== undefined) {
+        lines.push(`- AI citation and summarization: ${aiInput ? 'allowed' : 'not allowed'}`);
+    }
+    if (aiTrain !== undefined) {
+        lines.push(`- AI training data collection: ${aiTrain ? 'allowed' : 'not allowed'}`);
+    }
+    lines.push('');
+    const seenIds = new Set();
+    const allowedBots = [];
+    const blockedBots = [];
+    for (const bot of policy.botRules) {
+        if (seenIds.has(bot.id))
+            continue;
+        seenIds.add(bot.id);
+        (bot.action === 'allow' ? allowedBots : blockedBots).push(bot);
+    }
+    if (allowedBots.length > 0 || blockedBots.length > 0) {
+        lines.push('## AI Systems');
+        lines.push('');
+        if (allowedBots.length > 0) {
+            lines.push('### Allowed');
+            lines.push('');
+            for (const bot of allowedBots) {
+                lines.push(`- ${bot.id} (${bot.provider})`);
+            }
+            lines.push('');
+        }
+        if (blockedBots.length > 0) {
+            lines.push('### Blocked');
+            lines.push('');
+            for (const bot of blockedBots) {
+                lines.push(`- ${bot.id} (${bot.provider})`);
+            }
+            lines.push('');
+        }
+    }
+    if (policy.sitemaps.length > 0) {
+        lines.push('## Sitemaps');
+        lines.push('');
+        for (const sitemap of policy.sitemaps) {
+            lines.push(`- ${sitemap}`);
+        }
+        lines.push('');
+    }
+    return lines.join('\n').trimEnd() + '\n';
+}
+function renderContentSignals(contentSignals) {
+    const parts = [];
+    if (contentSignals.search !== undefined) {
+        parts.push(`search=${contentSignals.search ? 'yes' : 'no'}`);
+    }
+    if (contentSignals.aiInput !== undefined) {
+        parts.push(`ai-input=${contentSignals.aiInput ? 'yes' : 'no'}`);
+    }
+    if (contentSignals.aiTrain !== undefined) {
+        parts.push(`ai-train=${contentSignals.aiTrain ? 'yes' : 'no'}`);
+    }
+    return parts.length ? [`Content-signal: ${parts.join(', ')}`] : [];
+}
+function renderRule(rule) {
+    const lines = [];
+    for (const userAgent of rule.userAgent) {
+        lines.push(`User-agent: ${userAgent}`);
+    }
+    if (rule.comment) {
+        lines.push(`# ${rule.comment}`);
+    }
+    for (const allow of rule.allow ?? []) {
+        lines.push(`Allow: ${allow}`);
+    }
+    for (const disallow of rule.disallow ?? []) {
+        lines.push(`Disallow: ${disallow}`);
+    }
+    if (rule.crawlDelay !== undefined) {
+        lines.push(`Crawl-delay: ${rule.crawlDelay}`);
+    }
+    return lines;
+}
+export function renderRobotsTxt(policy) {
+    const lines = [
+        '# Generated by astro-crawler-policy',
+        `# preset: ${policy.preset}`,
+        ''
+    ];
+    const wildcardRule = policy.rules.find((rule) => rule.userAgent.includes('*')) ?? {
+        userAgent: ['*'],
+        allow: ['/'],
+        disallow: []
+    };
+    lines.push(...renderRule(wildcardRule));
+    lines.push(...renderContentSignals(policy.contentSignals));
+    lines.push('');
+    for (const rule of policy.rules) {
+        if (rule === wildcardRule) {
+            continue;
+        }
+        lines.push(...renderRule(rule));
+        lines.push('');
+    }
+    for (const botRule of policy.botRules) {
+        lines.push(`User-agent: ${botRule.userAgent}`);
+        lines.push(botRule.action === 'allow' ? 'Allow: /' : 'Disallow: /');
+        lines.push('');
+    }
+    if (policy.host) {
+        lines.push(`Host: ${policy.host}`);
+    }
+    for (const sitemap of policy.sitemaps) {
+        lines.push(`Sitemap: ${sitemap}`);
+    }
+    return `${lines.join('\n').trim()}\n`;
+}