@casoon/astro-crawler-policy 0.1.0 → 0.1.2

package/README.md

@@ -53,7 +53,9 @@ Presets are the primary way to express intent. Each preset sets default content
 | `blockTraining` | allow | allow | disallow | disallow |
 | `lockdown` | disallow | disallow | disallow | disallow |
 
-`blockTraining` additionally adds explicit `Disallow` rules for GPTBot, Google-Extended, and CCBot on top of the group-level setting.
+`citationFriendly` allows bots that do citation or summarization but blocks bots whose only purpose is training data collection (GPTBot, Google-Extended, CCBot, Bytespider, Applebot-Extended). Bots with mixed roles like ClaudeBot are allowed.
+
+`blockTraining` goes further and blocks every bot with any training category, including mixed bots like ClaudeBot and meta-externalagent.
 
 `lockdown` adds a global `User-agent: * / Disallow: /` rule, overriding everything.
 
@@ -62,7 +64,9 @@ Presets are the primary way to express intent. Each preset sets default content
 Content signals are non-standard directives appended to the wildcard `User-agent: *` block:
 
 ```
-Content-signal: search=yes, ai-input=yes, ai-train=no
+User-agent: *
+Content-Signal: search=yes, ai-input=yes, ai-train=no
+Allow: /
 ```
 
 They communicate intent to crawlers that support them. The three signals map to:
@@ -73,7 +77,7 @@ They communicate intent to crawlers that support them. The three signals map to:
 | `aiInput` | Using content as input for AI responses (citation, summarization) |
 | `aiTrain` | Using content as AI training data |
 
-Content signals are not yet a web standard. Google Search Console may flag them as unrecognised directives — the audit system emits an `info` message when they are present.
+The directive name and signal keys follow the [contentsignals.org](https://contentsignals.org) specification (proposed IETF aipref standard). Google Search Console may flag them as unrecognised directives — the audit system emits an `info` message when they are present.
 
 Each preset sets default values for all three signals. You can override them individually:
 
@@ -379,8 +383,8 @@ crawlerPolicy({
 # preset: citationFriendly
 
 User-agent: *
+Content-Signal: search=yes, ai-input=yes, ai-train=no
 Allow: /
-Content-signal: search=yes, ai-input=yes, ai-train=no
 
 User-agent: GPTBot
 Disallow: /
@@ -435,8 +439,8 @@ crawlerPolicy({ preset: 'seoOnly' })
 # preset: seoOnly
 
 User-agent: *
+Content-Signal: search=yes, ai-input=no, ai-train=no
 Allow: /
-Content-signal: search=yes, ai-input=no, ai-train=no
 
 User-agent: GPTBot
 Disallow: /
@@ -496,6 +500,10 @@ When `CONTEXT=staging` or `NODE_ENV=staging`:
 # preset: lockdown
 
 User-agent: *
+Content-Signal: search=no, ai-input=no, ai-train=no
 Disallow: /
-Content-signal: search=no, ai-input=no, ai-train=no
 ```
+
+---
+
+> This tool only works for crawlers and AI bots that actually respect robots.txt. Respect, however, is rare these days.

package/dist/presets.js

@@ -21,6 +21,13 @@ export const presetDefaults = {
             searchEngines: 'allow',
             verifiedAi: 'allow',
             unknownAi: 'disallow'
+        },
+        bots: {
+            GPTBot: 'disallow',
+            'Google-Extended': 'disallow',
+            CCBot: 'disallow',
+            Bytespider: 'disallow',
+            'Applebot-Extended': 'disallow'
         }
     },
     openToAi: {
@@ -48,8 +55,12 @@ export const presetDefaults = {
         },
         bots: {
             GPTBot: 'disallow',
+            ClaudeBot: 'disallow',
             'Google-Extended': 'disallow',
-            CCBot: 'disallow'
+            CCBot: 'disallow',
+            Bytespider: 'disallow',
+            'meta-externalagent': 'disallow',
+            'Applebot-Extended': 'disallow'
         }
     },
     lockdown: {

package/dist/render.js

@@ -77,7 +77,7 @@ function renderContentSignals(contentSignals) {
     if (contentSignals.aiTrain !== undefined) {
         parts.push(`ai-train=${contentSignals.aiTrain ? 'yes' : 'no'}`);
     }
-    return parts.length ? [`Content-signal: ${parts.join(', ')}`] : [];
+    return parts.length ? [`Content-Signal: ${parts.join(', ')}`] : [];
 }
 function renderRule(rule) {
     const lines = [];
@@ -109,8 +109,23 @@ export function renderRobotsTxt(policy) {
         allow: ['/'],
         disallow: []
     };
-    lines.push(...renderRule(wildcardRule));
+    // Render wildcard block with Content-Signal before Allow/Disallow (per spec)
+    for (const userAgent of wildcardRule.userAgent) {
+        lines.push(`User-agent: ${userAgent}`);
+    }
+    if (wildcardRule.comment) {
+        lines.push(`# ${wildcardRule.comment}`);
+    }
     lines.push(...renderContentSignals(policy.contentSignals));
+    for (const allow of wildcardRule.allow ?? []) {
+        lines.push(`Allow: ${allow}`);
+    }
+    for (const disallow of wildcardRule.disallow ?? []) {
+        lines.push(`Disallow: ${disallow}`);
+    }
+    if (wildcardRule.crawlDelay !== undefined) {
+        lines.push(`Crawl-delay: ${wildcardRule.crawlDelay}`);
+    }
     lines.push('');
     for (const rule of policy.rules) {
         if (rule === wildcardRule) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@casoon/astro-crawler-policy",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Policy-first crawler control for Astro — generates robots.txt and llms.txt with presets, per-bot rules, AI crawler registry, and build-time audits.",
   "type": "module",
   "main": "./dist/index.js",