@jimhoyd/urlcode 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (254) hide show
  1. package/CONTRIBUTING.md +80 -0
  2. package/LICENSE +202 -0
  3. package/NOTICE +20 -0
  4. package/README.md +342 -0
  5. package/ROADMAP.md +248 -0
  6. package/SECURITY.md +40 -0
  7. package/data/agents/LICENSES/ai-robots-txt.txt +24 -0
  8. package/data/agents/LICENSES/crawler-user-agents.txt +23 -0
  9. package/data/agents/ai-crawlers.json +1240 -0
  10. package/data/agents/crawlers.json +10515 -0
  11. package/data/agents/index.d.ts +3 -0
  12. package/data/agents/index.js +2453 -0
  13. package/data/agents/monitoring.json +1520 -0
  14. package/data/agents/seo.json +3865 -0
  15. package/dist/BUILD-MANIFEST.json +60 -0
  16. package/dist/adapters.js +59 -0
  17. package/dist/agent-lists.js +59 -0
  18. package/dist/assets.js +129 -0
  19. package/dist/authoring.js +68 -0
  20. package/dist/aws.js +110 -0
  21. package/dist/build-cloudflare.js +164 -0
  22. package/dist/cli.js +185 -0
  23. package/dist/client-address.js +85 -0
  24. package/dist/cloudflare.js +165 -0
  25. package/dist/compliance-rules/baseline.js +98 -0
  26. package/dist/compliance-rules/privacy.js +38 -0
  27. package/dist/compliance-rules/shared.js +62 -0
  28. package/dist/compliance-rules/strict.js +61 -0
  29. package/dist/compliance.js +219 -0
  30. package/dist/config-worker.js +9 -0
  31. package/dist/config.js +172 -0
  32. package/dist/errors.js +8 -0
  33. package/dist/function-sources.js +54 -0
  34. package/dist/function-worker.js +105 -0
  35. package/dist/functions.js +174 -0
  36. package/dist/guest-api.js +114 -0
  37. package/dist/header-validation.js +18 -0
  38. package/dist/http-policy.js +74 -0
  39. package/dist/http-response.js +84 -0
  40. package/dist/index.js +11 -0
  41. package/dist/link-api.js +136 -0
  42. package/dist/link-cli.js +141 -0
  43. package/dist/link-events.js +76 -0
  44. package/dist/link-records.js +31 -0
  45. package/dist/link-store-worker.js +150 -0
  46. package/dist/link-store.js +250 -0
  47. package/dist/logging.js +22 -0
  48. package/dist/management-policy.js +41 -0
  49. package/dist/match.js +124 -0
  50. package/dist/observability.js +242 -0
  51. package/dist/plugins.js +74 -0
  52. package/dist/policies/agents.js +248 -0
  53. package/dist/policies/cache.js +297 -0
  54. package/dist/policies/compression.js +187 -0
  55. package/dist/policies/security.js +0 -0
  56. package/dist/policies/throttle.js +131 -0
  57. package/dist/policies.js +142 -0
  58. package/dist/policy.js +55 -0
  59. package/dist/prerender.js +165 -0
  60. package/dist/project-tests.js +23 -0
  61. package/dist/readiness.js +216 -0
  62. package/dist/router.js +166 -0
  63. package/dist/runtime.js +244 -0
  64. package/dist/scaffold.js +0 -0
  65. package/dist/scripts/operational-drills.js +75 -0
  66. package/dist/server.js +270 -0
  67. package/dist/site.js +0 -0
  68. package/dist/sqlite-version.js +6 -0
  69. package/dist/types/adapters.d.ts +12 -0
  70. package/dist/types/agent-lists.d.ts +5 -0
  71. package/dist/types/assets.d.ts +13 -0
  72. package/dist/types/authoring.d.ts +2 -0
  73. package/dist/types/aws.d.ts +35 -0
  74. package/dist/types/build-cloudflare.d.ts +14 -0
  75. package/dist/types/cli.d.ts +2 -0
  76. package/dist/types/client-address.d.ts +9 -0
  77. package/dist/types/cloudflare.d.ts +75 -0
  78. package/dist/types/compliance-rules/baseline.d.ts +12 -0
  79. package/dist/types/compliance-rules/privacy.d.ts +6 -0
  80. package/dist/types/compliance-rules/shared.d.ts +24 -0
  81. package/dist/types/compliance-rules/strict.d.ts +9 -0
  82. package/dist/types/compliance.d.ts +119 -0
  83. package/dist/types/config-worker.d.ts +1 -0
  84. package/dist/types/config.d.ts +21 -0
  85. package/dist/types/errors.d.ts +7 -0
  86. package/dist/types/function-sources.d.ts +21 -0
  87. package/dist/types/function-worker.d.ts +1 -0
  88. package/dist/types/functions.d.ts +92 -0
  89. package/dist/types/guest-api.d.ts +16 -0
  90. package/dist/types/header-validation.d.ts +2 -0
  91. package/dist/types/http-policy.d.ts +39 -0
  92. package/dist/types/http-response.d.ts +43 -0
  93. package/dist/types/index.d.ts +11 -0
  94. package/dist/types/link-api.d.ts +30 -0
  95. package/dist/types/link-cli.d.ts +36 -0
  96. package/dist/types/link-events.d.ts +27 -0
  97. package/dist/types/link-records.d.ts +11 -0
  98. package/dist/types/link-store-worker.d.ts +1 -0
  99. package/dist/types/link-store.d.ts +130 -0
  100. package/dist/types/logging.d.ts +9 -0
  101. package/dist/types/management-policy.d.ts +9 -0
  102. package/dist/types/match.d.ts +81 -0
  103. package/dist/types/observability.d.ts +104 -0
  104. package/dist/types/plugins.d.ts +27 -0
  105. package/dist/types/policies/agents.d.ts +86 -0
  106. package/dist/types/policies/cache.d.ts +82 -0
  107. package/dist/types/policies/compression.d.ts +41 -0
  108. package/dist/types/policies/security.d.ts +37 -0
  109. package/dist/types/policies/throttle.d.ts +55 -0
  110. package/dist/types/policies.d.ts +36 -0
  111. package/dist/types/policy.d.ts +21 -0
  112. package/dist/types/prerender.d.ts +31 -0
  113. package/dist/types/project-tests.d.ts +13 -0
  114. package/dist/types/readiness.d.ts +121 -0
  115. package/dist/types/router.d.ts +11 -0
  116. package/dist/types/runtime.d.ts +91 -0
  117. package/dist/types/scaffold.d.ts +17 -0
  118. package/dist/types/server.d.ts +37 -0
  119. package/dist/types/site.d.ts +12 -0
  120. package/dist/types/sqlite-version.d.ts +1 -0
  121. package/dist/types/types.d.ts +332 -0
  122. package/dist/types/vercel.d.ts +12 -0
  123. package/dist/types.js +177 -0
  124. package/dist/vercel.js +65 -0
  125. package/docs/AI-AUTHORING.md +113 -0
  126. package/docs/ASSETS.md +106 -0
  127. package/docs/AWS.md +77 -0
  128. package/docs/BEST-PRACTICES.md +267 -0
  129. package/docs/CAPACITY.md +208 -0
  130. package/docs/CLOUDFLARE.md +110 -0
  131. package/docs/COMPLIANCE.md +242 -0
  132. package/docs/DYNAMIC-LINKS.md +561 -0
  133. package/docs/FUNCTION-SECURITY.md +113 -0
  134. package/docs/HTTP.md +129 -0
  135. package/docs/INSTALL.md +90 -0
  136. package/docs/LOAD-TESTING.md +91 -0
  137. package/docs/LOCAL-DEVELOPMENT.md +99 -0
  138. package/docs/MANAGEMENT-SECURITY.md +82 -0
  139. package/docs/MIDDLEWARE.md +83 -0
  140. package/docs/MONITORING.md +132 -0
  141. package/docs/OBSERVABILITY.md +229 -0
  142. package/docs/OPERATIONAL-PROOF.md +42 -0
  143. package/docs/OPERATIONS.md +207 -0
  144. package/docs/ORGANIZATION.md +135 -0
  145. package/docs/PERFORMANCE.md +72 -0
  146. package/docs/PLUGINS.md +234 -0
  147. package/docs/POLICIES.md +489 -0
  148. package/docs/PRERENDER.md +193 -0
  149. package/docs/PROJECT-DIRECTION.md +51 -0
  150. package/docs/READINESS.md +125 -0
  151. package/docs/README.md +58 -0
  152. package/docs/RELEASE-READINESS.md +95 -0
  153. package/docs/RELEASE-SECURITY.md +72 -0
  154. package/docs/RESILIENCE.md +160 -0
  155. package/docs/ROUTING.md +95 -0
  156. package/docs/SANDBOX-REVIEW.md +59 -0
  157. package/docs/SCAFFOLDING.md +72 -0
  158. package/docs/SECURITY-AUDIT.md +124 -0
  159. package/docs/SITE.md +150 -0
  160. package/docs/SPECIFICATION.md +232 -0
  161. package/docs/SPIKE-EXTENSIONS.md +475 -0
  162. package/docs/STANDARDS.md +303 -0
  163. package/docs/STARTERS.md +51 -0
  164. package/docs/TUNNELS.md +75 -0
  165. package/docs/TYPESCRIPT.md +102 -0
  166. package/docs/VERCEL.md +107 -0
  167. package/docs/YAML-GUIDE.md +521 -0
  168. package/docs/YAML-REFERENCE.md +328 -0
  169. package/docs/policies/agents.md +182 -0
  170. package/docs/policies/cache.md +139 -0
  171. package/docs/policies/compression.md +168 -0
  172. package/docs/policies/security.md +160 -0
  173. package/docs/policies/throttle.md +103 -0
  174. package/examples/assets/.env.example +1 -0
  175. package/examples/assets/.gitattributes +2 -0
  176. package/examples/assets/Makefile +30 -0
  177. package/examples/assets/README.md +18 -0
  178. package/examples/assets/functions/hello.mjs +3 -0
  179. package/examples/assets/gitignore.template +7 -0
  180. package/examples/assets/public/about.html +2 -0
  181. package/examples/assets/public/assets/example.txt +1 -0
  182. package/examples/assets/public/guide.txt +1 -0
  183. package/examples/assets/tests/requests.json +44 -0
  184. package/examples/assets/urlcode.yaml +27 -0
  185. package/examples/aws/README.md +13 -0
  186. package/examples/aws/handler.mjs +5 -0
  187. package/examples/aws/public/index.html +3 -0
  188. package/examples/aws/public/notes.txt +1 -0
  189. package/examples/aws/template.yaml +33 -0
  190. package/examples/aws/tests/requests.json +9 -0
  191. package/examples/aws/urlcode.yaml +19 -0
  192. package/examples/cloudflare/README.md +18 -0
  193. package/examples/cloudflare/package.json +14 -0
  194. package/examples/cloudflare/tests/requests.json +9 -0
  195. package/examples/cloudflare/urlcode.yaml +24 -0
  196. package/examples/cloudflare/wrangler.toml +5 -0
  197. package/examples/compliance/README.md +20 -0
  198. package/examples/compliance/rules.mjs +65 -0
  199. package/examples/cookbook/README.md +21 -0
  200. package/examples/cookbook/functions/choice.mjs +4 -0
  201. package/examples/cookbook/functions/echo.mjs +3 -0
  202. package/examples/cookbook/functions/hello.mjs +3 -0
  203. package/examples/cookbook/functions/text.mjs +3 -0
  204. package/examples/cookbook/middleware/headers.mjs +6 -0
  205. package/examples/cookbook/public/about.html +1 -0
  206. package/examples/cookbook/public/assets/index.html +1 -0
  207. package/examples/cookbook/public/assets/site.css +1 -0
  208. package/examples/cookbook/public/favicon.svg +1 -0
  209. package/examples/cookbook/public/guide.txt +1 -0
  210. package/examples/cookbook/public/llms.txt +6 -0
  211. package/examples/cookbook/routes/code.yaml +55 -0
  212. package/examples/cookbook/routes/files.yaml +17 -0
  213. package/examples/cookbook/routes/policies.yaml +28 -0
  214. package/examples/cookbook/routes/redirects.yaml +38 -0
  215. package/examples/cookbook/routes/responses.yaml +26 -0
  216. package/examples/cookbook/tests/requests.json +254 -0
  217. package/examples/cookbook/urlcode.yaml +22 -0
  218. package/examples/live-links/README.md +11 -0
  219. package/examples/live-links/tests/requests.json +6 -0
  220. package/examples/live-links/urlcode.yaml +16 -0
  221. package/examples/monitoring/blackbox-jobs.yaml +37 -0
  222. package/examples/monitoring/prometheus-rules.yaml +88 -0
  223. package/examples/monitoring/prometheus-scrape.yaml +20 -0
  224. package/examples/monitoring/vector.toml +65 -0
  225. package/examples/prerender/README.md +48 -0
  226. package/examples/prerender/functions/page.mjs +6 -0
  227. package/examples/prerender/middleware/template.mjs +29 -0
  228. package/examples/prerender/prerender.d.mts +4 -0
  229. package/examples/prerender/prerender.mjs +63 -0
  230. package/examples/prerender/tests/requests.json +39 -0
  231. package/examples/prerender/urlcode.yaml +50 -0
  232. package/examples/tunnel/dev-with-ngrok.sh +61 -0
  233. package/examples/vercel/README.md +13 -0
  234. package/examples/vercel/api/index.js +5 -0
  235. package/examples/vercel/package.json +8 -0
  236. package/examples/vercel/public/index.html +3 -0
  237. package/examples/vercel/public/notes.txt +1 -0
  238. package/examples/vercel/tests/requests.json +48 -0
  239. package/examples/vercel/urlcode.yaml +19 -0
  240. package/examples/vercel/vercel.json +10 -0
  241. package/llms.txt +46 -0
  242. package/package.json +123 -0
  243. package/schemas/urlcode.schema.json +1230 -0
  244. package/starters/default/.gitattributes +1 -0
  245. package/starters/default/Makefile +30 -0
  246. package/starters/default/README.md +34 -0
  247. package/starters/default/functions/hello.mjs +3 -0
  248. package/starters/default/gitignore.template +8 -0
  249. package/starters/default/middleware/headers.mjs +6 -0
  250. package/starters/default/routes/functions.yaml +20 -0
  251. package/starters/default/routes/marketing/links.yaml +7 -0
  252. package/starters/default/starter.json +5 -0
  253. package/starters/default/tests/requests.json +56 -0
  254. package/starters/default/urlcode.yaml +8 -0
package/data/agents/ai-crawlers.json ADDED
@@ -0,0 +1,1240 @@
1
+ {
2
+ "name": "ai-crawlers",
3
+ "description": "AI training, search and assistant crawlers from ai.robots.txt (every agent in robots.json).",
4
+ "source": {
5
+ "repository": "ai-robots-txt/ai.robots.txt",
6
+ "url": "https://github.com/ai-robots-txt/ai.robots.txt",
7
+ "license": "MIT",
8
+ "tag": "v1.52",
9
+ "commit": "2acefa38cce025b5cf9911a4415f26b7285c42e8",
10
+ "file": "robots.json",
11
+ "fetchedAt": "2026-09-17"
12
+ },
13
+ "entries": [
14
+ {
15
+ "name": "AddSearchBot",
16
+ "pattern": "AddSearchBot",
17
+ "source": "ai-robots-txt/ai.robots.txt",
18
+ "sourceRevision": "v1.52@2acefa38cce0",
19
+ "addedAt": "2026-09-17"
20
+ },
21
+ {
22
+ "name": "AgentTimes",
23
+ "pattern": "AgentTimes",
24
+ "source": "ai-robots-txt/ai.robots.txt",
25
+ "sourceRevision": "v1.52@2acefa38cce0",
26
+ "addedAt": "2026-09-17"
27
+ },
28
+ {
29
+ "name": "AI2Bot",
30
+ "pattern": "AI2Bot",
31
+ "source": "ai-robots-txt/ai.robots.txt",
32
+ "sourceRevision": "v1.52@2acefa38cce0",
33
+ "addedAt": "2026-09-17"
34
+ },
35
+ {
36
+ "name": "AI2Bot-DeepResearchEval",
37
+ "pattern": "AI2Bot\\-DeepResearchEval",
38
+ "source": "ai-robots-txt/ai.robots.txt",
39
+ "sourceRevision": "v1.52@2acefa38cce0",
40
+ "addedAt": "2026-09-17"
41
+ },
42
+ {
43
+ "name": "Ai2Bot-Dolma",
44
+ "pattern": "Ai2Bot\\-Dolma",
45
+ "source": "ai-robots-txt/ai.robots.txt",
46
+ "sourceRevision": "v1.52@2acefa38cce0",
47
+ "addedAt": "2026-09-17"
48
+ },
49
+ {
50
+ "name": "aiHitBot",
51
+ "pattern": "aiHitBot",
52
+ "source": "ai-robots-txt/ai.robots.txt",
53
+ "sourceRevision": "v1.52@2acefa38cce0",
54
+ "addedAt": "2026-09-17"
55
+ },
56
+ {
57
+ "name": "AIWebIndex",
58
+ "pattern": "AIWebIndex",
59
+ "source": "ai-robots-txt/ai.robots.txt",
60
+ "sourceRevision": "v1.52@2acefa38cce0",
61
+ "addedAt": "2026-09-17"
62
+ },
63
+ {
64
+ "name": "amazon-kendra",
65
+ "pattern": "amazon\\-kendra",
66
+ "source": "ai-robots-txt/ai.robots.txt",
67
+ "sourceRevision": "v1.52@2acefa38cce0",
68
+ "addedAt": "2026-09-17"
69
+ },
70
+ {
71
+ "name": "amazon-QBusiness",
72
+ "pattern": "amazon\\-QBusiness",
73
+ "source": "ai-robots-txt/ai.robots.txt",
74
+ "sourceRevision": "v1.52@2acefa38cce0",
75
+ "addedAt": "2026-09-17"
76
+ },
77
+ {
78
+ "name": "Amazonbot",
79
+ "pattern": "Amazonbot",
80
+ "source": "ai-robots-txt/ai.robots.txt",
81
+ "sourceRevision": "v1.52@2acefa38cce0",
82
+ "addedAt": "2026-09-17"
83
+ },
84
+ {
85
+ "name": "AmazonBuyForMe",
86
+ "pattern": "AmazonBuyForMe",
87
+ "source": "ai-robots-txt/ai.robots.txt",
88
+ "sourceRevision": "v1.52@2acefa38cce0",
89
+ "addedAt": "2026-09-17"
90
+ },
91
+ {
92
+ "name": "Amzn-SearchBot",
93
+ "pattern": "Amzn\\-SearchBot",
94
+ "source": "ai-robots-txt/ai.robots.txt",
95
+ "sourceRevision": "v1.52@2acefa38cce0",
96
+ "addedAt": "2026-09-17"
97
+ },
98
+ {
99
+ "name": "Amzn-User",
100
+ "pattern": "Amzn\\-User",
101
+ "source": "ai-robots-txt/ai.robots.txt",
102
+ "sourceRevision": "v1.52@2acefa38cce0",
103
+ "addedAt": "2026-09-17"
104
+ },
105
+ {
106
+ "name": "Andibot",
107
+ "pattern": "Andibot",
108
+ "source": "ai-robots-txt/ai.robots.txt",
109
+ "sourceRevision": "v1.52@2acefa38cce0",
110
+ "addedAt": "2026-09-17"
111
+ },
112
+ {
113
+ "name": "Anomura",
114
+ "pattern": "Anomura",
115
+ "source": "ai-robots-txt/ai.robots.txt",
116
+ "sourceRevision": "v1.52@2acefa38cce0",
117
+ "addedAt": "2026-09-17"
118
+ },
119
+ {
120
+ "name": "anthropic-ai",
121
+ "pattern": "anthropic\\-ai",
122
+ "source": "ai-robots-txt/ai.robots.txt",
123
+ "sourceRevision": "v1.52@2acefa38cce0",
124
+ "addedAt": "2026-09-17"
125
+ },
126
+ {
127
+ "name": "ApifyBot",
128
+ "pattern": "ApifyBot",
129
+ "source": "ai-robots-txt/ai.robots.txt",
130
+ "sourceRevision": "v1.52@2acefa38cce0",
131
+ "addedAt": "2026-09-17"
132
+ },
133
+ {
134
+ "name": "ApifyWebsiteContentCrawler",
135
+ "pattern": "ApifyWebsiteContentCrawler",
136
+ "source": "ai-robots-txt/ai.robots.txt",
137
+ "sourceRevision": "v1.52@2acefa38cce0",
138
+ "addedAt": "2026-09-17"
139
+ },
140
+ {
141
+ "name": "Applebot",
142
+ "pattern": "Applebot",
143
+ "source": "ai-robots-txt/ai.robots.txt",
144
+ "sourceRevision": "v1.52@2acefa38cce0",
145
+ "addedAt": "2026-09-17"
146
+ },
147
+ {
148
+ "name": "Applebot-Extended",
149
+ "pattern": "Applebot\\-Extended",
150
+ "source": "ai-robots-txt/ai.robots.txt",
151
+ "sourceRevision": "v1.52@2acefa38cce0",
152
+ "addedAt": "2026-09-17"
153
+ },
154
+ {
155
+ "name": "Aranet-SearchBot",
156
+ "pattern": "Aranet\\-SearchBot",
157
+ "source": "ai-robots-txt/ai.robots.txt",
158
+ "sourceRevision": "v1.52@2acefa38cce0",
159
+ "addedAt": "2026-09-17"
160
+ },
161
+ {
162
+ "name": "atlassian-bot",
163
+ "pattern": "atlassian\\-bot",
164
+ "source": "ai-robots-txt/ai.robots.txt",
165
+ "sourceRevision": "v1.52@2acefa38cce0",
166
+ "addedAt": "2026-09-17"
167
+ },
168
+ {
169
+ "name": "Awario",
170
+ "pattern": "Awario",
171
+ "source": "ai-robots-txt/ai.robots.txt",
172
+ "sourceRevision": "v1.52@2acefa38cce0",
173
+ "addedAt": "2026-09-17"
174
+ },
175
+ {
176
+ "name": "AzureAI-SearchBot",
177
+ "pattern": "AzureAI\\-SearchBot",
178
+ "source": "ai-robots-txt/ai.robots.txt",
179
+ "sourceRevision": "v1.52@2acefa38cce0",
180
+ "addedAt": "2026-09-17"
181
+ },
182
+ {
183
+ "name": "bedrockbot",
184
+ "pattern": "bedrockbot",
185
+ "source": "ai-robots-txt/ai.robots.txt",
186
+ "sourceRevision": "v1.52@2acefa38cce0",
187
+ "addedAt": "2026-09-17"
188
+ },
189
+ {
190
+ "name": "bigsur.ai",
191
+ "pattern": "bigsur\\.ai",
192
+ "source": "ai-robots-txt/ai.robots.txt",
193
+ "sourceRevision": "v1.52@2acefa38cce0",
194
+ "addedAt": "2026-09-17"
195
+ },
196
+ {
197
+ "name": "Bravebot",
198
+ "pattern": "Bravebot",
199
+ "source": "ai-robots-txt/ai.robots.txt",
200
+ "sourceRevision": "v1.52@2acefa38cce0",
201
+ "addedAt": "2026-09-17"
202
+ },
203
+ {
204
+ "name": "Brightbot",
205
+ "pattern": "Brightbot",
206
+ "source": "ai-robots-txt/ai.robots.txt",
207
+ "sourceRevision": "v1.52@2acefa38cce0",
208
+ "addedAt": "2026-09-17"
209
+ },
210
+ {
211
+ "name": "Brightbot 1.0",
212
+ "pattern": "Brightbot 1\\.0",
213
+ "source": "ai-robots-txt/ai.robots.txt",
214
+ "sourceRevision": "v1.52@2acefa38cce0",
215
+ "addedAt": "2026-09-17"
216
+ },
217
+ {
218
+ "name": "BuddyBot",
219
+ "pattern": "BuddyBot",
220
+ "source": "ai-robots-txt/ai.robots.txt",
221
+ "sourceRevision": "v1.52@2acefa38cce0",
222
+ "addedAt": "2026-09-17"
223
+ },
224
+ {
225
+ "name": "Bytespider",
226
+ "pattern": "Bytespider",
227
+ "source": "ai-robots-txt/ai.robots.txt",
228
+ "sourceRevision": "v1.52@2acefa38cce0",
229
+ "addedAt": "2026-09-17"
230
+ },
231
+ {
232
+ "name": "CCBot",
233
+ "pattern": "CCBot",
234
+ "source": "ai-robots-txt/ai.robots.txt",
235
+ "sourceRevision": "v1.52@2acefa38cce0",
236
+ "addedAt": "2026-09-17"
237
+ },
238
+ {
239
+ "name": "Channel3Bot",
240
+ "pattern": "Channel3Bot",
241
+ "source": "ai-robots-txt/ai.robots.txt",
242
+ "sourceRevision": "v1.52@2acefa38cce0",
243
+ "addedAt": "2026-09-17"
244
+ },
245
+ {
246
+ "name": "ChatGLM-Spider",
247
+ "pattern": "ChatGLM\\-Spider",
248
+ "source": "ai-robots-txt/ai.robots.txt",
249
+ "sourceRevision": "v1.52@2acefa38cce0",
250
+ "addedAt": "2026-09-17"
251
+ },
252
+ {
253
+ "name": "ChatGPT Agent",
254
+ "pattern": "ChatGPT Agent",
255
+ "source": "ai-robots-txt/ai.robots.txt",
256
+ "sourceRevision": "v1.52@2acefa38cce0",
257
+ "addedAt": "2026-09-17"
258
+ },
259
+ {
260
+ "name": "ChatGPT-User",
261
+ "pattern": "ChatGPT\\-User",
262
+ "source": "ai-robots-txt/ai.robots.txt",
263
+ "sourceRevision": "v1.52@2acefa38cce0",
264
+ "addedAt": "2026-09-17"
265
+ },
266
+ {
267
+ "name": "Claude-Code",
268
+ "pattern": "Claude\\-Code",
269
+ "source": "ai-robots-txt/ai.robots.txt",
270
+ "sourceRevision": "v1.52@2acefa38cce0",
271
+ "addedAt": "2026-09-17"
272
+ },
273
+ {
274
+ "name": "Claude-SearchBot",
275
+ "pattern": "Claude\\-SearchBot",
276
+ "source": "ai-robots-txt/ai.robots.txt",
277
+ "sourceRevision": "v1.52@2acefa38cce0",
278
+ "addedAt": "2026-09-17"
279
+ },
280
+ {
281
+ "name": "Claude-User",
282
+ "pattern": "Claude\\-User",
283
+ "source": "ai-robots-txt/ai.robots.txt",
284
+ "sourceRevision": "v1.52@2acefa38cce0",
285
+ "addedAt": "2026-09-17"
286
+ },
287
+ {
288
+ "name": "Claude-Web",
289
+ "pattern": "Claude\\-Web",
290
+ "source": "ai-robots-txt/ai.robots.txt",
291
+ "sourceRevision": "v1.52@2acefa38cce0",
292
+ "addedAt": "2026-09-17"
293
+ },
294
+ {
295
+ "name": "ClaudeBot",
296
+ "pattern": "ClaudeBot",
297
+ "source": "ai-robots-txt/ai.robots.txt",
298
+ "sourceRevision": "v1.52@2acefa38cce0",
299
+ "addedAt": "2026-09-17"
300
+ },
301
+ {
302
+ "name": "Cloudflare-AutoRAG",
303
+ "pattern": "Cloudflare\\-AutoRAG",
304
+ "source": "ai-robots-txt/ai.robots.txt",
305
+ "sourceRevision": "v1.52@2acefa38cce0",
306
+ "addedAt": "2026-09-17"
307
+ },
308
+ {
309
+ "name": "CloudVertexBot",
310
+ "pattern": "CloudVertexBot",
311
+ "source": "ai-robots-txt/ai.robots.txt",
312
+ "sourceRevision": "v1.52@2acefa38cce0",
313
+ "addedAt": "2026-09-17"
314
+ },
315
+ {
316
+ "name": "Code",
317
+ "pattern": "Code",
318
+ "source": "ai-robots-txt/ai.robots.txt",
319
+ "sourceRevision": "v1.52@2acefa38cce0",
320
+ "addedAt": "2026-09-17"
321
+ },
322
+ {
323
+ "name": "cohere-ai",
324
+ "pattern": "cohere\\-ai",
325
+ "source": "ai-robots-txt/ai.robots.txt",
326
+ "sourceRevision": "v1.52@2acefa38cce0",
327
+ "addedAt": "2026-09-17"
328
+ },
329
+ {
330
+ "name": "cohere-training-data-crawler",
331
+ "pattern": "cohere\\-training\\-data\\-crawler",
332
+ "source": "ai-robots-txt/ai.robots.txt",
333
+ "sourceRevision": "v1.52@2acefa38cce0",
334
+ "addedAt": "2026-09-17"
335
+ },
336
+ {
337
+ "name": "Cotoyogi",
338
+ "pattern": "Cotoyogi",
339
+ "source": "ai-robots-txt/ai.robots.txt",
340
+ "sourceRevision": "v1.52@2acefa38cce0",
341
+ "addedAt": "2026-09-17"
342
+ },
343
+ {
344
+ "name": "CragCrawler",
345
+ "pattern": "CragCrawler",
346
+ "source": "ai-robots-txt/ai.robots.txt",
347
+ "sourceRevision": "v1.52@2acefa38cce0",
348
+ "addedAt": "2026-09-17"
349
+ },
350
+ {
351
+ "name": "Crawl4AI",
352
+ "pattern": "Crawl4AI",
353
+ "source": "ai-robots-txt/ai.robots.txt",
354
+ "sourceRevision": "v1.52@2acefa38cce0",
355
+ "addedAt": "2026-09-17"
356
+ },
357
+ {
358
+ "name": "Crawlspace",
359
+ "pattern": "Crawlspace",
360
+ "source": "ai-robots-txt/ai.robots.txt",
361
+ "sourceRevision": "v1.52@2acefa38cce0",
362
+ "addedAt": "2026-09-17"
363
+ },
364
+ {
365
+ "name": "Cursor",
366
+ "pattern": "Cursor",
367
+ "source": "ai-robots-txt/ai.robots.txt",
368
+ "sourceRevision": "v1.52@2acefa38cce0",
369
+ "addedAt": "2026-09-17"
370
+ },
371
+ {
372
+ "name": "Datenbank Crawler",
373
+ "pattern": "Datenbank Crawler",
374
+ "source": "ai-robots-txt/ai.robots.txt",
375
+ "sourceRevision": "v1.52@2acefa38cce0",
376
+ "addedAt": "2026-09-17"
377
+ },
378
+ {
379
+ "name": "DeepSeekBot",
380
+ "pattern": "DeepSeekBot",
381
+ "source": "ai-robots-txt/ai.robots.txt",
382
+ "sourceRevision": "v1.52@2acefa38cce0",
383
+ "addedAt": "2026-09-17"
384
+ },
385
+ {
386
+ "name": "Devin",
387
+ "pattern": "Devin",
388
+ "source": "ai-robots-txt/ai.robots.txt",
389
+ "sourceRevision": "v1.52@2acefa38cce0",
390
+ "addedAt": "2026-09-17"
391
+ },
392
+ {
393
+ "name": "Diffbot",
394
+ "pattern": "Diffbot",
395
+ "source": "ai-robots-txt/ai.robots.txt",
396
+ "sourceRevision": "v1.52@2acefa38cce0",
397
+ "addedAt": "2026-09-17"
398
+ },
399
+ {
400
+ "name": "Diffbot-User",
401
+ "pattern": "Diffbot\\-User",
402
+ "source": "ai-robots-txt/ai.robots.txt",
403
+ "sourceRevision": "v1.52@2acefa38cce0",
404
+ "addedAt": "2026-09-17"
405
+ },
406
+ {
407
+ "name": "DoubaoBot",
408
+ "pattern": "DoubaoBot",
409
+ "source": "ai-robots-txt/ai.robots.txt",
410
+ "sourceRevision": "v1.52@2acefa38cce0",
411
+ "addedAt": "2026-09-17"
412
+ },
413
+ {
414
+ "name": "DuckAssistBot",
415
+ "pattern": "DuckAssistBot",
416
+ "source": "ai-robots-txt/ai.robots.txt",
417
+ "sourceRevision": "v1.52@2acefa38cce0",
418
+ "addedAt": "2026-09-17"
419
+ },
420
+ {
421
+ "name": "Echobot Bot",
422
+ "pattern": "Echobot Bot",
423
+ "source": "ai-robots-txt/ai.robots.txt",
424
+ "sourceRevision": "v1.52@2acefa38cce0",
425
+ "addedAt": "2026-09-17"
426
+ },
427
+ {
428
+ "name": "EchoboxBot",
429
+ "pattern": "EchoboxBot",
430
+ "source": "ai-robots-txt/ai.robots.txt",
431
+ "sourceRevision": "v1.52@2acefa38cce0",
432
+ "addedAt": "2026-09-17"
433
+ },
434
+ {
435
+ "name": "ERNIEBot",
436
+ "pattern": "ERNIEBot",
437
+ "source": "ai-robots-txt/ai.robots.txt",
438
+ "sourceRevision": "v1.52@2acefa38cce0",
439
+ "addedAt": "2026-09-17"
440
+ },
441
+ {
442
+ "name": "ExaBot",
443
+ "pattern": "ExaBot",
444
+ "source": "ai-robots-txt/ai.robots.txt",
445
+ "sourceRevision": "v1.52@2acefa38cce0",
446
+ "addedAt": "2026-09-17"
447
+ },
448
+ {
449
+ "name": "ExaSearchBot",
450
+ "pattern": "ExaSearchBot",
451
+ "source": "ai-robots-txt/ai.robots.txt",
452
+ "sourceRevision": "v1.52@2acefa38cce0",
453
+ "addedAt": "2026-09-17"
454
+ },
455
+ {
456
+ "name": "FacebookBot",
457
+ "pattern": "FacebookBot",
458
+ "source": "ai-robots-txt/ai.robots.txt",
459
+ "sourceRevision": "v1.52@2acefa38cce0",
460
+ "addedAt": "2026-09-17"
461
+ },
462
+ {
463
+ "name": "facebookexternalhit",
464
+ "pattern": "facebookexternalhit",
465
+ "source": "ai-robots-txt/ai.robots.txt",
466
+ "sourceRevision": "v1.52@2acefa38cce0",
467
+ "addedAt": "2026-09-17"
468
+ },
469
+ {
470
+ "name": "Factset_spyderbot",
471
+ "pattern": "Factset_spyderbot",
472
+ "source": "ai-robots-txt/ai.robots.txt",
473
+ "sourceRevision": "v1.52@2acefa38cce0",
474
+ "addedAt": "2026-09-17"
475
+ },
476
+ {
477
+ "name": "FirecrawlAgent",
478
+ "pattern": "FirecrawlAgent",
479
+ "source": "ai-robots-txt/ai.robots.txt",
480
+ "sourceRevision": "v1.52@2acefa38cce0",
481
+ "addedAt": "2026-09-17"
482
+ },
483
+ {
484
+ "name": "FriendlyCrawler",
485
+ "pattern": "FriendlyCrawler",
486
+ "source": "ai-robots-txt/ai.robots.txt",
487
+ "sourceRevision": "v1.52@2acefa38cce0",
488
+ "addedAt": "2026-09-17"
489
+ },
490
+ {
491
+ "name": "GeistHaus-PageFetcher",
492
+ "pattern": "GeistHaus\\-PageFetcher",
493
+ "source": "ai-robots-txt/ai.robots.txt",
494
+ "sourceRevision": "v1.52@2acefa38cce0",
495
+ "addedAt": "2026-09-17"
496
+ },
497
+ {
498
+ "name": "Gemini-Deep-Research",
499
+ "pattern": "Gemini\\-Deep\\-Research",
500
+ "source": "ai-robots-txt/ai.robots.txt",
501
+ "sourceRevision": "v1.52@2acefa38cce0",
502
+ "addedAt": "2026-09-17"
503
+ },
504
+ {
505
+ "name": "Google-Agent",
506
+ "pattern": "Google\\-Agent",
507
+ "source": "ai-robots-txt/ai.robots.txt",
508
+ "sourceRevision": "v1.52@2acefa38cce0",
509
+ "addedAt": "2026-09-17"
510
+ },
511
+ {
512
+ "name": "Google-CloudVertexBot",
513
+ "pattern": "Google\\-CloudVertexBot",
514
+ "source": "ai-robots-txt/ai.robots.txt",
515
+ "sourceRevision": "v1.52@2acefa38cce0",
516
+ "addedAt": "2026-09-17"
517
+ },
518
+ {
519
+ "name": "Google-Extended",
520
+ "pattern": "Google\\-Extended",
521
+ "source": "ai-robots-txt/ai.robots.txt",
522
+ "sourceRevision": "v1.52@2acefa38cce0",
523
+ "addedAt": "2026-09-17"
524
+ },
525
+ {
526
+ "name": "Google-Firebase",
527
+ "pattern": "Google\\-Firebase",
528
+ "source": "ai-robots-txt/ai.robots.txt",
529
+ "sourceRevision": "v1.52@2acefa38cce0",
530
+ "addedAt": "2026-09-17"
531
+ },
532
+ {
533
+ "name": "Google-Gemini-CLI",
534
+ "pattern": "Google\\-Gemini\\-CLI",
535
+ "source": "ai-robots-txt/ai.robots.txt",
536
+ "sourceRevision": "v1.52@2acefa38cce0",
537
+ "addedAt": "2026-09-17"
538
+ },
539
+ {
540
+ "name": "Google-NotebookLM",
541
+ "pattern": "Google\\-NotebookLM",
542
+ "source": "ai-robots-txt/ai.robots.txt",
543
+ "sourceRevision": "v1.52@2acefa38cce0",
544
+ "addedAt": "2026-09-17"
545
+ },
546
+ {
547
+ "name": "GoogleAgent-Mariner",
548
+ "pattern": "GoogleAgent\\-Mariner",
549
+ "source": "ai-robots-txt/ai.robots.txt",
550
+ "sourceRevision": "v1.52@2acefa38cce0",
551
+ "addedAt": "2026-09-17"
552
+ },
553
+ {
554
+ "name": "GoogleAgent-URLContext",
555
+ "pattern": "GoogleAgent\\-URLContext",
556
+ "source": "ai-robots-txt/ai.robots.txt",
557
+ "sourceRevision": "v1.52@2acefa38cce0",
558
+ "addedAt": "2026-09-17"
559
+ },
560
+ {
561
+ "name": "GoogleOther",
562
+ "pattern": "GoogleOther",
563
+ "source": "ai-robots-txt/ai.robots.txt",
564
+ "sourceRevision": "v1.52@2acefa38cce0",
565
+ "addedAt": "2026-09-17"
566
+ },
567
+ {
568
+ "name": "GoogleOther-Image",
569
+ "pattern": "GoogleOther\\-Image",
570
+ "source": "ai-robots-txt/ai.robots.txt",
571
+ "sourceRevision": "v1.52@2acefa38cce0",
572
+ "addedAt": "2026-09-17"
573
+ },
574
+ {
575
+ "name": "GoogleOther-Video",
576
+ "pattern": "GoogleOther\\-Video",
577
+ "source": "ai-robots-txt/ai.robots.txt",
578
+ "sourceRevision": "v1.52@2acefa38cce0",
579
+ "addedAt": "2026-09-17"
580
+ },
581
+ {
582
+ "name": "GPTBot",
583
+ "pattern": "GPTBot",
584
+ "source": "ai-robots-txt/ai.robots.txt",
585
+ "sourceRevision": "v1.52@2acefa38cce0",
586
+ "addedAt": "2026-09-17"
587
+ },
588
+ {
589
+ "name": "HenkBot",
590
+ "pattern": "HenkBot",
591
+ "source": "ai-robots-txt/ai.robots.txt",
592
+ "sourceRevision": "v1.52@2acefa38cce0",
593
+ "addedAt": "2026-09-17"
594
+ },
595
+ {
596
+ "name": "iAskBot",
597
+ "pattern": "iAskBot",
598
+ "source": "ai-robots-txt/ai.robots.txt",
599
+ "sourceRevision": "v1.52@2acefa38cce0",
600
+ "addedAt": "2026-09-17"
601
+ },
602
+ {
603
+ "name": "iaskspider",
604
+ "pattern": "iaskspider",
605
+ "source": "ai-robots-txt/ai.robots.txt",
606
+ "sourceRevision": "v1.52@2acefa38cce0",
607
+ "addedAt": "2026-09-17"
608
+ },
609
+ {
610
+ "name": "iaskspider/2.0",
611
+ "pattern": "iaskspider\\/2\\.0",
612
+ "source": "ai-robots-txt/ai.robots.txt",
613
+ "sourceRevision": "v1.52@2acefa38cce0",
614
+ "addedAt": "2026-09-17"
615
+ },
616
+ {
617
+ "name": "ICC-Crawler",
618
+ "pattern": "ICC\\-Crawler",
619
+ "source": "ai-robots-txt/ai.robots.txt",
620
+ "sourceRevision": "v1.52@2acefa38cce0",
621
+ "addedAt": "2026-09-17"
622
+ },
623
+ {
624
+ "name": "ImagesiftBot",
625
+ "pattern": "ImagesiftBot",
626
+ "source": "ai-robots-txt/ai.robots.txt",
627
+ "sourceRevision": "v1.52@2acefa38cce0",
628
+ "addedAt": "2026-09-17"
629
+ },
630
+ {
631
+ "name": "imageSpider",
632
+ "pattern": "imageSpider",
633
+ "source": "ai-robots-txt/ai.robots.txt",
634
+ "sourceRevision": "v1.52@2acefa38cce0",
635
+ "addedAt": "2026-09-17"
636
+ },
637
+ {
638
+ "name": "img2dataset",
639
+ "pattern": "img2dataset",
640
+ "source": "ai-robots-txt/ai.robots.txt",
641
+ "sourceRevision": "v1.52@2acefa38cce0",
642
+ "addedAt": "2026-09-17"
643
+ },
644
+ {
645
+ "name": "ISSCyberRiskCrawler",
646
+ "pattern": "ISSCyberRiskCrawler",
647
+ "source": "ai-robots-txt/ai.robots.txt",
648
+ "sourceRevision": "v1.52@2acefa38cce0",
649
+ "addedAt": "2026-09-17"
650
+ },
651
+ {
652
+ "name": "kagi-fetcher",
653
+ "pattern": "kagi\\-fetcher",
654
+ "source": "ai-robots-txt/ai.robots.txt",
655
+ "sourceRevision": "v1.52@2acefa38cce0",
656
+ "addedAt": "2026-09-17"
657
+ },
658
+ {
659
+ "name": "Kangaroo Bot",
660
+ "pattern": "Kangaroo Bot",
661
+ "source": "ai-robots-txt/ai.robots.txt",
662
+ "sourceRevision": "v1.52@2acefa38cce0",
663
+ "addedAt": "2026-09-17"
664
+ },
665
+ {
666
+ "name": "Kimi-SearchBot",
667
+ "pattern": "Kimi\\-SearchBot",
668
+ "source": "ai-robots-txt/ai.robots.txt",
669
+ "sourceRevision": "v1.52@2acefa38cce0",
670
+ "addedAt": "2026-09-17"
671
+ },
672
+ {
673
+ "name": "Kimi-User",
674
+ "pattern": "Kimi\\-User",
675
+ "source": "ai-robots-txt/ai.robots.txt",
676
+ "sourceRevision": "v1.52@2acefa38cce0",
677
+ "addedAt": "2026-09-17"
678
+ },
679
+ {
680
+ "name": "KimiBot",
681
+ "pattern": "KimiBot",
682
+ "source": "ai-robots-txt/ai.robots.txt",
683
+ "sourceRevision": "v1.52@2acefa38cce0",
684
+ "addedAt": "2026-09-17"
685
+ },
686
+ {
687
+ "name": "KlaviyoAIBot",
688
+ "pattern": "KlaviyoAIBot",
689
+ "source": "ai-robots-txt/ai.robots.txt",
690
+ "sourceRevision": "v1.52@2acefa38cce0",
691
+ "addedAt": "2026-09-17"
692
+ },
693
+ {
694
+ "name": "KunatoCrawler",
695
+ "pattern": "KunatoCrawler",
696
+ "source": "ai-robots-txt/ai.robots.txt",
697
+ "sourceRevision": "v1.52@2acefa38cce0",
698
+ "addedAt": "2026-09-17"
699
+ },
700
+ {
701
+ "name": "laion-huggingface-processor",
702
+ "pattern": "laion\\-huggingface\\-processor",
703
+ "source": "ai-robots-txt/ai.robots.txt",
704
+ "sourceRevision": "v1.52@2acefa38cce0",
705
+ "addedAt": "2026-09-17"
706
+ },
707
+ {
708
+ "name": "LAIONDownloader",
709
+ "pattern": "LAIONDownloader",
710
+ "source": "ai-robots-txt/ai.robots.txt",
711
+ "sourceRevision": "v1.52@2acefa38cce0",
712
+ "addedAt": "2026-09-17"
713
+ },
714
+ {
715
+ "name": "LCC",
716
+ "pattern": "LCC",
717
+ "source": "ai-robots-txt/ai.robots.txt",
718
+ "sourceRevision": "v1.52@2acefa38cce0",
719
+ "addedAt": "2026-09-17"
720
+ },
721
+ {
722
+ "name": "Lightpanda",
723
+ "pattern": "Lightpanda",
724
+ "source": "ai-robots-txt/ai.robots.txt",
725
+ "sourceRevision": "v1.52@2acefa38cce0",
726
+ "addedAt": "2026-09-17"
727
+ },
728
+ {
729
+ "name": "LinerBot",
730
+ "pattern": "LinerBot",
731
+ "source": "ai-robots-txt/ai.robots.txt",
732
+ "sourceRevision": "v1.52@2acefa38cce0",
733
+ "addedAt": "2026-09-17"
734
+ },
735
+ {
736
+ "name": "Linguee Bot",
737
+ "pattern": "Linguee Bot",
738
+ "source": "ai-robots-txt/ai.robots.txt",
739
+ "sourceRevision": "v1.52@2acefa38cce0",
740
+ "addedAt": "2026-09-17"
741
+ },
742
+ {
743
+ "name": "LinkupBot",
744
+ "pattern": "LinkupBot",
745
+ "source": "ai-robots-txt/ai.robots.txt",
746
+ "sourceRevision": "v1.52@2acefa38cce0",
747
+ "addedAt": "2026-09-17"
748
+ },
749
+ {
750
+ "name": "Manus-User",
751
+ "pattern": "Manus\\-User",
752
+ "source": "ai-robots-txt/ai.robots.txt",
753
+ "sourceRevision": "v1.52@2acefa38cce0",
754
+ "addedAt": "2026-09-17"
755
+ },
756
+ {
757
+ "name": "meta-externalagent",
758
+ "pattern": "meta\\-externalagent",
759
+ "source": "ai-robots-txt/ai.robots.txt",
760
+ "sourceRevision": "v1.52@2acefa38cce0",
761
+ "addedAt": "2026-09-17"
762
+ },
763
+ {
764
+ "name": "Meta-ExternalAgent",
765
+ "pattern": "Meta\\-ExternalAgent",
766
+ "source": "ai-robots-txt/ai.robots.txt",
767
+ "sourceRevision": "v1.52@2acefa38cce0",
768
+ "addedAt": "2026-09-17"
769
+ },
770
+ {
771
+ "name": "meta-externalfetcher",
772
+ "pattern": "meta\\-externalfetcher",
773
+ "source": "ai-robots-txt/ai.robots.txt",
774
+ "sourceRevision": "v1.52@2acefa38cce0",
775
+ "addedAt": "2026-09-17"
776
+ },
777
+ {
778
+ "name": "Meta-ExternalFetcher",
779
+ "pattern": "Meta\\-ExternalFetcher",
780
+ "source": "ai-robots-txt/ai.robots.txt",
781
+ "sourceRevision": "v1.52@2acefa38cce0",
782
+ "addedAt": "2026-09-17"
783
+ },
784
+ {
785
+ "name": "meta-webindexer",
786
+ "pattern": "meta\\-webindexer",
787
+ "source": "ai-robots-txt/ai.robots.txt",
788
+ "sourceRevision": "v1.52@2acefa38cce0",
789
+ "addedAt": "2026-09-17"
790
+ },
791
+ {
792
+ "name": "MistralAI-Index",
793
+ "pattern": "MistralAI\\-Index",
794
+ "source": "ai-robots-txt/ai.robots.txt",
795
+ "sourceRevision": "v1.52@2acefa38cce0",
796
+ "addedAt": "2026-09-17"
797
+ },
798
+ {
799
+ "name": "MistralAI-Training",
800
+ "pattern": "MistralAI\\-Training",
801
+ "source": "ai-robots-txt/ai.robots.txt",
802
+ "sourceRevision": "v1.52@2acefa38cce0",
803
+ "addedAt": "2026-09-17"
804
+ },
805
+ {
806
+ "name": "MistralAI-User",
807
+ "pattern": "MistralAI\\-User",
808
+ "source": "ai-robots-txt/ai.robots.txt",
809
+ "sourceRevision": "v1.52@2acefa38cce0",
810
+ "addedAt": "2026-09-17"
811
+ },
812
+ {
813
+ "name": "MistralAI-User/1.0",
814
+ "pattern": "MistralAI\\-User\\/1\\.0",
815
+ "source": "ai-robots-txt/ai.robots.txt",
816
+ "sourceRevision": "v1.52@2acefa38cce0",
817
+ "addedAt": "2026-09-17"
818
+ },
819
+ {
820
+ "name": "Mozilla-Tabstack",
821
+ "pattern": "Mozilla\\-Tabstack",
822
+ "source": "ai-robots-txt/ai.robots.txt",
823
+ "sourceRevision": "v1.52@2acefa38cce0",
824
+ "addedAt": "2026-09-17"
825
+ },
826
+ {
827
+ "name": "MyCentralAIScraperBot",
828
+ "pattern": "MyCentralAIScraperBot",
829
+ "source": "ai-robots-txt/ai.robots.txt",
830
+ "sourceRevision": "v1.52@2acefa38cce0",
831
+ "addedAt": "2026-09-17"
832
+ },
833
+ {
834
+ "name": "NagetBot",
835
+ "pattern": "NagetBot",
836
+ "source": "ai-robots-txt/ai.robots.txt",
837
+ "sourceRevision": "v1.52@2acefa38cce0",
838
+ "addedAt": "2026-09-17"
839
+ },
840
+ {
841
+ "name": "netEstate Imprint Crawler",
842
+ "pattern": "netEstate Imprint Crawler",
843
+ "source": "ai-robots-txt/ai.robots.txt",
844
+ "sourceRevision": "v1.52@2acefa38cce0",
845
+ "addedAt": "2026-09-17"
846
+ },
847
+ {
848
+ "name": "newsai",
849
+ "pattern": "newsai",
850
+ "source": "ai-robots-txt/ai.robots.txt",
851
+ "sourceRevision": "v1.52@2acefa38cce0",
852
+ "addedAt": "2026-09-17"
853
+ },
854
+ {
855
+ "name": "NotebookLM",
856
+ "pattern": "NotebookLM",
857
+ "source": "ai-robots-txt/ai.robots.txt",
858
+ "sourceRevision": "v1.52@2acefa38cce0",
859
+ "addedAt": "2026-09-17"
860
+ },
861
+ {
862
+ "name": "NovaAct",
863
+ "pattern": "NovaAct",
864
+ "source": "ai-robots-txt/ai.robots.txt",
865
+ "sourceRevision": "v1.52@2acefa38cce0",
866
+ "addedAt": "2026-09-17"
867
+ },
868
+ {
869
+ "name": "OAI-AdsBot",
870
+ "pattern": "OAI\\-AdsBot",
871
+ "source": "ai-robots-txt/ai.robots.txt",
872
+ "sourceRevision": "v1.52@2acefa38cce0",
873
+ "addedAt": "2026-09-17"
874
+ },
875
+ {
876
+ "name": "OAI-SearchBot",
877
+ "pattern": "OAI\\-SearchBot",
878
+ "source": "ai-robots-txt/ai.robots.txt",
879
+ "sourceRevision": "v1.52@2acefa38cce0",
880
+ "addedAt": "2026-09-17"
881
+ },
882
+ {
883
+ "name": "omgili",
884
+ "pattern": "omgili",
885
+ "source": "ai-robots-txt/ai.robots.txt",
886
+ "sourceRevision": "v1.52@2acefa38cce0",
887
+ "addedAt": "2026-09-17"
888
+ },
889
+ {
890
+ "name": "omgilibot",
891
+ "pattern": "omgilibot",
892
+ "source": "ai-robots-txt/ai.robots.txt",
893
+ "sourceRevision": "v1.52@2acefa38cce0",
894
+ "addedAt": "2026-09-17"
895
+ },
896
+ {
897
+ "name": "OpenAI",
898
+ "pattern": "OpenAI",
899
+ "source": "ai-robots-txt/ai.robots.txt",
900
+ "sourceRevision": "v1.52@2acefa38cce0",
901
+ "addedAt": "2026-09-17"
902
+ },
903
+ {
904
+ "name": "opencode",
905
+ "pattern": "opencode",
906
+ "source": "ai-robots-txt/ai.robots.txt",
907
+ "sourceRevision": "v1.52@2acefa38cce0",
908
+ "addedAt": "2026-09-17"
909
+ },
910
+ {
911
+ "name": "Operator",
912
+ "pattern": "Operator",
913
+ "source": "ai-robots-txt/ai.robots.txt",
914
+ "sourceRevision": "v1.52@2acefa38cce0",
915
+ "addedAt": "2026-09-17"
916
+ },
917
+ {
918
+ "name": "PanguBot",
919
+ "pattern": "PanguBot",
920
+ "source": "ai-robots-txt/ai.robots.txt",
921
+ "sourceRevision": "v1.52@2acefa38cce0",
922
+ "addedAt": "2026-09-17"
923
+ },
924
+ {
925
+ "name": "Panscient",
926
+ "pattern": "Panscient",
927
+ "source": "ai-robots-txt/ai.robots.txt",
928
+ "sourceRevision": "v1.52@2acefa38cce0",
929
+ "addedAt": "2026-09-17"
930
+ },
931
+ {
932
+ "name": "panscient.com",
933
+ "pattern": "panscient\\.com",
934
+ "source": "ai-robots-txt/ai.robots.txt",
935
+ "sourceRevision": "v1.52@2acefa38cce0",
936
+ "addedAt": "2026-09-17"
937
+ },
938
+ {
939
+ "name": "Perplexity-User",
940
+ "pattern": "Perplexity\\-User",
941
+ "source": "ai-robots-txt/ai.robots.txt",
942
+ "sourceRevision": "v1.52@2acefa38cce0",
943
+ "addedAt": "2026-09-17"
944
+ },
945
+ {
946
+ "name": "PerplexityBot",
947
+ "pattern": "PerplexityBot",
948
+ "source": "ai-robots-txt/ai.robots.txt",
949
+ "sourceRevision": "v1.52@2acefa38cce0",
950
+ "addedAt": "2026-09-17"
951
+ },
952
+ {
953
+ "name": "PetalBot",
954
+ "pattern": "PetalBot",
955
+ "source": "ai-robots-txt/ai.robots.txt",
956
+ "sourceRevision": "v1.52@2acefa38cce0",
957
+ "addedAt": "2026-09-17"
958
+ },
959
+ {
960
+ "name": "PhindBot",
961
+ "pattern": "PhindBot",
962
+ "source": "ai-robots-txt/ai.robots.txt",
963
+ "sourceRevision": "v1.52@2acefa38cce0",
964
+ "addedAt": "2026-09-17"
965
+ },
966
+ {
967
+ "name": "Poggio-Citations",
968
+ "pattern": "Poggio\\-Citations",
969
+ "source": "ai-robots-txt/ai.robots.txt",
970
+ "sourceRevision": "v1.52@2acefa38cce0",
971
+ "addedAt": "2026-09-17"
972
+ },
973
+ {
974
+ "name": "Poseidon Research Crawler",
975
+ "pattern": "Poseidon Research Crawler",
976
+ "source": "ai-robots-txt/ai.robots.txt",
977
+ "sourceRevision": "v1.52@2acefa38cce0",
978
+ "addedAt": "2026-09-17"
979
+ },
980
+ {
981
+ "name": "QualifiedBot",
982
+ "pattern": "QualifiedBot",
983
+ "source": "ai-robots-txt/ai.robots.txt",
984
+ "sourceRevision": "v1.52@2acefa38cce0",
985
+ "addedAt": "2026-09-17"
986
+ },
987
+ {
988
+ "name": "Querit-SearchBot",
989
+ "pattern": "Querit\\-SearchBot",
990
+ "source": "ai-robots-txt/ai.robots.txt",
991
+ "sourceRevision": "v1.52@2acefa38cce0",
992
+ "addedAt": "2026-09-17"
993
+ },
994
+ {
995
+ "name": "QueritBot",
996
+ "pattern": "QueritBot",
997
+ "source": "ai-robots-txt/ai.robots.txt",
998
+ "sourceRevision": "v1.52@2acefa38cce0",
999
+ "addedAt": "2026-09-17"
1000
+ },
1001
+ {
1002
+ "name": "QuillBot",
1003
+ "pattern": "QuillBot",
1004
+ "source": "ai-robots-txt/ai.robots.txt",
1005
+ "sourceRevision": "v1.52@2acefa38cce0",
1006
+ "addedAt": "2026-09-17"
1007
+ },
1008
+ {
1009
+ "name": "quillbot.com",
1010
+ "pattern": "quillbot\\.com",
1011
+ "source": "ai-robots-txt/ai.robots.txt",
1012
+ "sourceRevision": "v1.52@2acefa38cce0",
1013
+ "addedAt": "2026-09-17"
1014
+ },
1015
+ {
1016
+ "name": "QwenBot",
1017
+ "pattern": "QwenBot",
1018
+ "source": "ai-robots-txt/ai.robots.txt",
1019
+ "sourceRevision": "v1.52@2acefa38cce0",
1020
+ "addedAt": "2026-09-17"
1021
+ },
1022
+ {
1023
+ "name": "Reflectionbot",
1024
+ "pattern": "Reflectionbot",
1025
+ "source": "ai-robots-txt/ai.robots.txt",
1026
+ "sourceRevision": "v1.52@2acefa38cce0",
1027
+ "addedAt": "2026-09-17"
1028
+ },
1029
+ {
1030
+ "name": "SBIntuitionsBot",
1031
+ "pattern": "SBIntuitionsBot",
1032
+ "source": "ai-robots-txt/ai.robots.txt",
1033
+ "sourceRevision": "v1.52@2acefa38cce0",
1034
+ "addedAt": "2026-09-17"
1035
+ },
1036
+ {
1037
+ "name": "Scrapy",
1038
+ "pattern": "Scrapy",
1039
+ "source": "ai-robots-txt/ai.robots.txt",
1040
+ "sourceRevision": "v1.52@2acefa38cce0",
1041
+ "addedAt": "2026-09-17"
1042
+ },
1043
+ {
1044
+ "name": "SemrushBot-OCOB",
1045
+ "pattern": "SemrushBot\\-OCOB",
1046
+ "source": "ai-robots-txt/ai.robots.txt",
1047
+ "sourceRevision": "v1.52@2acefa38cce0",
1048
+ "addedAt": "2026-09-17"
1049
+ },
1050
+ {
1051
+ "name": "SemrushBot-SWA",
1052
+ "pattern": "SemrushBot\\-SWA",
1053
+ "source": "ai-robots-txt/ai.robots.txt",
1054
+ "sourceRevision": "v1.52@2acefa38cce0",
1055
+ "addedAt": "2026-09-17"
1056
+ },
1057
+ {
1058
+ "name": "Shap-User",
1059
+ "pattern": "Shap\\-User",
1060
+ "source": "ai-robots-txt/ai.robots.txt",
1061
+ "sourceRevision": "v1.52@2acefa38cce0",
1062
+ "addedAt": "2026-09-17"
1063
+ },
1064
+ {
1065
+ "name": "ShapBot",
1066
+ "pattern": "ShapBot",
1067
+ "source": "ai-robots-txt/ai.robots.txt",
1068
+ "sourceRevision": "v1.52@2acefa38cce0",
1069
+ "addedAt": "2026-09-17"
1070
+ },
1071
+ {
1072
+ "name": "Sidetrade indexer bot",
1073
+ "pattern": "Sidetrade indexer bot",
1074
+ "source": "ai-robots-txt/ai.robots.txt",
1075
+ "sourceRevision": "v1.52@2acefa38cce0",
1076
+ "addedAt": "2026-09-17"
1077
+ },
1078
+ {
1079
+ "name": "Spider",
1080
+ "pattern": "Spider",
1081
+ "source": "ai-robots-txt/ai.robots.txt",
1082
+ "sourceRevision": "v1.52@2acefa38cce0",
1083
+ "addedAt": "2026-09-17"
1084
+ },
1085
+ {
1086
+ "name": "TavilyBot",
1087
+ "pattern": "TavilyBot",
1088
+ "source": "ai-robots-txt/ai.robots.txt",
1089
+ "sourceRevision": "v1.52@2acefa38cce0",
1090
+ "addedAt": "2026-09-17"
1091
+ },
1092
+ {
1093
+ "name": "Terra Cotta",
1094
+ "pattern": "Terra Cotta",
1095
+ "source": "ai-robots-txt/ai.robots.txt",
1096
+ "sourceRevision": "v1.52@2acefa38cce0",
1097
+ "addedAt": "2026-09-17"
1098
+ },
1099
+ {
1100
+ "name": "TerraCotta",
1101
+ "pattern": "TerraCotta",
1102
+ "source": "ai-robots-txt/ai.robots.txt",
1103
+ "sourceRevision": "v1.52@2acefa38cce0",
1104
+ "addedAt": "2026-09-17"
1105
+ },
1106
+ {
1107
+ "name": "Thinkbot",
1108
+ "pattern": "Thinkbot",
1109
+ "source": "ai-robots-txt/ai.robots.txt",
1110
+ "sourceRevision": "v1.52@2acefa38cce0",
1111
+ "addedAt": "2026-09-17"
1112
+ },
1113
+ {
1114
+ "name": "TikTokSpider",
1115
+ "pattern": "TikTokSpider",
1116
+ "source": "ai-robots-txt/ai.robots.txt",
1117
+ "sourceRevision": "v1.52@2acefa38cce0",
1118
+ "addedAt": "2026-09-17"
1119
+ },
1120
+ {
1121
+ "name": "Timpibot",
1122
+ "pattern": "Timpibot",
1123
+ "source": "ai-robots-txt/ai.robots.txt",
1124
+ "sourceRevision": "v1.52@2acefa38cce0",
1125
+ "addedAt": "2026-09-17"
1126
+ },
1127
+ {
1128
+ "name": "TongyiBot",
1129
+ "pattern": "TongyiBot",
1130
+ "source": "ai-robots-txt/ai.robots.txt",
1131
+ "sourceRevision": "v1.52@2acefa38cce0",
1132
+ "addedAt": "2026-09-17"
1133
+ },
1134
+ {
1135
+ "name": "Trae",
1136
+ "pattern": "Trae",
1137
+ "source": "ai-robots-txt/ai.robots.txt",
1138
+ "sourceRevision": "v1.52@2acefa38cce0",
1139
+ "addedAt": "2026-09-17"
1140
+ },
1141
+ {
1142
+ "name": "TwinAgent",
1143
+ "pattern": "TwinAgent",
1144
+ "source": "ai-robots-txt/ai.robots.txt",
1145
+ "sourceRevision": "v1.52@2acefa38cce0",
1146
+ "addedAt": "2026-09-17"
1147
+ },
1148
+ {
1149
+ "name": "UseAI",
1150
+ "pattern": "UseAI",
1151
+ "source": "ai-robots-txt/ai.robots.txt",
1152
+ "sourceRevision": "v1.52@2acefa38cce0",
1153
+ "addedAt": "2026-09-17"
1154
+ },
1155
+ {
1156
+ "name": "VelenPublicWebCrawler",
1157
+ "pattern": "VelenPublicWebCrawler",
1158
+ "source": "ai-robots-txt/ai.robots.txt",
1159
+ "sourceRevision": "v1.52@2acefa38cce0",
1160
+ "addedAt": "2026-09-17"
1161
+ },
1162
+ {
1163
+ "name": "WARDBot",
1164
+ "pattern": "WARDBot",
1165
+ "source": "ai-robots-txt/ai.robots.txt",
1166
+ "sourceRevision": "v1.52@2acefa38cce0",
1167
+ "addedAt": "2026-09-17"
1168
+ },
1169
+ {
1170
+ "name": "webzio-extended",
1171
+ "pattern": "webzio\\-extended",
1172
+ "source": "ai-robots-txt/ai.robots.txt",
1173
+ "sourceRevision": "v1.52@2acefa38cce0",
1174
+ "addedAt": "2026-09-17"
1175
+ },
1176
+ {
1177
+ "name": "Webzio-Extended",
1178
+ "pattern": "Webzio\\-Extended",
1179
+ "source": "ai-robots-txt/ai.robots.txt",
1180
+ "sourceRevision": "v1.52@2acefa38cce0",
1181
+ "addedAt": "2026-09-17"
1182
+ },
1183
+ {
1184
+ "name": "wpbot",
1185
+ "pattern": "wpbot",
1186
+ "source": "ai-robots-txt/ai.robots.txt",
1187
+ "sourceRevision": "v1.52@2acefa38cce0",
1188
+ "addedAt": "2026-09-17"
1189
+ },
1190
+ {
1191
+ "name": "WRTNBot",
1192
+ "pattern": "WRTNBot",
1193
+ "source": "ai-robots-txt/ai.robots.txt",
1194
+ "sourceRevision": "v1.52@2acefa38cce0",
1195
+ "addedAt": "2026-09-17"
1196
+ },
1197
+ {
1198
+ "name": "YaK",
1199
+ "pattern": "YaK",
1200
+ "source": "ai-robots-txt/ai.robots.txt",
1201
+ "sourceRevision": "v1.52@2acefa38cce0",
1202
+ "addedAt": "2026-09-17"
1203
+ },
1204
+ {
1205
+ "name": "YandexAdditional",
1206
+ "pattern": "YandexAdditional",
1207
+ "source": "ai-robots-txt/ai.robots.txt",
1208
+ "sourceRevision": "v1.52@2acefa38cce0",
1209
+ "addedAt": "2026-09-17"
1210
+ },
1211
+ {
1212
+ "name": "YandexAdditionalBot",
1213
+ "pattern": "YandexAdditionalBot",
1214
+ "source": "ai-robots-txt/ai.robots.txt",
1215
+ "sourceRevision": "v1.52@2acefa38cce0",
1216
+ "addedAt": "2026-09-17"
1217
+ },
1218
+ {
1219
+ "name": "YiyanBot",
1220
+ "pattern": "YiyanBot",
1221
+ "source": "ai-robots-txt/ai.robots.txt",
1222
+ "sourceRevision": "v1.52@2acefa38cce0",
1223
+ "addedAt": "2026-09-17"
1224
+ },
1225
+ {
1226
+ "name": "YouBot",
1227
+ "pattern": "YouBot",
1228
+ "source": "ai-robots-txt/ai.robots.txt",
1229
+ "sourceRevision": "v1.52@2acefa38cce0",
1230
+ "addedAt": "2026-09-17"
1231
+ },
1232
+ {
1233
+ "name": "ZanistaBot",
1234
+ "pattern": "ZanistaBot",
1235
+ "source": "ai-robots-txt/ai.robots.txt",
1236
+ "sourceRevision": "v1.52@2acefa38cce0",
1237
+ "addedAt": "2026-09-17"
1238
+ }
1239
+ ]
1240
+ }