@pug-sh/crawlers 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +205 -2
- package/dist/agents.generated.d.ts +3 -0
- package/dist/agents.generated.js +791 -0
- package/dist/crawlers.d.ts +55 -0
- package/dist/crawlers.js +219 -0
- package/dist/events.generated.d.ts +3 -0
- package/dist/events.generated.js +6 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +1 -0
- package/dist/match.d.ts +13 -0
- package/dist/match.js +59 -0
- package/package.json +62 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Pug
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,206 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @pug-sh/crawlers
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
See which AI products, search engines and SEO tools read your site, and why: to
|
|
4
|
+
answer someone's question, to build a search index, or to train a model.
|
|
5
|
+
|
|
6
|
+
Crawlers fetch HTML and don't run JavaScript, so a browser analytics script never
|
|
7
|
+
sees them. This package runs in your site's own request handling instead. When a
|
|
8
|
+
request comes from a known crawler, it sends one small report to
|
|
9
|
+
[Pug](https://pug.sh) without delaying the response. Pug checks the request's IP
|
|
10
|
+
against the ranges the crawler's operator publishes, so a scraper borrowing
|
|
11
|
+
GPTBot's name reads as `failed` rather than `verified`.
|
|
12
|
+
|
|
13
|
+
It has no dependencies and runs on Cloudflare Workers, Vercel, Next.js, Hono,
|
|
14
|
+
Bun, Deno, Express and plain Node.
|
|
15
|
+
|
|
16
|
+
## Install
|
|
17
|
+
|
|
18
|
+
```sh
|
|
19
|
+
npm install @pug-sh/crawlers
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
You need a private key (`prv_...`) from your project's settings in Pug. Keep it
|
|
23
|
+
on the server, in an environment variable: it can also read your analytics.
|
|
24
|
+
|
|
25
|
+
## Cloudflare Workers
|
|
26
|
+
|
|
27
|
+
```ts
|
|
28
|
+
import { crawlers } from '@pug-sh/crawlers'
|
|
29
|
+
|
|
30
|
+
export default {
|
|
31
|
+
async fetch(request, env, ctx) {
|
|
32
|
+
const response = await env.ASSETS.fetch(request)
|
|
33
|
+
crawlers({ apiKey: env.PUG_PRIVATE_KEY }).track(request, response, ctx)
|
|
34
|
+
return response
|
|
35
|
+
},
|
|
36
|
+
} satisfies ExportedHandler<Env>
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Passing `ctx` lets the Worker finish the report after the response is sent.
|
|
40
|
+
Cloudflare serves a static asset without running the Worker, so turn on
|
|
41
|
+
`run_worker_first`, or crawlers' page fetches never reach `track`:
|
|
42
|
+
|
|
43
|
+
```jsonc
|
|
44
|
+
// wrangler.jsonc
|
|
45
|
+
{
|
|
46
|
+
"assets": { "directory": "./dist", "binding": "ASSETS", "run_worker_first": true }
|
|
47
|
+
}
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
On Pages, track from `functions/_middleware.ts`: take the response from
|
|
51
|
+
`await context.next()` and pass `context` as the third argument.
|
|
52
|
+
|
|
53
|
+
## Hono, Bun and Deno
|
|
54
|
+
|
|
55
|
+
```ts
|
|
56
|
+
import { crawlers } from '@pug-sh/crawlers'
|
|
57
|
+
|
|
58
|
+
const pug = crawlers({ apiKey: process.env.PUG_PRIVATE_KEY })
|
|
59
|
+
|
|
60
|
+
app.use(async (c, next) => {
|
|
61
|
+
await next()
|
|
62
|
+
pug.track(c.req.raw, c.res)
|
|
63
|
+
})
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
On Cloudflare, create it inside the middleware from `c.env.PUG_PRIVATE_KEY`, and
|
|
67
|
+
pass `c.executionCtx` as the third argument. On Vercel, pass the `waitUntil` of
|
|
68
|
+
`@vercel/functions` as an option. Elsewhere, see
|
|
69
|
+
[the client's address](#the-clients-address).
|
|
70
|
+
|
|
71
|
+
## Next.js and Vercel
|
|
72
|
+
|
|
73
|
+
```ts
|
|
74
|
+
// proxy.ts
|
|
75
|
+
import { crawlers } from '@pug-sh/crawlers'
|
|
76
|
+
import { type NextFetchEvent, type NextRequest, NextResponse } from 'next/server'
|
|
77
|
+
|
|
78
|
+
const pug = crawlers({ apiKey: process.env.PUG_PRIVATE_KEY })
|
|
79
|
+
|
|
80
|
+
export function proxy(request: NextRequest, event: NextFetchEvent) {
|
|
81
|
+
pug.track(request, undefined, event)
|
|
82
|
+
return NextResponse.next()
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export const config = {
|
|
86
|
+
matcher: '/((?!_next/static|_next/image|favicon.ico).*)',
|
|
87
|
+
}
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
The proxy runs before the page, so these reports carry no status code. The
|
|
91
|
+
matcher keeps it off Next.js' own files.
|
|
92
|
+
|
|
93
|
+
## Express and node:http
|
|
94
|
+
|
|
95
|
+
```ts
|
|
96
|
+
import { crawlers } from '@pug-sh/crawlers'
|
|
97
|
+
|
|
98
|
+
app.use(crawlers({ apiKey: process.env.PUG_PRIVATE_KEY }).express())
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
The middleware reports once the response closes. With plain `node:http`, call
|
|
102
|
+
it at the start of your handler: `middleware(req, res)`. Behind a reverse proxy,
|
|
103
|
+
forward the public host (nginx: `proxy_set_header Host $host`), or reports name
|
|
104
|
+
the upstream. On serverless functions, pass your platform's `waitUntil` as an
|
|
105
|
+
option, such as the one from `@vercel/functions`, so the function lives until
|
|
106
|
+
the report is sent.
|
|
107
|
+
|
|
108
|
+
## Any other server
|
|
109
|
+
|
|
110
|
+
Report a request yourself with one POST, from any language:
|
|
111
|
+
|
|
112
|
+
```sh
|
|
113
|
+
curl https://api.pugs.dev/sdk.events.v1.EventsService/BatchCreate \
|
|
114
|
+
-H 'x-api-key: prv_...' \
|
|
115
|
+
-H 'content-type: application/json' \
|
|
116
|
+
-d '{"events": [{"eventId": "0193b2c4-8f1e-7a2b-9c3d-4e5f6a7b8c9d", "occurTime": "2026-11-14T09:30:00Z",
|
|
117
|
+
"kind": "crawler_request",
|
|
118
|
+
"client": {"userAgent": "... GPTBot/1.2 ...", "ip": "20.171.207.5"},
|
|
119
|
+
"autoProperties": {"$url": {"stringValue": "https://example.com/pricing"},
|
|
120
|
+
"$status_code": {"intValue": 200}}}]}'
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
One call takes up to 1000 events. Make `eventId` and `occurTime` once and resend
|
|
124
|
+
both unchanged on a retry, so Pug stores the request once. A 200 can still drop
|
|
125
|
+
an event: the response's `dropped`, `droppedByReason` and `invalidEvents` say
|
|
126
|
+
which and why.
|
|
127
|
+
|
|
128
|
+
## Options
|
|
129
|
+
|
|
130
|
+
| Option | Default | What it does |
|
|
131
|
+
| --------------- | ---------------------- | ---------------------------------------------------------------------------------- |
|
|
132
|
+
| `apiKey` | required | A private key, `prv_...`. Without one, nothing is reported. |
|
|
133
|
+
| `endpoint` | `https://api.pugs.dev` | Where Pug runs. A self-hosted install sets its own. |
|
|
134
|
+
| `categories` | all five | Which kinds of crawler to report: `answers`, `search`, `training`, `seo`, `other`. |
|
|
135
|
+
| `skip` | none | Agents to leave out, by the name Pug shows, in any case, such as `['Googlebot']`. |
|
|
136
|
+
| `includeAssets` | `false` | Also report requests for scripts, styles, images, fonts and `/_next/` files. |
|
|
137
|
+
| `getIP` | the platform's | Returns the crawler's address for a request, in place of the platform's. |
|
|
138
|
+
| `waitUntil` | none | Keeps a serverless function alive until a report is sent, when there's no `ctx`. |
|
|
139
|
+
|
|
140
|
+
## The categories
|
|
141
|
+
|
|
142
|
+
| Category | Meaning | Examples |
|
|
143
|
+
| ---------- | --------------------------------------------------------- | ------------------------------------------ |
|
|
144
|
+
| `answers` | Fetched live because a person asked an AI product | ChatGPT-User, Claude-User, Perplexity-User |
|
|
145
|
+
| `search` | Builds a search index that results and AI answers draw on | OAI-SearchBot, Googlebot, bingbot |
|
|
146
|
+
| `training` | Collects pages to train models | GPTBot, ClaudeBot, CCBot |
|
|
147
|
+
| `seo` | Crawls for an SEO tool | AhrefsBot, SemrushBot, MJ12bot |
|
|
148
|
+
| `other` | AI-related, but none of the above | OAI-AdsBot, GoogleOther |
|
|
149
|
+
|
|
150
|
+
The AI agents and the major search engines come from Pug's own table, where
|
|
151
|
+
each has its company and, where the operator publishes them, its IP ranges.
|
|
152
|
+
Other search engines and SEO tools come from the
|
|
153
|
+
[crawler-user-agents](https://github.com/monperrus/crawler-user-agents) list and
|
|
154
|
+
get no IP check. The package bundles both, so a new crawler is reported once you
|
|
155
|
+
upgrade to a release that includes it.
|
|
156
|
+
|
|
157
|
+
## What it sends
|
|
158
|
+
|
|
159
|
+
Only requests whose user agent names a known crawler, so your visitors' requests
|
|
160
|
+
never leave your server. For each one it sends:
|
|
161
|
+
|
|
162
|
+
| Field | Value |
|
|
163
|
+
| ------------------ | ------------------------------------------------------------ |
|
|
164
|
+
| `$url` | The full URL, without its query string or fragment |
|
|
165
|
+
| `client.userAgent` | The User-Agent header |
|
|
166
|
+
| `client.ip` | The client's address, for the IP check. Pug never stores it. |
|
|
167
|
+
| `$status_code` | The response's status, when the adapter sees it |
|
|
168
|
+
|
|
169
|
+
It never sends query strings, cookies, other headers, bodies, requests other
|
|
170
|
+
than GET and HEAD, or (unless `includeAssets` is on) scripts, styles, images,
|
|
171
|
+
fonts and Next.js' `/_next/` files.
|
|
172
|
+
|
|
173
|
+
## Delivery
|
|
174
|
+
|
|
175
|
+
One POST per crawler request, without delaying the response, with a 2-second
|
|
176
|
+
timeout and one retry on a network error, a 409 or a server error. It never
|
|
177
|
+
throws, so it can't break your site. It logs each kind of failure once per
|
|
178
|
+
process, including reports Pug drops, with the reason.
|
|
179
|
+
|
|
180
|
+
## The client's address
|
|
181
|
+
|
|
182
|
+
The package reads `CF-Connecting-IP` on Cloudflare Workers, `x-real-ip` on other
|
|
183
|
+
fetch-style servers (Vercel sets it), `req.ip` on Express (which follows its
|
|
184
|
+
`trust proxy` setting) and the socket on plain Node. It sends an address only
|
|
185
|
+
when it is a single IP. Pass `getIP` when that's wrong:
|
|
186
|
+
|
|
187
|
+
- **Behind a CDN**, your server sees the CDN's address, so every request reads
|
|
188
|
+
`failed`. Read the CDN's client-IP header in `getIP`.
|
|
189
|
+
- **A fetch-style server that Vercel doesn't front** can't trust `x-real-ip`
|
|
190
|
+
unless your own proxy sets it, since anyone can send it. Return the socket's
|
|
191
|
+
address, such as `getIP: (req: Request) => server.requestIP(req)?.address` on
|
|
192
|
+
Bun.
|
|
193
|
+
|
|
194
|
+
Running at the edge (Vercel, or Workers with `run_worker_first`) sees every
|
|
195
|
+
request, including those a CDN would answer from its cache.
|
|
196
|
+
|
|
197
|
+
## Volume
|
|
198
|
+
|
|
199
|
+
Crawler requests count toward your Pug usage like any event, and a busy site
|
|
200
|
+
can get more of them than page views. SEO tools are among the busiest crawlers,
|
|
201
|
+
so `seo` is the first category to leave out.
|
|
202
|
+
|
|
203
|
+
## License
|
|
204
|
+
|
|
205
|
+
MIT. The bundled crawler-user-agents entries are MIT too, © 2017 Martin
|
|
206
|
+
Monperrus.
|