@pug-sh/crawlers 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/agents.generated.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// AUTOGENERATED: do not edit. Run `make
|
|
1
|
+
// AUTOGENERATED: do not edit. Run `make codegen` to regenerate.
|
|
2
2
|
// table: pug's crawler table (proto/common/crawlers/v1), as [name, category, ...lowercased tokens].
|
|
3
3
|
// listed: crawler-user-agents@1.66.0, its search-engine and seo entries, as [name, category, pattern],
|
|
4
4
|
// under its license:
|
package/dist/crawlers.js
CHANGED
|
@@ -1,10 +1,7 @@
|
|
|
1
|
+
import { maxPropertyLength, maxUserAgentBytes, procedure } from './events.generated.js';
|
|
1
2
|
import { categories, matchAgent, truncate } from './match.js';
|
|
2
3
|
const defaultEndpoint = 'https://api.pugs.dev';
|
|
3
|
-
const procedure = '/sdk.events.v1.EventsService/BatchCreate';
|
|
4
4
|
const timeoutMs = 2000;
|
|
5
|
-
// The server's limits: $url in characters, a user agent in bytes.
|
|
6
|
-
const maxURLLength = 1024;
|
|
7
|
-
const maxUserAgentBytes = 2048;
|
|
8
5
|
// Framework internals, scripts, styles, images and fonts: crawlers fetch them too, which adds volume and no insight.
|
|
9
6
|
const asset = /^\/_next\/|\.(?:js|mjs|cjs|css|map|png|jpe?g|gif|svg|ico|webp|avif|bmp|tiff?|woff2?|ttf|otf|eot)$/i;
|
|
10
7
|
const ipv4 = /^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$/;
|
|
@@ -66,7 +63,7 @@ function collector(options) {
|
|
|
66
63
|
if (!options.includeAssets && asset.test(page.pathname))
|
|
67
64
|
return undefined;
|
|
68
65
|
const url = page.origin + page.pathname;
|
|
69
|
-
if (url.length >
|
|
66
|
+
if (url.length > maxPropertyLength)
|
|
70
67
|
return undefined;
|
|
71
68
|
const agent = matchAgent(userAgent);
|
|
72
69
|
if (!agent || !wanted.has(agent.category) || skip.has(agent.name.toLowerCase()))
|
|
@@ -149,10 +146,10 @@ async function send(endpoint, apiKey, report, ip, status) {
|
|
|
149
146
|
eventId: report.eventId,
|
|
150
147
|
occurTime: report.occurTime,
|
|
151
148
|
kind: 'crawler_request',
|
|
152
|
-
client: { userAgent: report.userAgent,
|
|
149
|
+
client: { userAgent: report.userAgent, ip },
|
|
153
150
|
autoProperties: {
|
|
154
151
|
$url: { stringValue: report.url },
|
|
155
|
-
...(statusCode ? { $status_code: { intValue: statusCode } } : {}),
|
|
152
|
+
...(statusCode ? { $status_code: { intValue: String(statusCode) } } : {}),
|
|
156
153
|
},
|
|
157
154
|
},
|
|
158
155
|
],
|
|
@@ -165,7 +162,9 @@ async function send(endpoint, apiKey, report, ip, status) {
|
|
|
165
162
|
method: 'POST',
|
|
166
163
|
headers: { 'content-type': 'application/json', 'x-api-key': apiKey },
|
|
167
164
|
body,
|
|
168
|
-
redirect
|
|
165
|
+
// Never follow a redirect, which would hand the key to another host. Workers throw on
|
|
166
|
+
// 'error', so a 3xx comes back as a refusal instead.
|
|
167
|
+
redirect: 'manual',
|
|
169
168
|
signal: AbortSignal.timeout(timeoutMs),
|
|
170
169
|
});
|
|
171
170
|
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
// AUTOGENERATED: do not edit. Run `make codegen` to regenerate.
|
|
2
|
+
// From pug's protos: the procedure a report goes to, the max_len of an auto property's
|
|
3
|
+
// string_value, in code points, and the max_bytes of the client's user_agent.
|
|
4
|
+
export const procedure = "/sdk.events.v1.EventsService/BatchCreate";
|
|
5
|
+
export const maxPropertyLength = 1024;
|
|
6
|
+
export const maxUserAgentBytes = 2048;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pug-sh/crawlers",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2",
|
|
4
4
|
"description": "Report the AI crawlers, search engines and SEO tools that read your site to Pug, from your own request handling.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -53,6 +53,8 @@
|
|
|
53
53
|
"license": "MIT",
|
|
54
54
|
"devDependencies": {
|
|
55
55
|
"@biomejs/biome": "2.6.3",
|
|
56
|
+
"@bufbuild/protobuf": "^2.17.0",
|
|
57
|
+
"@bufbuild/protoc-gen-es": "^2.17.0",
|
|
56
58
|
"@types/node": "^26.6.5",
|
|
57
59
|
"crawler-user-agents": "1.66.0",
|
|
58
60
|
"knip": "^6.41.0",
|