@pug-sh/crawlers 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +205 -2
- package/dist/agents.generated.d.ts +3 -0
- package/dist/agents.generated.js +791 -0
- package/dist/crawlers.d.ts +55 -0
- package/dist/crawlers.js +219 -0
- package/dist/events.generated.d.ts +3 -0
- package/dist/events.generated.js +6 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +1 -0
- package/dist/match.d.ts +13 -0
- package/dist/match.js +59 -0
- package/package.json +62 -4
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { type Category } from './match.js';
|
|
2
|
+
/** A request as Express or node:http hands it over. */
|
|
3
|
+
export interface NodeRequest {
|
|
4
|
+
method?: string | undefined;
|
|
5
|
+
url?: string | undefined;
|
|
6
|
+
originalUrl?: string | undefined;
|
|
7
|
+
protocol?: string | undefined;
|
|
8
|
+
ip?: string | undefined;
|
|
9
|
+
headers: Record<string, string | string[] | undefined>;
|
|
10
|
+
socket?: {
|
|
11
|
+
remoteAddress?: string | undefined;
|
|
12
|
+
encrypted?: boolean | undefined;
|
|
13
|
+
} | undefined;
|
|
14
|
+
}
|
|
15
|
+
/** A response as Express or node:http hands it over: the report goes once it closes. */
|
|
16
|
+
export interface NodeResponse {
|
|
17
|
+
statusCode: number;
|
|
18
|
+
writableFinished?: boolean | undefined;
|
|
19
|
+
once(event: 'close', listener: () => void): unknown;
|
|
20
|
+
}
|
|
21
|
+
/** What keeps a platform running until the report is sent: Workers' ctx, Next.js' event. */
|
|
22
|
+
export interface WaitUntil {
|
|
23
|
+
waitUntil(promise: Promise<unknown>): void;
|
|
24
|
+
}
|
|
25
|
+
export interface CrawlersOptions {
|
|
26
|
+
/** A private (prv_) key; without one nothing is reported. A public key won't do: it ships inside web pages. */
|
|
27
|
+
apiKey: string | undefined;
|
|
28
|
+
/** Where pug runs. Defaults to https://api.pugs.dev; a self-hosted install sets its own. */
|
|
29
|
+
endpoint?: string | undefined;
|
|
30
|
+
/** The categories to report. Defaults to all five; SEO tools are among the busiest crawlers. */
|
|
31
|
+
categories?: readonly Category[] | undefined;
|
|
32
|
+
/** Agents to leave out, by the name pug shows, in any case, such as 'Googlebot'. */
|
|
33
|
+
skip?: readonly string[] | undefined;
|
|
34
|
+
/** Report requests for scripts, styles, images, fonts and /_next/ too. */
|
|
35
|
+
includeAssets?: boolean | undefined;
|
|
36
|
+
/** Replaces the platform's client address, such as with a CDN's client-IP header. Return a single IP. */
|
|
37
|
+
getIP?(request: Request | NodeRequest): string | null | undefined;
|
|
38
|
+
/** Keeps a serverless function alive until a report is sent, such as Vercel's waitUntil. A ctx wins. */
|
|
39
|
+
waitUntil?(promise: Promise<unknown>): void;
|
|
40
|
+
}
|
|
41
|
+
export interface Crawlers {
|
|
42
|
+
/**
|
|
43
|
+
* Fetch-style servers: Cloudflare Workers, Hono, Bun, Deno. Pass the response to report its
|
|
44
|
+
* status, and the platform's ctx so it waits for the report. A Next.js proxy has no response yet.
|
|
45
|
+
*/
|
|
46
|
+
track(request: Request, response?: Response, ctx?: WaitUntil): void;
|
|
47
|
+
/** Express or node:http middleware. It reports once the response closes. */
|
|
48
|
+
express(): (req: NodeRequest, res: NodeResponse, next?: (err?: unknown) => void) => void;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Reports requests from known crawlers to pug: one POST per request, without delaying the response.
|
|
52
|
+
* People's requests never leave the site, and neither do query strings, cookies or other headers. It
|
|
53
|
+
* never throws, so it can't break the site it runs in.
|
|
54
|
+
*/
|
|
55
|
+
export declare function crawlers(options: CrawlersOptions): Crawlers;
|
package/dist/crawlers.js
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
import { maxPropertyLength, maxUserAgentBytes, procedure } from './events.generated.js';
|
|
2
|
+
import { categories, matchAgent, truncate } from './match.js';
|
|
3
|
+
const defaultEndpoint = 'https://api.pugs.dev';
|
|
4
|
+
const timeoutMs = 2000;
|
|
5
|
+
// Framework internals, scripts, styles, images and fonts: crawlers fetch them too, which adds volume and no insight.
|
|
6
|
+
const asset = /^\/_next\/|\.(?:js|mjs|cjs|css|map|png|jpe?g|gif|svg|ico|webp|avif|bmp|tiff?|woff2?|ttf|otf|eot)$/i;
|
|
7
|
+
const ipv4 = /^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$/;
|
|
8
|
+
const warned = new Set();
|
|
9
|
+
function warnOnce(kind, message, detail) {
|
|
10
|
+
if (warned.has(kind))
|
|
11
|
+
return;
|
|
12
|
+
warned.add(kind);
|
|
13
|
+
// A detail that can't be printed, or a console that throws, mustn't break the site.
|
|
14
|
+
try {
|
|
15
|
+
console.warn(`@pug-sh/crawlers: ${message}${detail === undefined ? '' : ` (${String(detail)})`}`);
|
|
16
|
+
}
|
|
17
|
+
catch { }
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Reports requests from known crawlers to pug: one POST per request, without delaying the response.
|
|
21
|
+
* People's requests never leave the site, and neither do query strings, cookies or other headers. It
|
|
22
|
+
* never throws, so it can't break the site it runs in.
|
|
23
|
+
*/
|
|
24
|
+
export function crawlers(options) {
|
|
25
|
+
try {
|
|
26
|
+
return collector(options);
|
|
27
|
+
}
|
|
28
|
+
catch (err) {
|
|
29
|
+
warnOnce('options', 'invalid options, so nothing is reported', err);
|
|
30
|
+
return { track() { }, express: () => (_req, _res, next) => next?.() };
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
function collector(options) {
|
|
34
|
+
const apiKey = typeof options.apiKey === 'string' ? options.apiKey : '';
|
|
35
|
+
const usable = apiKey.startsWith('prv_');
|
|
36
|
+
if (!usable)
|
|
37
|
+
warnOnce('apiKey', 'apiKey must be a private key (prv_...), so nothing is reported');
|
|
38
|
+
const base = String(options.endpoint || defaultEndpoint).replace(/\/+$/, '');
|
|
39
|
+
if (!/^https?:\/\/[^/]/.test(base))
|
|
40
|
+
throw new TypeError(`endpoint must be an http(s) URL, not '${base}'`);
|
|
41
|
+
const endpoint = base + procedure;
|
|
42
|
+
const wanted = new Set(options.categories ?? categories);
|
|
43
|
+
for (const category of wanted) {
|
|
44
|
+
if (!categories.includes(category))
|
|
45
|
+
throw new TypeError(`unknown category '${category}'`);
|
|
46
|
+
}
|
|
47
|
+
const skip = new Set((options.skip ?? []).map(name => name.toLowerCase()));
|
|
48
|
+
const describe = (method, href, userAgent) => {
|
|
49
|
+
if (!usable || !userAgent)
|
|
50
|
+
return undefined;
|
|
51
|
+
method = method.toUpperCase();
|
|
52
|
+
if (method !== 'GET' && method !== 'HEAD')
|
|
53
|
+
return undefined;
|
|
54
|
+
let page;
|
|
55
|
+
try {
|
|
56
|
+
page = new URL(href);
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
return undefined;
|
|
60
|
+
}
|
|
61
|
+
if (page.protocol !== 'http:' && page.protocol !== 'https:')
|
|
62
|
+
return undefined;
|
|
63
|
+
if (!options.includeAssets && asset.test(page.pathname))
|
|
64
|
+
return undefined;
|
|
65
|
+
const url = page.origin + page.pathname;
|
|
66
|
+
if (url.length > maxPropertyLength)
|
|
67
|
+
return undefined;
|
|
68
|
+
const agent = matchAgent(userAgent);
|
|
69
|
+
if (!agent || !wanted.has(agent.category) || skip.has(agent.name.toLowerCase()))
|
|
70
|
+
return undefined;
|
|
71
|
+
return {
|
|
72
|
+
eventId: crypto.randomUUID(),
|
|
73
|
+
occurTime: new Date().toISOString(),
|
|
74
|
+
url,
|
|
75
|
+
userAgent: truncate(userAgent, maxUserAgentBytes),
|
|
76
|
+
};
|
|
77
|
+
};
|
|
78
|
+
const clientIP = (request, platform) => {
|
|
79
|
+
let ip = platform;
|
|
80
|
+
if (options.getIP) {
|
|
81
|
+
try {
|
|
82
|
+
ip = options.getIP(request);
|
|
83
|
+
}
|
|
84
|
+
catch (err) {
|
|
85
|
+
warnOnce('getIP', 'getIP threw, so reports go without an address', err);
|
|
86
|
+
return undefined;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
if (ip == null || ip === '')
|
|
90
|
+
return undefined;
|
|
91
|
+
if (typeof ip === 'string' && isIP(ip))
|
|
92
|
+
return ip;
|
|
93
|
+
warnOnce('ip', "the client's address isn't a single IP, so reports go without one");
|
|
94
|
+
return undefined;
|
|
95
|
+
};
|
|
96
|
+
return {
|
|
97
|
+
track(request, response, ctx) {
|
|
98
|
+
try {
|
|
99
|
+
const report = describe(request.method, request.url, request.headers.get('user-agent'));
|
|
100
|
+
if (!report)
|
|
101
|
+
return;
|
|
102
|
+
// Only Workers' requests have cf, and there CF-Connecting-IP can't be forged; Vercel sets x-real-ip.
|
|
103
|
+
const platform = 'cf' in request ? request.headers.get('cf-connecting-ip') : request.headers.get('x-real-ip');
|
|
104
|
+
const sending = send(endpoint, apiKey, report, clientIP(request, platform), () => response?.status);
|
|
105
|
+
if (ctx)
|
|
106
|
+
ctx.waitUntil(sending);
|
|
107
|
+
else
|
|
108
|
+
options.waitUntil?.(sending);
|
|
109
|
+
}
|
|
110
|
+
catch (err) {
|
|
111
|
+
warnOnce('error', 'could not report a request', err);
|
|
112
|
+
}
|
|
113
|
+
},
|
|
114
|
+
express() {
|
|
115
|
+
return (req, res, next) => {
|
|
116
|
+
try {
|
|
117
|
+
const path = req.originalUrl ?? req.url ?? '/';
|
|
118
|
+
const host = first(req.headers.host) ?? first(req.headers[':authority']);
|
|
119
|
+
const protocol = req.protocol ?? (req.socket?.encrypted ? 'https' : 'http');
|
|
120
|
+
// A target that isn't a path is already a full URL, as a request through a proxy sends it.
|
|
121
|
+
const href = path.startsWith('/') ? host && `${protocol}://${host}${path}` : path;
|
|
122
|
+
const report = href && describe(req.method ?? 'GET', href, first(req.headers['user-agent']));
|
|
123
|
+
if (report) {
|
|
124
|
+
const ip = clientIP(req, req.ip ?? req.socket?.remoteAddress);
|
|
125
|
+
const status = () => (res.writableFinished ? res.statusCode : undefined);
|
|
126
|
+
const sending = new Promise(resolve => res.once('close', () => resolve(send(endpoint, apiKey, report, ip, status)))).catch(err => warnOnce('error', 'could not report a request', err));
|
|
127
|
+
options.waitUntil?.(sending);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
catch (err) {
|
|
131
|
+
warnOnce('error', 'could not report a request', err);
|
|
132
|
+
}
|
|
133
|
+
next?.();
|
|
134
|
+
};
|
|
135
|
+
},
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
// One POST, with one retry on a network error, the timeout, a 409 or a 5xx. The retry resends the same
|
|
139
|
+
// body, so pug stores the request once. It never rejects: nothing awaits it on Bun or Node.
|
|
140
|
+
async function send(endpoint, apiKey, report, ip, status) {
|
|
141
|
+
try {
|
|
142
|
+
const statusCode = status();
|
|
143
|
+
const body = JSON.stringify({
|
|
144
|
+
events: [
|
|
145
|
+
{
|
|
146
|
+
eventId: report.eventId,
|
|
147
|
+
occurTime: report.occurTime,
|
|
148
|
+
kind: 'crawler_request',
|
|
149
|
+
client: { userAgent: report.userAgent, ip },
|
|
150
|
+
autoProperties: {
|
|
151
|
+
$url: { stringValue: report.url },
|
|
152
|
+
...(statusCode ? { $status_code: { intValue: String(statusCode) } } : {}),
|
|
153
|
+
},
|
|
154
|
+
},
|
|
155
|
+
],
|
|
156
|
+
});
|
|
157
|
+
let failure;
|
|
158
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
159
|
+
let res;
|
|
160
|
+
try {
|
|
161
|
+
res = await fetch(endpoint, {
|
|
162
|
+
method: 'POST',
|
|
163
|
+
headers: { 'content-type': 'application/json', 'x-api-key': apiKey },
|
|
164
|
+
body,
|
|
165
|
+
redirect: 'error',
|
|
166
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
catch (err) {
|
|
170
|
+
failure = err instanceof Error && err.cause ? err.cause : err;
|
|
171
|
+
continue;
|
|
172
|
+
}
|
|
173
|
+
if (res.ok) {
|
|
174
|
+
warnDropped((await res.json().catch(() => null)));
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
failure = `${res.status}: ${await reason(res)}`;
|
|
178
|
+
if (res.status < 500 && res.status !== 409) {
|
|
179
|
+
warnOnce('refused', 'pug refused a report', failure);
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
warnOnce('undelivered', `could not deliver reports to ${endpoint}`, failure);
|
|
184
|
+
}
|
|
185
|
+
catch (err) {
|
|
186
|
+
warnOnce('error', 'could not report a request', err);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
// A 200 can still drop the event, as when it breaks a rule or names no crawler the server knows.
|
|
190
|
+
function warnDropped(result) {
|
|
191
|
+
if (!result?.dropped)
|
|
192
|
+
return;
|
|
193
|
+
warnOnce('dropped', 'pug dropped a report', result.invalidEvents?.[0]?.violations ?? Object.keys(result.droppedByReason ?? {}));
|
|
194
|
+
}
|
|
195
|
+
async function reason(res) {
|
|
196
|
+
try {
|
|
197
|
+
return JSON.parse(await res.text()).message ?? res.statusText;
|
|
198
|
+
}
|
|
199
|
+
catch {
|
|
200
|
+
return res.statusText;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
function isIP(value) {
|
|
204
|
+
if (ipv4.test(value))
|
|
205
|
+
return true;
|
|
206
|
+
if (!value.includes(':') || !/^[\da-f:.]+$/i.test(value))
|
|
207
|
+
return false;
|
|
208
|
+
// URL's IPv6 parser, since edge runtimes have no node:net.
|
|
209
|
+
try {
|
|
210
|
+
new URL(`http://[${value}]`);
|
|
211
|
+
return true;
|
|
212
|
+
}
|
|
213
|
+
catch {
|
|
214
|
+
return false;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
function first(value) {
|
|
218
|
+
return Array.isArray(value) ? value[0] : value;
|
|
219
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
// AUTOGENERATED: do not edit. Run `make codegen` to regenerate.
|
|
2
|
+
// From pug's protos: the procedure a report goes to, the max_len of an auto property's
|
|
3
|
+
// string_value, in code points, and the max_bytes of the client's user_agent.
|
|
4
|
+
export const procedure = "/sdk.events.v1.EventsService/BatchCreate";
|
|
5
|
+
export const maxPropertyLength = 1024;
|
|
6
|
+
export const maxUserAgentBytes = 2048;
|
package/dist/index.d.ts
ADDED
package/dist/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { crawlers } from './crawlers.js';
|
package/dist/match.d.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export declare const categories: readonly ['answers', 'search', 'training', 'seo', 'other'];
|
|
2
|
+
/** Why a crawler fetches a page. */
|
|
3
|
+
export type Category = (typeof categories)[number];
|
|
4
|
+
export declare function truncate(s: string, bytes: number): string;
|
|
5
|
+
/**
|
|
6
|
+
* The crawler a user agent belongs to, as the server picks it: the table row whose token it holds as
|
|
7
|
+
* a whole word, any case, the longest token winning, or else the crawler-user-agents search-engine or
|
|
8
|
+
* seo entry that matches earliest in it, the first listed winning a tie. The server has the last word.
|
|
9
|
+
*/
|
|
10
|
+
export declare function matchAgent(userAgent: string): {
|
|
11
|
+
name: string;
|
|
12
|
+
category: Category;
|
|
13
|
+
} | undefined;
|
package/dist/match.js
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { listed, table } from './agents.generated.js';
|
|
2
|
+
export const categories = ['answers', 'search', 'training', 'seo', 'other'];
|
|
3
|
+
// The server reads only this much of a user agent.
|
|
4
|
+
const maxUserAgentBytes = 512;
|
|
5
|
+
const encoder = new TextEncoder();
|
|
6
|
+
let compiled;
|
|
7
|
+
const isWord = (c) => (c >= 48 && c <= 57) || (c >= 65 && c <= 90) || (c >= 97 && c <= 122) || c === 95;
|
|
8
|
+
function containsWord(s, token) {
|
|
9
|
+
for (let from = 0;;) {
|
|
10
|
+
const i = s.indexOf(token, from);
|
|
11
|
+
if (i < 0)
|
|
12
|
+
return false;
|
|
13
|
+
const end = i + token.length;
|
|
14
|
+
if ((i === 0 || !isWord(s.charCodeAt(i - 1))) && (end === s.length || !isWord(s.charCodeAt(end))))
|
|
15
|
+
return true;
|
|
16
|
+
from = i + 1;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
// Cut at a whole character, so the server never sees a partial one.
|
|
20
|
+
export function truncate(s, bytes) {
|
|
21
|
+
const { read } = encoder.encodeInto(s, new Uint8Array(bytes));
|
|
22
|
+
return read === s.length ? s : s.slice(0, read);
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* The crawler a user agent belongs to, as the server picks it: the table row whose token it holds as
|
|
26
|
+
* a whole word, any case, the longest token winning, or else the crawler-user-agents search-engine or
|
|
27
|
+
* seo entry that matches earliest in it, the first listed winning a tie. The server has the last word.
|
|
28
|
+
*/
|
|
29
|
+
export function matchAgent(userAgent) {
|
|
30
|
+
let cut = truncate(userAgent, maxUserAgentBytes).length;
|
|
31
|
+
// Drop the word the bound cuts, as the server does, or "GPTBotX" cut after its T reads as GPTBot.
|
|
32
|
+
if (cut < userAgent.length)
|
|
33
|
+
while (cut > 0 && isWord(userAgent.charCodeAt(cut)))
|
|
34
|
+
cut--;
|
|
35
|
+
const head = userAgent.slice(0, cut);
|
|
36
|
+
const lower = head.toLowerCase();
|
|
37
|
+
let best;
|
|
38
|
+
let bestLength = 0;
|
|
39
|
+
for (const [name, category, ...tokens] of table) {
|
|
40
|
+
for (const token of tokens) {
|
|
41
|
+
if (token.length > bestLength && containsWord(lower, token)) {
|
|
42
|
+
best = { name, category };
|
|
43
|
+
bestLength = token.length;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
if (best)
|
|
48
|
+
return best;
|
|
49
|
+
compiled ??= listed.map(([name, category, pattern]) => [name, category, new RegExp(pattern)]);
|
|
50
|
+
let at = Number.POSITIVE_INFINITY;
|
|
51
|
+
for (const [name, category, re] of compiled) {
|
|
52
|
+
const i = head.search(re);
|
|
53
|
+
if (i >= 0 && i < at) {
|
|
54
|
+
best = { name, category };
|
|
55
|
+
at = i;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return best;
|
|
59
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,64 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pug-sh/crawlers",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.1",
|
|
4
|
+
"description": "Report the AI crawlers, search engines and SEO tools that read your site to Pug, from your own request handling.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"types": "./dist/index.d.ts",
|
|
11
|
+
"default": "./dist/index.js"
|
|
12
|
+
}
|
|
13
|
+
},
|
|
14
|
+
"files": [
|
|
15
|
+
"dist",
|
|
16
|
+
"README.md",
|
|
17
|
+
"LICENSE"
|
|
18
|
+
],
|
|
19
|
+
"sideEffects": false,
|
|
20
|
+
"engines": {
|
|
21
|
+
"node": ">=20"
|
|
22
|
+
},
|
|
23
|
+
"homepage": "https://pug.sh",
|
|
24
|
+
"repository": {
|
|
25
|
+
"type": "git",
|
|
26
|
+
"url": "git+https://github.com/pug-sh/sdk-crawlers.git"
|
|
27
|
+
},
|
|
28
|
+
"bugs": {
|
|
29
|
+
"url": "https://github.com/pug-sh/sdk-crawlers/issues"
|
|
30
|
+
},
|
|
31
|
+
"publishConfig": {
|
|
32
|
+
"access": "public"
|
|
33
|
+
},
|
|
34
|
+
"scripts": {
|
|
35
|
+
"build": "tsc -p tsconfig.build.json",
|
|
36
|
+
"prepublishOnly": "npm run build",
|
|
37
|
+
"lint": "biome check --write .",
|
|
38
|
+
"format": "biome format --write .",
|
|
39
|
+
"typecheck": "tsc --noEmit",
|
|
40
|
+
"knip": "knip",
|
|
41
|
+
"test": "vitest run",
|
|
42
|
+
"test:watch": "vitest"
|
|
43
|
+
},
|
|
44
|
+
"keywords": [
|
|
45
|
+
"analytics",
|
|
46
|
+
"crawlers",
|
|
47
|
+
"ai",
|
|
48
|
+
"bots",
|
|
49
|
+
"seo",
|
|
50
|
+
"pug"
|
|
51
|
+
],
|
|
52
|
+
"author": "pug.sh",
|
|
53
|
+
"license": "MIT",
|
|
54
|
+
"devDependencies": {
|
|
55
|
+
"@biomejs/biome": "2.6.3",
|
|
56
|
+
"@bufbuild/protobuf": "^2.17.0",
|
|
57
|
+
"@bufbuild/protoc-gen-es": "^2.17.0",
|
|
58
|
+
"@types/node": "^26.6.5",
|
|
59
|
+
"crawler-user-agents": "1.66.0",
|
|
60
|
+
"knip": "^6.41.0",
|
|
61
|
+
"typescript": "^7.0.2",
|
|
62
|
+
"vitest": "^5.0.2"
|
|
63
|
+
}
|
|
64
|
+
}
|