@pug-sh/crawlers 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
- // AUTOGENERATED: do not edit. Run `make agents` to regenerate.
1
+ // AUTOGENERATED: do not edit. Run `make codegen` to regenerate.
2
2
  // table: pug's crawler table (proto/common/crawlers/v1), as [name, category, ...lowercased tokens].
3
3
  // listed: crawler-user-agents@1.66.0, its search-engine and seo entries, as [name, category, pattern],
4
4
  // under its license:
package/dist/crawlers.js CHANGED
@@ -1,10 +1,7 @@
1
+ import { maxPropertyLength, maxUserAgentBytes, procedure } from './events.generated.js';
1
2
  import { categories, matchAgent, truncate } from './match.js';
2
3
  const defaultEndpoint = 'https://api.pugs.dev';
3
- const procedure = '/sdk.events.v1.EventsService/BatchCreate';
4
4
  const timeoutMs = 2000;
5
- // The server's limits: $url in characters, a user agent in bytes.
6
- const maxURLLength = 1024;
7
- const maxUserAgentBytes = 2048;
8
5
  // Framework internals, scripts, styles, images and fonts: crawlers fetch them too, which adds volume and no insight.
9
6
  const asset = /^\/_next\/|\.(?:js|mjs|cjs|css|map|png|jpe?g|gif|svg|ico|webp|avif|bmp|tiff?|woff2?|ttf|otf|eot)$/i;
10
7
  const ipv4 = /^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$/;
@@ -66,7 +63,7 @@ function collector(options) {
66
63
  if (!options.includeAssets && asset.test(page.pathname))
67
64
  return undefined;
68
65
  const url = page.origin + page.pathname;
69
- if (url.length > maxURLLength)
66
+ if (url.length > maxPropertyLength)
70
67
  return undefined;
71
68
  const agent = matchAgent(userAgent);
72
69
  if (!agent || !wanted.has(agent.category) || skip.has(agent.name.toLowerCase()))
@@ -149,10 +146,10 @@ async function send(endpoint, apiKey, report, ip, status) {
149
146
  eventId: report.eventId,
150
147
  occurTime: report.occurTime,
151
148
  kind: 'crawler_request',
152
- client: { userAgent: report.userAgent, ...(ip ? { ip } : {}) },
149
+ client: { userAgent: report.userAgent, ip },
153
150
  autoProperties: {
154
151
  $url: { stringValue: report.url },
155
- ...(statusCode ? { $status_code: { intValue: statusCode } } : {}),
152
+ ...(statusCode ? { $status_code: { intValue: String(statusCode) } } : {}),
156
153
  },
157
154
  },
158
155
  ],
@@ -0,0 +1,3 @@
1
+ export declare const procedure = "/sdk.events.v1.EventsService/BatchCreate";
2
+ export declare const maxPropertyLength = 1024;
3
+ export declare const maxUserAgentBytes = 2048;
@@ -0,0 +1,6 @@
1
+ // AUTOGENERATED: do not edit. Run `make codegen` to regenerate.
2
+ // From pug's protos: the procedure a report goes to, the max_len of an auto property's
3
+ // string_value, in code points, and the max_bytes of the client's user_agent.
4
+ export const procedure = "/sdk.events.v1.EventsService/BatchCreate";
5
+ export const maxPropertyLength = 1024;
6
+ export const maxUserAgentBytes = 2048;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@pug-sh/crawlers",
3
- "version": "0.1.0",
3
+ "version": "0.1.1",
4
4
  "description": "Report the AI crawlers, search engines and SEO tools that read your site to Pug, from your own request handling.",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",
@@ -53,6 +53,8 @@
53
53
  "license": "MIT",
54
54
  "devDependencies": {
55
55
  "@biomejs/biome": "2.6.3",
56
+ "@bufbuild/protobuf": "^2.17.0",
57
+ "@bufbuild/protoc-gen-es": "^2.17.0",
56
58
  "@types/node": "^26.6.5",
57
59
  "crawler-user-agents": "1.66.0",
58
60
  "knip": "^6.41.0",