@growthgenie/ai-traffic 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -16,11 +16,9 @@ the response. Aborted responses have unknown status. No sampling is performed.
16
16
 
17
17
  ## Setup
18
18
 
19
- After publication: `pnpm add @growthgenie/ai-traffic`. In this monorepo use a
20
- `workspace:*` dependency. Create a **generic** connection using the existing A1b
21
- control plane; the generic route authenticates generic keys. Do not use a native
22
- `node` connector key against it. Configure exact allowed hosts, production/staging
23
- scope and traffic mode at connection creation. Keep the full issued
19
+ `npm install @growthgenie/ai-traffic`. In Growth Genie, open AI traffic → Connect your
20
+ site and choose **Next.js or Node** (or Vercel → **Middleware**); that creates the
21
+ connection with your site's hosts and shows its key once. Keep the full issued
24
22
  `gg_ai_<uuid>.<tag>_<secret>` key in server secret configuration. The server still
25
23
  owns classification, tenant binding, verification, policy and accounting.
26
24
 
@@ -91,7 +89,12 @@ unverified fetches. No country is inferred.
91
89
  Request-only referral candidates require GET, a recognized AI referrer,
92
90
  `Sec-Fetch-Mode: navigate` **and** `Sec-Fetch-Dest: document`, an eligible path,
93
91
  and no prefetch/preview purpose. API/framework/static asset/private/account paths
94
- are excluded (including encoded forms). No navigation evidence means no referral
92
+ are excluded from referrals (including encoded forms). A request that names a
93
+ catalog AI crawler is sent on any path except private ones: since catalog
94
+ `ai-traffic/3` the receiver counts a crawler's files (images, CSS, JS, fonts…)
95
+ and calls scanners probing paths such as `/.env` or `/wp-login.php` fake bots.
96
+ Static files a CDN or your host serves without running Node never reach the
97
+ package, so only an edge method sees those. No navigation evidence means no referral
95
98
  candidate. **A1b v1 does not transmit navigation headers, and its referral rule
96
99
  requires a successful final status and document evidence. Consequently Proxy-only
97
100
  referral candidates are dropped centrally, not counted. Referral measurement is
@@ -263,7 +266,7 @@ Owner's unexecuted live G-node gate:
263
266
  Deploy on actual Vercel Hobby **and** Pro. Record Next/Node/package versions,
264
267
  plan, host limits, region, timestamp and sanitized request captures.
265
268
  2. Create separate generic staging connections. Run `/v1/ai-traffic/test`; confirm
266
- test health changes without analytics/allowance. Rotate/revoke keys and check
269
+ test health changes without analytics or usage. Rotate/revoke keys and check
267
270
  failures without recording keys or payloads in logs.
268
271
  3. Send a known-UA fetch to a dynamic page through Proxy; reconcile one accepted
269
272
  unverified observation, null status/type, no duplicate charges on ambiguous
@@ -277,6 +280,6 @@ Owner's unexecuted live G-node gate:
277
280
  measure which requests execute Proxy/Node and disclose every bypass. Test
278
281
  static export build rejection with `withAiTrafficConfig`.
279
282
  6. Measure delivery success, loss, overhead and shutdown at host execution limits
280
- and reconcile recent rows, daily aggregates and workspace allowance. Save
283
+ and reconcile recent rows, daily aggregates and workspace usage. Save
281
284
  sanitized captures/negative cases plus source/version/date notes. Do not claim
282
285
  full G-node or universal plan/version support until these steps pass.
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  AiTrafficRequestEvent
3
- } from "./chunk-RYXHXHXU.js";
3
+ } from "./chunk-QHDUB2NG.js";
4
4
 
5
5
  // src/delivery.ts
6
6
  var deliveryLimits = Object.freeze({ concurrency: 2, queueEvents: 256, timeoutMs: 3e3, retries: 3, baseDelayMs: 1e3, maxDelayMs: 6e4, maxDeliveryMs: 3e4, shutdownMs: 5e3 });
@@ -1,11 +1,11 @@
1
1
  import {
2
2
  makeEvent,
3
3
  validatePolicy
4
- } from "./chunk-HAXY7PYH.js";
4
+ } from "./chunk-PFHZMJQO.js";
5
5
  import {
6
6
  __commonJS,
7
7
  __toESM
8
- } from "./chunk-RYXHXHXU.js";
8
+ } from "./chunk-QHDUB2NG.js";
9
9
 
10
10
  // ../../node_modules/.pnpm/ipaddr.js@2.3.0/node_modules/ipaddr.js/lib/ipaddr.js
11
11
  var require_ipaddr = __commonJS({
@@ -3,10 +3,10 @@ import {
3
3
  normalizeAiTrafficHostname,
4
4
  normalizeAiTrafficPath,
5
5
  normalizeAiTrafficReferrer
6
- } from "./chunk-RYXHXHXU.js";
6
+ } from "./chunk-QHDUB2NG.js";
7
7
 
8
8
  // ../contracts/src/aiTrafficCatalog.ts
9
- var AI_TRAFFIC_CLASSIFIER_VERSION = "ai-traffic/2";
9
+ var AI_TRAFFIC_CLASSIFIER_VERSION = "ai-traffic/3";
10
10
  var DOCS = {
11
11
  openai: "https://developers.openai.com/api/docs/bots",
12
12
  anthropic: "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
@@ -14,6 +14,7 @@ var DOCS = {
14
14
  duckduckgo: "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot",
15
15
  mistral: "https://docs.mistral.ai/robots",
16
16
  meta: "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
17
+ facebook: "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
17
18
  commoncrawl: "https://commoncrawl.org/ccbot",
18
19
  growth_genie: "https://growthgenie.ai",
19
20
  opencode: "https://opencode.ai",
@@ -39,6 +40,9 @@ var AI_TRAFFIC_AGENTS = [
39
40
  agent("Meta-ExternalFetcher", "meta", "answer_fetch", null),
40
41
  agent("Meta-WebIndexer", "meta", "search_index", null),
41
42
  agent("Meta-ExternalAgent", "meta", "model_training", null),
43
+ // Facebook's link-preview fetcher (shares, Messenger and WhatsApp cards; reads og:image), not one of Meta's AI
44
+ // crawlers, so it is its own provider outside the Meta group. Meta publishes no agent-specific ranges for it.
45
+ agent("facebookexternalhit", "facebook", "link_preview", null),
42
46
  agent("CCBot", "commoncrawl", "model_training", "ccbot"),
43
47
  agent("GrowthGenieBot/1.0", "growth_genie", "growth_genie", null, true, "GrowthGenieBot/1.0"),
44
48
  // Page-fetch sources and exclusions: apps/mcp/AI-TRAFFIC-CODING-AGENTS.md.
@@ -88,18 +92,56 @@ function classifyAiTrafficReferrer(referrer) {
88
92
  const entry = AI_TRAFFIC_REFERRERS.find((row) => row.hosts.some((listed) => host === listed || host.endsWith(`.${listed}`)));
89
93
  return entry ? { source: entry.source, host, canonicalHost: entry.hosts[0] } : null;
90
94
  }
91
- function isAiTrafficExcludedPath(raw) {
95
+ var AI_TRAFFIC_ASSET_DIRS = ["_next", "_nuxt", "_astro", "_vercel", "_framework", "cdn-cgi"];
96
+ var AI_TRAFFIC_ENDPOINT_DIRS = ["api", "wp-json"];
97
+ var AI_TRAFFIC_FILE_EXTENSIONS = [
98
+ "avif",
99
+ "bmp",
100
+ "css",
101
+ "csv",
102
+ "eot",
103
+ "gif",
104
+ "ico",
105
+ "jpeg",
106
+ "jpg",
107
+ "js",
108
+ "json",
109
+ "map",
110
+ "mjs",
111
+ "mp3",
112
+ "mp4",
113
+ "ogg",
114
+ "otf",
115
+ "pdf",
116
+ "png",
117
+ "svg",
118
+ "ttf",
119
+ "webm",
120
+ "webp",
121
+ "woff",
122
+ "woff2",
123
+ "zip"
124
+ ];
125
+ var EXCLUDED_DIRS = /* @__PURE__ */ new Set([...AI_TRAFFIC_ASSET_DIRS, ...AI_TRAFFIC_ENDPOINT_DIRS, "wp-admin", "graphql"]);
126
+ var FILE_EXTENSIONS = new Set(AI_TRAFFIC_FILE_EXTENSIONS);
127
+ function pathParts(raw) {
92
128
  const path = normalizeAiTrafficPath(raw);
93
- if (path === null) return true;
94
- const decoded = decodeURIComponent(path).toLowerCase();
95
- return /^\/(?:api|_next|_nuxt|_astro|_vercel|_framework|cdn-cgi|wp-json|wp-admin|graphql)(?:\/|$)/.test(decoded) || /\.(?:avif|bmp|css|csv|eot|gif|ico|jpe?g|js|json|map|mjs|mp[34]|ogg|otf|pdf|png|svg|ttf|webm|webp|woff2?|zip)$/i.test(decoded);
129
+ if (path === null) return null;
130
+ const segments = decodeURIComponent(path).toLowerCase().split("/").slice(1);
131
+ const name = segments.at(-1) ?? "";
132
+ const dot = name.lastIndexOf(".");
133
+ return { segments, first: segments[0] ?? "", extension: dot === -1 ? "" : name.slice(dot + 1) };
134
+ }
135
+ function isAiTrafficExcludedPath(raw) {
136
+ const parts = pathParts(raw);
137
+ return parts === null || EXCLUDED_DIRS.has(parts.first) || FILE_EXTENSIONS.has(parts.extension);
96
138
  }
97
139
  function isAiTrafficCandidate(input) {
98
- if (isAiTrafficExcludedPath(input.path)) return false;
140
+ if (normalizeAiTrafficPath(input.path) === null) return false;
99
141
  const agents = findAiTrafficAgents(input.userAgent);
100
142
  if (agents.some((agent2) => !agent2.included)) return false;
101
143
  if (agents.length > 0 && (input.method === "GET" || input.method === "HEAD")) return true;
102
- return input.method === "GET" && classifyAiTrafficReferrer(input.referrer) !== null;
144
+ return input.method === "GET" && !isAiTrafficExcludedPath(input.path) && classifyAiTrafficReferrer(input.referrer) !== null;
103
145
  }
104
146
 
105
147
  // src/event.ts
@@ -119,7 +161,7 @@ function makeEvent(input, hosts) {
119
161
  const host = normalizeAiTrafficHostname(url.hostname);
120
162
  if (!host || !hosts.has(host)) return null;
121
163
  const path = url.pathname;
122
- if (isAiTrafficExcludedPath(path) || /^\/(?:admin|private|account|reset-password|password-reset|verify-email)(?:\/|$)/i.test(decodeURIComponent(path))) return null;
164
+ if (normalizeAiTrafficPath(path) === null || /^\/(?:admin|private|account|reset-password|password-reset|verify-email)(?:\/|$)/i.test(decodeURIComponent(path))) return null;
123
165
  const userAgent = input.headers.get("user-agent") ?? "";
124
166
  const referrer = input.headers.get("referer");
125
167
  const method = input.method.toUpperCase();
@@ -19029,7 +19029,9 @@ var AiTrafficVerification = external_exports.enum([
19029
19029
  "unverified_ip_outside_range",
19030
19030
  "unverified_proxy_ip",
19031
19031
  "unverified_proxy_ranges_unavailable",
19032
- "self_claimed"
19032
+ "self_claimed",
19033
+ // ai-traffic/3: a claimed crawler without a verified IP asked for a path scanners probe (a fake bot).
19034
+ "unverified_scanner_path"
19033
19035
  ]);
19034
19036
  var AI_TRAFFIC_FETCH_VERIFICATIONS = AiTrafficVerification.exclude(["self_claimed"]).options;
19035
19037
  var AI_TRAFFIC_UNVERIFIED_REASONS = AiTrafficVerification.options.filter((value) => value.startsWith("unverified_"));
@@ -1,7 +1,7 @@
1
1
  import {
2
2
  makeEvent,
3
3
  validatePolicy
4
- } from "./chunk-HAXY7PYH.js";
4
+ } from "./chunk-PFHZMJQO.js";
5
5
 
6
6
  // src/next.ts
7
7
  function assertNextOutput(output) {
package/dist/index.js CHANGED
@@ -1,12 +1,12 @@
1
1
  import "./chunk-LI2LYV3R.js";
2
2
  import {
3
3
  catalogVersion
4
- } from "./chunk-HAXY7PYH.js";
4
+ } from "./chunk-PFHZMJQO.js";
5
5
  import {
6
6
  createTrafficClient,
7
7
  deliveryLimits
8
- } from "./chunk-5PNSCBAB.js";
9
- import "./chunk-RYXHXHXU.js";
8
+ } from "./chunk-5WVWPJBS.js";
9
+ import "./chunk-QHDUB2NG.js";
10
10
  export {
11
11
  catalogVersion,
12
12
  createTrafficClient,
@@ -1,7 +1,7 @@
1
1
  import {
2
2
  createTrafficClient
3
- } from "./chunk-5PNSCBAB.js";
4
- import "./chunk-RYXHXHXU.js";
3
+ } from "./chunk-5WVWPJBS.js";
4
+ import "./chunk-QHDUB2NG.js";
5
5
 
6
6
  // src/next-server.ts
7
7
  import "server-only";
package/dist/next.d.ts CHANGED
@@ -21,6 +21,8 @@ export declare function createNextProxy(options: NextOptions): (request: Request
21
21
  /** Wrap only a final Web Response handler, never Proxy's NextResponse.next() placeholder. */
22
22
  export declare function createResponseWrapper(options: NextOptions): <Args extends unknown[]>(handler: (request: Request, ...args: Args) => Response | Promise<Response>) => (request: Request, event: LifetimeEvent, ...args: Args) => Promise<Response>;
23
23
  /** Call in next.config so static export fails even when Proxy would never be loaded. */
24
- export declare function withAiTrafficConfig<T extends {
24
+ /** Returns the same config, typed with `output` so `config.output` compiles for any config, including an empty one.
25
+ * Any object is accepted at compile time (Next configs rarely set `output`); a static export still throws at run time. */
26
+ export declare function withAiTrafficConfig<T extends object>(config: T): T & {
25
27
  output?: "standalone" | "export";
26
- }>(config: T): T;
28
+ };
package/dist/next.js CHANGED
@@ -4,9 +4,9 @@ import {
4
4
  createResponseWrapper,
5
5
  proxyCoverage,
6
6
  withAiTrafficConfig
7
- } from "./chunk-54YHPRVG.js";
8
- import "./chunk-HAXY7PYH.js";
9
- import "./chunk-RYXHXHXU.js";
7
+ } from "./chunk-XIHA3LLI.js";
8
+ import "./chunk-PFHZMJQO.js";
9
+ import "./chunk-QHDUB2NG.js";
10
10
  export {
11
11
  assertNextOutput,
12
12
  createNextProxy,
package/dist/node.js CHANGED
@@ -1,9 +1,9 @@
1
1
  import {
2
2
  createNodeMiddleware,
3
3
  nodeCoverage
4
- } from "./chunk-EGXH5ES5.js";
5
- import "./chunk-HAXY7PYH.js";
6
- import "./chunk-RYXHXHXU.js";
4
+ } from "./chunk-GACSUWSG.js";
5
+ import "./chunk-PFHZMJQO.js";
6
+ import "./chunk-QHDUB2NG.js";
7
7
  export {
8
8
  createNodeMiddleware,
9
9
  nodeCoverage
@@ -1,12 +1,12 @@
1
1
  import "./chunk-LI2LYV3R.js";
2
2
  import {
3
3
  catalogVersion
4
- } from "./chunk-HAXY7PYH.js";
4
+ } from "./chunk-PFHZMJQO.js";
5
5
  import {
6
6
  createTrafficClient,
7
7
  deliveryLimits
8
- } from "./chunk-5PNSCBAB.js";
9
- import "./chunk-RYXHXHXU.js";
8
+ } from "./chunk-5WVWPJBS.js";
9
+ import "./chunk-QHDUB2NG.js";
10
10
 
11
11
  // src/server-index.ts
12
12
  import "server-only";
@@ -4,9 +4,9 @@ import {
4
4
  createResponseWrapper,
5
5
  proxyCoverage,
6
6
  withAiTrafficConfig
7
- } from "./chunk-54YHPRVG.js";
8
- import "./chunk-HAXY7PYH.js";
9
- import "./chunk-RYXHXHXU.js";
7
+ } from "./chunk-XIHA3LLI.js";
8
+ import "./chunk-PFHZMJQO.js";
9
+ import "./chunk-QHDUB2NG.js";
10
10
 
11
11
  // src/server-next.ts
12
12
  import "server-only";
@@ -1,9 +1,9 @@
1
1
  import {
2
2
  createNodeMiddleware,
3
3
  nodeCoverage
4
- } from "./chunk-EGXH5ES5.js";
5
- import "./chunk-HAXY7PYH.js";
6
- import "./chunk-RYXHXHXU.js";
4
+ } from "./chunk-GACSUWSG.js";
5
+ import "./chunk-PFHZMJQO.js";
6
+ import "./chunk-QHDUB2NG.js";
7
7
 
8
8
  // src/server-node.ts
9
9
  import "server-only";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@growthgenie/ai-traffic",
3
- "version": "0.1.0",
3
+ "version": "0.1.2",
4
4
  "type": "module",
5
5
  "engines": {
6
6
  "node": ">=24"