@growthgenie/ai-traffic 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -8
- package/dist/{chunk-5PNSCBAB.js → chunk-5WVWPJBS.js} +1 -1
- package/dist/{chunk-EGXH5ES5.js → chunk-GACSUWSG.js} +2 -2
- package/dist/{chunk-HAXY7PYH.js → chunk-PFHZMJQO.js} +51 -9
- package/dist/{chunk-RYXHXHXU.js → chunk-QHDUB2NG.js} +3 -1
- package/dist/{chunk-54YHPRVG.js → chunk-XIHA3LLI.js} +1 -1
- package/dist/index.js +3 -3
- package/dist/next-server.js +2 -2
- package/dist/next.d.ts +4 -2
- package/dist/next.js +3 -3
- package/dist/node.js +3 -3
- package/dist/server-index.js +3 -3
- package/dist/server-next.js +3 -3
- package/dist/server-node.js +3 -3
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -16,11 +16,9 @@ the response. Aborted responses have unknown status. No sampling is performed.
|
|
|
16
16
|
|
|
17
17
|
## Setup
|
|
18
18
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
`node` connector key against it. Configure exact allowed hosts, production/staging
|
|
23
|
-
scope and traffic mode at connection creation. Keep the full issued
|
|
19
|
+
`npm install @growthgenie/ai-traffic`. In Growth Genie, open AI traffic → Connect your
|
|
20
|
+
site and choose **Next.js or Node** (or Vercel → **Middleware**); that creates the
|
|
21
|
+
connection with your site's hosts and shows its key once. Keep the full issued
|
|
24
22
|
`gg_ai_<uuid>.<tag>_<secret>` key in server secret configuration. The server still
|
|
25
23
|
owns classification, tenant binding, verification, policy and accounting.
|
|
26
24
|
|
|
@@ -91,7 +89,12 @@ unverified fetches. No country is inferred.
|
|
|
91
89
|
Request-only referral candidates require GET, a recognized AI referrer,
|
|
92
90
|
`Sec-Fetch-Mode: navigate` **and** `Sec-Fetch-Dest: document`, an eligible path,
|
|
93
91
|
and no prefetch/preview purpose. API/framework/static asset/private/account paths
|
|
94
|
-
are excluded (including encoded forms).
|
|
92
|
+
are excluded from referrals (including encoded forms). A request that names a
|
|
93
|
+
catalog AI crawler is sent on any path except private ones: since catalog
|
|
94
|
+
`ai-traffic/3` the receiver counts a crawler's files (images, CSS, JS, fonts…)
|
|
95
|
+
and calls scanners probing paths such as `/.env` or `/wp-login.php` fake bots.
|
|
96
|
+
Static files a CDN or your host serves without running Node never reach the
|
|
97
|
+
package, so only an edge method sees those. No navigation evidence means no referral
|
|
95
98
|
candidate. **A1b v1 does not transmit navigation headers, and its referral rule
|
|
96
99
|
requires a successful final status and document evidence. Consequently Proxy-only
|
|
97
100
|
referral candidates are dropped centrally, not counted. Referral measurement is
|
|
@@ -263,7 +266,7 @@ Owner's unexecuted live G-node gate:
|
|
|
263
266
|
Deploy on actual Vercel Hobby **and** Pro. Record Next/Node/package versions,
|
|
264
267
|
plan, host limits, region, timestamp and sanitized request captures.
|
|
265
268
|
2. Create separate generic staging connections. Run `/v1/ai-traffic/test`; confirm
|
|
266
|
-
test health changes without analytics
|
|
269
|
+
test health changes without analytics or usage. Rotate/revoke keys and check
|
|
267
270
|
failures without recording keys or payloads in logs.
|
|
268
271
|
3. Send a known-UA fetch to a dynamic page through Proxy; reconcile one accepted
|
|
269
272
|
unverified observation, null status/type, no duplicate charges on ambiguous
|
|
@@ -277,6 +280,6 @@ Owner's unexecuted live G-node gate:
|
|
|
277
280
|
measure which requests execute Proxy/Node and disclose every bypass. Test
|
|
278
281
|
static export build rejection with `withAiTrafficConfig`.
|
|
279
282
|
6. Measure delivery success, loss, overhead and shutdown at host execution limits
|
|
280
|
-
and reconcile recent rows, daily aggregates and workspace
|
|
283
|
+
and reconcile recent rows, daily aggregates and workspace usage. Save
|
|
281
284
|
sanitized captures/negative cases plus source/version/date notes. Do not claim
|
|
282
285
|
full G-node or universal plan/version support until these steps pass.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
AiTrafficRequestEvent
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-QHDUB2NG.js";
|
|
4
4
|
|
|
5
5
|
// src/delivery.ts
|
|
6
6
|
var deliveryLimits = Object.freeze({ concurrency: 2, queueEvents: 256, timeoutMs: 3e3, retries: 3, baseDelayMs: 1e3, maxDelayMs: 6e4, maxDeliveryMs: 3e4, shutdownMs: 5e3 });
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import {
|
|
2
2
|
makeEvent,
|
|
3
3
|
validatePolicy
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-PFHZMJQO.js";
|
|
5
5
|
import {
|
|
6
6
|
__commonJS,
|
|
7
7
|
__toESM
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-QHDUB2NG.js";
|
|
9
9
|
|
|
10
10
|
// ../../node_modules/.pnpm/ipaddr.js@2.3.0/node_modules/ipaddr.js/lib/ipaddr.js
|
|
11
11
|
var require_ipaddr = __commonJS({
|
|
@@ -3,10 +3,10 @@ import {
|
|
|
3
3
|
normalizeAiTrafficHostname,
|
|
4
4
|
normalizeAiTrafficPath,
|
|
5
5
|
normalizeAiTrafficReferrer
|
|
6
|
-
} from "./chunk-
|
|
6
|
+
} from "./chunk-QHDUB2NG.js";
|
|
7
7
|
|
|
8
8
|
// ../contracts/src/aiTrafficCatalog.ts
|
|
9
|
-
var AI_TRAFFIC_CLASSIFIER_VERSION = "ai-traffic/
|
|
9
|
+
var AI_TRAFFIC_CLASSIFIER_VERSION = "ai-traffic/3";
|
|
10
10
|
var DOCS = {
|
|
11
11
|
openai: "https://developers.openai.com/api/docs/bots",
|
|
12
12
|
anthropic: "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
|
|
@@ -14,6 +14,7 @@ var DOCS = {
|
|
|
14
14
|
duckduckgo: "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot",
|
|
15
15
|
mistral: "https://docs.mistral.ai/robots",
|
|
16
16
|
meta: "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
|
|
17
|
+
facebook: "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
|
|
17
18
|
commoncrawl: "https://commoncrawl.org/ccbot",
|
|
18
19
|
growth_genie: "https://growthgenie.ai",
|
|
19
20
|
opencode: "https://opencode.ai",
|
|
@@ -39,6 +40,9 @@ var AI_TRAFFIC_AGENTS = [
|
|
|
39
40
|
agent("Meta-ExternalFetcher", "meta", "answer_fetch", null),
|
|
40
41
|
agent("Meta-WebIndexer", "meta", "search_index", null),
|
|
41
42
|
agent("Meta-ExternalAgent", "meta", "model_training", null),
|
|
43
|
+
// Facebook's link-preview fetcher (shares, Messenger and WhatsApp cards; reads og:image), not one of Meta's AI
|
|
44
|
+
// crawlers, so it is its own provider outside the Meta group. Meta publishes no agent-specific ranges for it.
|
|
45
|
+
agent("facebookexternalhit", "facebook", "link_preview", null),
|
|
42
46
|
agent("CCBot", "commoncrawl", "model_training", "ccbot"),
|
|
43
47
|
agent("GrowthGenieBot/1.0", "growth_genie", "growth_genie", null, true, "GrowthGenieBot/1.0"),
|
|
44
48
|
// Page-fetch sources and exclusions: apps/mcp/AI-TRAFFIC-CODING-AGENTS.md.
|
|
@@ -88,18 +92,56 @@ function classifyAiTrafficReferrer(referrer) {
|
|
|
88
92
|
const entry = AI_TRAFFIC_REFERRERS.find((row) => row.hosts.some((listed) => host === listed || host.endsWith(`.${listed}`)));
|
|
89
93
|
return entry ? { source: entry.source, host, canonicalHost: entry.hosts[0] } : null;
|
|
90
94
|
}
|
|
91
|
-
|
|
95
|
+
var AI_TRAFFIC_ASSET_DIRS = ["_next", "_nuxt", "_astro", "_vercel", "_framework", "cdn-cgi"];
|
|
96
|
+
var AI_TRAFFIC_ENDPOINT_DIRS = ["api", "wp-json"];
|
|
97
|
+
var AI_TRAFFIC_FILE_EXTENSIONS = [
|
|
98
|
+
"avif",
|
|
99
|
+
"bmp",
|
|
100
|
+
"css",
|
|
101
|
+
"csv",
|
|
102
|
+
"eot",
|
|
103
|
+
"gif",
|
|
104
|
+
"ico",
|
|
105
|
+
"jpeg",
|
|
106
|
+
"jpg",
|
|
107
|
+
"js",
|
|
108
|
+
"json",
|
|
109
|
+
"map",
|
|
110
|
+
"mjs",
|
|
111
|
+
"mp3",
|
|
112
|
+
"mp4",
|
|
113
|
+
"ogg",
|
|
114
|
+
"otf",
|
|
115
|
+
"pdf",
|
|
116
|
+
"png",
|
|
117
|
+
"svg",
|
|
118
|
+
"ttf",
|
|
119
|
+
"webm",
|
|
120
|
+
"webp",
|
|
121
|
+
"woff",
|
|
122
|
+
"woff2",
|
|
123
|
+
"zip"
|
|
124
|
+
];
|
|
125
|
+
var EXCLUDED_DIRS = /* @__PURE__ */ new Set([...AI_TRAFFIC_ASSET_DIRS, ...AI_TRAFFIC_ENDPOINT_DIRS, "wp-admin", "graphql"]);
|
|
126
|
+
var FILE_EXTENSIONS = new Set(AI_TRAFFIC_FILE_EXTENSIONS);
|
|
127
|
+
function pathParts(raw) {
|
|
92
128
|
const path = normalizeAiTrafficPath(raw);
|
|
93
|
-
if (path === null) return
|
|
94
|
-
const
|
|
95
|
-
|
|
129
|
+
if (path === null) return null;
|
|
130
|
+
const segments = decodeURIComponent(path).toLowerCase().split("/").slice(1);
|
|
131
|
+
const name = segments.at(-1) ?? "";
|
|
132
|
+
const dot = name.lastIndexOf(".");
|
|
133
|
+
return { segments, first: segments[0] ?? "", extension: dot === -1 ? "" : name.slice(dot + 1) };
|
|
134
|
+
}
|
|
135
|
+
function isAiTrafficExcludedPath(raw) {
|
|
136
|
+
const parts = pathParts(raw);
|
|
137
|
+
return parts === null || EXCLUDED_DIRS.has(parts.first) || FILE_EXTENSIONS.has(parts.extension);
|
|
96
138
|
}
|
|
97
139
|
function isAiTrafficCandidate(input) {
|
|
98
|
-
if (
|
|
140
|
+
if (normalizeAiTrafficPath(input.path) === null) return false;
|
|
99
141
|
const agents = findAiTrafficAgents(input.userAgent);
|
|
100
142
|
if (agents.some((agent2) => !agent2.included)) return false;
|
|
101
143
|
if (agents.length > 0 && (input.method === "GET" || input.method === "HEAD")) return true;
|
|
102
|
-
return input.method === "GET" && classifyAiTrafficReferrer(input.referrer) !== null;
|
|
144
|
+
return input.method === "GET" && !isAiTrafficExcludedPath(input.path) && classifyAiTrafficReferrer(input.referrer) !== null;
|
|
103
145
|
}
|
|
104
146
|
|
|
105
147
|
// src/event.ts
|
|
@@ -119,7 +161,7 @@ function makeEvent(input, hosts) {
|
|
|
119
161
|
const host = normalizeAiTrafficHostname(url.hostname);
|
|
120
162
|
if (!host || !hosts.has(host)) return null;
|
|
121
163
|
const path = url.pathname;
|
|
122
|
-
if (
|
|
164
|
+
if (normalizeAiTrafficPath(path) === null || /^\/(?:admin|private|account|reset-password|password-reset|verify-email)(?:\/|$)/i.test(decodeURIComponent(path))) return null;
|
|
123
165
|
const userAgent = input.headers.get("user-agent") ?? "";
|
|
124
166
|
const referrer = input.headers.get("referer");
|
|
125
167
|
const method = input.method.toUpperCase();
|
|
@@ -19029,7 +19029,9 @@ var AiTrafficVerification = external_exports.enum([
|
|
|
19029
19029
|
"unverified_ip_outside_range",
|
|
19030
19030
|
"unverified_proxy_ip",
|
|
19031
19031
|
"unverified_proxy_ranges_unavailable",
|
|
19032
|
-
"self_claimed"
|
|
19032
|
+
"self_claimed",
|
|
19033
|
+
// ai-traffic/3: a claimed crawler without a verified IP asked for a path scanners probe (a fake bot).
|
|
19034
|
+
"unverified_scanner_path"
|
|
19033
19035
|
]);
|
|
19034
19036
|
var AI_TRAFFIC_FETCH_VERIFICATIONS = AiTrafficVerification.exclude(["self_claimed"]).options;
|
|
19035
19037
|
var AI_TRAFFIC_UNVERIFIED_REASONS = AiTrafficVerification.options.filter((value) => value.startsWith("unverified_"));
|
package/dist/index.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import "./chunk-LI2LYV3R.js";
|
|
2
2
|
import {
|
|
3
3
|
catalogVersion
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-PFHZMJQO.js";
|
|
5
5
|
import {
|
|
6
6
|
createTrafficClient,
|
|
7
7
|
deliveryLimits
|
|
8
|
-
} from "./chunk-
|
|
9
|
-
import "./chunk-
|
|
8
|
+
} from "./chunk-5WVWPJBS.js";
|
|
9
|
+
import "./chunk-QHDUB2NG.js";
|
|
10
10
|
export {
|
|
11
11
|
catalogVersion,
|
|
12
12
|
createTrafficClient,
|
package/dist/next-server.js
CHANGED
package/dist/next.d.ts
CHANGED
|
@@ -21,6 +21,8 @@ export declare function createNextProxy(options: NextOptions): (request: Request
|
|
|
21
21
|
/** Wrap only a final Web Response handler, never Proxy's NextResponse.next() placeholder. */
|
|
22
22
|
export declare function createResponseWrapper(options: NextOptions): <Args extends unknown[]>(handler: (request: Request, ...args: Args) => Response | Promise<Response>) => (request: Request, event: LifetimeEvent, ...args: Args) => Promise<Response>;
|
|
23
23
|
/** Call in next.config so static export fails even when Proxy would never be loaded. */
|
|
24
|
-
|
|
24
|
+
/** Returns the same config, typed with `output` so `config.output` compiles for any config, including an empty one.
|
|
25
|
+
* Any object is accepted at compile time (Next configs rarely set `output`); a static export still throws at run time. */
|
|
26
|
+
export declare function withAiTrafficConfig<T extends object>(config: T): T & {
|
|
25
27
|
output?: "standalone" | "export";
|
|
26
|
-
}
|
|
28
|
+
};
|
package/dist/next.js
CHANGED
|
@@ -4,9 +4,9 @@ import {
|
|
|
4
4
|
createResponseWrapper,
|
|
5
5
|
proxyCoverage,
|
|
6
6
|
withAiTrafficConfig
|
|
7
|
-
} from "./chunk-
|
|
8
|
-
import "./chunk-
|
|
9
|
-
import "./chunk-
|
|
7
|
+
} from "./chunk-XIHA3LLI.js";
|
|
8
|
+
import "./chunk-PFHZMJQO.js";
|
|
9
|
+
import "./chunk-QHDUB2NG.js";
|
|
10
10
|
export {
|
|
11
11
|
assertNextOutput,
|
|
12
12
|
createNextProxy,
|
package/dist/node.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createNodeMiddleware,
|
|
3
3
|
nodeCoverage
|
|
4
|
-
} from "./chunk-
|
|
5
|
-
import "./chunk-
|
|
6
|
-
import "./chunk-
|
|
4
|
+
} from "./chunk-GACSUWSG.js";
|
|
5
|
+
import "./chunk-PFHZMJQO.js";
|
|
6
|
+
import "./chunk-QHDUB2NG.js";
|
|
7
7
|
export {
|
|
8
8
|
createNodeMiddleware,
|
|
9
9
|
nodeCoverage
|
package/dist/server-index.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import "./chunk-LI2LYV3R.js";
|
|
2
2
|
import {
|
|
3
3
|
catalogVersion
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-PFHZMJQO.js";
|
|
5
5
|
import {
|
|
6
6
|
createTrafficClient,
|
|
7
7
|
deliveryLimits
|
|
8
|
-
} from "./chunk-
|
|
9
|
-
import "./chunk-
|
|
8
|
+
} from "./chunk-5WVWPJBS.js";
|
|
9
|
+
import "./chunk-QHDUB2NG.js";
|
|
10
10
|
|
|
11
11
|
// src/server-index.ts
|
|
12
12
|
import "server-only";
|
package/dist/server-next.js
CHANGED
|
@@ -4,9 +4,9 @@ import {
|
|
|
4
4
|
createResponseWrapper,
|
|
5
5
|
proxyCoverage,
|
|
6
6
|
withAiTrafficConfig
|
|
7
|
-
} from "./chunk-
|
|
8
|
-
import "./chunk-
|
|
9
|
-
import "./chunk-
|
|
7
|
+
} from "./chunk-XIHA3LLI.js";
|
|
8
|
+
import "./chunk-PFHZMJQO.js";
|
|
9
|
+
import "./chunk-QHDUB2NG.js";
|
|
10
10
|
|
|
11
11
|
// src/server-next.ts
|
|
12
12
|
import "server-only";
|
package/dist/server-node.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createNodeMiddleware,
|
|
3
3
|
nodeCoverage
|
|
4
|
-
} from "./chunk-
|
|
5
|
-
import "./chunk-
|
|
6
|
-
import "./chunk-
|
|
4
|
+
} from "./chunk-GACSUWSG.js";
|
|
5
|
+
import "./chunk-PFHZMJQO.js";
|
|
6
|
+
import "./chunk-QHDUB2NG.js";
|
|
7
7
|
|
|
8
8
|
// src/server-node.ts
|
|
9
9
|
import "server-only";
|