@growthgenie/ai-traffic 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -3
- package/dist/{chunk-5PNSCBAB.js → chunk-5WVWPJBS.js} +1 -1
- package/dist/{chunk-EGXH5ES5.js → chunk-GACSUWSG.js} +2 -2
- package/dist/{chunk-HAXY7PYH.js → chunk-PFHZMJQO.js} +51 -9
- package/dist/{chunk-RYXHXHXU.js → chunk-QHDUB2NG.js} +3 -1
- package/dist/{chunk-54YHPRVG.js → chunk-XIHA3LLI.js} +1 -1
- package/dist/index.js +3 -3
- package/dist/next-server.js +2 -2
- package/dist/next.js +3 -3
- package/dist/node.js +3 -3
- package/dist/server-index.js +3 -3
- package/dist/server-next.js +3 -3
- package/dist/server-node.js +3 -3
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -89,7 +89,12 @@ unverified fetches. No country is inferred.
|
|
|
89
89
|
Request-only referral candidates require GET, a recognized AI referrer,
|
|
90
90
|
`Sec-Fetch-Mode: navigate` **and** `Sec-Fetch-Dest: document`, an eligible path,
|
|
91
91
|
and no prefetch/preview purpose. API/framework/static asset/private/account paths
|
|
92
|
-
are excluded (including encoded forms).
|
|
92
|
+
are excluded from referrals (including encoded forms). A request that names a
|
|
93
|
+
catalog AI crawler is sent on any path except private ones: since catalog
|
|
94
|
+
`ai-traffic/3` the receiver counts a crawler's files (images, CSS, JS, fonts…)
|
|
95
|
+
and calls scanners probing paths such as `/.env` or `/wp-login.php` fake bots.
|
|
96
|
+
Static files a CDN or your host serves without running Node never reach the
|
|
97
|
+
package, so only an edge method sees those. No navigation evidence means no referral
|
|
93
98
|
candidate. **A1b v1 does not transmit navigation headers, and its referral rule
|
|
94
99
|
requires a successful final status and document evidence. Consequently Proxy-only
|
|
95
100
|
referral candidates are dropped centrally, not counted. Referral measurement is
|
|
@@ -261,7 +266,7 @@ Owner's unexecuted live G-node gate:
|
|
|
261
266
|
Deploy on actual Vercel Hobby **and** Pro. Record Next/Node/package versions,
|
|
262
267
|
plan, host limits, region, timestamp and sanitized request captures.
|
|
263
268
|
2. Create separate generic staging connections. Run `/v1/ai-traffic/test`; confirm
|
|
264
|
-
test health changes without analytics
|
|
269
|
+
test health changes without analytics or usage. Rotate/revoke keys and check
|
|
265
270
|
failures without recording keys or payloads in logs.
|
|
266
271
|
3. Send a known-UA fetch to a dynamic page through Proxy; reconcile one accepted
|
|
267
272
|
unverified observation, null status/type, no duplicate charges on ambiguous
|
|
@@ -275,6 +280,6 @@ Owner's unexecuted live G-node gate:
|
|
|
275
280
|
measure which requests execute Proxy/Node and disclose every bypass. Test
|
|
276
281
|
static export build rejection with `withAiTrafficConfig`.
|
|
277
282
|
6. Measure delivery success, loss, overhead and shutdown at host execution limits
|
|
278
|
-
and reconcile recent rows, daily aggregates and workspace
|
|
283
|
+
and reconcile recent rows, daily aggregates and workspace usage. Save
|
|
279
284
|
sanitized captures/negative cases plus source/version/date notes. Do not claim
|
|
280
285
|
full G-node or universal plan/version support until these steps pass.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
AiTrafficRequestEvent
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-QHDUB2NG.js";
|
|
4
4
|
|
|
5
5
|
// src/delivery.ts
|
|
6
6
|
var deliveryLimits = Object.freeze({ concurrency: 2, queueEvents: 256, timeoutMs: 3e3, retries: 3, baseDelayMs: 1e3, maxDelayMs: 6e4, maxDeliveryMs: 3e4, shutdownMs: 5e3 });
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import {
|
|
2
2
|
makeEvent,
|
|
3
3
|
validatePolicy
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-PFHZMJQO.js";
|
|
5
5
|
import {
|
|
6
6
|
__commonJS,
|
|
7
7
|
__toESM
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-QHDUB2NG.js";
|
|
9
9
|
|
|
10
10
|
// ../../node_modules/.pnpm/ipaddr.js@2.3.0/node_modules/ipaddr.js/lib/ipaddr.js
|
|
11
11
|
var require_ipaddr = __commonJS({
|
|
@@ -3,10 +3,10 @@ import {
|
|
|
3
3
|
normalizeAiTrafficHostname,
|
|
4
4
|
normalizeAiTrafficPath,
|
|
5
5
|
normalizeAiTrafficReferrer
|
|
6
|
-
} from "./chunk-
|
|
6
|
+
} from "./chunk-QHDUB2NG.js";
|
|
7
7
|
|
|
8
8
|
// ../contracts/src/aiTrafficCatalog.ts
|
|
9
|
-
var AI_TRAFFIC_CLASSIFIER_VERSION = "ai-traffic/
|
|
9
|
+
var AI_TRAFFIC_CLASSIFIER_VERSION = "ai-traffic/3";
|
|
10
10
|
var DOCS = {
|
|
11
11
|
openai: "https://developers.openai.com/api/docs/bots",
|
|
12
12
|
anthropic: "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
|
|
@@ -14,6 +14,7 @@ var DOCS = {
|
|
|
14
14
|
duckduckgo: "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot",
|
|
15
15
|
mistral: "https://docs.mistral.ai/robots",
|
|
16
16
|
meta: "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
|
|
17
|
+
facebook: "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
|
|
17
18
|
commoncrawl: "https://commoncrawl.org/ccbot",
|
|
18
19
|
growth_genie: "https://growthgenie.ai",
|
|
19
20
|
opencode: "https://opencode.ai",
|
|
@@ -39,6 +40,9 @@ var AI_TRAFFIC_AGENTS = [
|
|
|
39
40
|
agent("Meta-ExternalFetcher", "meta", "answer_fetch", null),
|
|
40
41
|
agent("Meta-WebIndexer", "meta", "search_index", null),
|
|
41
42
|
agent("Meta-ExternalAgent", "meta", "model_training", null),
|
|
43
|
+
// Facebook's link-preview fetcher (shares, Messenger and WhatsApp cards; reads og:image), not one of Meta's AI
|
|
44
|
+
// crawlers, so it is its own provider outside the Meta group. Meta publishes no agent-specific ranges for it.
|
|
45
|
+
agent("facebookexternalhit", "facebook", "link_preview", null),
|
|
42
46
|
agent("CCBot", "commoncrawl", "model_training", "ccbot"),
|
|
43
47
|
agent("GrowthGenieBot/1.0", "growth_genie", "growth_genie", null, true, "GrowthGenieBot/1.0"),
|
|
44
48
|
// Page-fetch sources and exclusions: apps/mcp/AI-TRAFFIC-CODING-AGENTS.md.
|
|
@@ -88,18 +92,56 @@ function classifyAiTrafficReferrer(referrer) {
|
|
|
88
92
|
const entry = AI_TRAFFIC_REFERRERS.find((row) => row.hosts.some((listed) => host === listed || host.endsWith(`.${listed}`)));
|
|
89
93
|
return entry ? { source: entry.source, host, canonicalHost: entry.hosts[0] } : null;
|
|
90
94
|
}
|
|
91
|
-
|
|
95
|
+
var AI_TRAFFIC_ASSET_DIRS = ["_next", "_nuxt", "_astro", "_vercel", "_framework", "cdn-cgi"];
|
|
96
|
+
var AI_TRAFFIC_ENDPOINT_DIRS = ["api", "wp-json"];
|
|
97
|
+
var AI_TRAFFIC_FILE_EXTENSIONS = [
|
|
98
|
+
"avif",
|
|
99
|
+
"bmp",
|
|
100
|
+
"css",
|
|
101
|
+
"csv",
|
|
102
|
+
"eot",
|
|
103
|
+
"gif",
|
|
104
|
+
"ico",
|
|
105
|
+
"jpeg",
|
|
106
|
+
"jpg",
|
|
107
|
+
"js",
|
|
108
|
+
"json",
|
|
109
|
+
"map",
|
|
110
|
+
"mjs",
|
|
111
|
+
"mp3",
|
|
112
|
+
"mp4",
|
|
113
|
+
"ogg",
|
|
114
|
+
"otf",
|
|
115
|
+
"pdf",
|
|
116
|
+
"png",
|
|
117
|
+
"svg",
|
|
118
|
+
"ttf",
|
|
119
|
+
"webm",
|
|
120
|
+
"webp",
|
|
121
|
+
"woff",
|
|
122
|
+
"woff2",
|
|
123
|
+
"zip"
|
|
124
|
+
];
|
|
125
|
+
var EXCLUDED_DIRS = /* @__PURE__ */ new Set([...AI_TRAFFIC_ASSET_DIRS, ...AI_TRAFFIC_ENDPOINT_DIRS, "wp-admin", "graphql"]);
|
|
126
|
+
var FILE_EXTENSIONS = new Set(AI_TRAFFIC_FILE_EXTENSIONS);
|
|
127
|
+
function pathParts(raw) {
|
|
92
128
|
const path = normalizeAiTrafficPath(raw);
|
|
93
|
-
if (path === null) return
|
|
94
|
-
const
|
|
95
|
-
|
|
129
|
+
if (path === null) return null;
|
|
130
|
+
const segments = decodeURIComponent(path).toLowerCase().split("/").slice(1);
|
|
131
|
+
const name = segments.at(-1) ?? "";
|
|
132
|
+
const dot = name.lastIndexOf(".");
|
|
133
|
+
return { segments, first: segments[0] ?? "", extension: dot === -1 ? "" : name.slice(dot + 1) };
|
|
134
|
+
}
|
|
135
|
+
function isAiTrafficExcludedPath(raw) {
|
|
136
|
+
const parts = pathParts(raw);
|
|
137
|
+
return parts === null || EXCLUDED_DIRS.has(parts.first) || FILE_EXTENSIONS.has(parts.extension);
|
|
96
138
|
}
|
|
97
139
|
function isAiTrafficCandidate(input) {
|
|
98
|
-
if (
|
|
140
|
+
if (normalizeAiTrafficPath(input.path) === null) return false;
|
|
99
141
|
const agents = findAiTrafficAgents(input.userAgent);
|
|
100
142
|
if (agents.some((agent2) => !agent2.included)) return false;
|
|
101
143
|
if (agents.length > 0 && (input.method === "GET" || input.method === "HEAD")) return true;
|
|
102
|
-
return input.method === "GET" && classifyAiTrafficReferrer(input.referrer) !== null;
|
|
144
|
+
return input.method === "GET" && !isAiTrafficExcludedPath(input.path) && classifyAiTrafficReferrer(input.referrer) !== null;
|
|
103
145
|
}
|
|
104
146
|
|
|
105
147
|
// src/event.ts
|
|
@@ -119,7 +161,7 @@ function makeEvent(input, hosts) {
|
|
|
119
161
|
const host = normalizeAiTrafficHostname(url.hostname);
|
|
120
162
|
if (!host || !hosts.has(host)) return null;
|
|
121
163
|
const path = url.pathname;
|
|
122
|
-
if (
|
|
164
|
+
if (normalizeAiTrafficPath(path) === null || /^\/(?:admin|private|account|reset-password|password-reset|verify-email)(?:\/|$)/i.test(decodeURIComponent(path))) return null;
|
|
123
165
|
const userAgent = input.headers.get("user-agent") ?? "";
|
|
124
166
|
const referrer = input.headers.get("referer");
|
|
125
167
|
const method = input.method.toUpperCase();
|
|
@@ -19029,7 +19029,9 @@ var AiTrafficVerification = external_exports.enum([
|
|
|
19029
19029
|
"unverified_ip_outside_range",
|
|
19030
19030
|
"unverified_proxy_ip",
|
|
19031
19031
|
"unverified_proxy_ranges_unavailable",
|
|
19032
|
-
"self_claimed"
|
|
19032
|
+
"self_claimed",
|
|
19033
|
+
// ai-traffic/3: a claimed crawler without a verified IP asked for a path scanners probe (a fake bot).
|
|
19034
|
+
"unverified_scanner_path"
|
|
19033
19035
|
]);
|
|
19034
19036
|
var AI_TRAFFIC_FETCH_VERIFICATIONS = AiTrafficVerification.exclude(["self_claimed"]).options;
|
|
19035
19037
|
var AI_TRAFFIC_UNVERIFIED_REASONS = AiTrafficVerification.options.filter((value) => value.startsWith("unverified_"));
|
package/dist/index.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import "./chunk-LI2LYV3R.js";
|
|
2
2
|
import {
|
|
3
3
|
catalogVersion
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-PFHZMJQO.js";
|
|
5
5
|
import {
|
|
6
6
|
createTrafficClient,
|
|
7
7
|
deliveryLimits
|
|
8
|
-
} from "./chunk-
|
|
9
|
-
import "./chunk-
|
|
8
|
+
} from "./chunk-5WVWPJBS.js";
|
|
9
|
+
import "./chunk-QHDUB2NG.js";
|
|
10
10
|
export {
|
|
11
11
|
catalogVersion,
|
|
12
12
|
createTrafficClient,
|
package/dist/next-server.js
CHANGED
package/dist/next.js
CHANGED
|
@@ -4,9 +4,9 @@ import {
|
|
|
4
4
|
createResponseWrapper,
|
|
5
5
|
proxyCoverage,
|
|
6
6
|
withAiTrafficConfig
|
|
7
|
-
} from "./chunk-
|
|
8
|
-
import "./chunk-
|
|
9
|
-
import "./chunk-
|
|
7
|
+
} from "./chunk-XIHA3LLI.js";
|
|
8
|
+
import "./chunk-PFHZMJQO.js";
|
|
9
|
+
import "./chunk-QHDUB2NG.js";
|
|
10
10
|
export {
|
|
11
11
|
assertNextOutput,
|
|
12
12
|
createNextProxy,
|
package/dist/node.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createNodeMiddleware,
|
|
3
3
|
nodeCoverage
|
|
4
|
-
} from "./chunk-
|
|
5
|
-
import "./chunk-
|
|
6
|
-
import "./chunk-
|
|
4
|
+
} from "./chunk-GACSUWSG.js";
|
|
5
|
+
import "./chunk-PFHZMJQO.js";
|
|
6
|
+
import "./chunk-QHDUB2NG.js";
|
|
7
7
|
export {
|
|
8
8
|
createNodeMiddleware,
|
|
9
9
|
nodeCoverage
|
package/dist/server-index.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import "./chunk-LI2LYV3R.js";
|
|
2
2
|
import {
|
|
3
3
|
catalogVersion
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-PFHZMJQO.js";
|
|
5
5
|
import {
|
|
6
6
|
createTrafficClient,
|
|
7
7
|
deliveryLimits
|
|
8
|
-
} from "./chunk-
|
|
9
|
-
import "./chunk-
|
|
8
|
+
} from "./chunk-5WVWPJBS.js";
|
|
9
|
+
import "./chunk-QHDUB2NG.js";
|
|
10
10
|
|
|
11
11
|
// src/server-index.ts
|
|
12
12
|
import "server-only";
|
package/dist/server-next.js
CHANGED
|
@@ -4,9 +4,9 @@ import {
|
|
|
4
4
|
createResponseWrapper,
|
|
5
5
|
proxyCoverage,
|
|
6
6
|
withAiTrafficConfig
|
|
7
|
-
} from "./chunk-
|
|
8
|
-
import "./chunk-
|
|
9
|
-
import "./chunk-
|
|
7
|
+
} from "./chunk-XIHA3LLI.js";
|
|
8
|
+
import "./chunk-PFHZMJQO.js";
|
|
9
|
+
import "./chunk-QHDUB2NG.js";
|
|
10
10
|
|
|
11
11
|
// src/server-next.ts
|
|
12
12
|
import "server-only";
|
package/dist/server-node.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createNodeMiddleware,
|
|
3
3
|
nodeCoverage
|
|
4
|
-
} from "./chunk-
|
|
5
|
-
import "./chunk-
|
|
6
|
-
import "./chunk-
|
|
4
|
+
} from "./chunk-GACSUWSG.js";
|
|
5
|
+
import "./chunk-PFHZMJQO.js";
|
|
6
|
+
import "./chunk-QHDUB2NG.js";
|
|
7
7
|
|
|
8
8
|
// src/server-node.ts
|
|
9
9
|
import "server-only";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@growthgenie/ai-traffic",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=24"
|
|
@@ -51,8 +51,8 @@
|
|
|
51
51
|
"react-dom": "19.2.8",
|
|
52
52
|
"typescript": "6.0.3",
|
|
53
53
|
"vitest": "5.0.0",
|
|
54
|
-
"@growthwand/
|
|
55
|
-
"@growthwand/
|
|
54
|
+
"@growthwand/config": "0.0.0",
|
|
55
|
+
"@growthwand/contracts": "0.0.0"
|
|
56
56
|
},
|
|
57
57
|
"license": "MIT",
|
|
58
58
|
"peerDependencies": {
|