@canonry/canonry 7.84.0 → 7.85.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agent-workspace/skills/aero/references/ai-traffic-tracking.md +2 -1
- package/assets/agent-workspace/skills/canonry/references/ai-traffic-tracking.md +2 -1
- package/assets/agent-workspace/skills/canonry/references/server-side-traffic.md +31 -7
- package/assets/assets/{AgentSetupPage-B9NOQP8a.js → AgentSetupPage-NdqmQkYa.js} +1 -1
- package/assets/assets/{AuditHistoryPanel-Fvh4WFjq.js → AuditHistoryPanel-CuqdFCb9.js} +1 -1
- package/assets/assets/{BacklinksPage-D1UHFLoV.js → BacklinksPage-CVjZiG1i.js} +1 -1
- package/assets/assets/{HistoryPage-CBcacDgP.js → HistoryPage-DRvdgemW.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-ZuZpurwr.js → MeasurementPropertyPage-Bc5EpJFV.js} +1 -1
- package/assets/assets/{ProjectPage-BoqDyoGi.js → ProjectPage-CGGBSiK7.js} +1 -1
- package/assets/assets/{RunsPage-tlCk_gOw.js → RunsPage-B6_o81G3.js} +1 -1
- package/assets/assets/{SettingsPage-7mRTFLFQ.js → SettingsPage-DjmVapHO.js} +1 -1
- package/assets/assets/{SiteHealthSection-l_A_AmY_.js → SiteHealthSection-BPPB0D9D.js} +2 -2
- package/assets/assets/{ToneBadge-CXyUIWzB.js → ToneBadge-DwUBSAjL.js} +1 -1
- package/assets/assets/{TrafficPage-DQbBT3TP.js → TrafficPage-BvbjxOw3.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-D3DwE6Qd.js → TrafficSourceDetailPage-BGJwxS_Q.js} +1 -1
- package/assets/assets/{account-context-Bhgg1lEH.js → account-context-D7M4W4wP.js} +1 -1
- package/assets/assets/agent-trust-copy-WXXeDvDs.js +1 -0
- package/assets/assets/{card-B16Dvxau.js → card-DULVRmzI.js} +1 -1
- package/assets/assets/{extract-error-message-B5_v_-U8.js → extract-error-message-B9-6WgLm.js} +1 -1
- package/assets/assets/{format-helpers-Bs9rrO74.js → format-helpers-Cgq0VzXz.js} +1 -1
- package/assets/assets/{index-3aBRkQpI.js → index-A-4JN4Pv.js} +4 -4
- package/assets/assets/{query-client-D-tt1sn3.js → query-client-1Q6EwPba.js} +1 -1
- package/assets/assets/{react-query-DQr9PoNm.js → react-query-D0OiMBt3.js} +1 -1
- package/assets/assets/{react-sigma_core.esm.min-BfZVKh7v.js → react-sigma_core.esm.min-CR09uPhZ.js} +1 -1
- package/assets/assets/{service-Cmt36bhL.js → service-Bu-7icpb.js} +1 -1
- package/assets/index.html +7 -7
- package/dist/{ads-KBXA3LV6.js → ads-ETZIGLL5.js} +3 -3
- package/dist/{agent-OMNIUTNZ.js → agent-WG2FUC7C.js} +7 -7
- package/dist/{agent-ask-3JQNG2PE.js → agent-ask-775XBFOH.js} +3 -3
- package/dist/{agent-conversations-KO5PDSM2.js → agent-conversations-AYZ3LQJQ.js} +3 -3
- package/dist/{agent-journeys-QL6NG3BE.js → agent-journeys-GWNZG73C.js} +3 -3
- package/dist/{agent-memory-2WZ6LVAN.js → agent-memory-E5MRRS5P.js} +3 -3
- package/dist/{agent-providers-TABMSD4J.js → agent-providers-DEQZWO2M.js} +3 -3
- package/dist/{agent-transcript-YNF63MPR.js → agent-transcript-4AKVVHQC.js} +3 -3
- package/dist/{analytics-5FGY3FHC.js → analytics-HSLTWKDA.js} +3 -3
- package/dist/{apply-2HX4KYT6.js → apply-4GD7VGG7.js} +3 -3
- package/dist/{backfill-ABSW6J3S.js → backfill-LNE2BW34.js} +10 -10
- package/dist/{backlinks-7CRPWNNN.js → backlinks-TGE3DWA2.js} +3 -3
- package/dist/{bing-IT46EUFH.js → bing-56GNRC55.js} +3 -3
- package/dist/{bootstrap-CHOXLC7Q.js → bootstrap-42XCWZTS.js} +10 -10
- package/dist/{cdp-HQWKZGVQ.js → cdp-UODIMKE3.js} +7 -7
- package/dist/{chunk-EMSZDVDP.js → chunk-2O5XBKEO.js} +5 -5
- package/dist/{chunk-O7TWL6ZH.js → chunk-2R6WI237.js} +333 -203
- package/dist/{chunk-B42R2SPU.js → chunk-2X6UAPAC.js} +191 -9
- package/dist/{chunk-VCUNNS4R.js → chunk-4KK3OTHE.js} +1 -1
- package/dist/{chunk-6TLHA6XA.js → chunk-5NEZO3SC.js} +3 -3
- package/dist/{chunk-WJITVHAU.js → chunk-6SGHSSWQ.js} +2 -2
- package/dist/{chunk-Q57GN464.js → chunk-AICFUZ6X.js} +1 -1
- package/dist/{chunk-2MROJL32.js → chunk-BQTXXXUG.js} +1 -1
- package/dist/{chunk-5PEVGTNX.js → chunk-C2H5TPKD.js} +338 -338
- package/dist/{chunk-LWYALZ6F.js → chunk-D45AMUYA.js} +4 -4
- package/dist/{chunk-WR4PA335.js → chunk-DWK6NIJF.js} +1 -1
- package/dist/{chunk-YPR2GVJC.js → chunk-GR7VVRNN.js} +1 -1
- package/dist/{chunk-3H6FSUKM.js → chunk-IOW23XNK.js} +3 -3
- package/dist/{chunk-6SVAUILQ.js → chunk-JVO5GQZH.js} +2912 -2560
- package/dist/{chunk-TZCUMC7U.js → chunk-KBLMQDUZ.js} +16 -8
- package/dist/{chunk-K3OY52MI.js → chunk-KNKOTLVW.js} +3 -1
- package/dist/{chunk-BIGIIVGV.js → chunk-L5R5S57T.js} +55 -1
- package/dist/{chunk-GJE3OG4Y.js → chunk-NY252PIY.js} +3 -3
- package/dist/{chunk-CEOIGQTS.js → chunk-OHDSI5XS.js} +1 -1
- package/dist/{chunk-I642VITF.js → chunk-OS5BVALI.js} +2 -2
- package/dist/{chunk-OMUINNSO.js → chunk-PQWZCAVS.js} +1 -1
- package/dist/{chunk-D37PJYLD.js → chunk-PX2KVGOJ.js} +1 -1
- package/dist/{chunk-ZRL5FO5G.js → chunk-Q7354YI3.js} +3 -3
- package/dist/{chunk-LYNYNNOD.js → chunk-QEATYB7C.js} +1 -1
- package/dist/{chunk-OBGP3DRQ.js → chunk-RCAB7M6A.js} +2 -2
- package/dist/{chunk-43CPBLRJ.js → chunk-SS33RWKG.js} +1 -1
- package/dist/{chunk-LHAD6EK3.js → chunk-TPUBPXBN.js} +2 -2
- package/dist/{chunk-MZDIWY3K.js → chunk-TVZMFCX2.js} +6 -6
- package/dist/{chunk-3EI4FHR5.js → chunk-UMEQ7EYN.js} +4 -4
- package/dist/{chunk-AAKRCOQV.js → chunk-UMV4HYHD.js} +2 -2
- package/dist/{chunk-6Y75QV2S.js → chunk-VFSHZHZS.js} +2 -2
- package/dist/{chunk-LE6ZODCY.js → chunk-X7XKMARG.js} +2 -2
- package/dist/{chunk-QLKG7G4Y.js → chunk-YDC2V5FQ.js} +21 -21
- package/dist/{chunk-HYEPL5UY.js → chunk-Z53CKLDS.js} +3 -3
- package/dist/{chunk-UYR5JI6R.js → chunk-ZXMDIWW6.js} +4 -4
- package/dist/{citations-YRBUXEWU.js → citations-RXBPR5MX.js} +3 -3
- package/dist/{cli-catalog-JTIC72BY.js → cli-catalog-QVCJWPCI.js} +3 -3
- package/dist/{cli-commands-LUGJZGJU.js → cli-commands-ZXR4Z2RW.js} +7 -7
- package/dist/cli.js +13 -13
- package/dist/{competitor-ZKNDADMJ.js → competitor-MDNNHRSD.js} +3 -3
- package/dist/{content-ANRSBG3Y.js → content-D3JZZQ64.js} +3 -3
- package/dist/{dashboard-HTJBWAPW.js → dashboard-5S2A46GM.js} +6 -6
- package/dist/{demo-V3ODU4JK.js → demo-DQHWRIKR.js} +1 -1
- package/dist/{demo-server-QYMT3KT7.js → demo-server-FY2XBSD2.js} +12 -12
- package/dist/{discover-DX7KIWHU.js → discover-NVNWHOTZ.js} +3 -3
- package/dist/{discover-eval-PXWJYPHE.js → discover-eval-CJB6GYUS.js} +3 -3
- package/dist/{doctor-W2MTLKX7.js → doctor-DLSKNC4W.js} +3 -3
- package/dist/{evidence-PKKLSVNG.js → evidence-QNJULIXM.js} +3 -3
- package/dist/{export-cmd-3PRWU7NH.js → export-cmd-Z2YG2WLK.js} +3 -3
- package/dist/{ga-QPH56JDO.js → ga-RSNHBP2B.js} +3 -3
- package/dist/{gbp-MJPIQZYZ.js → gbp-ZEV62LSZ.js} +3 -3
- package/dist/{google-2XZC7DNJ.js → google-RZHEIKW5.js} +4 -4
- package/dist/{google-marketing-EDXL7ODG.js → google-marketing-DYQXPTD3.js} +1 -1
- package/dist/{health-cmd-YR2MFYNK.js → health-cmd-6ATPYUYG.js} +3 -3
- package/dist/{history-OM4OXDOZ.js → history-KLX6XRY4.js} +3 -3
- package/dist/index.d.ts +301 -95
- package/dist/index.js +25 -25
- package/dist/{init-5FAQPNH4.js → init-5BALV6SU.js} +12 -12
- package/dist/{insights-YGMPBDVA.js → insights-ARCEFFK5.js} +3 -3
- package/dist/intelligence-service-YJFZ6IBI.js +16 -0
- package/dist/{journeys-ZPQCEHXY.js → journeys-WVRQRZQJ.js} +3 -3
- package/dist/{keys-YZBRRGYE.js → keys-5KAJFRZZ.js} +3 -3
- package/dist/{keyword-HSPNKAJQ.js → keyword-PTUH3TFD.js} +3 -3
- package/dist/{logs-NO2N7W34.js → logs-NM7ACXGQ.js} +3 -3
- package/dist/{mcp-EAGJ2Y2D.js → mcp-ALRQXYL3.js} +4 -4
- package/dist/mcp.js +9 -9
- package/dist/{measurement-plan-6VCBAM5B.js → measurement-plan-T52JHK2J.js} +5 -5
- package/dist/{notify-SS5H5GRB.js → notify-MJBMRDXD.js} +3 -3
- package/dist/{organic-evidence-XOZQXTHF.js → organic-evidence-ZUCQFWD5.js} +3 -3
- package/dist/{overview-3PLQYWDY.js → overview-74IHPU2A.js} +4 -4
- package/dist/{project-ARUHBFCG.js → project-DKQOPMH6.js} +3 -3
- package/dist/{query-4KCFDJ5V.js → query-YDJ5T46L.js} +3 -3
- package/dist/{research-3NTYXGWN.js → research-5DU6DFZW.js} +3 -3
- package/dist/{results-export-YOQQXBMP.js → results-export-EDVVXKVW.js} +3 -3
- package/dist/{run-64UJAPPG.js → run-HRHGSIYE.js} +4 -4
- package/dist/{schedule-JN5JE4EK.js → schedule-GZVKQGMP.js} +3 -3
- package/dist/{search-2QIHUMR7.js → search-M37GRKVC.js} +3 -3
- package/dist/{serve-V2NPNYMP.js → serve-DW6PPZPJ.js} +26 -26
- package/dist/{session-347VV5OG.js → session-H427H2SK.js} +13 -13
- package/dist/{settings-XM5NGHFW.js → settings-WGS7GP7C.js} +3 -3
- package/dist/{skills-4MWWCA36.js → skills-CDDUPMAU.js} +2 -2
- package/dist/{snapshot-RWHR3VJ4.js → snapshot-5MAK7PWE.js} +3 -3
- package/dist/{src-HUF2NBUG.js → src-FEV4KYXN.js} +6 -2
- package/dist/{status-3OI7PTQ5.js → status-EHFOYTXQ.js} +4 -4
- package/dist/{technical-aeo-RXHIFLHD.js → technical-aeo-KVY4RQWT.js} +3 -3
- package/dist/{telemetry-DZSITJKQ.js → telemetry-7XRRIHVC.js} +4 -4
- package/dist/{tools-ZUUOSB75.js → tools-EHONHGB6.js} +11 -11
- package/dist/{traffic-KIQBUHJW.js → traffic-TBR6GOWN.js} +5 -5
- package/dist/{users-ID4O7NXQ.js → users-YQS4X47Q.js} +3 -3
- package/dist/{visibility-stats-X5MTD47P.js → visibility-stats-JXIOSJXC.js} +3 -3
- package/dist/{wordpress-WU2HLWIW.js → wordpress-HVOLEPQY.js} +3 -3
- package/package.json +1 -1
- package/assets/assets/agent-trust-copy-CKZsKhc0.js +0 -1
- package/dist/intelligence-service-GCN4S6QT.js +0 -16
|
@@ -12,7 +12,139 @@ import {
|
|
|
12
12
|
classifyTrafficPath,
|
|
13
13
|
hostMatchesDomain,
|
|
14
14
|
hostOf
|
|
15
|
-
} from "./chunk-
|
|
15
|
+
} from "./chunk-KNKOTLVW.js";
|
|
16
|
+
|
|
17
|
+
// ../integration-traffic/src/agent-trust.ts
|
|
18
|
+
var AGENT_TRUST_RULES_VERSION = 2;
|
|
19
|
+
var SCANNER_SEGMENTS = /* @__PURE__ */ new Set([
|
|
20
|
+
"wp-login.php",
|
|
21
|
+
"xmlrpc.php",
|
|
22
|
+
"phpmyadmin",
|
|
23
|
+
"phpunit",
|
|
24
|
+
"eval-stdin.php",
|
|
25
|
+
"phpinfo.php",
|
|
26
|
+
// Debug consoles of web frameworks, and the Vite dev server's file access.
|
|
27
|
+
"_profiler",
|
|
28
|
+
"_debugbar",
|
|
29
|
+
"_ignition",
|
|
30
|
+
"__debug__",
|
|
31
|
+
"elmah.axd",
|
|
32
|
+
"trace.axd",
|
|
33
|
+
"@fs"
|
|
34
|
+
]);
|
|
35
|
+
var ENCODED_TRAVERSAL = /(?:%2e%2e|%2e\.|\.%2e)(?:\/|\\|%2f|%5c|$)|\.\.(?:%2f|%5c)|%252e%252e/i;
|
|
36
|
+
var NETLIFY_PUBLIC = /* @__PURE__ */ new Set(["functions", "edge-functions", "images"]);
|
|
37
|
+
var CONFIG_FOLDERS = /* @__PURE__ */ new Set(["config", "configs", "conf", "keys", "secrets", "credentials", "private", "certs", "ssl", "backup", "backups"]);
|
|
38
|
+
var SECRET_EXTENSIONS = /* @__PURE__ */ new Set(["pem", "key", "p12", "pfx", "jks", "keystore", "ppk", "kdbx", "ovpn", "tfstate", "tfvars", "sql", "bak", "swp"]);
|
|
39
|
+
var CREDENTIAL_FILE = /(?:^|[-_.])(?:credentials?|secrets?|service[-_]?account(?:[-_]?key)?|private[-_]?key|access[-_]?keys?|adminsdk(?:[-_][\w-]+)?)\.(?:json|ya?ml|toml|ini|conf|cfg|xml|txt|js|env|properties)$/;
|
|
40
|
+
var SSH_KEY_FILE = /^id_(?:rsa|dsa|ecdsa|ed25519)(?:\.pub)?$/;
|
|
41
|
+
var SECRET_FILES = /* @__PURE__ */ new Set([
|
|
42
|
+
"key.json",
|
|
43
|
+
"keyfile.json",
|
|
44
|
+
"sa.json",
|
|
45
|
+
"gcp-key.json",
|
|
46
|
+
"gcp-service.json",
|
|
47
|
+
"gc-service.json",
|
|
48
|
+
"firebase-admin.json",
|
|
49
|
+
"docker-compose.yml",
|
|
50
|
+
"docker-compose.yaml",
|
|
51
|
+
"serverless.yml",
|
|
52
|
+
"serverless.yaml",
|
|
53
|
+
"rclone.conf",
|
|
54
|
+
"appsettings.json",
|
|
55
|
+
"local.settings.json",
|
|
56
|
+
"application.properties",
|
|
57
|
+
"application.yml",
|
|
58
|
+
"application.yaml",
|
|
59
|
+
"values.yaml",
|
|
60
|
+
"web.config",
|
|
61
|
+
"configuration.php",
|
|
62
|
+
"settings.py",
|
|
63
|
+
"database.yml",
|
|
64
|
+
"parameters.yml",
|
|
65
|
+
"env.json",
|
|
66
|
+
"env.js"
|
|
67
|
+
]);
|
|
68
|
+
var ENVIRONMENT_CONFIG = /^(?:production|prod|development|dev|staging|default|local|test)\.(?:json|ya?ml|js|toml)$/;
|
|
69
|
+
var REASON_ORDER = [
|
|
70
|
+
AgentTrustReasons["scanner-path"],
|
|
71
|
+
AgentTrustReasons["scanner-burst"],
|
|
72
|
+
AgentTrustReasons["unusual-user-agent"],
|
|
73
|
+
AgentTrustReasons["ip-in-published-list"],
|
|
74
|
+
AgentTrustReasons["ip-outside-published-list"],
|
|
75
|
+
AgentTrustReasons["no-ip"],
|
|
76
|
+
AgentTrustReasons["no-published-list"],
|
|
77
|
+
AgentTrustReasons["documented-user-agent"]
|
|
78
|
+
];
|
|
79
|
+
var FAKE_REASONS = /* @__PURE__ */ new Set([
|
|
80
|
+
AgentTrustReasons["scanner-path"],
|
|
81
|
+
AgentTrustReasons["scanner-burst"],
|
|
82
|
+
AgentTrustReasons["unusual-user-agent"]
|
|
83
|
+
]);
|
|
84
|
+
function isHiddenSegment(segment, next) {
|
|
85
|
+
if (!segment.startsWith(".") || segment === "." || segment === ".well-known") return false;
|
|
86
|
+
return !(segment === ".netlify" && next !== void 0 && NETLIFY_PUBLIC.has(next));
|
|
87
|
+
}
|
|
88
|
+
function isSecretFile(folders, file) {
|
|
89
|
+
if (!folders.every((folder) => CONFIG_FOLDERS.has(folder))) return false;
|
|
90
|
+
if (folders.length > 0 && ENVIRONMENT_CONFIG.test(file)) return true;
|
|
91
|
+
const dot = file.lastIndexOf(".");
|
|
92
|
+
const extension = dot > 0 ? file.slice(dot + 1) : "";
|
|
93
|
+
return SECRET_EXTENSIONS.has(extension) || SECRET_FILES.has(file) || SSH_KEY_FILE.test(file) || CREDENTIAL_FILE.test(file) || /^appsettings\.[\w-]+\.json$/.test(file);
|
|
94
|
+
}
|
|
95
|
+
function isScannerPath(rawPath) {
|
|
96
|
+
const pathOnly = (rawPath.split(/[?#]/)[0] ?? "").toLowerCase();
|
|
97
|
+
if (ENCODED_TRAVERSAL.test(pathOnly)) return true;
|
|
98
|
+
let decoded = pathOnly;
|
|
99
|
+
try {
|
|
100
|
+
decoded = decodeURIComponent(pathOnly);
|
|
101
|
+
} catch {
|
|
102
|
+
}
|
|
103
|
+
const segments = decoded.split(/[\\/]+/).filter(Boolean);
|
|
104
|
+
for (const [index, segment] of segments.entries()) {
|
|
105
|
+
if (segment === ".." || segment.endsWith("..")) return true;
|
|
106
|
+
if (SCANNER_SEGMENTS.has(segment) || segment.startsWith("wp-config") || segment.endsWith(".env")) return true;
|
|
107
|
+
if (isHiddenSegment(segment, segments[index + 1])) return true;
|
|
108
|
+
if (segment === "proc" && segments[index + 1] === "self") return true;
|
|
109
|
+
if (segment === "var" && segments[index + 1] === "run" && segments[index + 2] === "secrets") return true;
|
|
110
|
+
}
|
|
111
|
+
const file = segments.at(-1);
|
|
112
|
+
if (file === void 0) return false;
|
|
113
|
+
if (segments[0] === "userfiles" && !file.includes(".")) return true;
|
|
114
|
+
return isSecretFile(segments.slice(0, -1), file);
|
|
115
|
+
}
|
|
116
|
+
function sortAgentTrustReasons(reasons) {
|
|
117
|
+
const present = new Set(reasons);
|
|
118
|
+
return REASON_ORDER.filter((reason) => present.has(reason));
|
|
119
|
+
}
|
|
120
|
+
function withScannerBurst(assessment) {
|
|
121
|
+
if (assessment.trust === AgentTrustLevels.verified || assessment.trust === AgentTrustLevels.likely_fake) return assessment;
|
|
122
|
+
return {
|
|
123
|
+
...assessment,
|
|
124
|
+
trust: AgentTrustLevels.likely_fake,
|
|
125
|
+
trustReasons: sortAgentTrustReasons([...assessment.trustReasons, AgentTrustReasons["scanner-burst"]])
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
function ipReason(input) {
|
|
129
|
+
if (input.verification.verified) return AgentTrustReasons["ip-in-published-list"];
|
|
130
|
+
if (input.verification.manifest) return AgentTrustReasons["ip-outside-published-list"];
|
|
131
|
+
return input.listPublished ? AgentTrustReasons["no-ip"] : AgentTrustReasons["no-published-list"];
|
|
132
|
+
}
|
|
133
|
+
function assessAgentTrust(input) {
|
|
134
|
+
const reasons = /* @__PURE__ */ new Set([ipReason(input)]);
|
|
135
|
+
const documented = input.rule.documentedUserAgents;
|
|
136
|
+
if (documented && documented.length > 0) {
|
|
137
|
+
reasons.add(documented.some((pattern) => pattern.test(input.userAgent.trim())) ? AgentTrustReasons["documented-user-agent"] : AgentTrustReasons["unusual-user-agent"]);
|
|
138
|
+
}
|
|
139
|
+
if (isScannerPath(input.path)) reasons.add(AgentTrustReasons["scanner-path"]);
|
|
140
|
+
return { trust: trustFor(reasons), trustReasons: sortAgentTrustReasons(reasons) };
|
|
141
|
+
}
|
|
142
|
+
function trustFor(reasons) {
|
|
143
|
+
if (reasons.has(AgentTrustReasons["ip-in-published-list"])) return AgentTrustLevels.verified;
|
|
144
|
+
for (const reason of reasons) if (FAKE_REASONS.has(reason)) return AgentTrustLevels.likely_fake;
|
|
145
|
+
const unchecked = reasons.has(AgentTrustReasons["no-ip"]) || reasons.has(AgentTrustReasons["no-published-list"]);
|
|
146
|
+
return unchecked && reasons.has(AgentTrustReasons["documented-user-agent"]) ? AgentTrustLevels.likely_real : AgentTrustLevels.unknown;
|
|
147
|
+
}
|
|
16
148
|
|
|
17
149
|
// ../integration-traffic/src/rules.ts
|
|
18
150
|
var SELF_TRAFFIC_USER_AGENT_PATTERNS = [
|
|
@@ -451,81 +583,6 @@ var DEFAULT_AI_REFERRER_RULES = [
|
|
|
451
583
|
{ domain: AI_ENGINE_DOMAINS.metaAi, operator: "Meta", product: "Meta AI" }
|
|
452
584
|
];
|
|
453
585
|
|
|
454
|
-
// ../integration-traffic/src/agent-trust.ts
|
|
455
|
-
var SCANNER_SEGMENTS = /* @__PURE__ */ new Set([
|
|
456
|
-
".git",
|
|
457
|
-
".svn",
|
|
458
|
-
".hg",
|
|
459
|
-
".github",
|
|
460
|
-
".aws",
|
|
461
|
-
".ssh",
|
|
462
|
-
".docker",
|
|
463
|
-
".kube",
|
|
464
|
-
".npmrc",
|
|
465
|
-
".htpasswd",
|
|
466
|
-
".htaccess",
|
|
467
|
-
".ds_store",
|
|
468
|
-
"wp-login.php",
|
|
469
|
-
"xmlrpc.php",
|
|
470
|
-
"phpmyadmin",
|
|
471
|
-
"phpunit",
|
|
472
|
-
"eval-stdin.php"
|
|
473
|
-
]);
|
|
474
|
-
var ENCODED_TRAVERSAL = /(?:%2e%2e|%2e\.|\.%2e)(?:\/|\\|%2f|%5c|$)|\.\.(?:%2f|%5c)|%252e%252e/i;
|
|
475
|
-
var REASON_ORDER = [
|
|
476
|
-
AgentTrustReasons["scanner-path"],
|
|
477
|
-
AgentTrustReasons["unusual-user-agent"],
|
|
478
|
-
AgentTrustReasons["ip-in-published-list"],
|
|
479
|
-
AgentTrustReasons["ip-outside-published-list"],
|
|
480
|
-
AgentTrustReasons["no-ip"],
|
|
481
|
-
AgentTrustReasons["no-published-list"],
|
|
482
|
-
AgentTrustReasons["documented-user-agent"]
|
|
483
|
-
];
|
|
484
|
-
var FAKE_REASONS = /* @__PURE__ */ new Set([
|
|
485
|
-
AgentTrustReasons["scanner-path"],
|
|
486
|
-
AgentTrustReasons["unusual-user-agent"]
|
|
487
|
-
]);
|
|
488
|
-
function isScannerPath(rawPath) {
|
|
489
|
-
const pathOnly = (rawPath.split(/[?#]/)[0] ?? "").toLowerCase();
|
|
490
|
-
if (ENCODED_TRAVERSAL.test(pathOnly)) return true;
|
|
491
|
-
let decoded = pathOnly;
|
|
492
|
-
try {
|
|
493
|
-
decoded = decodeURIComponent(pathOnly);
|
|
494
|
-
} catch {
|
|
495
|
-
}
|
|
496
|
-
for (const segment of decoded.split(/[\\/]+/)) {
|
|
497
|
-
if (!segment) continue;
|
|
498
|
-
if (segment === ".." || segment.endsWith("..")) return true;
|
|
499
|
-
if (SCANNER_SEGMENTS.has(segment) || segment.startsWith("wp-config")) return true;
|
|
500
|
-
if (segment === ".env" || segment.startsWith(".env.") || segment.endsWith(".env")) return true;
|
|
501
|
-
}
|
|
502
|
-
return false;
|
|
503
|
-
}
|
|
504
|
-
function sortAgentTrustReasons(reasons) {
|
|
505
|
-
const present = new Set(reasons);
|
|
506
|
-
return REASON_ORDER.filter((reason) => present.has(reason));
|
|
507
|
-
}
|
|
508
|
-
function ipReason(input) {
|
|
509
|
-
if (input.verification.verified) return AgentTrustReasons["ip-in-published-list"];
|
|
510
|
-
if (input.verification.manifest) return AgentTrustReasons["ip-outside-published-list"];
|
|
511
|
-
return input.listPublished ? AgentTrustReasons["no-ip"] : AgentTrustReasons["no-published-list"];
|
|
512
|
-
}
|
|
513
|
-
function assessAgentTrust(input) {
|
|
514
|
-
const reasons = /* @__PURE__ */ new Set([ipReason(input)]);
|
|
515
|
-
const documented = input.rule.documentedUserAgents;
|
|
516
|
-
if (documented && documented.length > 0) {
|
|
517
|
-
reasons.add(documented.some((pattern) => pattern.test(input.userAgent.trim())) ? AgentTrustReasons["documented-user-agent"] : AgentTrustReasons["unusual-user-agent"]);
|
|
518
|
-
}
|
|
519
|
-
if (isScannerPath(input.path)) reasons.add(AgentTrustReasons["scanner-path"]);
|
|
520
|
-
return { trust: trustFor(reasons), trustReasons: sortAgentTrustReasons(reasons) };
|
|
521
|
-
}
|
|
522
|
-
function trustFor(reasons) {
|
|
523
|
-
if (reasons.has(AgentTrustReasons["ip-in-published-list"])) return AgentTrustLevels.verified;
|
|
524
|
-
for (const reason of reasons) if (FAKE_REASONS.has(reason)) return AgentTrustLevels.likely_fake;
|
|
525
|
-
const unchecked = reasons.has(AgentTrustReasons["no-ip"]) || reasons.has(AgentTrustReasons["no-published-list"]);
|
|
526
|
-
return unchecked && reasons.has(AgentTrustReasons["documented-user-agent"]) ? AgentTrustLevels.likely_real : AgentTrustLevels.unknown;
|
|
527
|
-
}
|
|
528
|
-
|
|
529
586
|
// ../integration-traffic/src/ip-ranges/amazonbot.json
|
|
530
587
|
var amazonbot_default = {
|
|
531
588
|
_source: "https://developer.amazon.com/amazonbot/ip-addresses/",
|
|
@@ -18016,97 +18073,8 @@ function classifyAiReferral(event) {
|
|
|
18016
18073
|
return null;
|
|
18017
18074
|
}
|
|
18018
18075
|
|
|
18019
|
-
// ../integration-traffic/src/unlisted-agents.ts
|
|
18020
|
-
var UNLISTED_AGENT_SKIPPED_EXTENSIONS = [
|
|
18021
|
-
"avif",
|
|
18022
|
-
"bmp",
|
|
18023
|
-
"css",
|
|
18024
|
-
"gif",
|
|
18025
|
-
"ico",
|
|
18026
|
-
"jpeg",
|
|
18027
|
-
"jpg",
|
|
18028
|
-
"js",
|
|
18029
|
-
"map",
|
|
18030
|
-
"mjs",
|
|
18031
|
-
"mp3",
|
|
18032
|
-
"mp4",
|
|
18033
|
-
"otf",
|
|
18034
|
-
"png",
|
|
18035
|
-
"svg",
|
|
18036
|
-
"ttf",
|
|
18037
|
-
"webm",
|
|
18038
|
-
"webmanifest",
|
|
18039
|
-
"webp",
|
|
18040
|
-
"woff",
|
|
18041
|
-
"woff2",
|
|
18042
|
-
"zip"
|
|
18043
|
-
];
|
|
18044
|
-
var GENERIC_HTTP_CLIENTS = [
|
|
18045
|
-
"curl",
|
|
18046
|
-
"wget",
|
|
18047
|
-
"python",
|
|
18048
|
-
"python-requests",
|
|
18049
|
-
"python-urllib",
|
|
18050
|
-
"python-httpx",
|
|
18051
|
-
"aiohttp",
|
|
18052
|
-
"httpx",
|
|
18053
|
-
"go-http-client",
|
|
18054
|
-
"okhttp",
|
|
18055
|
-
"java",
|
|
18056
|
-
"apache-httpclient",
|
|
18057
|
-
"axios",
|
|
18058
|
-
"node",
|
|
18059
|
-
"node-fetch",
|
|
18060
|
-
"undici",
|
|
18061
|
-
"got",
|
|
18062
|
-
"libwww-perl",
|
|
18063
|
-
"guzzlehttp",
|
|
18064
|
-
"ruby",
|
|
18065
|
-
"faraday",
|
|
18066
|
-
"http.rb",
|
|
18067
|
-
"php",
|
|
18068
|
-
"wordpress",
|
|
18069
|
-
"dalvik",
|
|
18070
|
-
"postmanruntime",
|
|
18071
|
-
"insomnia"
|
|
18072
|
-
];
|
|
18073
|
-
var BROWSER_SHAPED_AGENT_MARKERS = ["compatible;", "fetch", "agent", "headless", "+http"];
|
|
18074
|
-
var PRODUCT_NAME_END = /[/\s;(]/;
|
|
18075
|
-
var SKIPPED_PATH = new RegExp(`\\.(?:${UNLISTED_AGENT_SKIPPED_EXTENSIONS.join("|")})$`, "i");
|
|
18076
|
-
var COMPATIBLE_NAME = /compatible;\s*([a-z0-9][a-z0-9._-]*)/;
|
|
18077
|
-
var BOT_LIKE_NAME = /([a-z0-9][a-z0-9._-]*(?:bot|crawler|spider|fetcher|agent|-user))\//;
|
|
18078
|
-
var HEADLESS_NAME = /\b(headless[a-z]*)\//;
|
|
18079
|
-
var BROWSER_COMPATIBLE_NAMES = ["msie"];
|
|
18080
|
-
var MAX_AGENT_NAME_LENGTH = 64;
|
|
18081
|
-
var HEADER_NAME_PREFIX = /^user-agent\s*:\s*/;
|
|
18082
|
-
var AGENT_NAME = /^[a-z0-9][a-z0-9._-]*$/;
|
|
18083
|
-
var KNOWN_AGENT_NAMES = [
|
|
18084
|
-
{ marker: "facebookexternalhit/1.1 facebot twitterbot/1.0", name: "imessage-link-preview" }
|
|
18085
|
-
];
|
|
18086
|
-
function boundedAgentName(name) {
|
|
18087
|
-
const bounded = name.replace(/^[._-]+/, "").slice(0, MAX_AGENT_NAME_LENGTH).replace(/[._-]+$/, "");
|
|
18088
|
-
return AGENT_NAME.test(bounded) ? bounded : null;
|
|
18089
|
-
}
|
|
18090
|
-
var UNRECOGNIZED_AGENT_NAMING_VERSION = 2;
|
|
18091
|
-
function unrecognizedAgentName(request) {
|
|
18092
|
-
const method = (request.method ?? "GET").toUpperCase();
|
|
18093
|
-
if (method !== "GET" && method !== "HEAD") return null;
|
|
18094
|
-
if (request.path && SKIPPED_PATH.test(request.path.split("?")[0] ?? "")) return null;
|
|
18095
|
-
const ua = (request.userAgent ?? "").trim().toLowerCase().replace(HEADER_NAME_PREFIX, "");
|
|
18096
|
-
if (!ua) return null;
|
|
18097
|
-
const known = KNOWN_AGENT_NAMES.find((entry) => ua.includes(entry.marker));
|
|
18098
|
-
if (known) return known.name;
|
|
18099
|
-
const product = ua.split(PRODUCT_NAME_END)[0] ?? "";
|
|
18100
|
-
if (!product || GENERIC_HTTP_CLIENTS.includes(product)) return null;
|
|
18101
|
-
if (product !== "mozilla") {
|
|
18102
|
-
return boundedAgentName(ua.includes("/") ? product : ua.replace(/[^a-z0-9._-]+/g, "-"));
|
|
18103
|
-
}
|
|
18104
|
-
const name = COMPATIBLE_NAME.exec(ua)?.[1] ?? BOT_LIKE_NAME.exec(ua)?.[1] ?? HEADLESS_NAME.exec(ua)?.[1];
|
|
18105
|
-
if (!name || BROWSER_COMPATIBLE_NAMES.includes(name)) return null;
|
|
18106
|
-
return boundedAgentName(name);
|
|
18107
|
-
}
|
|
18108
|
-
|
|
18109
18076
|
// ../integration-traffic/src/journey.ts
|
|
18077
|
+
var JOURNEY_RULES_VERSION = 2;
|
|
18110
18078
|
var ASSET_EXTENSION_PATTERN = /\.(?:avif|bmp|css|gif|ico|jpe?g|js|map|mjs|mp3|mp4|otf|png|svg|ttf|txt|webm|webmanifest|webp|woff2?|xml)$/i;
|
|
18111
18079
|
var ASSET_PATH_PREFIXES = ["/_next/static/", "/_nuxt/", "/_ipx/", "/_vercel/", "/assets/", "/build/", "/dist/", "/fonts/", "/images/", "/img/", "/static/", "/wp-content/", "/wp-includes/", "/cdn-cgi/"];
|
|
18112
18080
|
var DATA_PATH_PATTERN = /\.json$|^\/api\/|^\/wp-json\/|^\/graphql/i;
|
|
@@ -18321,10 +18289,10 @@ function sessionizeJourneyRequests(requests, state, options) {
|
|
|
18321
18289
|
let dropped = 0;
|
|
18322
18290
|
let capped = 0;
|
|
18323
18291
|
for (const [visitorKey, list] of byVisitor) {
|
|
18324
|
-
list.sort((a, b) => a.observedAt < b.observedAt ? -1 : a.observedAt > b.observedAt ? 1 :
|
|
18292
|
+
list.sort((a, b) => a.observedAt < b.observedAt ? -1 : a.observedAt > b.observedAt ? 1 : Number(a.kind === "data") - Number(b.kind === "data"));
|
|
18325
18293
|
const prior = state.get(visitorKey);
|
|
18326
|
-
const isWeakAi = (r) => Boolean(r.signal.aiProduct) && r.stepReferer;
|
|
18327
|
-
const isStrongAi = (r) => Boolean(r.signal.aiProduct) && !r.stepReferer;
|
|
18294
|
+
const isWeakAi = (r) => r.kind === "page" && Boolean(r.signal.aiProduct) && r.stepReferer;
|
|
18295
|
+
const isStrongAi = (r) => r.kind === "page" && Boolean(r.signal.aiProduct) && !r.stepReferer;
|
|
18328
18296
|
const aiTimes = [...new Set([
|
|
18329
18297
|
prior?.firstAiAt,
|
|
18330
18298
|
prior?.lastAiAt,
|
|
@@ -18355,12 +18323,11 @@ function sessionizeJourneyRequests(requests, state, options) {
|
|
|
18355
18323
|
}
|
|
18356
18324
|
if (aiTouch && request.signal.aiProduct && !engineAt.has(request.observedAt)) engineAt.set(request.observedAt, request.signal.aiProduct);
|
|
18357
18325
|
const entry = aiTouch ? classifyJourneyEntry(request.signal) : request.stepReferer ? DIRECT_ENTRY : classifyJourneyEntry({ ...request.signal, aiProduct: null, aiDomain: null, aiPaid: false });
|
|
18358
|
-
const outside = aiTouch || hasEntryEvidence(request);
|
|
18326
|
+
const outside = request.kind === "page" && (aiTouch || hasEntryEvidence(request));
|
|
18359
18327
|
const placement = placeRequest(known, t, outside ? entryKey(entry) : null, options.idleMs);
|
|
18360
18328
|
const continues = placement.visit !== void 0 && !placement.head;
|
|
18361
18329
|
const kept = aiTouch || withinLookback || continues;
|
|
18362
|
-
|
|
18363
|
-
if (!kept || opensVisit && request.kind === "data" && !outside) {
|
|
18330
|
+
if (!kept || request.kind === "data" && !continues) {
|
|
18364
18331
|
dropped += 1;
|
|
18365
18332
|
continue;
|
|
18366
18333
|
}
|
|
@@ -18428,39 +18395,156 @@ function sessionizeJourneyRequests(requests, state, options) {
|
|
|
18428
18395
|
return { visits: [...visits.values()], events, visitors, dropped, capped };
|
|
18429
18396
|
}
|
|
18430
18397
|
|
|
18431
|
-
// ../integration-traffic/src/
|
|
18432
|
-
|
|
18433
|
-
|
|
18398
|
+
// ../integration-traffic/src/scanner-bursts.ts
|
|
18399
|
+
var SCANNER_BURST_WINDOW_MS = 10 * 6e4;
|
|
18400
|
+
function mergeRuns(runs, windowMs) {
|
|
18401
|
+
runs.sort((a, b) => a.firstProbeAt - b.firstProbeAt);
|
|
18402
|
+
const merged = [];
|
|
18403
|
+
for (const run of runs) {
|
|
18404
|
+
const last = merged.at(-1);
|
|
18405
|
+
if (last && run.firstProbeAt <= last.lastProbeAt + windowMs) last.lastProbeAt = Math.max(last.lastProbeAt, run.lastProbeAt);
|
|
18406
|
+
else merged.push({ ...run });
|
|
18407
|
+
}
|
|
18408
|
+
return merged;
|
|
18409
|
+
}
|
|
18410
|
+
function findScannerBursts(events, options) {
|
|
18411
|
+
const windowMs = options.windowMs ?? SCANNER_BURST_WINDOW_MS;
|
|
18412
|
+
const keys = /* @__PURE__ */ new Map();
|
|
18413
|
+
const probesByClient = /* @__PURE__ */ new Map();
|
|
18414
|
+
const candidates = [];
|
|
18434
18415
|
for (const event of events) {
|
|
18435
|
-
|
|
18416
|
+
const remoteIp = event.remoteIp?.trim();
|
|
18417
|
+
if (!remoteIp || isSelfTraffic(event)) continue;
|
|
18436
18418
|
const agent = classifyAiUserFetch(event);
|
|
18437
|
-
if (!agent || agent.trust === AgentTrustLevels.
|
|
18438
|
-
const
|
|
18439
|
-
if (!Number.isFinite(
|
|
18440
|
-
|
|
18441
|
-
if (
|
|
18442
|
-
|
|
18443
|
-
|
|
18444
|
-
|
|
18445
|
-
|
|
18446
|
-
|
|
18447
|
-
|
|
18448
|
-
|
|
18449
|
-
|
|
18450
|
-
|
|
18451
|
-
|
|
18452
|
-
|
|
18453
|
-
verificationManifest: agent.verificationManifest,
|
|
18454
|
-
trust: agent.trust,
|
|
18455
|
-
trustReasons: agent.trustReasons,
|
|
18456
|
-
host: hostOf(event.host) ?? hostOf(event.requestUrl),
|
|
18457
|
-
path: retainedPath,
|
|
18458
|
-
method: event.method?.trim().toUpperCase() || null,
|
|
18459
|
-
status: event.status,
|
|
18460
|
-
latencyMs: event.latencyMs !== null && Number.isFinite(event.latencyMs) && event.latencyMs >= 0 ? event.latencyMs : null
|
|
18461
|
-
});
|
|
18419
|
+
if (!agent || agent.trust === AgentTrustLevels.verified) continue;
|
|
18420
|
+
const at = Date.parse(event.observedAt);
|
|
18421
|
+
if (!Number.isFinite(at)) continue;
|
|
18422
|
+
let client = keys.get(remoteIp);
|
|
18423
|
+
if (client === void 0) {
|
|
18424
|
+
client = options.clientKey(remoteIp);
|
|
18425
|
+
keys.set(remoteIp, client);
|
|
18426
|
+
}
|
|
18427
|
+
if (client === null) continue;
|
|
18428
|
+
if (agent.trustReasons.includes(AgentTrustReasons["scanner-path"])) {
|
|
18429
|
+
const probes = probesByClient.get(client);
|
|
18430
|
+
if (probes) probes.push(at);
|
|
18431
|
+
else probesByClient.set(client, [at]);
|
|
18432
|
+
} else if (agent.trust !== AgentTrustLevels.likely_fake) {
|
|
18433
|
+
candidates.push({ eventId: event.eventId, client, at });
|
|
18434
|
+
}
|
|
18462
18435
|
}
|
|
18463
|
-
|
|
18436
|
+
const clients = [.../* @__PURE__ */ new Set([...probesByClient.keys(), ...candidates.map((candidate) => candidate.client)])];
|
|
18437
|
+
const known = clients.length > 0 && options.known ? options.known(clients) : /* @__PURE__ */ new Map();
|
|
18438
|
+
const runsByClient = /* @__PURE__ */ new Map();
|
|
18439
|
+
const windows = /* @__PURE__ */ new Map();
|
|
18440
|
+
for (const client of clients) {
|
|
18441
|
+
const stored = known.get(client);
|
|
18442
|
+
const probes = probesByClient.get(client) ?? [];
|
|
18443
|
+
const runs = mergeRuns([...stored ? [stored] : [], ...probes.map((at) => ({ firstProbeAt: at, lastProbeAt: at }))], windowMs);
|
|
18444
|
+
if (runs.length === 0) continue;
|
|
18445
|
+
runsByClient.set(client, runs);
|
|
18446
|
+
const latest = runs.at(-1);
|
|
18447
|
+
if (probes.length > 0 && (latest.firstProbeAt !== stored?.firstProbeAt || latest.lastProbeAt !== stored.lastProbeAt)) windows.set(client, latest);
|
|
18448
|
+
}
|
|
18449
|
+
const eventIds = /* @__PURE__ */ new Set();
|
|
18450
|
+
for (const candidate of candidates) {
|
|
18451
|
+
const runs = runsByClient.get(candidate.client);
|
|
18452
|
+
if (runs?.some((run) => candidate.at >= run.firstProbeAt - windowMs && candidate.at <= run.lastProbeAt + windowMs)) eventIds.add(candidate.eventId);
|
|
18453
|
+
}
|
|
18454
|
+
return { eventIds, windows };
|
|
18455
|
+
}
|
|
18456
|
+
function findScannerBurstsInBatch(events) {
|
|
18457
|
+
return findScannerBursts(events, { clientKey: (remoteIp) => remoteIp }).eventIds;
|
|
18458
|
+
}
|
|
18459
|
+
|
|
18460
|
+
// ../integration-traffic/src/unlisted-agents.ts
|
|
18461
|
+
var UNLISTED_AGENT_SKIPPED_EXTENSIONS = [
|
|
18462
|
+
"avif",
|
|
18463
|
+
"bmp",
|
|
18464
|
+
"css",
|
|
18465
|
+
"gif",
|
|
18466
|
+
"ico",
|
|
18467
|
+
"jpeg",
|
|
18468
|
+
"jpg",
|
|
18469
|
+
"js",
|
|
18470
|
+
"map",
|
|
18471
|
+
"mjs",
|
|
18472
|
+
"mp3",
|
|
18473
|
+
"mp4",
|
|
18474
|
+
"otf",
|
|
18475
|
+
"png",
|
|
18476
|
+
"svg",
|
|
18477
|
+
"ttf",
|
|
18478
|
+
"webm",
|
|
18479
|
+
"webmanifest",
|
|
18480
|
+
"webp",
|
|
18481
|
+
"woff",
|
|
18482
|
+
"woff2",
|
|
18483
|
+
"zip"
|
|
18484
|
+
];
|
|
18485
|
+
var GENERIC_HTTP_CLIENTS = [
|
|
18486
|
+
"curl",
|
|
18487
|
+
"wget",
|
|
18488
|
+
"python",
|
|
18489
|
+
"python-requests",
|
|
18490
|
+
"python-urllib",
|
|
18491
|
+
"python-httpx",
|
|
18492
|
+
"aiohttp",
|
|
18493
|
+
"httpx",
|
|
18494
|
+
"go-http-client",
|
|
18495
|
+
"okhttp",
|
|
18496
|
+
"java",
|
|
18497
|
+
"apache-httpclient",
|
|
18498
|
+
"axios",
|
|
18499
|
+
"node",
|
|
18500
|
+
"node-fetch",
|
|
18501
|
+
"undici",
|
|
18502
|
+
"got",
|
|
18503
|
+
"libwww-perl",
|
|
18504
|
+
"guzzlehttp",
|
|
18505
|
+
"ruby",
|
|
18506
|
+
"faraday",
|
|
18507
|
+
"http.rb",
|
|
18508
|
+
"php",
|
|
18509
|
+
"wordpress",
|
|
18510
|
+
"dalvik",
|
|
18511
|
+
"postmanruntime",
|
|
18512
|
+
"insomnia"
|
|
18513
|
+
];
|
|
18514
|
+
var BROWSER_SHAPED_AGENT_MARKERS = ["compatible;", "fetch", "agent", "headless", "+http"];
|
|
18515
|
+
var PRODUCT_NAME_END = /[/\s;(]/;
|
|
18516
|
+
var SKIPPED_PATH = new RegExp(`\\.(?:${UNLISTED_AGENT_SKIPPED_EXTENSIONS.join("|")})$`, "i");
|
|
18517
|
+
var COMPATIBLE_NAME = /compatible;\s*([a-z0-9][a-z0-9._-]*)/;
|
|
18518
|
+
var BOT_LIKE_NAME = /([a-z0-9][a-z0-9._-]*(?:bot|crawler|spider|fetcher|agent|-user))\//;
|
|
18519
|
+
var HEADLESS_NAME = /\b(headless[a-z]*)\//;
|
|
18520
|
+
var BROWSER_COMPATIBLE_NAMES = ["msie"];
|
|
18521
|
+
var MAX_AGENT_NAME_LENGTH = 64;
|
|
18522
|
+
var HEADER_NAME_PREFIX = /^user-agent\s*:\s*/;
|
|
18523
|
+
var AGENT_NAME = /^[a-z0-9][a-z0-9._-]*$/;
|
|
18524
|
+
var KNOWN_AGENT_NAMES = [
|
|
18525
|
+
{ marker: "facebookexternalhit/1.1 facebot twitterbot/1.0", name: "imessage-link-preview" }
|
|
18526
|
+
];
|
|
18527
|
+
function boundedAgentName(name) {
|
|
18528
|
+
const bounded = name.replace(/^[._-]+/, "").slice(0, MAX_AGENT_NAME_LENGTH).replace(/[._-]+$/, "");
|
|
18529
|
+
return AGENT_NAME.test(bounded) ? bounded : null;
|
|
18530
|
+
}
|
|
18531
|
+
var UNRECOGNIZED_AGENT_NAMING_VERSION = 2;
|
|
18532
|
+
function unrecognizedAgentName(request) {
|
|
18533
|
+
const method = (request.method ?? "GET").toUpperCase();
|
|
18534
|
+
if (method !== "GET" && method !== "HEAD") return null;
|
|
18535
|
+
if (request.path && SKIPPED_PATH.test(request.path.split("?")[0] ?? "")) return null;
|
|
18536
|
+
const ua = (request.userAgent ?? "").trim().toLowerCase().replace(HEADER_NAME_PREFIX, "");
|
|
18537
|
+
if (!ua) return null;
|
|
18538
|
+
const known = KNOWN_AGENT_NAMES.find((entry) => ua.includes(entry.marker));
|
|
18539
|
+
if (known) return known.name;
|
|
18540
|
+
const product = ua.split(PRODUCT_NAME_END)[0] ?? "";
|
|
18541
|
+
if (!product || GENERIC_HTTP_CLIENTS.includes(product)) return null;
|
|
18542
|
+
if (product !== "mozilla") {
|
|
18543
|
+
return boundedAgentName(ua.includes("/") ? product : ua.replace(/[^a-z0-9._-]+/g, "-"));
|
|
18544
|
+
}
|
|
18545
|
+
const name = COMPATIBLE_NAME.exec(ua)?.[1] ?? BOT_LIKE_NAME.exec(ua)?.[1] ?? HEADLESS_NAME.exec(ua)?.[1];
|
|
18546
|
+
if (!name || BROWSER_COMPATIBLE_NAMES.includes(name)) return null;
|
|
18547
|
+
return boundedAgentName(name);
|
|
18464
18548
|
}
|
|
18465
18549
|
|
|
18466
18550
|
// ../integration-traffic/src/rollup.ts
|
|
@@ -18594,6 +18678,7 @@ function buildTrafficProbeReport(events, options = {}) {
|
|
|
18594
18678
|
const topAiReferralLandingPaths = /* @__PURE__ */ new Map();
|
|
18595
18679
|
const includedEvents = events.filter((event) => !isSelfTraffic(event));
|
|
18596
18680
|
const selfTrafficExcluded = events.length - includedEvents.length;
|
|
18681
|
+
const scannerBursts = options.scannerBurstEventIds ?? findScannerBurstsInBatch(includedEvents);
|
|
18597
18682
|
let crawlerHits = 0;
|
|
18598
18683
|
let aiUserFetchHits = 0;
|
|
18599
18684
|
let aiUserFetchLikelyFakeHits = 0;
|
|
@@ -18606,7 +18691,8 @@ function buildTrafficProbeReport(events, options = {}) {
|
|
|
18606
18691
|
const tsHour = hourBucket(event.observedAt);
|
|
18607
18692
|
const pathNormalized = normalizeTrafficPathPattern(event.path);
|
|
18608
18693
|
const crawler = classifyCrawler(event);
|
|
18609
|
-
const
|
|
18694
|
+
const classifiedFetch = classifyAiUserFetch(event);
|
|
18695
|
+
const aiUserFetch = classifiedFetch && scannerBursts.has(event.eventId) ? withScannerBurst(classifiedFetch) : classifiedFetch;
|
|
18610
18696
|
const aiReferral = classifyAiReferral(event);
|
|
18611
18697
|
if (crawler) {
|
|
18612
18698
|
crawlerHits += 1;
|
|
@@ -18806,6 +18892,43 @@ function incrementBucket(map, key, fields) {
|
|
|
18806
18892
|
else map.set(key, { fields, hits: 1 });
|
|
18807
18893
|
}
|
|
18808
18894
|
|
|
18895
|
+
// ../integration-traffic/src/agent-journeys.ts
|
|
18896
|
+
function extractAgentJourneyRequests(events, options = {}) {
|
|
18897
|
+
const bursts = options.scannerBurstEventIds ?? findScannerBurstsInBatch(events);
|
|
18898
|
+
const requests = [];
|
|
18899
|
+
for (const event of events) {
|
|
18900
|
+
if (isSelfTraffic(event)) continue;
|
|
18901
|
+
const classified = classifyAiUserFetch(event);
|
|
18902
|
+
const agent = classified && bursts.has(event.eventId) ? withScannerBurst(classified) : classified;
|
|
18903
|
+
if (!agent || agent.trust === AgentTrustLevels.likely_fake) continue;
|
|
18904
|
+
const timestamp = Date.parse(event.observedAt);
|
|
18905
|
+
if (!Number.isFinite(timestamp)) continue;
|
|
18906
|
+
const path = event.path.split(/[?#]/)[0] || "/";
|
|
18907
|
+
if (classifyTrafficPath(path) === TrafficPathClasses.asset && !/\.json\/?$/i.test(path)) continue;
|
|
18908
|
+
const maskedPath = maskJourneyPath(path);
|
|
18909
|
+
const retainedPath = maskedPath.length > 512 ? `${maskedPath.slice(0, 505)}/:token` : maskedPath;
|
|
18910
|
+
requests.push({
|
|
18911
|
+
sourceEventId: event.eventId,
|
|
18912
|
+
ts: new Date(timestamp).toISOString(),
|
|
18913
|
+
remoteIp: event.remoteIp?.trim() || null,
|
|
18914
|
+
userAgent: agent.matchedUserAgent,
|
|
18915
|
+
agentId: agent.botId,
|
|
18916
|
+
operator: agent.operator,
|
|
18917
|
+
product: agent.product,
|
|
18918
|
+
verificationStatus: agent.verificationStatus,
|
|
18919
|
+
verificationManifest: agent.verificationManifest,
|
|
18920
|
+
trust: agent.trust,
|
|
18921
|
+
trustReasons: agent.trustReasons,
|
|
18922
|
+
host: hostOf(event.host) ?? hostOf(event.requestUrl),
|
|
18923
|
+
path: retainedPath,
|
|
18924
|
+
method: event.method?.trim().toUpperCase() || null,
|
|
18925
|
+
status: event.status,
|
|
18926
|
+
latencyMs: event.latencyMs !== null && Number.isFinite(event.latencyMs) && event.latencyMs >= 0 ? event.latencyMs : null
|
|
18927
|
+
});
|
|
18928
|
+
}
|
|
18929
|
+
return requests;
|
|
18930
|
+
}
|
|
18931
|
+
|
|
18809
18932
|
// ../integration-cloudflare-worker/src/types.ts
|
|
18810
18933
|
var CLOUDFLARE_WORKER_BINDINGS = {
|
|
18811
18934
|
deliveryMode: "CANONRY_DELIVERY_MODE",
|
|
@@ -19406,19 +19529,26 @@ function verifyRequestSignature(opts) {
|
|
|
19406
19529
|
}
|
|
19407
19530
|
|
|
19408
19531
|
export {
|
|
19532
|
+
AGENT_TRUST_RULES_VERSION,
|
|
19533
|
+
isScannerPath,
|
|
19409
19534
|
SELF_TRAFFIC_USER_AGENT_PATTERNS,
|
|
19410
19535
|
DEFAULT_AI_CRAWLER_RULES,
|
|
19411
19536
|
isSelfTraffic,
|
|
19412
19537
|
classifyCrawler,
|
|
19413
19538
|
classifyAiUserFetch,
|
|
19539
|
+
classifyAiReferral,
|
|
19540
|
+
JOURNEY_RULES_VERSION,
|
|
19414
19541
|
extractJourneyRequests,
|
|
19415
19542
|
sessionizeJourneyRequests,
|
|
19543
|
+
SCANNER_BURST_WINDOW_MS,
|
|
19544
|
+
findScannerBursts,
|
|
19416
19545
|
extractAgentJourneyRequests,
|
|
19417
19546
|
UNLISTED_AGENT_SKIPPED_EXTENSIONS,
|
|
19418
19547
|
GENERIC_HTTP_CLIENTS,
|
|
19419
19548
|
BROWSER_SHAPED_AGENT_MARKERS,
|
|
19420
19549
|
UNRECOGNIZED_AGENT_NAMING_VERSION,
|
|
19421
19550
|
unrecognizedAgentName,
|
|
19551
|
+
normalizeTrafficPathPattern,
|
|
19422
19552
|
buildTrafficProbeReport,
|
|
19423
19553
|
normalizeCloudflareWorkerEvent,
|
|
19424
19554
|
CLOUDFLARE_WORKER_BINDINGS,
|