@canonry/canonry 4.163.0 → 4.164.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/assets/{AuditHistoryPanel-FDQx9TS5.js → AuditHistoryPanel-CD1GVkS9.js} +1 -1
- package/assets/assets/{BacklinksPage-DVvsQsdk.js → BacklinksPage-B0Ijhntp.js} +1 -1
- package/assets/assets/{HistoryPage-31HR6TIJ.js → HistoryPage-DIKtpT_z.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-Dps7MF7r.js → MeasurementPropertyPage-BfPwwIsT.js} +1 -1
- package/assets/assets/{ProjectPage-BijTUrpk.js → ProjectPage-3czWArKE.js} +1 -1
- package/assets/assets/{RunRow-9DiFrBlc.js → RunRow-CHyd0gQH.js} +1 -1
- package/assets/assets/{RunsPage-DQlAc4J9.js → RunsPage-Dujt0SQ5.js} +1 -1
- package/assets/assets/{SettingsPage-c1nxJsbL.js → SettingsPage-CJE9Oeg9.js} +1 -1
- package/assets/assets/SiteHealthSection-DvuvziPJ.js +4 -0
- package/assets/assets/{TrafficPage-HzwQyLzl.js → TrafficPage-B0tru9ek.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-BTVAQjRd.js → TrafficSourceDetailPage-DshEUtQX.js} +1 -1
- package/assets/assets/{extract-error-message-B5bhJRmf.js → extract-error-message-O2dirpiH.js} +1 -1
- package/assets/assets/{index-CQsvCUM0.js → index-C_y0aGQL.js} +19 -19
- package/assets/assets/{react-sigma_core.esm.min-6BxcniCA.js → react-sigma_core.esm.min-sVSSzmzq.js} +1 -1
- package/assets/index.html +1 -1
- package/dist/{chunk-ESH3LPYQ.js → chunk-BFGHZ2GU.js} +15 -0
- package/dist/{chunk-INXI6PO7.js → chunk-C4WKYU5C.js} +1 -1
- package/dist/{chunk-G5UZTODF.js → chunk-GCQPFCP2.js} +786 -95
- package/dist/{chunk-MRHYSEZB.js → chunk-RULHNUBH.js} +66 -62
- package/dist/cli.js +17 -5
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-AFAFGLCM.js → intelligence-service-Z3XS2UXQ.js} +2 -2
- package/dist/mcp.js +2 -2
- package/package.json +9 -9
- package/assets/assets/SiteHealthSection-0B0oUUQx.js +0 -4
|
@@ -581,7 +581,7 @@ import {
|
|
|
581
581
|
wordpressSchemaDeployResultDtoSchema,
|
|
582
582
|
wordpressSchemaStatusResultDtoSchema,
|
|
583
583
|
wordpressStatusDtoSchema
|
|
584
|
-
} from "./chunk-
|
|
584
|
+
} from "./chunk-BFGHZ2GU.js";
|
|
585
585
|
|
|
586
586
|
// src/intelligence-service.ts
|
|
587
587
|
import { eq as eq59, desc as desc26, asc as asc11, and as and47, ne as ne7, or as or13, inArray as inArray21, gte as gte14, lte as lte12 } from "drizzle-orm";
|
|
@@ -608,6 +608,7 @@ __export(schema_exports, {
|
|
|
608
608
|
agentToolEvents: () => agentToolEvents,
|
|
609
609
|
aiReferralEventsHourly: () => aiReferralEventsHourly,
|
|
610
610
|
aiUserFetchEventsHourly: () => aiUserFetchEventsHourly,
|
|
611
|
+
aiUserFetchVerificationManifestsHourly: () => aiUserFetchVerificationManifestsHourly,
|
|
611
612
|
apiKeys: () => apiKeys,
|
|
612
613
|
auditLog: () => auditLog,
|
|
613
614
|
backlinkDomains: () => backlinkDomains,
|
|
@@ -620,6 +621,7 @@ __export(schema_exports, {
|
|
|
620
621
|
competitors: () => competitors,
|
|
621
622
|
contentTargetDismissals: () => contentTargetDismissals,
|
|
622
623
|
crawlerEventsHourly: () => crawlerEventsHourly,
|
|
624
|
+
crawlerVerificationManifestsHourly: () => crawlerVerificationManifestsHourly,
|
|
623
625
|
discoveryProbes: () => discoveryProbes,
|
|
624
626
|
discoverySessions: () => discoverySessions,
|
|
625
627
|
doctorHealthState: () => doctorHealthState,
|
|
@@ -2242,6 +2244,55 @@ var crawlerEventsHourly = sqliteTable("crawler_events_hourly", {
|
|
|
2242
2244
|
index("idx_crawler_hourly_project_ts").on(table.projectId, table.tsHour),
|
|
2243
2245
|
index("idx_crawler_hourly_path").on(table.projectId, table.pathNormalized)
|
|
2244
2246
|
]);
|
|
2247
|
+
var crawlerVerificationManifestsHourly = sqliteTable("crawler_verification_manifests_hourly", {
|
|
2248
|
+
projectId: text("project_id").notNull(),
|
|
2249
|
+
sourceId: text("source_id").notNull(),
|
|
2250
|
+
tsHour: text("ts_hour").notNull(),
|
|
2251
|
+
botId: text("bot_id").notNull(),
|
|
2252
|
+
verificationStatus: text("verification_status").notNull(),
|
|
2253
|
+
pathNormalized: text("path_normalized").notNull(),
|
|
2254
|
+
status: integer("status").notNull(),
|
|
2255
|
+
manifestId: text("manifest_id").notNull(),
|
|
2256
|
+
manifestJson: text("manifest_json", { mode: "json" }).$type(),
|
|
2257
|
+
hits: integer("hits").notNull().default(0),
|
|
2258
|
+
createdAt: text("created_at").notNull(),
|
|
2259
|
+
updatedAt: text("updated_at").notNull()
|
|
2260
|
+
}, (table) => [
|
|
2261
|
+
primaryKey({
|
|
2262
|
+
columns: [
|
|
2263
|
+
table.projectId,
|
|
2264
|
+
table.sourceId,
|
|
2265
|
+
table.tsHour,
|
|
2266
|
+
table.botId,
|
|
2267
|
+
table.verificationStatus,
|
|
2268
|
+
table.pathNormalized,
|
|
2269
|
+
table.status,
|
|
2270
|
+
table.manifestId
|
|
2271
|
+
]
|
|
2272
|
+
}),
|
|
2273
|
+
foreignKey({
|
|
2274
|
+
name: "crawler_verification_manifests_parent_fk",
|
|
2275
|
+
columns: [
|
|
2276
|
+
table.projectId,
|
|
2277
|
+
table.sourceId,
|
|
2278
|
+
table.tsHour,
|
|
2279
|
+
table.botId,
|
|
2280
|
+
table.verificationStatus,
|
|
2281
|
+
table.pathNormalized,
|
|
2282
|
+
table.status
|
|
2283
|
+
],
|
|
2284
|
+
foreignColumns: [
|
|
2285
|
+
crawlerEventsHourly.projectId,
|
|
2286
|
+
crawlerEventsHourly.sourceId,
|
|
2287
|
+
crawlerEventsHourly.tsHour,
|
|
2288
|
+
crawlerEventsHourly.botId,
|
|
2289
|
+
crawlerEventsHourly.verificationStatus,
|
|
2290
|
+
crawlerEventsHourly.pathNormalized,
|
|
2291
|
+
crawlerEventsHourly.status
|
|
2292
|
+
]
|
|
2293
|
+
}).onDelete("cascade"),
|
|
2294
|
+
index("idx_crawler_verification_manifests_project_ts").on(table.projectId, table.tsHour)
|
|
2295
|
+
]);
|
|
2245
2296
|
var aiUserFetchEventsHourly = sqliteTable("ai_user_fetch_events_hourly", {
|
|
2246
2297
|
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
2247
2298
|
sourceId: text("source_id").notNull().references(() => trafficSources.id, { onDelete: "cascade" }),
|
|
@@ -2270,6 +2321,55 @@ var aiUserFetchEventsHourly = sqliteTable("ai_user_fetch_events_hourly", {
|
|
|
2270
2321
|
index("idx_ai_user_fetch_hourly_project_ts").on(table.projectId, table.tsHour),
|
|
2271
2322
|
index("idx_ai_user_fetch_hourly_path").on(table.projectId, table.pathNormalized)
|
|
2272
2323
|
]);
|
|
2324
|
+
var aiUserFetchVerificationManifestsHourly = sqliteTable("ai_user_fetch_verification_manifests_hourly", {
|
|
2325
|
+
projectId: text("project_id").notNull(),
|
|
2326
|
+
sourceId: text("source_id").notNull(),
|
|
2327
|
+
tsHour: text("ts_hour").notNull(),
|
|
2328
|
+
botId: text("bot_id").notNull(),
|
|
2329
|
+
verificationStatus: text("verification_status").notNull(),
|
|
2330
|
+
pathNormalized: text("path_normalized").notNull(),
|
|
2331
|
+
status: integer("status").notNull(),
|
|
2332
|
+
manifestId: text("manifest_id").notNull(),
|
|
2333
|
+
manifestJson: text("manifest_json", { mode: "json" }).$type(),
|
|
2334
|
+
hits: integer("hits").notNull().default(0),
|
|
2335
|
+
createdAt: text("created_at").notNull(),
|
|
2336
|
+
updatedAt: text("updated_at").notNull()
|
|
2337
|
+
}, (table) => [
|
|
2338
|
+
primaryKey({
|
|
2339
|
+
columns: [
|
|
2340
|
+
table.projectId,
|
|
2341
|
+
table.sourceId,
|
|
2342
|
+
table.tsHour,
|
|
2343
|
+
table.botId,
|
|
2344
|
+
table.verificationStatus,
|
|
2345
|
+
table.pathNormalized,
|
|
2346
|
+
table.status,
|
|
2347
|
+
table.manifestId
|
|
2348
|
+
]
|
|
2349
|
+
}),
|
|
2350
|
+
foreignKey({
|
|
2351
|
+
name: "ai_user_fetch_verification_manifests_parent_fk",
|
|
2352
|
+
columns: [
|
|
2353
|
+
table.projectId,
|
|
2354
|
+
table.sourceId,
|
|
2355
|
+
table.tsHour,
|
|
2356
|
+
table.botId,
|
|
2357
|
+
table.verificationStatus,
|
|
2358
|
+
table.pathNormalized,
|
|
2359
|
+
table.status
|
|
2360
|
+
],
|
|
2361
|
+
foreignColumns: [
|
|
2362
|
+
aiUserFetchEventsHourly.projectId,
|
|
2363
|
+
aiUserFetchEventsHourly.sourceId,
|
|
2364
|
+
aiUserFetchEventsHourly.tsHour,
|
|
2365
|
+
aiUserFetchEventsHourly.botId,
|
|
2366
|
+
aiUserFetchEventsHourly.verificationStatus,
|
|
2367
|
+
aiUserFetchEventsHourly.pathNormalized,
|
|
2368
|
+
aiUserFetchEventsHourly.status
|
|
2369
|
+
]
|
|
2370
|
+
}).onDelete("cascade"),
|
|
2371
|
+
index("idx_ai_user_fetch_verification_manifests_project_ts").on(table.projectId, table.tsHour)
|
|
2372
|
+
]);
|
|
2273
2373
|
var aiReferralEventsHourly = sqliteTable("ai_referral_events_hourly", {
|
|
2274
2374
|
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
2275
2375
|
sourceId: text("source_id").notNull().references(() => trafficSources.id, { onDelete: "cascade" }),
|
|
@@ -6264,6 +6364,71 @@ var MIGRATION_VERSIONS = [
|
|
|
6264
6364
|
`ALTER TABLE site_crawl_edges ADD COLUMN placement_unknown_occurrences INTEGER`,
|
|
6265
6365
|
`ALTER TABLE site_crawl_snapshots ADD COLUMN link_placement_ruleset_version TEXT`
|
|
6266
6366
|
]
|
|
6367
|
+
},
|
|
6368
|
+
{
|
|
6369
|
+
version: 139,
|
|
6370
|
+
name: "traffic-verification-manifest-provenance",
|
|
6371
|
+
// Verification is evidence produced by one exact range manifest. Store
|
|
6372
|
+
// that provenance in additive sidecars: the existing rollup keys stay
|
|
6373
|
+
// unchanged so an older binary can still upsert them after a rollback.
|
|
6374
|
+
// Historical rows deliberately receive no sidecar row because the old
|
|
6375
|
+
// schema never recorded which bundle classified them.
|
|
6376
|
+
statements: [
|
|
6377
|
+
`CREATE TABLE IF NOT EXISTS crawler_verification_manifests_hourly (
|
|
6378
|
+
project_id TEXT NOT NULL,
|
|
6379
|
+
source_id TEXT NOT NULL,
|
|
6380
|
+
ts_hour TEXT NOT NULL,
|
|
6381
|
+
bot_id TEXT NOT NULL,
|
|
6382
|
+
verification_status TEXT NOT NULL,
|
|
6383
|
+
path_normalized TEXT NOT NULL,
|
|
6384
|
+
status INTEGER NOT NULL,
|
|
6385
|
+
manifest_id TEXT NOT NULL,
|
|
6386
|
+
manifest_json TEXT,
|
|
6387
|
+
hits INTEGER NOT NULL DEFAULT 0,
|
|
6388
|
+
created_at TEXT NOT NULL,
|
|
6389
|
+
updated_at TEXT NOT NULL,
|
|
6390
|
+
PRIMARY KEY (
|
|
6391
|
+
project_id, source_id, ts_hour, bot_id, verification_status,
|
|
6392
|
+
path_normalized, status, manifest_id
|
|
6393
|
+
),
|
|
6394
|
+
FOREIGN KEY (
|
|
6395
|
+
project_id, source_id, ts_hour, bot_id, verification_status,
|
|
6396
|
+
path_normalized, status
|
|
6397
|
+
) REFERENCES crawler_events_hourly (
|
|
6398
|
+
project_id, source_id, ts_hour, bot_id, verification_status,
|
|
6399
|
+
path_normalized, status
|
|
6400
|
+
) ON DELETE CASCADE
|
|
6401
|
+
)`,
|
|
6402
|
+
`CREATE INDEX IF NOT EXISTS idx_crawler_verification_manifests_project_ts
|
|
6403
|
+
ON crawler_verification_manifests_hourly(project_id, ts_hour)`,
|
|
6404
|
+
`CREATE TABLE IF NOT EXISTS ai_user_fetch_verification_manifests_hourly (
|
|
6405
|
+
project_id TEXT NOT NULL,
|
|
6406
|
+
source_id TEXT NOT NULL,
|
|
6407
|
+
ts_hour TEXT NOT NULL,
|
|
6408
|
+
bot_id TEXT NOT NULL,
|
|
6409
|
+
verification_status TEXT NOT NULL,
|
|
6410
|
+
path_normalized TEXT NOT NULL,
|
|
6411
|
+
status INTEGER NOT NULL,
|
|
6412
|
+
manifest_id TEXT NOT NULL,
|
|
6413
|
+
manifest_json TEXT,
|
|
6414
|
+
hits INTEGER NOT NULL DEFAULT 0,
|
|
6415
|
+
created_at TEXT NOT NULL,
|
|
6416
|
+
updated_at TEXT NOT NULL,
|
|
6417
|
+
PRIMARY KEY (
|
|
6418
|
+
project_id, source_id, ts_hour, bot_id, verification_status,
|
|
6419
|
+
path_normalized, status, manifest_id
|
|
6420
|
+
),
|
|
6421
|
+
FOREIGN KEY (
|
|
6422
|
+
project_id, source_id, ts_hour, bot_id, verification_status,
|
|
6423
|
+
path_normalized, status
|
|
6424
|
+
) REFERENCES ai_user_fetch_events_hourly (
|
|
6425
|
+
project_id, source_id, ts_hour, bot_id, verification_status,
|
|
6426
|
+
path_normalized, status
|
|
6427
|
+
) ON DELETE CASCADE
|
|
6428
|
+
)`,
|
|
6429
|
+
`CREATE INDEX IF NOT EXISTS idx_ai_user_fetch_verification_manifests_project_ts
|
|
6430
|
+
ON ai_user_fetch_verification_manifests_hourly(project_id, ts_hour)`
|
|
6431
|
+
]
|
|
6267
6432
|
}
|
|
6268
6433
|
];
|
|
6269
6434
|
function addRunsMeasurementPlanVersionForeignKey(tx) {
|
|
@@ -48518,13 +48683,78 @@ async function listCloudRunTrafficEvents(accessToken, options) {
|
|
|
48518
48683
|
|
|
48519
48684
|
// ../integration-traffic/src/ip-ranges/anthropic.json
|
|
48520
48685
|
var anthropic_default = {
|
|
48521
|
-
_source: "
|
|
48522
|
-
|
|
48523
|
-
_notes: "Anthropic does not publish a clean machine-readable IP-range JSON like Google or OpenAI do. These prefixes are taken straight from ARIN's authoritative allocation records (NOT bgp.tools' AS-based view, which can include misleading announcements \u2014 e.g. AS60808 is registered to Anthropic but announces 209.249.57.0/24 which actually belongs to Mitel Networks). 216.73.216.0/22 (AWS-ANTHROPIC) is the load-bearing prefix \u2014 empirical Cloud Run logs show 100% of real ClaudeBot traffic comes from there; the other two are Anthropic's own allocations for non-crawler infra (kept since the verification gate still requires a Claude-* UA match). Refresh by re-running the ARIN RDAP query above and replacing the prefix list.",
|
|
48686
|
+
_source: "https://claude.com/crawling/bots.json",
|
|
48687
|
+
creationTime: "2026-08-13T20:38:01Z",
|
|
48524
48688
|
prefixes: [
|
|
48525
|
-
{
|
|
48526
|
-
|
|
48527
|
-
|
|
48689
|
+
{
|
|
48690
|
+
ipv4Prefix: "216.73.216.0/22"
|
|
48691
|
+
},
|
|
48692
|
+
{
|
|
48693
|
+
ipv4Prefix: "34.162.230.222/32"
|
|
48694
|
+
},
|
|
48695
|
+
{
|
|
48696
|
+
ipv4Prefix: "34.162.244.71/32"
|
|
48697
|
+
},
|
|
48698
|
+
{
|
|
48699
|
+
ipv4Prefix: "34.162.191.81/32"
|
|
48700
|
+
},
|
|
48701
|
+
{
|
|
48702
|
+
ipv4Prefix: "34.150.241.79/32"
|
|
48703
|
+
},
|
|
48704
|
+
{
|
|
48705
|
+
ipv4Prefix: "34.85.172.162/32"
|
|
48706
|
+
},
|
|
48707
|
+
{
|
|
48708
|
+
ipv4Prefix: "35.245.175.129/32"
|
|
48709
|
+
},
|
|
48710
|
+
{
|
|
48711
|
+
ipv4Prefix: "34.182.222.37/32"
|
|
48712
|
+
},
|
|
48713
|
+
{
|
|
48714
|
+
ipv4Prefix: "34.186.108.163/32"
|
|
48715
|
+
},
|
|
48716
|
+
{
|
|
48717
|
+
ipv4Prefix: "35.245.89.239/32"
|
|
48718
|
+
},
|
|
48719
|
+
{
|
|
48720
|
+
ipv4Prefix: "34.182.161.143/32"
|
|
48721
|
+
},
|
|
48722
|
+
{
|
|
48723
|
+
ipv4Prefix: "34.182.218.27/32"
|
|
48724
|
+
},
|
|
48725
|
+
{
|
|
48726
|
+
ipv4Prefix: "34.182.220.85/32"
|
|
48727
|
+
},
|
|
48728
|
+
{
|
|
48729
|
+
ipv4Prefix: "34.11.34.31/32"
|
|
48730
|
+
},
|
|
48731
|
+
{
|
|
48732
|
+
ipv4Prefix: "34.182.140.95/32"
|
|
48733
|
+
},
|
|
48734
|
+
{
|
|
48735
|
+
ipv4Prefix: "136.107.176.208/32"
|
|
48736
|
+
},
|
|
48737
|
+
{
|
|
48738
|
+
ipv4Prefix: "34.182.226.151/32"
|
|
48739
|
+
},
|
|
48740
|
+
{
|
|
48741
|
+
ipv4Prefix: "34.182.226.221/32"
|
|
48742
|
+
},
|
|
48743
|
+
{
|
|
48744
|
+
ipv4Prefix: "35.221.29.174/32"
|
|
48745
|
+
},
|
|
48746
|
+
{
|
|
48747
|
+
ipv4Prefix: "34.182.225.167/32"
|
|
48748
|
+
},
|
|
48749
|
+
{
|
|
48750
|
+
ipv4Prefix: "40.124.101.48/28"
|
|
48751
|
+
},
|
|
48752
|
+
{
|
|
48753
|
+
ipv4Prefix: "20.102.46.224/28"
|
|
48754
|
+
},
|
|
48755
|
+
{
|
|
48756
|
+
ipv4Prefix: "20.64.57.208/28"
|
|
48757
|
+
}
|
|
48528
48758
|
]
|
|
48529
48759
|
};
|
|
48530
48760
|
|
|
@@ -50623,6 +50853,141 @@ var perplexitybot_default = {
|
|
|
50623
50853
|
]
|
|
50624
50854
|
};
|
|
50625
50855
|
|
|
50856
|
+
// ../integration-traffic/src/ip-range-manifest.ts
|
|
50857
|
+
import { createHash as createHash4 } from "crypto";
|
|
50858
|
+
function isRecord(value) {
|
|
50859
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
50860
|
+
}
|
|
50861
|
+
function stableJson2(value, seen = /* @__PURE__ */ new Set()) {
|
|
50862
|
+
if (value === null) return "null";
|
|
50863
|
+
if (typeof value === "string" || typeof value === "boolean") return JSON.stringify(value);
|
|
50864
|
+
if (typeof value === "number") {
|
|
50865
|
+
return Number.isFinite(value) ? JSON.stringify(value) : JSON.stringify(String(value));
|
|
50866
|
+
}
|
|
50867
|
+
if (typeof value !== "object") return JSON.stringify(`[${typeof value}]`);
|
|
50868
|
+
if (seen.has(value)) return '"[circular]"';
|
|
50869
|
+
seen.add(value);
|
|
50870
|
+
const result = Array.isArray(value) ? `[${value.map((entry) => stableJson2(entry, seen)).join(",")}]` : `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableJson2(value[key], seen)}`).join(",")}}`;
|
|
50871
|
+
seen.delete(value);
|
|
50872
|
+
return result;
|
|
50873
|
+
}
|
|
50874
|
+
function parseIp(ip) {
|
|
50875
|
+
if (typeof ip !== "string" || ip.length === 0) return null;
|
|
50876
|
+
const mappedMatch = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/i.exec(ip);
|
|
50877
|
+
if (mappedMatch) return parseIp(mappedMatch[1]);
|
|
50878
|
+
if (ip.includes(":")) {
|
|
50879
|
+
const sides = ip.split("::");
|
|
50880
|
+
if (sides.length > 2) return null;
|
|
50881
|
+
const left = sides[0].length > 0 ? sides[0].split(":") : [];
|
|
50882
|
+
const right = sides.length === 2 && sides[1].length > 0 ? sides[1].split(":") : [];
|
|
50883
|
+
const groupCount = left.length + right.length;
|
|
50884
|
+
if (sides.length === 1 && groupCount !== 8) return null;
|
|
50885
|
+
if (sides.length === 2 && groupCount >= 8) return null;
|
|
50886
|
+
const fill = 8 - groupCount;
|
|
50887
|
+
const groups = [...left, ...new Array(fill).fill("0"), ...right];
|
|
50888
|
+
let addr2 = 0n;
|
|
50889
|
+
for (const group of groups) {
|
|
50890
|
+
if (!/^[0-9a-f]{1,4}$/i.test(group)) return null;
|
|
50891
|
+
addr2 = addr2 << 16n | BigInt(Number.parseInt(group, 16));
|
|
50892
|
+
}
|
|
50893
|
+
return { version: 6, addr: addr2 };
|
|
50894
|
+
}
|
|
50895
|
+
const octets = ip.split(".");
|
|
50896
|
+
if (octets.length !== 4) return null;
|
|
50897
|
+
let addr = 0n;
|
|
50898
|
+
for (const octet of octets) {
|
|
50899
|
+
if (!/^(?:0|[1-9]\d{0,2})$/.test(octet)) return null;
|
|
50900
|
+
const value = Number(octet);
|
|
50901
|
+
if (value > 255) return null;
|
|
50902
|
+
addr = addr << 8n | BigInt(value);
|
|
50903
|
+
}
|
|
50904
|
+
return { version: 4, addr };
|
|
50905
|
+
}
|
|
50906
|
+
function parseCidr(cidr) {
|
|
50907
|
+
if (typeof cidr !== "string") return null;
|
|
50908
|
+
const slash = cidr.indexOf("/");
|
|
50909
|
+
if (slash <= 0 || slash !== cidr.lastIndexOf("/") || slash === cidr.length - 1) return null;
|
|
50910
|
+
const ipPart = cidr.slice(0, slash);
|
|
50911
|
+
const prefixStr = cidr.slice(slash + 1);
|
|
50912
|
+
if (!/^(?:0|[1-9]\d{0,2})$/.test(prefixStr)) return null;
|
|
50913
|
+
const parsed = parseIp(ipPart);
|
|
50914
|
+
if (!parsed) return null;
|
|
50915
|
+
const prefix = Number(prefixStr);
|
|
50916
|
+
const totalBits = parsed.version === 4 ? 32 : 128;
|
|
50917
|
+
if (prefix > totalBits) return null;
|
|
50918
|
+
const allOnes = (1n << BigInt(totalBits)) - 1n;
|
|
50919
|
+
const mask = allOnes >> BigInt(totalBits - prefix) << BigInt(totalBits - prefix);
|
|
50920
|
+
return {
|
|
50921
|
+
version: parsed.version,
|
|
50922
|
+
network: parsed.addr & mask,
|
|
50923
|
+
mask
|
|
50924
|
+
};
|
|
50925
|
+
}
|
|
50926
|
+
function canonicalPrefixKey(entry) {
|
|
50927
|
+
if (isRecord(entry)) {
|
|
50928
|
+
const hasIpv4 = Object.hasOwn(entry, "ipv4Prefix");
|
|
50929
|
+
const hasIpv6 = Object.hasOwn(entry, "ipv6Prefix");
|
|
50930
|
+
if (hasIpv4 !== hasIpv6) {
|
|
50931
|
+
const value = entry[hasIpv4 ? "ipv4Prefix" : "ipv6Prefix"];
|
|
50932
|
+
if (typeof value === "string") {
|
|
50933
|
+
const parsed = parseCidr(value);
|
|
50934
|
+
const expectedVersion = hasIpv4 ? 4 : 6;
|
|
50935
|
+
if (parsed?.version === expectedVersion) {
|
|
50936
|
+
const hexWidth = parsed.version === 4 ? 8 : 32;
|
|
50937
|
+
const network = parsed.network.toString(16).padStart(hexWidth, "0");
|
|
50938
|
+
const mask = parsed.mask.toString(16).padStart(hexWidth, "0");
|
|
50939
|
+
return `cidr:${parsed.version}:${network}:${mask}`;
|
|
50940
|
+
}
|
|
50941
|
+
}
|
|
50942
|
+
}
|
|
50943
|
+
}
|
|
50944
|
+
return `invalid:${stableJson2(entry)}`;
|
|
50945
|
+
}
|
|
50946
|
+
function ipRangeManifestContentHash(value) {
|
|
50947
|
+
const prefixes = isRecord(value) ? value.prefixes : void 0;
|
|
50948
|
+
const canonical = Array.isArray(prefixes) ? prefixes.map(canonicalPrefixKey).sort() : [`invalid-prefixes:${stableJson2(prefixes)}`];
|
|
50949
|
+
return createHash4("sha256").update(JSON.stringify(canonical)).digest("hex");
|
|
50950
|
+
}
|
|
50951
|
+
function validateIpRangeManifestPayload(value) {
|
|
50952
|
+
if (!isRecord(value)) return { ok: false, error: "response must be a JSON object" };
|
|
50953
|
+
const creationTime = value.creationTime;
|
|
50954
|
+
if (typeof creationTime !== "string" || creationTime.trim().length === 0) {
|
|
50955
|
+
return { ok: false, error: "response missing `creationTime` version" };
|
|
50956
|
+
}
|
|
50957
|
+
if (!Array.isArray(value.prefixes)) {
|
|
50958
|
+
return { ok: false, error: "response missing `prefixes` array" };
|
|
50959
|
+
}
|
|
50960
|
+
const prefixes = [];
|
|
50961
|
+
for (const [index2, entry] of value.prefixes.entries()) {
|
|
50962
|
+
if (!isRecord(entry)) {
|
|
50963
|
+
return { ok: false, error: `prefixes[${index2}] must be an object` };
|
|
50964
|
+
}
|
|
50965
|
+
const hasIpv4 = Object.hasOwn(entry, "ipv4Prefix");
|
|
50966
|
+
const hasIpv6 = Object.hasOwn(entry, "ipv6Prefix");
|
|
50967
|
+
if (hasIpv4 === hasIpv6) {
|
|
50968
|
+
return { ok: false, error: `prefixes[${index2}] must contain exactly one IP prefix` };
|
|
50969
|
+
}
|
|
50970
|
+
const field = hasIpv4 ? "ipv4Prefix" : "ipv6Prefix";
|
|
50971
|
+
const cidr = entry[field];
|
|
50972
|
+
if (typeof cidr !== "string") {
|
|
50973
|
+
return { ok: false, error: `prefixes[${index2}].${field} must be a string` };
|
|
50974
|
+
}
|
|
50975
|
+
const parsed = parseCidr(cidr);
|
|
50976
|
+
const expectedVersion = hasIpv4 ? 4 : 6;
|
|
50977
|
+
if (!parsed || parsed.version !== expectedVersion) {
|
|
50978
|
+
return { ok: false, error: `prefixes[${index2}].${field} is not a valid IPv${expectedVersion} CIDR` };
|
|
50979
|
+
}
|
|
50980
|
+
prefixes.push(hasIpv4 ? { ipv4Prefix: cidr } : { ipv6Prefix: cidr });
|
|
50981
|
+
}
|
|
50982
|
+
return {
|
|
50983
|
+
ok: true,
|
|
50984
|
+
value: {
|
|
50985
|
+
creationTime: creationTime.trim(),
|
|
50986
|
+
prefixes
|
|
50987
|
+
}
|
|
50988
|
+
};
|
|
50989
|
+
}
|
|
50990
|
+
|
|
50626
50991
|
// ../integration-traffic/src/ip-verify.ts
|
|
50627
50992
|
var RULE_ID_TO_RANGES = {
|
|
50628
50993
|
// OpenAI — three separate published lists (training crawler vs
|
|
@@ -50656,94 +51021,56 @@ var RULE_ID_TO_RANGES = {
|
|
|
50656
51021
|
"perplexity-bot": perplexitybot_default,
|
|
50657
51022
|
// src: https://www.perplexity.ai/perplexity-user.json
|
|
50658
51023
|
"perplexity-user": perplexity_user_default,
|
|
50659
|
-
// Anthropic
|
|
50660
|
-
//
|
|
50661
|
-
//
|
|
50662
|
-
//
|
|
50663
|
-
//
|
|
50664
|
-
// logs show all real ClaudeBot traffic comes from there. The same
|
|
50665
|
-
// raw set is shared across every Claude-* UA the classifier emits:
|
|
50666
|
-
// both the training crawler and the per-user fetcher map here.
|
|
50667
|
-
// src: https://rdap.arin.net/registry/entity/AP-2440
|
|
51024
|
+
// Anthropic publishes one shared crawler-origin manifest. It confirms
|
|
51025
|
+
// Anthropic origin but does not attribute a prefix to ClaudeBot,
|
|
51026
|
+
// Claude-SearchBot, or Claude-User individually, so both classifier rules
|
|
51027
|
+
// map to the same file and the UA remains the product discriminator.
|
|
51028
|
+
// src: https://claude.com/crawling/bots.json
|
|
50668
51029
|
"anthropic-claudebot": anthropic_default,
|
|
50669
51030
|
"claude-user": anthropic_default
|
|
50670
51031
|
};
|
|
51032
|
+
function manifestMetadata(raw) {
|
|
51033
|
+
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) return null;
|
|
51034
|
+
const record = raw;
|
|
51035
|
+
const source = typeof record._source === "string" ? record._source.trim() : "";
|
|
51036
|
+
const version = typeof record.creationTime === "string" ? record.creationTime.trim() : "";
|
|
51037
|
+
if (!source || !version) return null;
|
|
51038
|
+
const contentHash2 = ipRangeManifestContentHash(raw);
|
|
51039
|
+
return { id: `${source}#${version}#sha256:${contentHash2}`, source, version };
|
|
51040
|
+
}
|
|
51041
|
+
function verificationDataFromRaw(raw) {
|
|
51042
|
+
const validation = validateIpRangeManifestPayload(raw);
|
|
51043
|
+
const ranges = [];
|
|
51044
|
+
if (validation.ok) {
|
|
51045
|
+
for (const entry of validation.value.prefixes) {
|
|
51046
|
+
const parsed = parseCidr(entry.ipv4Prefix ?? entry.ipv6Prefix);
|
|
51047
|
+
if (parsed) ranges.push(parsed);
|
|
51048
|
+
}
|
|
51049
|
+
}
|
|
51050
|
+
return { ranges, manifest: manifestMetadata(raw) };
|
|
51051
|
+
}
|
|
51052
|
+
function decideIpVerification(ip, data) {
|
|
51053
|
+
if (!ip || data.ranges.length === 0) return { verified: false, manifest: null };
|
|
51054
|
+
const parsed = parseIp(ip);
|
|
51055
|
+
if (!parsed) return { verified: false, manifest: null };
|
|
51056
|
+
for (const cidr of data.ranges) {
|
|
51057
|
+
if (parsed.version !== cidr.version) continue;
|
|
51058
|
+
if ((parsed.addr & cidr.mask) === cidr.network) {
|
|
51059
|
+
return { verified: true, manifest: data.manifest };
|
|
51060
|
+
}
|
|
51061
|
+
}
|
|
51062
|
+
return { verified: false, manifest: data.manifest };
|
|
51063
|
+
}
|
|
50671
51064
|
var CACHE = (() => {
|
|
50672
51065
|
const cache = /* @__PURE__ */ new Map();
|
|
50673
51066
|
for (const [ruleId, raw] of Object.entries(RULE_ID_TO_RANGES)) {
|
|
50674
|
-
|
|
50675
|
-
for (const entry of raw.prefixes) {
|
|
50676
|
-
const cidr = entry.ipv4Prefix ?? entry.ipv6Prefix;
|
|
50677
|
-
if (!cidr) continue;
|
|
50678
|
-
const p = parseCidr(cidr);
|
|
50679
|
-
if (p) parsed.push(p);
|
|
50680
|
-
}
|
|
50681
|
-
cache.set(ruleId, parsed);
|
|
51067
|
+
cache.set(ruleId, verificationDataFromRaw(raw));
|
|
50682
51068
|
}
|
|
50683
51069
|
return cache;
|
|
50684
51070
|
})();
|
|
50685
|
-
function
|
|
50686
|
-
|
|
50687
|
-
|
|
50688
|
-
if (mappedMatch) return parseIp(mappedMatch[1]);
|
|
50689
|
-
if (ip.includes(":")) {
|
|
50690
|
-
const sides = ip.split("::");
|
|
50691
|
-
if (sides.length > 2) return null;
|
|
50692
|
-
const left = sides[0].length > 0 ? sides[0].split(":") : [];
|
|
50693
|
-
const right = sides.length === 2 && sides[1].length > 0 ? sides[1].split(":") : [];
|
|
50694
|
-
const groupCount = left.length + right.length;
|
|
50695
|
-
if (groupCount > 8) return null;
|
|
50696
|
-
if (sides.length === 1 && groupCount !== 8) return null;
|
|
50697
|
-
const fill = 8 - groupCount;
|
|
50698
|
-
const groups = [...left, ...new Array(fill).fill("0"), ...right];
|
|
50699
|
-
let addr2 = 0n;
|
|
50700
|
-
for (const g of groups) {
|
|
50701
|
-
if (g.length === 0 || g.length > 4) return null;
|
|
50702
|
-
const n = Number.parseInt(g, 16);
|
|
50703
|
-
if (!Number.isFinite(n) || n < 0 || n > 65535) return null;
|
|
50704
|
-
addr2 = addr2 << 16n | BigInt(n);
|
|
50705
|
-
}
|
|
50706
|
-
return { version: 6, addr: addr2 };
|
|
50707
|
-
}
|
|
50708
|
-
const octets = ip.split(".");
|
|
50709
|
-
if (octets.length !== 4) return null;
|
|
50710
|
-
let addr = 0n;
|
|
50711
|
-
for (const o of octets) {
|
|
50712
|
-
if (o.length === 0 || o.length > 3) return null;
|
|
50713
|
-
const n = Number.parseInt(o, 10);
|
|
50714
|
-
if (!Number.isInteger(n) || n < 0 || n > 255) return null;
|
|
50715
|
-
addr = addr << 8n | BigInt(n);
|
|
50716
|
-
}
|
|
50717
|
-
return { version: 4, addr };
|
|
50718
|
-
}
|
|
50719
|
-
function parseCidr(cidr) {
|
|
50720
|
-
const [ipPart, prefixStr] = cidr.split("/");
|
|
50721
|
-
if (!ipPart || !prefixStr) return null;
|
|
50722
|
-
const prefix = Number.parseInt(prefixStr, 10);
|
|
50723
|
-
if (!Number.isInteger(prefix)) return null;
|
|
50724
|
-
const parsed = parseIp(ipPart);
|
|
50725
|
-
if (!parsed) return null;
|
|
50726
|
-
const totalBits = parsed.version === 4 ? 32 : 128;
|
|
50727
|
-
if (prefix < 0 || prefix > totalBits) return null;
|
|
50728
|
-
const allOnes = (1n << BigInt(totalBits)) - 1n;
|
|
50729
|
-
const mask = allOnes >> BigInt(totalBits - prefix) << BigInt(totalBits - prefix);
|
|
50730
|
-
return {
|
|
50731
|
-
version: parsed.version,
|
|
50732
|
-
network: parsed.addr & mask,
|
|
50733
|
-
mask
|
|
50734
|
-
};
|
|
50735
|
-
}
|
|
50736
|
-
function verifyIpForRule(ip, ruleId) {
|
|
50737
|
-
if (!ip) return false;
|
|
50738
|
-
const ranges = CACHE.get(ruleId);
|
|
50739
|
-
if (!ranges || ranges.length === 0) return false;
|
|
50740
|
-
const parsed = parseIp(ip);
|
|
50741
|
-
if (!parsed) return false;
|
|
50742
|
-
for (const cidr of ranges) {
|
|
50743
|
-
if (parsed.version !== cidr.version) continue;
|
|
50744
|
-
if ((parsed.addr & cidr.mask) === cidr.network) return true;
|
|
50745
|
-
}
|
|
50746
|
-
return false;
|
|
51071
|
+
function verifyIpForRuleDecision(ip, ruleId) {
|
|
51072
|
+
const data = CACHE.get(ruleId);
|
|
51073
|
+
return data ? decideIpVerification(ip, data) : { verified: false, manifest: null };
|
|
50747
51074
|
}
|
|
50748
51075
|
|
|
50749
51076
|
// ../integration-traffic/src/rules.ts
|
|
@@ -51113,13 +51440,14 @@ function classifyCrawler(event) {
|
|
|
51113
51440
|
for (const rule of DEFAULT_AI_CRAWLER_RULES) {
|
|
51114
51441
|
if (rule.purpose === USER_FETCH_PURPOSE) continue;
|
|
51115
51442
|
if (rule.userAgentPatterns.some((pattern) => pattern.test(userAgent))) {
|
|
51116
|
-
const
|
|
51443
|
+
const decision = verifyIpForRuleDecision(event.remoteIp, rule.id);
|
|
51117
51444
|
return {
|
|
51118
51445
|
botId: rule.id,
|
|
51119
51446
|
operator: rule.operator,
|
|
51120
51447
|
product: rule.product,
|
|
51121
51448
|
purpose: rule.purpose,
|
|
51122
|
-
verificationStatus: verified ? "verified" : "claimed_unverified",
|
|
51449
|
+
verificationStatus: decision.verified ? "verified" : "claimed_unverified",
|
|
51450
|
+
verificationManifest: decision.manifest,
|
|
51123
51451
|
matchedUserAgent: userAgent
|
|
51124
51452
|
};
|
|
51125
51453
|
}
|
|
@@ -51132,12 +51460,13 @@ function classifyAiUserFetch(event) {
|
|
|
51132
51460
|
for (const rule of DEFAULT_AI_CRAWLER_RULES) {
|
|
51133
51461
|
if (rule.purpose !== USER_FETCH_PURPOSE) continue;
|
|
51134
51462
|
if (rule.userAgentPatterns.some((pattern) => pattern.test(userAgent))) {
|
|
51135
|
-
const
|
|
51463
|
+
const decision = verifyIpForRuleDecision(event.remoteIp, rule.id);
|
|
51136
51464
|
return {
|
|
51137
51465
|
botId: rule.id,
|
|
51138
51466
|
operator: rule.operator,
|
|
51139
51467
|
product: rule.product,
|
|
51140
|
-
verificationStatus: verified ? "verified" : "claimed_unverified",
|
|
51468
|
+
verificationStatus: decision.verified ? "verified" : "claimed_unverified",
|
|
51469
|
+
verificationManifest: decision.manifest,
|
|
51141
51470
|
matchedUserAgent: userAgent
|
|
51142
51471
|
};
|
|
51143
51472
|
}
|
|
@@ -51286,10 +51615,10 @@ function mergeReferralSession(current, next) {
|
|
|
51286
51615
|
return { ...winner, trafficClass: paid ? AiReferralTrafficClasses.paid : winner.trafficClass };
|
|
51287
51616
|
}
|
|
51288
51617
|
function sortCrawlerBuckets(a, b) {
|
|
51289
|
-
return a.tsHour.localeCompare(b.tsHour) || a.botId.localeCompare(b.botId) || a.pathNormalized.localeCompare(b.pathNormalized) || String(a.status).localeCompare(String(b.status));
|
|
51618
|
+
return a.tsHour.localeCompare(b.tsHour) || a.botId.localeCompare(b.botId) || a.verificationStatus.localeCompare(b.verificationStatus) || (a.verificationManifest?.id ?? "").localeCompare(b.verificationManifest?.id ?? "") || a.pathNormalized.localeCompare(b.pathNormalized) || String(a.status).localeCompare(String(b.status));
|
|
51290
51619
|
}
|
|
51291
51620
|
function sortAiUserFetchBuckets(a, b) {
|
|
51292
|
-
return a.tsHour.localeCompare(b.tsHour) || a.botId.localeCompare(b.botId) || a.pathNormalized.localeCompare(b.pathNormalized) || String(a.status).localeCompare(String(b.status));
|
|
51621
|
+
return a.tsHour.localeCompare(b.tsHour) || a.botId.localeCompare(b.botId) || a.verificationStatus.localeCompare(b.verificationStatus) || (a.verificationManifest?.id ?? "").localeCompare(b.verificationManifest?.id ?? "") || a.pathNormalized.localeCompare(b.pathNormalized) || String(a.status).localeCompare(String(b.status));
|
|
51293
51622
|
}
|
|
51294
51623
|
function sortReferralBuckets(a, b) {
|
|
51295
51624
|
return a.tsHour.localeCompare(b.tsHour) || a.product.localeCompare(b.product) || a.sourceDomain.localeCompare(b.sourceDomain) || a.landingPathNormalized.localeCompare(b.landingPathNormalized) || String(a.status).localeCompare(String(b.status));
|
|
@@ -51332,6 +51661,7 @@ function buildTrafficProbeReport(events, options = {}) {
|
|
|
51332
51661
|
tsHour,
|
|
51333
51662
|
crawler.botId,
|
|
51334
51663
|
crawler.verificationStatus,
|
|
51664
|
+
crawler.verificationManifest?.id ?? "no-manifest",
|
|
51335
51665
|
pathNormalized,
|
|
51336
51666
|
event.status ?? "null"
|
|
51337
51667
|
].join(" ");
|
|
@@ -51345,6 +51675,7 @@ function buildTrafficProbeReport(events, options = {}) {
|
|
|
51345
51675
|
operator: crawler.operator,
|
|
51346
51676
|
product: crawler.product,
|
|
51347
51677
|
verificationStatus: crawler.verificationStatus,
|
|
51678
|
+
verificationManifest: crawler.verificationManifest,
|
|
51348
51679
|
pathNormalized,
|
|
51349
51680
|
status: event.status,
|
|
51350
51681
|
hits: 1,
|
|
@@ -51363,6 +51694,7 @@ function buildTrafficProbeReport(events, options = {}) {
|
|
|
51363
51694
|
tsHour,
|
|
51364
51695
|
aiUserFetch.botId,
|
|
51365
51696
|
aiUserFetch.verificationStatus,
|
|
51697
|
+
aiUserFetch.verificationManifest?.id ?? "no-manifest",
|
|
51366
51698
|
pathNormalized,
|
|
51367
51699
|
event.status ?? "null"
|
|
51368
51700
|
].join(" ");
|
|
@@ -51376,6 +51708,7 @@ function buildTrafficProbeReport(events, options = {}) {
|
|
|
51376
51708
|
operator: aiUserFetch.operator,
|
|
51377
51709
|
product: aiUserFetch.product,
|
|
51378
51710
|
verificationStatus: aiUserFetch.verificationStatus,
|
|
51711
|
+
verificationManifest: aiUserFetch.verificationManifest,
|
|
51379
51712
|
pathNormalized,
|
|
51380
51713
|
status: event.status,
|
|
51381
51714
|
hits: 1,
|
|
@@ -53042,6 +53375,38 @@ function writeTrafficEventBatch(opts) {
|
|
|
53042
53375
|
updatedAt: opts.receivedAt
|
|
53043
53376
|
}
|
|
53044
53377
|
}).run();
|
|
53378
|
+
if (bucket.verificationManifest) {
|
|
53379
|
+
tx.insert(crawlerVerificationManifestsHourly).values({
|
|
53380
|
+
projectId: opts.projectId,
|
|
53381
|
+
sourceId: opts.sourceId,
|
|
53382
|
+
tsHour: bucket.tsHour,
|
|
53383
|
+
botId: bucket.botId,
|
|
53384
|
+
verificationStatus: bucket.verificationStatus,
|
|
53385
|
+
pathNormalized: bucket.pathNormalized,
|
|
53386
|
+
status,
|
|
53387
|
+
manifestId: bucket.verificationManifest.id,
|
|
53388
|
+
manifestJson: bucket.verificationManifest,
|
|
53389
|
+
hits: bucket.hits,
|
|
53390
|
+
createdAt: opts.receivedAt,
|
|
53391
|
+
updatedAt: opts.receivedAt
|
|
53392
|
+
}).onConflictDoUpdate({
|
|
53393
|
+
target: [
|
|
53394
|
+
crawlerVerificationManifestsHourly.projectId,
|
|
53395
|
+
crawlerVerificationManifestsHourly.sourceId,
|
|
53396
|
+
crawlerVerificationManifestsHourly.tsHour,
|
|
53397
|
+
crawlerVerificationManifestsHourly.botId,
|
|
53398
|
+
crawlerVerificationManifestsHourly.verificationStatus,
|
|
53399
|
+
crawlerVerificationManifestsHourly.pathNormalized,
|
|
53400
|
+
crawlerVerificationManifestsHourly.status,
|
|
53401
|
+
crawlerVerificationManifestsHourly.manifestId
|
|
53402
|
+
],
|
|
53403
|
+
set: {
|
|
53404
|
+
hits: sql18`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
53405
|
+
manifestJson: bucket.verificationManifest,
|
|
53406
|
+
updatedAt: opts.receivedAt
|
|
53407
|
+
}
|
|
53408
|
+
}).run();
|
|
53409
|
+
}
|
|
53045
53410
|
crawlerBucketRows += 1;
|
|
53046
53411
|
}
|
|
53047
53412
|
for (const bucket of report.aiUserFetchEventsHourly) {
|
|
@@ -53075,6 +53440,38 @@ function writeTrafficEventBatch(opts) {
|
|
|
53075
53440
|
updatedAt: opts.receivedAt
|
|
53076
53441
|
}
|
|
53077
53442
|
}).run();
|
|
53443
|
+
if (bucket.verificationManifest) {
|
|
53444
|
+
tx.insert(aiUserFetchVerificationManifestsHourly).values({
|
|
53445
|
+
projectId: opts.projectId,
|
|
53446
|
+
sourceId: opts.sourceId,
|
|
53447
|
+
tsHour: bucket.tsHour,
|
|
53448
|
+
botId: bucket.botId,
|
|
53449
|
+
verificationStatus: bucket.verificationStatus,
|
|
53450
|
+
pathNormalized: bucket.pathNormalized,
|
|
53451
|
+
status,
|
|
53452
|
+
manifestId: bucket.verificationManifest.id,
|
|
53453
|
+
manifestJson: bucket.verificationManifest,
|
|
53454
|
+
hits: bucket.hits,
|
|
53455
|
+
createdAt: opts.receivedAt,
|
|
53456
|
+
updatedAt: opts.receivedAt
|
|
53457
|
+
}).onConflictDoUpdate({
|
|
53458
|
+
target: [
|
|
53459
|
+
aiUserFetchVerificationManifestsHourly.projectId,
|
|
53460
|
+
aiUserFetchVerificationManifestsHourly.sourceId,
|
|
53461
|
+
aiUserFetchVerificationManifestsHourly.tsHour,
|
|
53462
|
+
aiUserFetchVerificationManifestsHourly.botId,
|
|
53463
|
+
aiUserFetchVerificationManifestsHourly.verificationStatus,
|
|
53464
|
+
aiUserFetchVerificationManifestsHourly.pathNormalized,
|
|
53465
|
+
aiUserFetchVerificationManifestsHourly.status,
|
|
53466
|
+
aiUserFetchVerificationManifestsHourly.manifestId
|
|
53467
|
+
],
|
|
53468
|
+
set: {
|
|
53469
|
+
hits: sql18`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
53470
|
+
manifestJson: bucket.verificationManifest,
|
|
53471
|
+
updatedAt: opts.receivedAt
|
|
53472
|
+
}
|
|
53473
|
+
}).run();
|
|
53474
|
+
}
|
|
53078
53475
|
aiUserFetchBucketRows += 1;
|
|
53079
53476
|
}
|
|
53080
53477
|
for (const bucket of report.aiReferralEventsHourly) {
|
|
@@ -53393,6 +53790,117 @@ function trafficSeriesPoint(pointsByBucket, bucket) {
|
|
|
53393
53790
|
pointsByBucket.set(bucket, created);
|
|
53394
53791
|
return created;
|
|
53395
53792
|
}
|
|
53793
|
+
function verificationEventKey(row) {
|
|
53794
|
+
return JSON.stringify([
|
|
53795
|
+
row.sourceId,
|
|
53796
|
+
row.tsHour,
|
|
53797
|
+
row.botId,
|
|
53798
|
+
row.verificationStatus,
|
|
53799
|
+
row.pathNormalized,
|
|
53800
|
+
row.status
|
|
53801
|
+
]);
|
|
53802
|
+
}
|
|
53803
|
+
function verificationSelectedRowsJson(projectId, rows) {
|
|
53804
|
+
return JSON.stringify(rows.map((row) => [
|
|
53805
|
+
projectId,
|
|
53806
|
+
row.sourceId,
|
|
53807
|
+
row.tsHour,
|
|
53808
|
+
row.botId,
|
|
53809
|
+
row.verificationStatus,
|
|
53810
|
+
row.pathNormalized,
|
|
53811
|
+
row.status
|
|
53812
|
+
]));
|
|
53813
|
+
}
|
|
53814
|
+
function buildVerificationProvenance(events, rows) {
|
|
53815
|
+
const manifestsByEvent = /* @__PURE__ */ new Map();
|
|
53816
|
+
for (const row of rows) {
|
|
53817
|
+
const key = verificationEventKey(row);
|
|
53818
|
+
const manifests = manifestsByEvent.get(key);
|
|
53819
|
+
const usage = { manifestId: row.manifestId, manifest: row.manifest, hits: row.hits };
|
|
53820
|
+
if (manifests) manifests.push(usage);
|
|
53821
|
+
else manifestsByEvent.set(key, [usage]);
|
|
53822
|
+
}
|
|
53823
|
+
const provenanceByEvent = /* @__PURE__ */ new Map();
|
|
53824
|
+
for (const event of events) {
|
|
53825
|
+
const key = verificationEventKey(event);
|
|
53826
|
+
const verificationManifests = manifestsByEvent.get(key) ?? [];
|
|
53827
|
+
verificationManifests.sort((a, b) => a.manifestId.localeCompare(b.manifestId));
|
|
53828
|
+
const attributedHits = verificationManifests.reduce((sum, manifest) => sum + manifest.hits, 0);
|
|
53829
|
+
provenanceByEvent.set(key, {
|
|
53830
|
+
verificationManifests,
|
|
53831
|
+
verificationUnattributedHits: Math.max(0, event.hits - attributedHits)
|
|
53832
|
+
});
|
|
53833
|
+
}
|
|
53834
|
+
return provenanceByEvent;
|
|
53835
|
+
}
|
|
53836
|
+
function crawlerVerificationProvenance(db, projectId, events) {
|
|
53837
|
+
if (events.length === 0) return /* @__PURE__ */ new Map();
|
|
53838
|
+
const selectedRows = verificationSelectedRowsJson(projectId, events);
|
|
53839
|
+
const rows = db.select({
|
|
53840
|
+
sourceId: crawlerVerificationManifestsHourly.sourceId,
|
|
53841
|
+
tsHour: crawlerVerificationManifestsHourly.tsHour,
|
|
53842
|
+
botId: crawlerVerificationManifestsHourly.botId,
|
|
53843
|
+
verificationStatus: crawlerVerificationManifestsHourly.verificationStatus,
|
|
53844
|
+
pathNormalized: crawlerVerificationManifestsHourly.pathNormalized,
|
|
53845
|
+
status: crawlerVerificationManifestsHourly.status,
|
|
53846
|
+
manifestId: crawlerVerificationManifestsHourly.manifestId,
|
|
53847
|
+
manifest: crawlerVerificationManifestsHourly.manifestJson,
|
|
53848
|
+
hits: crawlerVerificationManifestsHourly.hits
|
|
53849
|
+
}).from(crawlerVerificationManifestsHourly).where(sql19`(
|
|
53850
|
+
${crawlerVerificationManifestsHourly.projectId},
|
|
53851
|
+
${crawlerVerificationManifestsHourly.sourceId},
|
|
53852
|
+
${crawlerVerificationManifestsHourly.tsHour},
|
|
53853
|
+
${crawlerVerificationManifestsHourly.botId},
|
|
53854
|
+
${crawlerVerificationManifestsHourly.verificationStatus},
|
|
53855
|
+
${crawlerVerificationManifestsHourly.pathNormalized},
|
|
53856
|
+
${crawlerVerificationManifestsHourly.status}
|
|
53857
|
+
) IN (
|
|
53858
|
+
SELECT
|
|
53859
|
+
json_extract(value, '$[0]'),
|
|
53860
|
+
json_extract(value, '$[1]'),
|
|
53861
|
+
json_extract(value, '$[2]'),
|
|
53862
|
+
json_extract(value, '$[3]'),
|
|
53863
|
+
json_extract(value, '$[4]'),
|
|
53864
|
+
json_extract(value, '$[5]'),
|
|
53865
|
+
json_extract(value, '$[6]')
|
|
53866
|
+
FROM json_each(${selectedRows})
|
|
53867
|
+
)`).orderBy(crawlerVerificationManifestsHourly.manifestId).all();
|
|
53868
|
+
return buildVerificationProvenance(events, rows);
|
|
53869
|
+
}
|
|
53870
|
+
function aiUserFetchVerificationProvenance(db, projectId, events) {
|
|
53871
|
+
if (events.length === 0) return /* @__PURE__ */ new Map();
|
|
53872
|
+
const selectedRows = verificationSelectedRowsJson(projectId, events);
|
|
53873
|
+
const rows = db.select({
|
|
53874
|
+
sourceId: aiUserFetchVerificationManifestsHourly.sourceId,
|
|
53875
|
+
tsHour: aiUserFetchVerificationManifestsHourly.tsHour,
|
|
53876
|
+
botId: aiUserFetchVerificationManifestsHourly.botId,
|
|
53877
|
+
verificationStatus: aiUserFetchVerificationManifestsHourly.verificationStatus,
|
|
53878
|
+
pathNormalized: aiUserFetchVerificationManifestsHourly.pathNormalized,
|
|
53879
|
+
status: aiUserFetchVerificationManifestsHourly.status,
|
|
53880
|
+
manifestId: aiUserFetchVerificationManifestsHourly.manifestId,
|
|
53881
|
+
manifest: aiUserFetchVerificationManifestsHourly.manifestJson,
|
|
53882
|
+
hits: aiUserFetchVerificationManifestsHourly.hits
|
|
53883
|
+
}).from(aiUserFetchVerificationManifestsHourly).where(sql19`(
|
|
53884
|
+
${aiUserFetchVerificationManifestsHourly.projectId},
|
|
53885
|
+
${aiUserFetchVerificationManifestsHourly.sourceId},
|
|
53886
|
+
${aiUserFetchVerificationManifestsHourly.tsHour},
|
|
53887
|
+
${aiUserFetchVerificationManifestsHourly.botId},
|
|
53888
|
+
${aiUserFetchVerificationManifestsHourly.verificationStatus},
|
|
53889
|
+
${aiUserFetchVerificationManifestsHourly.pathNormalized},
|
|
53890
|
+
${aiUserFetchVerificationManifestsHourly.status}
|
|
53891
|
+
) IN (
|
|
53892
|
+
SELECT
|
|
53893
|
+
json_extract(value, '$[0]'),
|
|
53894
|
+
json_extract(value, '$[1]'),
|
|
53895
|
+
json_extract(value, '$[2]'),
|
|
53896
|
+
json_extract(value, '$[3]'),
|
|
53897
|
+
json_extract(value, '$[4]'),
|
|
53898
|
+
json_extract(value, '$[5]'),
|
|
53899
|
+
json_extract(value, '$[6]')
|
|
53900
|
+
FROM json_each(${selectedRows})
|
|
53901
|
+
)`).orderBy(aiUserFetchVerificationManifestsHourly.manifestId).all();
|
|
53902
|
+
return buildVerificationProvenance(events, rows);
|
|
53903
|
+
}
|
|
53396
53904
|
function parseSourceConfig(row) {
|
|
53397
53905
|
return row.configJson;
|
|
53398
53906
|
}
|
|
@@ -53570,6 +54078,7 @@ async function runBackfillTask(options) {
|
|
|
53570
54078
|
)
|
|
53571
54079
|
).run();
|
|
53572
54080
|
for (const bucket of report.crawlerEventsHourly) {
|
|
54081
|
+
const status = bucket.status ?? 0;
|
|
53573
54082
|
tx.insert(crawlerEventsHourly).values({
|
|
53574
54083
|
projectId: project.id,
|
|
53575
54084
|
sourceId: sourceRow2.id,
|
|
@@ -53578,14 +54087,62 @@ async function runBackfillTask(options) {
|
|
|
53578
54087
|
operator: bucket.operator,
|
|
53579
54088
|
verificationStatus: bucket.verificationStatus,
|
|
53580
54089
|
pathNormalized: bucket.pathNormalized,
|
|
53581
|
-
status
|
|
54090
|
+
status,
|
|
53582
54091
|
hits: bucket.hits,
|
|
53583
54092
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
53584
54093
|
createdAt: finishedAt,
|
|
53585
54094
|
updatedAt: finishedAt
|
|
54095
|
+
}).onConflictDoUpdate({
|
|
54096
|
+
target: [
|
|
54097
|
+
crawlerEventsHourly.projectId,
|
|
54098
|
+
crawlerEventsHourly.sourceId,
|
|
54099
|
+
crawlerEventsHourly.tsHour,
|
|
54100
|
+
crawlerEventsHourly.botId,
|
|
54101
|
+
crawlerEventsHourly.verificationStatus,
|
|
54102
|
+
crawlerEventsHourly.pathNormalized,
|
|
54103
|
+
crawlerEventsHourly.status
|
|
54104
|
+
],
|
|
54105
|
+
set: {
|
|
54106
|
+
hits: sql19`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
54107
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
54108
|
+
updatedAt: finishedAt
|
|
54109
|
+
}
|
|
53586
54110
|
}).run();
|
|
54111
|
+
if (bucket.verificationManifest) {
|
|
54112
|
+
tx.insert(crawlerVerificationManifestsHourly).values({
|
|
54113
|
+
projectId: project.id,
|
|
54114
|
+
sourceId: sourceRow2.id,
|
|
54115
|
+
tsHour: bucket.tsHour,
|
|
54116
|
+
botId: bucket.botId,
|
|
54117
|
+
verificationStatus: bucket.verificationStatus,
|
|
54118
|
+
pathNormalized: bucket.pathNormalized,
|
|
54119
|
+
status,
|
|
54120
|
+
manifestId: bucket.verificationManifest.id,
|
|
54121
|
+
manifestJson: bucket.verificationManifest,
|
|
54122
|
+
hits: bucket.hits,
|
|
54123
|
+
createdAt: finishedAt,
|
|
54124
|
+
updatedAt: finishedAt
|
|
54125
|
+
}).onConflictDoUpdate({
|
|
54126
|
+
target: [
|
|
54127
|
+
crawlerVerificationManifestsHourly.projectId,
|
|
54128
|
+
crawlerVerificationManifestsHourly.sourceId,
|
|
54129
|
+
crawlerVerificationManifestsHourly.tsHour,
|
|
54130
|
+
crawlerVerificationManifestsHourly.botId,
|
|
54131
|
+
crawlerVerificationManifestsHourly.verificationStatus,
|
|
54132
|
+
crawlerVerificationManifestsHourly.pathNormalized,
|
|
54133
|
+
crawlerVerificationManifestsHourly.status,
|
|
54134
|
+
crawlerVerificationManifestsHourly.manifestId
|
|
54135
|
+
],
|
|
54136
|
+
set: {
|
|
54137
|
+
hits: sql19`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
54138
|
+
manifestJson: bucket.verificationManifest,
|
|
54139
|
+
updatedAt: finishedAt
|
|
54140
|
+
}
|
|
54141
|
+
}).run();
|
|
54142
|
+
}
|
|
53587
54143
|
}
|
|
53588
54144
|
for (const bucket of report.aiUserFetchEventsHourly) {
|
|
54145
|
+
const status = bucket.status ?? 0;
|
|
53589
54146
|
tx.insert(aiUserFetchEventsHourly).values({
|
|
53590
54147
|
projectId: project.id,
|
|
53591
54148
|
sourceId: sourceRow2.id,
|
|
@@ -53594,12 +54151,59 @@ async function runBackfillTask(options) {
|
|
|
53594
54151
|
operator: bucket.operator,
|
|
53595
54152
|
verificationStatus: bucket.verificationStatus,
|
|
53596
54153
|
pathNormalized: bucket.pathNormalized,
|
|
53597
|
-
status
|
|
54154
|
+
status,
|
|
53598
54155
|
hits: bucket.hits,
|
|
53599
54156
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
53600
54157
|
createdAt: finishedAt,
|
|
53601
54158
|
updatedAt: finishedAt
|
|
54159
|
+
}).onConflictDoUpdate({
|
|
54160
|
+
target: [
|
|
54161
|
+
aiUserFetchEventsHourly.projectId,
|
|
54162
|
+
aiUserFetchEventsHourly.sourceId,
|
|
54163
|
+
aiUserFetchEventsHourly.tsHour,
|
|
54164
|
+
aiUserFetchEventsHourly.botId,
|
|
54165
|
+
aiUserFetchEventsHourly.verificationStatus,
|
|
54166
|
+
aiUserFetchEventsHourly.pathNormalized,
|
|
54167
|
+
aiUserFetchEventsHourly.status
|
|
54168
|
+
],
|
|
54169
|
+
set: {
|
|
54170
|
+
hits: sql19`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
54171
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
54172
|
+
updatedAt: finishedAt
|
|
54173
|
+
}
|
|
53602
54174
|
}).run();
|
|
54175
|
+
if (bucket.verificationManifest) {
|
|
54176
|
+
tx.insert(aiUserFetchVerificationManifestsHourly).values({
|
|
54177
|
+
projectId: project.id,
|
|
54178
|
+
sourceId: sourceRow2.id,
|
|
54179
|
+
tsHour: bucket.tsHour,
|
|
54180
|
+
botId: bucket.botId,
|
|
54181
|
+
verificationStatus: bucket.verificationStatus,
|
|
54182
|
+
pathNormalized: bucket.pathNormalized,
|
|
54183
|
+
status,
|
|
54184
|
+
manifestId: bucket.verificationManifest.id,
|
|
54185
|
+
manifestJson: bucket.verificationManifest,
|
|
54186
|
+
hits: bucket.hits,
|
|
54187
|
+
createdAt: finishedAt,
|
|
54188
|
+
updatedAt: finishedAt
|
|
54189
|
+
}).onConflictDoUpdate({
|
|
54190
|
+
target: [
|
|
54191
|
+
aiUserFetchVerificationManifestsHourly.projectId,
|
|
54192
|
+
aiUserFetchVerificationManifestsHourly.sourceId,
|
|
54193
|
+
aiUserFetchVerificationManifestsHourly.tsHour,
|
|
54194
|
+
aiUserFetchVerificationManifestsHourly.botId,
|
|
54195
|
+
aiUserFetchVerificationManifestsHourly.verificationStatus,
|
|
54196
|
+
aiUserFetchVerificationManifestsHourly.pathNormalized,
|
|
54197
|
+
aiUserFetchVerificationManifestsHourly.status,
|
|
54198
|
+
aiUserFetchVerificationManifestsHourly.manifestId
|
|
54199
|
+
],
|
|
54200
|
+
set: {
|
|
54201
|
+
hits: sql19`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
54202
|
+
manifestJson: bucket.verificationManifest,
|
|
54203
|
+
updatedAt: finishedAt
|
|
54204
|
+
}
|
|
54205
|
+
}).run();
|
|
54206
|
+
}
|
|
53603
54207
|
}
|
|
53604
54208
|
for (const bucket of report.aiReferralEventsHourly) {
|
|
53605
54209
|
tx.insert(aiReferralEventsHourly).values({
|
|
@@ -55082,6 +55686,38 @@ async function trafficRoutes(app, opts) {
|
|
|
55082
55686
|
updatedAt: finishedAt
|
|
55083
55687
|
}
|
|
55084
55688
|
}).run();
|
|
55689
|
+
if (bucket.verificationManifest) {
|
|
55690
|
+
tx.insert(crawlerVerificationManifestsHourly).values({
|
|
55691
|
+
projectId: project.id,
|
|
55692
|
+
sourceId: sourceRow2.id,
|
|
55693
|
+
tsHour: bucket.tsHour,
|
|
55694
|
+
botId: bucket.botId,
|
|
55695
|
+
verificationStatus: bucket.verificationStatus,
|
|
55696
|
+
pathNormalized: bucket.pathNormalized,
|
|
55697
|
+
status,
|
|
55698
|
+
manifestId: bucket.verificationManifest.id,
|
|
55699
|
+
manifestJson: bucket.verificationManifest,
|
|
55700
|
+
hits: bucket.hits,
|
|
55701
|
+
createdAt: finishedAt,
|
|
55702
|
+
updatedAt: finishedAt
|
|
55703
|
+
}).onConflictDoUpdate({
|
|
55704
|
+
target: [
|
|
55705
|
+
crawlerVerificationManifestsHourly.projectId,
|
|
55706
|
+
crawlerVerificationManifestsHourly.sourceId,
|
|
55707
|
+
crawlerVerificationManifestsHourly.tsHour,
|
|
55708
|
+
crawlerVerificationManifestsHourly.botId,
|
|
55709
|
+
crawlerVerificationManifestsHourly.verificationStatus,
|
|
55710
|
+
crawlerVerificationManifestsHourly.pathNormalized,
|
|
55711
|
+
crawlerVerificationManifestsHourly.status,
|
|
55712
|
+
crawlerVerificationManifestsHourly.manifestId
|
|
55713
|
+
],
|
|
55714
|
+
set: {
|
|
55715
|
+
hits: sql19`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
55716
|
+
manifestJson: bucket.verificationManifest,
|
|
55717
|
+
updatedAt: finishedAt
|
|
55718
|
+
}
|
|
55719
|
+
}).run();
|
|
55720
|
+
}
|
|
55085
55721
|
crawlerBucketRows += 1;
|
|
55086
55722
|
}
|
|
55087
55723
|
for (const bucket of report.aiUserFetchEventsHourly) {
|
|
@@ -55115,6 +55751,38 @@ async function trafficRoutes(app, opts) {
|
|
|
55115
55751
|
updatedAt: finishedAt
|
|
55116
55752
|
}
|
|
55117
55753
|
}).run();
|
|
55754
|
+
if (bucket.verificationManifest) {
|
|
55755
|
+
tx.insert(aiUserFetchVerificationManifestsHourly).values({
|
|
55756
|
+
projectId: project.id,
|
|
55757
|
+
sourceId: sourceRow2.id,
|
|
55758
|
+
tsHour: bucket.tsHour,
|
|
55759
|
+
botId: bucket.botId,
|
|
55760
|
+
verificationStatus: bucket.verificationStatus,
|
|
55761
|
+
pathNormalized: bucket.pathNormalized,
|
|
55762
|
+
status,
|
|
55763
|
+
manifestId: bucket.verificationManifest.id,
|
|
55764
|
+
manifestJson: bucket.verificationManifest,
|
|
55765
|
+
hits: bucket.hits,
|
|
55766
|
+
createdAt: finishedAt,
|
|
55767
|
+
updatedAt: finishedAt
|
|
55768
|
+
}).onConflictDoUpdate({
|
|
55769
|
+
target: [
|
|
55770
|
+
aiUserFetchVerificationManifestsHourly.projectId,
|
|
55771
|
+
aiUserFetchVerificationManifestsHourly.sourceId,
|
|
55772
|
+
aiUserFetchVerificationManifestsHourly.tsHour,
|
|
55773
|
+
aiUserFetchVerificationManifestsHourly.botId,
|
|
55774
|
+
aiUserFetchVerificationManifestsHourly.verificationStatus,
|
|
55775
|
+
aiUserFetchVerificationManifestsHourly.pathNormalized,
|
|
55776
|
+
aiUserFetchVerificationManifestsHourly.status,
|
|
55777
|
+
aiUserFetchVerificationManifestsHourly.manifestId
|
|
55778
|
+
],
|
|
55779
|
+
set: {
|
|
55780
|
+
hits: sql19`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
55781
|
+
manifestJson: bucket.verificationManifest,
|
|
55782
|
+
updatedAt: finishedAt
|
|
55783
|
+
}
|
|
55784
|
+
}).run();
|
|
55785
|
+
}
|
|
55118
55786
|
aiUserFetchBucketRows += 1;
|
|
55119
55787
|
}
|
|
55120
55788
|
for (const bucket of report.aiReferralEventsHourly) {
|
|
@@ -55691,6 +56359,8 @@ async function trafficRoutes(app, opts) {
|
|
|
55691
56359
|
botId: r.botId,
|
|
55692
56360
|
operator: r.operator,
|
|
55693
56361
|
verificationStatus: r.verificationStatus,
|
|
56362
|
+
verificationManifests: [],
|
|
56363
|
+
verificationUnattributedHits: r.hits,
|
|
55694
56364
|
pathNormalized: r.pathNormalized,
|
|
55695
56365
|
pathClass: classifyTrafficPath(r.pathNormalized),
|
|
55696
56366
|
status: r.status,
|
|
@@ -55729,6 +56399,8 @@ async function trafficRoutes(app, opts) {
|
|
|
55729
56399
|
botId: r.botId,
|
|
55730
56400
|
operator: r.operator,
|
|
55731
56401
|
verificationStatus: r.verificationStatus,
|
|
56402
|
+
verificationManifests: [],
|
|
56403
|
+
verificationUnattributedHits: r.hits,
|
|
55732
56404
|
pathNormalized: r.pathNormalized,
|
|
55733
56405
|
status: r.status,
|
|
55734
56406
|
hits: r.hits
|
|
@@ -55785,6 +56457,25 @@ async function trafficRoutes(app, opts) {
|
|
|
55785
56457
|
}
|
|
55786
56458
|
events.sort((a, b) => a.tsHour < b.tsHour ? 1 : a.tsHour > b.tsHour ? -1 : 0);
|
|
55787
56459
|
const trimmed = events.slice(0, limit);
|
|
56460
|
+
const trimmedCrawlerEvents = trimmed.filter(
|
|
56461
|
+
(event) => event.kind === TrafficEventKinds.crawler
|
|
56462
|
+
);
|
|
56463
|
+
const trimmedAiUserFetchEvents = trimmed.filter(
|
|
56464
|
+
(event) => event.kind === TrafficEventKinds["ai-user-fetch"]
|
|
56465
|
+
);
|
|
56466
|
+
const crawlerProvenance = crawlerVerificationProvenance(app.db, project.id, trimmedCrawlerEvents);
|
|
56467
|
+
const aiUserFetchProvenance = aiUserFetchVerificationProvenance(app.db, project.id, trimmedAiUserFetchEvents);
|
|
56468
|
+
const eventsWithProvenance = trimmed.map((event) => {
|
|
56469
|
+
if (event.kind === TrafficEventKinds.crawler) {
|
|
56470
|
+
const provenance = crawlerProvenance.get(verificationEventKey(event));
|
|
56471
|
+
return provenance ? { ...event, ...provenance } : event;
|
|
56472
|
+
}
|
|
56473
|
+
if (event.kind === TrafficEventKinds["ai-user-fetch"]) {
|
|
56474
|
+
const provenance = aiUserFetchProvenance.get(verificationEventKey(event));
|
|
56475
|
+
return provenance ? { ...event, ...provenance } : event;
|
|
56476
|
+
}
|
|
56477
|
+
return event;
|
|
56478
|
+
});
|
|
55788
56479
|
const response = {
|
|
55789
56480
|
windowStart: sinceIso,
|
|
55790
56481
|
windowEnd: untilIso,
|
|
@@ -55808,7 +56499,7 @@ async function trafficRoutes(app, opts) {
|
|
|
55808
56499
|
returned: trimmed.length,
|
|
55809
56500
|
truncated: trimmed.length < totalEventRows
|
|
55810
56501
|
},
|
|
55811
|
-
events:
|
|
56502
|
+
events: eventsWithProvenance
|
|
55812
56503
|
};
|
|
55813
56504
|
return response;
|
|
55814
56505
|
});
|