@canonry/canonry 4.163.0 → 4.164.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. package/assets/assets/{AuditHistoryPanel-FDQx9TS5.js → AuditHistoryPanel-CD1GVkS9.js} +1 -1
  2. package/assets/assets/{BacklinksPage-DVvsQsdk.js → BacklinksPage-B0Ijhntp.js} +1 -1
  3. package/assets/assets/{HistoryPage-31HR6TIJ.js → HistoryPage-DIKtpT_z.js} +1 -1
  4. package/assets/assets/{MeasurementPropertyPage-Dps7MF7r.js → MeasurementPropertyPage-BfPwwIsT.js} +1 -1
  5. package/assets/assets/{ProjectPage-BijTUrpk.js → ProjectPage-3czWArKE.js} +1 -1
  6. package/assets/assets/{RunRow-9DiFrBlc.js → RunRow-CHyd0gQH.js} +1 -1
  7. package/assets/assets/{RunsPage-DQlAc4J9.js → RunsPage-Dujt0SQ5.js} +1 -1
  8. package/assets/assets/{SettingsPage-c1nxJsbL.js → SettingsPage-CJE9Oeg9.js} +1 -1
  9. package/assets/assets/SiteHealthSection-DvuvziPJ.js +4 -0
  10. package/assets/assets/{TrafficPage-HzwQyLzl.js → TrafficPage-B0tru9ek.js} +1 -1
  11. package/assets/assets/{TrafficSourceDetailPage-BTVAQjRd.js → TrafficSourceDetailPage-DshEUtQX.js} +1 -1
  12. package/assets/assets/{extract-error-message-B5bhJRmf.js → extract-error-message-O2dirpiH.js} +1 -1
  13. package/assets/assets/{index-CQsvCUM0.js → index-C_y0aGQL.js} +19 -19
  14. package/assets/assets/{react-sigma_core.esm.min-6BxcniCA.js → react-sigma_core.esm.min-sVSSzmzq.js} +1 -1
  15. package/assets/index.html +1 -1
  16. package/dist/{chunk-ESH3LPYQ.js → chunk-BFGHZ2GU.js} +15 -0
  17. package/dist/{chunk-INXI6PO7.js → chunk-C4WKYU5C.js} +1 -1
  18. package/dist/{chunk-G5UZTODF.js → chunk-GCQPFCP2.js} +786 -95
  19. package/dist/{chunk-MRHYSEZB.js → chunk-RULHNUBH.js} +66 -62
  20. package/dist/cli.js +17 -5
  21. package/dist/index.js +4 -4
  22. package/dist/{intelligence-service-AFAFGLCM.js → intelligence-service-Z3XS2UXQ.js} +2 -2
  23. package/dist/mcp.js +2 -2
  24. package/package.json +9 -9
  25. package/assets/assets/SiteHealthSection-0B0oUUQx.js +0 -4
@@ -581,7 +581,7 @@ import {
581
581
  wordpressSchemaDeployResultDtoSchema,
582
582
  wordpressSchemaStatusResultDtoSchema,
583
583
  wordpressStatusDtoSchema
584
- } from "./chunk-ESH3LPYQ.js";
584
+ } from "./chunk-BFGHZ2GU.js";
585
585
 
586
586
  // src/intelligence-service.ts
587
587
  import { eq as eq59, desc as desc26, asc as asc11, and as and47, ne as ne7, or as or13, inArray as inArray21, gte as gte14, lte as lte12 } from "drizzle-orm";
@@ -608,6 +608,7 @@ __export(schema_exports, {
608
608
  agentToolEvents: () => agentToolEvents,
609
609
  aiReferralEventsHourly: () => aiReferralEventsHourly,
610
610
  aiUserFetchEventsHourly: () => aiUserFetchEventsHourly,
611
+ aiUserFetchVerificationManifestsHourly: () => aiUserFetchVerificationManifestsHourly,
611
612
  apiKeys: () => apiKeys,
612
613
  auditLog: () => auditLog,
613
614
  backlinkDomains: () => backlinkDomains,
@@ -620,6 +621,7 @@ __export(schema_exports, {
620
621
  competitors: () => competitors,
621
622
  contentTargetDismissals: () => contentTargetDismissals,
622
623
  crawlerEventsHourly: () => crawlerEventsHourly,
624
+ crawlerVerificationManifestsHourly: () => crawlerVerificationManifestsHourly,
623
625
  discoveryProbes: () => discoveryProbes,
624
626
  discoverySessions: () => discoverySessions,
625
627
  doctorHealthState: () => doctorHealthState,
@@ -2242,6 +2244,55 @@ var crawlerEventsHourly = sqliteTable("crawler_events_hourly", {
2242
2244
  index("idx_crawler_hourly_project_ts").on(table.projectId, table.tsHour),
2243
2245
  index("idx_crawler_hourly_path").on(table.projectId, table.pathNormalized)
2244
2246
  ]);
2247
+ var crawlerVerificationManifestsHourly = sqliteTable("crawler_verification_manifests_hourly", {
2248
+ projectId: text("project_id").notNull(),
2249
+ sourceId: text("source_id").notNull(),
2250
+ tsHour: text("ts_hour").notNull(),
2251
+ botId: text("bot_id").notNull(),
2252
+ verificationStatus: text("verification_status").notNull(),
2253
+ pathNormalized: text("path_normalized").notNull(),
2254
+ status: integer("status").notNull(),
2255
+ manifestId: text("manifest_id").notNull(),
2256
+ manifestJson: text("manifest_json", { mode: "json" }).$type(),
2257
+ hits: integer("hits").notNull().default(0),
2258
+ createdAt: text("created_at").notNull(),
2259
+ updatedAt: text("updated_at").notNull()
2260
+ }, (table) => [
2261
+ primaryKey({
2262
+ columns: [
2263
+ table.projectId,
2264
+ table.sourceId,
2265
+ table.tsHour,
2266
+ table.botId,
2267
+ table.verificationStatus,
2268
+ table.pathNormalized,
2269
+ table.status,
2270
+ table.manifestId
2271
+ ]
2272
+ }),
2273
+ foreignKey({
2274
+ name: "crawler_verification_manifests_parent_fk",
2275
+ columns: [
2276
+ table.projectId,
2277
+ table.sourceId,
2278
+ table.tsHour,
2279
+ table.botId,
2280
+ table.verificationStatus,
2281
+ table.pathNormalized,
2282
+ table.status
2283
+ ],
2284
+ foreignColumns: [
2285
+ crawlerEventsHourly.projectId,
2286
+ crawlerEventsHourly.sourceId,
2287
+ crawlerEventsHourly.tsHour,
2288
+ crawlerEventsHourly.botId,
2289
+ crawlerEventsHourly.verificationStatus,
2290
+ crawlerEventsHourly.pathNormalized,
2291
+ crawlerEventsHourly.status
2292
+ ]
2293
+ }).onDelete("cascade"),
2294
+ index("idx_crawler_verification_manifests_project_ts").on(table.projectId, table.tsHour)
2295
+ ]);
2245
2296
  var aiUserFetchEventsHourly = sqliteTable("ai_user_fetch_events_hourly", {
2246
2297
  projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
2247
2298
  sourceId: text("source_id").notNull().references(() => trafficSources.id, { onDelete: "cascade" }),
@@ -2270,6 +2321,55 @@ var aiUserFetchEventsHourly = sqliteTable("ai_user_fetch_events_hourly", {
2270
2321
  index("idx_ai_user_fetch_hourly_project_ts").on(table.projectId, table.tsHour),
2271
2322
  index("idx_ai_user_fetch_hourly_path").on(table.projectId, table.pathNormalized)
2272
2323
  ]);
2324
+ var aiUserFetchVerificationManifestsHourly = sqliteTable("ai_user_fetch_verification_manifests_hourly", {
2325
+ projectId: text("project_id").notNull(),
2326
+ sourceId: text("source_id").notNull(),
2327
+ tsHour: text("ts_hour").notNull(),
2328
+ botId: text("bot_id").notNull(),
2329
+ verificationStatus: text("verification_status").notNull(),
2330
+ pathNormalized: text("path_normalized").notNull(),
2331
+ status: integer("status").notNull(),
2332
+ manifestId: text("manifest_id").notNull(),
2333
+ manifestJson: text("manifest_json", { mode: "json" }).$type(),
2334
+ hits: integer("hits").notNull().default(0),
2335
+ createdAt: text("created_at").notNull(),
2336
+ updatedAt: text("updated_at").notNull()
2337
+ }, (table) => [
2338
+ primaryKey({
2339
+ columns: [
2340
+ table.projectId,
2341
+ table.sourceId,
2342
+ table.tsHour,
2343
+ table.botId,
2344
+ table.verificationStatus,
2345
+ table.pathNormalized,
2346
+ table.status,
2347
+ table.manifestId
2348
+ ]
2349
+ }),
2350
+ foreignKey({
2351
+ name: "ai_user_fetch_verification_manifests_parent_fk",
2352
+ columns: [
2353
+ table.projectId,
2354
+ table.sourceId,
2355
+ table.tsHour,
2356
+ table.botId,
2357
+ table.verificationStatus,
2358
+ table.pathNormalized,
2359
+ table.status
2360
+ ],
2361
+ foreignColumns: [
2362
+ aiUserFetchEventsHourly.projectId,
2363
+ aiUserFetchEventsHourly.sourceId,
2364
+ aiUserFetchEventsHourly.tsHour,
2365
+ aiUserFetchEventsHourly.botId,
2366
+ aiUserFetchEventsHourly.verificationStatus,
2367
+ aiUserFetchEventsHourly.pathNormalized,
2368
+ aiUserFetchEventsHourly.status
2369
+ ]
2370
+ }).onDelete("cascade"),
2371
+ index("idx_ai_user_fetch_verification_manifests_project_ts").on(table.projectId, table.tsHour)
2372
+ ]);
2273
2373
  var aiReferralEventsHourly = sqliteTable("ai_referral_events_hourly", {
2274
2374
  projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
2275
2375
  sourceId: text("source_id").notNull().references(() => trafficSources.id, { onDelete: "cascade" }),
@@ -6264,6 +6364,71 @@ var MIGRATION_VERSIONS = [
6264
6364
  `ALTER TABLE site_crawl_edges ADD COLUMN placement_unknown_occurrences INTEGER`,
6265
6365
  `ALTER TABLE site_crawl_snapshots ADD COLUMN link_placement_ruleset_version TEXT`
6266
6366
  ]
6367
+ },
6368
+ {
6369
+ version: 139,
6370
+ name: "traffic-verification-manifest-provenance",
6371
+ // Verification is evidence produced by one exact range manifest. Store
6372
+ // that provenance in additive sidecars: the existing rollup keys stay
6373
+ // unchanged so an older binary can still upsert them after a rollback.
6374
+ // Historical rows deliberately receive no sidecar row because the old
6375
+ // schema never recorded which bundle classified them.
6376
+ statements: [
6377
+ `CREATE TABLE IF NOT EXISTS crawler_verification_manifests_hourly (
6378
+ project_id TEXT NOT NULL,
6379
+ source_id TEXT NOT NULL,
6380
+ ts_hour TEXT NOT NULL,
6381
+ bot_id TEXT NOT NULL,
6382
+ verification_status TEXT NOT NULL,
6383
+ path_normalized TEXT NOT NULL,
6384
+ status INTEGER NOT NULL,
6385
+ manifest_id TEXT NOT NULL,
6386
+ manifest_json TEXT,
6387
+ hits INTEGER NOT NULL DEFAULT 0,
6388
+ created_at TEXT NOT NULL,
6389
+ updated_at TEXT NOT NULL,
6390
+ PRIMARY KEY (
6391
+ project_id, source_id, ts_hour, bot_id, verification_status,
6392
+ path_normalized, status, manifest_id
6393
+ ),
6394
+ FOREIGN KEY (
6395
+ project_id, source_id, ts_hour, bot_id, verification_status,
6396
+ path_normalized, status
6397
+ ) REFERENCES crawler_events_hourly (
6398
+ project_id, source_id, ts_hour, bot_id, verification_status,
6399
+ path_normalized, status
6400
+ ) ON DELETE CASCADE
6401
+ )`,
6402
+ `CREATE INDEX IF NOT EXISTS idx_crawler_verification_manifests_project_ts
6403
+ ON crawler_verification_manifests_hourly(project_id, ts_hour)`,
6404
+ `CREATE TABLE IF NOT EXISTS ai_user_fetch_verification_manifests_hourly (
6405
+ project_id TEXT NOT NULL,
6406
+ source_id TEXT NOT NULL,
6407
+ ts_hour TEXT NOT NULL,
6408
+ bot_id TEXT NOT NULL,
6409
+ verification_status TEXT NOT NULL,
6410
+ path_normalized TEXT NOT NULL,
6411
+ status INTEGER NOT NULL,
6412
+ manifest_id TEXT NOT NULL,
6413
+ manifest_json TEXT,
6414
+ hits INTEGER NOT NULL DEFAULT 0,
6415
+ created_at TEXT NOT NULL,
6416
+ updated_at TEXT NOT NULL,
6417
+ PRIMARY KEY (
6418
+ project_id, source_id, ts_hour, bot_id, verification_status,
6419
+ path_normalized, status, manifest_id
6420
+ ),
6421
+ FOREIGN KEY (
6422
+ project_id, source_id, ts_hour, bot_id, verification_status,
6423
+ path_normalized, status
6424
+ ) REFERENCES ai_user_fetch_events_hourly (
6425
+ project_id, source_id, ts_hour, bot_id, verification_status,
6426
+ path_normalized, status
6427
+ ) ON DELETE CASCADE
6428
+ )`,
6429
+ `CREATE INDEX IF NOT EXISTS idx_ai_user_fetch_verification_manifests_project_ts
6430
+ ON ai_user_fetch_verification_manifests_hourly(project_id, ts_hour)`
6431
+ ]
6267
6432
  }
6268
6433
  ];
6269
6434
  function addRunsMeasurementPlanVersionForeignKey(tx) {
@@ -48518,13 +48683,78 @@ async function listCloudRunTrafficEvents(accessToken, options) {
48518
48683
 
48519
48684
  // ../integration-traffic/src/ip-ranges/anthropic.json
48520
48685
  var anthropic_default = {
48521
- _source: "ARIN RDAP \u2014 every network registered to Anthropic, PBC (handle AP-2440)",
48522
- _query: "https://rdap.arin.net/registry/entity/AP-2440",
48523
- _notes: "Anthropic does not publish a clean machine-readable IP-range JSON like Google or OpenAI do. These prefixes are taken straight from ARIN's authoritative allocation records (NOT bgp.tools' AS-based view, which can include misleading announcements \u2014 e.g. AS60808 is registered to Anthropic but announces 209.249.57.0/24 which actually belongs to Mitel Networks). 216.73.216.0/22 (AWS-ANTHROPIC) is the load-bearing prefix \u2014 empirical Cloud Run logs show 100% of real ClaudeBot traffic comes from there; the other two are Anthropic's own allocations for non-crawler infra (kept since the verification gate still requires a Claude-* UA match). Refresh by re-running the ARIN RDAP query above and replacing the prefix list.",
48686
+ _source: "https://claude.com/crawling/bots.json",
48687
+ creationTime: "2026-08-13T20:38:01Z",
48524
48688
  prefixes: [
48525
- { ipv4Prefix: "216.73.216.0/22" },
48526
- { ipv4Prefix: "160.79.104.0/21" },
48527
- { ipv6Prefix: "2607:6bc0::/32" }
48689
+ {
48690
+ ipv4Prefix: "216.73.216.0/22"
48691
+ },
48692
+ {
48693
+ ipv4Prefix: "34.162.230.222/32"
48694
+ },
48695
+ {
48696
+ ipv4Prefix: "34.162.244.71/32"
48697
+ },
48698
+ {
48699
+ ipv4Prefix: "34.162.191.81/32"
48700
+ },
48701
+ {
48702
+ ipv4Prefix: "34.150.241.79/32"
48703
+ },
48704
+ {
48705
+ ipv4Prefix: "34.85.172.162/32"
48706
+ },
48707
+ {
48708
+ ipv4Prefix: "35.245.175.129/32"
48709
+ },
48710
+ {
48711
+ ipv4Prefix: "34.182.222.37/32"
48712
+ },
48713
+ {
48714
+ ipv4Prefix: "34.186.108.163/32"
48715
+ },
48716
+ {
48717
+ ipv4Prefix: "35.245.89.239/32"
48718
+ },
48719
+ {
48720
+ ipv4Prefix: "34.182.161.143/32"
48721
+ },
48722
+ {
48723
+ ipv4Prefix: "34.182.218.27/32"
48724
+ },
48725
+ {
48726
+ ipv4Prefix: "34.182.220.85/32"
48727
+ },
48728
+ {
48729
+ ipv4Prefix: "34.11.34.31/32"
48730
+ },
48731
+ {
48732
+ ipv4Prefix: "34.182.140.95/32"
48733
+ },
48734
+ {
48735
+ ipv4Prefix: "136.107.176.208/32"
48736
+ },
48737
+ {
48738
+ ipv4Prefix: "34.182.226.151/32"
48739
+ },
48740
+ {
48741
+ ipv4Prefix: "34.182.226.221/32"
48742
+ },
48743
+ {
48744
+ ipv4Prefix: "35.221.29.174/32"
48745
+ },
48746
+ {
48747
+ ipv4Prefix: "34.182.225.167/32"
48748
+ },
48749
+ {
48750
+ ipv4Prefix: "40.124.101.48/28"
48751
+ },
48752
+ {
48753
+ ipv4Prefix: "20.102.46.224/28"
48754
+ },
48755
+ {
48756
+ ipv4Prefix: "20.64.57.208/28"
48757
+ }
48528
48758
  ]
48529
48759
  };
48530
48760
 
@@ -50623,6 +50853,141 @@ var perplexitybot_default = {
50623
50853
  ]
50624
50854
  };
50625
50855
 
50856
+ // ../integration-traffic/src/ip-range-manifest.ts
50857
+ import { createHash as createHash4 } from "crypto";
50858
+ function isRecord(value) {
50859
+ return typeof value === "object" && value !== null && !Array.isArray(value);
50860
+ }
50861
+ function stableJson2(value, seen = /* @__PURE__ */ new Set()) {
50862
+ if (value === null) return "null";
50863
+ if (typeof value === "string" || typeof value === "boolean") return JSON.stringify(value);
50864
+ if (typeof value === "number") {
50865
+ return Number.isFinite(value) ? JSON.stringify(value) : JSON.stringify(String(value));
50866
+ }
50867
+ if (typeof value !== "object") return JSON.stringify(`[${typeof value}]`);
50868
+ if (seen.has(value)) return '"[circular]"';
50869
+ seen.add(value);
50870
+ const result = Array.isArray(value) ? `[${value.map((entry) => stableJson2(entry, seen)).join(",")}]` : `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableJson2(value[key], seen)}`).join(",")}}`;
50871
+ seen.delete(value);
50872
+ return result;
50873
+ }
50874
+ function parseIp(ip) {
50875
+ if (typeof ip !== "string" || ip.length === 0) return null;
50876
+ const mappedMatch = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/i.exec(ip);
50877
+ if (mappedMatch) return parseIp(mappedMatch[1]);
50878
+ if (ip.includes(":")) {
50879
+ const sides = ip.split("::");
50880
+ if (sides.length > 2) return null;
50881
+ const left = sides[0].length > 0 ? sides[0].split(":") : [];
50882
+ const right = sides.length === 2 && sides[1].length > 0 ? sides[1].split(":") : [];
50883
+ const groupCount = left.length + right.length;
50884
+ if (sides.length === 1 && groupCount !== 8) return null;
50885
+ if (sides.length === 2 && groupCount >= 8) return null;
50886
+ const fill = 8 - groupCount;
50887
+ const groups = [...left, ...new Array(fill).fill("0"), ...right];
50888
+ let addr2 = 0n;
50889
+ for (const group of groups) {
50890
+ if (!/^[0-9a-f]{1,4}$/i.test(group)) return null;
50891
+ addr2 = addr2 << 16n | BigInt(Number.parseInt(group, 16));
50892
+ }
50893
+ return { version: 6, addr: addr2 };
50894
+ }
50895
+ const octets = ip.split(".");
50896
+ if (octets.length !== 4) return null;
50897
+ let addr = 0n;
50898
+ for (const octet of octets) {
50899
+ if (!/^(?:0|[1-9]\d{0,2})$/.test(octet)) return null;
50900
+ const value = Number(octet);
50901
+ if (value > 255) return null;
50902
+ addr = addr << 8n | BigInt(value);
50903
+ }
50904
+ return { version: 4, addr };
50905
+ }
50906
+ function parseCidr(cidr) {
50907
+ if (typeof cidr !== "string") return null;
50908
+ const slash = cidr.indexOf("/");
50909
+ if (slash <= 0 || slash !== cidr.lastIndexOf("/") || slash === cidr.length - 1) return null;
50910
+ const ipPart = cidr.slice(0, slash);
50911
+ const prefixStr = cidr.slice(slash + 1);
50912
+ if (!/^(?:0|[1-9]\d{0,2})$/.test(prefixStr)) return null;
50913
+ const parsed = parseIp(ipPart);
50914
+ if (!parsed) return null;
50915
+ const prefix = Number(prefixStr);
50916
+ const totalBits = parsed.version === 4 ? 32 : 128;
50917
+ if (prefix > totalBits) return null;
50918
+ const allOnes = (1n << BigInt(totalBits)) - 1n;
50919
+ const mask = allOnes >> BigInt(totalBits - prefix) << BigInt(totalBits - prefix);
50920
+ return {
50921
+ version: parsed.version,
50922
+ network: parsed.addr & mask,
50923
+ mask
50924
+ };
50925
+ }
50926
+ function canonicalPrefixKey(entry) {
50927
+ if (isRecord(entry)) {
50928
+ const hasIpv4 = Object.hasOwn(entry, "ipv4Prefix");
50929
+ const hasIpv6 = Object.hasOwn(entry, "ipv6Prefix");
50930
+ if (hasIpv4 !== hasIpv6) {
50931
+ const value = entry[hasIpv4 ? "ipv4Prefix" : "ipv6Prefix"];
50932
+ if (typeof value === "string") {
50933
+ const parsed = parseCidr(value);
50934
+ const expectedVersion = hasIpv4 ? 4 : 6;
50935
+ if (parsed?.version === expectedVersion) {
50936
+ const hexWidth = parsed.version === 4 ? 8 : 32;
50937
+ const network = parsed.network.toString(16).padStart(hexWidth, "0");
50938
+ const mask = parsed.mask.toString(16).padStart(hexWidth, "0");
50939
+ return `cidr:${parsed.version}:${network}:${mask}`;
50940
+ }
50941
+ }
50942
+ }
50943
+ }
50944
+ return `invalid:${stableJson2(entry)}`;
50945
+ }
50946
+ function ipRangeManifestContentHash(value) {
50947
+ const prefixes = isRecord(value) ? value.prefixes : void 0;
50948
+ const canonical = Array.isArray(prefixes) ? prefixes.map(canonicalPrefixKey).sort() : [`invalid-prefixes:${stableJson2(prefixes)}`];
50949
+ return createHash4("sha256").update(JSON.stringify(canonical)).digest("hex");
50950
+ }
50951
+ function validateIpRangeManifestPayload(value) {
50952
+ if (!isRecord(value)) return { ok: false, error: "response must be a JSON object" };
50953
+ const creationTime = value.creationTime;
50954
+ if (typeof creationTime !== "string" || creationTime.trim().length === 0) {
50955
+ return { ok: false, error: "response missing `creationTime` version" };
50956
+ }
50957
+ if (!Array.isArray(value.prefixes)) {
50958
+ return { ok: false, error: "response missing `prefixes` array" };
50959
+ }
50960
+ const prefixes = [];
50961
+ for (const [index2, entry] of value.prefixes.entries()) {
50962
+ if (!isRecord(entry)) {
50963
+ return { ok: false, error: `prefixes[${index2}] must be an object` };
50964
+ }
50965
+ const hasIpv4 = Object.hasOwn(entry, "ipv4Prefix");
50966
+ const hasIpv6 = Object.hasOwn(entry, "ipv6Prefix");
50967
+ if (hasIpv4 === hasIpv6) {
50968
+ return { ok: false, error: `prefixes[${index2}] must contain exactly one IP prefix` };
50969
+ }
50970
+ const field = hasIpv4 ? "ipv4Prefix" : "ipv6Prefix";
50971
+ const cidr = entry[field];
50972
+ if (typeof cidr !== "string") {
50973
+ return { ok: false, error: `prefixes[${index2}].${field} must be a string` };
50974
+ }
50975
+ const parsed = parseCidr(cidr);
50976
+ const expectedVersion = hasIpv4 ? 4 : 6;
50977
+ if (!parsed || parsed.version !== expectedVersion) {
50978
+ return { ok: false, error: `prefixes[${index2}].${field} is not a valid IPv${expectedVersion} CIDR` };
50979
+ }
50980
+ prefixes.push(hasIpv4 ? { ipv4Prefix: cidr } : { ipv6Prefix: cidr });
50981
+ }
50982
+ return {
50983
+ ok: true,
50984
+ value: {
50985
+ creationTime: creationTime.trim(),
50986
+ prefixes
50987
+ }
50988
+ };
50989
+ }
50990
+
50626
50991
  // ../integration-traffic/src/ip-verify.ts
50627
50992
  var RULE_ID_TO_RANGES = {
50628
50993
  // OpenAI — three separate published lists (training crawler vs
@@ -50656,94 +51021,56 @@ var RULE_ID_TO_RANGES = {
50656
51021
  "perplexity-bot": perplexitybot_default,
50657
51022
  // src: https://www.perplexity.ai/perplexity-user.json
50658
51023
  "perplexity-user": perplexity_user_default,
50659
- // Anthropic — no machine-readable JSON published. The bundled
50660
- // anthropic.json is the set of networks registered to Anthropic,
50661
- // PBC at ARIN (the authoritative allocation record). Maintained by
50662
- // hand; refresh by re-querying the ARIN entity below. The crawler
50663
- // block is AWS-ANTHROPIC 216.73.216.0/22 — empirical Cloud Run
50664
- // logs show all real ClaudeBot traffic comes from there. The same
50665
- // raw set is shared across every Claude-* UA the classifier emits:
50666
- // both the training crawler and the per-user fetcher map here.
50667
- // src: https://rdap.arin.net/registry/entity/AP-2440
51024
+ // Anthropic publishes one shared crawler-origin manifest. It confirms
51025
+ // Anthropic origin but does not attribute a prefix to ClaudeBot,
51026
+ // Claude-SearchBot, or Claude-User individually, so both classifier rules
51027
+ // map to the same file and the UA remains the product discriminator.
51028
+ // src: https://claude.com/crawling/bots.json
50668
51029
  "anthropic-claudebot": anthropic_default,
50669
51030
  "claude-user": anthropic_default
50670
51031
  };
51032
+ function manifestMetadata(raw) {
51033
+ if (typeof raw !== "object" || raw === null || Array.isArray(raw)) return null;
51034
+ const record = raw;
51035
+ const source = typeof record._source === "string" ? record._source.trim() : "";
51036
+ const version = typeof record.creationTime === "string" ? record.creationTime.trim() : "";
51037
+ if (!source || !version) return null;
51038
+ const contentHash2 = ipRangeManifestContentHash(raw);
51039
+ return { id: `${source}#${version}#sha256:${contentHash2}`, source, version };
51040
+ }
51041
+ function verificationDataFromRaw(raw) {
51042
+ const validation = validateIpRangeManifestPayload(raw);
51043
+ const ranges = [];
51044
+ if (validation.ok) {
51045
+ for (const entry of validation.value.prefixes) {
51046
+ const parsed = parseCidr(entry.ipv4Prefix ?? entry.ipv6Prefix);
51047
+ if (parsed) ranges.push(parsed);
51048
+ }
51049
+ }
51050
+ return { ranges, manifest: manifestMetadata(raw) };
51051
+ }
51052
+ function decideIpVerification(ip, data) {
51053
+ if (!ip || data.ranges.length === 0) return { verified: false, manifest: null };
51054
+ const parsed = parseIp(ip);
51055
+ if (!parsed) return { verified: false, manifest: null };
51056
+ for (const cidr of data.ranges) {
51057
+ if (parsed.version !== cidr.version) continue;
51058
+ if ((parsed.addr & cidr.mask) === cidr.network) {
51059
+ return { verified: true, manifest: data.manifest };
51060
+ }
51061
+ }
51062
+ return { verified: false, manifest: data.manifest };
51063
+ }
50671
51064
  var CACHE = (() => {
50672
51065
  const cache = /* @__PURE__ */ new Map();
50673
51066
  for (const [ruleId, raw] of Object.entries(RULE_ID_TO_RANGES)) {
50674
- const parsed = [];
50675
- for (const entry of raw.prefixes) {
50676
- const cidr = entry.ipv4Prefix ?? entry.ipv6Prefix;
50677
- if (!cidr) continue;
50678
- const p = parseCidr(cidr);
50679
- if (p) parsed.push(p);
50680
- }
50681
- cache.set(ruleId, parsed);
51067
+ cache.set(ruleId, verificationDataFromRaw(raw));
50682
51068
  }
50683
51069
  return cache;
50684
51070
  })();
50685
- function parseIp(ip) {
50686
- if (!ip) return null;
50687
- const mappedMatch = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/i.exec(ip);
50688
- if (mappedMatch) return parseIp(mappedMatch[1]);
50689
- if (ip.includes(":")) {
50690
- const sides = ip.split("::");
50691
- if (sides.length > 2) return null;
50692
- const left = sides[0].length > 0 ? sides[0].split(":") : [];
50693
- const right = sides.length === 2 && sides[1].length > 0 ? sides[1].split(":") : [];
50694
- const groupCount = left.length + right.length;
50695
- if (groupCount > 8) return null;
50696
- if (sides.length === 1 && groupCount !== 8) return null;
50697
- const fill = 8 - groupCount;
50698
- const groups = [...left, ...new Array(fill).fill("0"), ...right];
50699
- let addr2 = 0n;
50700
- for (const g of groups) {
50701
- if (g.length === 0 || g.length > 4) return null;
50702
- const n = Number.parseInt(g, 16);
50703
- if (!Number.isFinite(n) || n < 0 || n > 65535) return null;
50704
- addr2 = addr2 << 16n | BigInt(n);
50705
- }
50706
- return { version: 6, addr: addr2 };
50707
- }
50708
- const octets = ip.split(".");
50709
- if (octets.length !== 4) return null;
50710
- let addr = 0n;
50711
- for (const o of octets) {
50712
- if (o.length === 0 || o.length > 3) return null;
50713
- const n = Number.parseInt(o, 10);
50714
- if (!Number.isInteger(n) || n < 0 || n > 255) return null;
50715
- addr = addr << 8n | BigInt(n);
50716
- }
50717
- return { version: 4, addr };
50718
- }
50719
- function parseCidr(cidr) {
50720
- const [ipPart, prefixStr] = cidr.split("/");
50721
- if (!ipPart || !prefixStr) return null;
50722
- const prefix = Number.parseInt(prefixStr, 10);
50723
- if (!Number.isInteger(prefix)) return null;
50724
- const parsed = parseIp(ipPart);
50725
- if (!parsed) return null;
50726
- const totalBits = parsed.version === 4 ? 32 : 128;
50727
- if (prefix < 0 || prefix > totalBits) return null;
50728
- const allOnes = (1n << BigInt(totalBits)) - 1n;
50729
- const mask = allOnes >> BigInt(totalBits - prefix) << BigInt(totalBits - prefix);
50730
- return {
50731
- version: parsed.version,
50732
- network: parsed.addr & mask,
50733
- mask
50734
- };
50735
- }
50736
- function verifyIpForRule(ip, ruleId) {
50737
- if (!ip) return false;
50738
- const ranges = CACHE.get(ruleId);
50739
- if (!ranges || ranges.length === 0) return false;
50740
- const parsed = parseIp(ip);
50741
- if (!parsed) return false;
50742
- for (const cidr of ranges) {
50743
- if (parsed.version !== cidr.version) continue;
50744
- if ((parsed.addr & cidr.mask) === cidr.network) return true;
50745
- }
50746
- return false;
51071
+ function verifyIpForRuleDecision(ip, ruleId) {
51072
+ const data = CACHE.get(ruleId);
51073
+ return data ? decideIpVerification(ip, data) : { verified: false, manifest: null };
50747
51074
  }
50748
51075
 
50749
51076
  // ../integration-traffic/src/rules.ts
@@ -51113,13 +51440,14 @@ function classifyCrawler(event) {
51113
51440
  for (const rule of DEFAULT_AI_CRAWLER_RULES) {
51114
51441
  if (rule.purpose === USER_FETCH_PURPOSE) continue;
51115
51442
  if (rule.userAgentPatterns.some((pattern) => pattern.test(userAgent))) {
51116
- const verified = verifyIpForRule(event.remoteIp, rule.id);
51443
+ const decision = verifyIpForRuleDecision(event.remoteIp, rule.id);
51117
51444
  return {
51118
51445
  botId: rule.id,
51119
51446
  operator: rule.operator,
51120
51447
  product: rule.product,
51121
51448
  purpose: rule.purpose,
51122
- verificationStatus: verified ? "verified" : "claimed_unverified",
51449
+ verificationStatus: decision.verified ? "verified" : "claimed_unverified",
51450
+ verificationManifest: decision.manifest,
51123
51451
  matchedUserAgent: userAgent
51124
51452
  };
51125
51453
  }
@@ -51132,12 +51460,13 @@ function classifyAiUserFetch(event) {
51132
51460
  for (const rule of DEFAULT_AI_CRAWLER_RULES) {
51133
51461
  if (rule.purpose !== USER_FETCH_PURPOSE) continue;
51134
51462
  if (rule.userAgentPatterns.some((pattern) => pattern.test(userAgent))) {
51135
- const verified = verifyIpForRule(event.remoteIp, rule.id);
51463
+ const decision = verifyIpForRuleDecision(event.remoteIp, rule.id);
51136
51464
  return {
51137
51465
  botId: rule.id,
51138
51466
  operator: rule.operator,
51139
51467
  product: rule.product,
51140
- verificationStatus: verified ? "verified" : "claimed_unverified",
51468
+ verificationStatus: decision.verified ? "verified" : "claimed_unverified",
51469
+ verificationManifest: decision.manifest,
51141
51470
  matchedUserAgent: userAgent
51142
51471
  };
51143
51472
  }
@@ -51286,10 +51615,10 @@ function mergeReferralSession(current, next) {
51286
51615
  return { ...winner, trafficClass: paid ? AiReferralTrafficClasses.paid : winner.trafficClass };
51287
51616
  }
51288
51617
  function sortCrawlerBuckets(a, b) {
51289
- return a.tsHour.localeCompare(b.tsHour) || a.botId.localeCompare(b.botId) || a.pathNormalized.localeCompare(b.pathNormalized) || String(a.status).localeCompare(String(b.status));
51618
+ return a.tsHour.localeCompare(b.tsHour) || a.botId.localeCompare(b.botId) || a.verificationStatus.localeCompare(b.verificationStatus) || (a.verificationManifest?.id ?? "").localeCompare(b.verificationManifest?.id ?? "") || a.pathNormalized.localeCompare(b.pathNormalized) || String(a.status).localeCompare(String(b.status));
51290
51619
  }
51291
51620
  function sortAiUserFetchBuckets(a, b) {
51292
- return a.tsHour.localeCompare(b.tsHour) || a.botId.localeCompare(b.botId) || a.pathNormalized.localeCompare(b.pathNormalized) || String(a.status).localeCompare(String(b.status));
51621
+ return a.tsHour.localeCompare(b.tsHour) || a.botId.localeCompare(b.botId) || a.verificationStatus.localeCompare(b.verificationStatus) || (a.verificationManifest?.id ?? "").localeCompare(b.verificationManifest?.id ?? "") || a.pathNormalized.localeCompare(b.pathNormalized) || String(a.status).localeCompare(String(b.status));
51293
51622
  }
51294
51623
  function sortReferralBuckets(a, b) {
51295
51624
  return a.tsHour.localeCompare(b.tsHour) || a.product.localeCompare(b.product) || a.sourceDomain.localeCompare(b.sourceDomain) || a.landingPathNormalized.localeCompare(b.landingPathNormalized) || String(a.status).localeCompare(String(b.status));
@@ -51332,6 +51661,7 @@ function buildTrafficProbeReport(events, options = {}) {
51332
51661
  tsHour,
51333
51662
  crawler.botId,
51334
51663
  crawler.verificationStatus,
51664
+ crawler.verificationManifest?.id ?? "no-manifest",
51335
51665
  pathNormalized,
51336
51666
  event.status ?? "null"
51337
51667
  ].join(" ");
@@ -51345,6 +51675,7 @@ function buildTrafficProbeReport(events, options = {}) {
51345
51675
  operator: crawler.operator,
51346
51676
  product: crawler.product,
51347
51677
  verificationStatus: crawler.verificationStatus,
51678
+ verificationManifest: crawler.verificationManifest,
51348
51679
  pathNormalized,
51349
51680
  status: event.status,
51350
51681
  hits: 1,
@@ -51363,6 +51694,7 @@ function buildTrafficProbeReport(events, options = {}) {
51363
51694
  tsHour,
51364
51695
  aiUserFetch.botId,
51365
51696
  aiUserFetch.verificationStatus,
51697
+ aiUserFetch.verificationManifest?.id ?? "no-manifest",
51366
51698
  pathNormalized,
51367
51699
  event.status ?? "null"
51368
51700
  ].join(" ");
@@ -51376,6 +51708,7 @@ function buildTrafficProbeReport(events, options = {}) {
51376
51708
  operator: aiUserFetch.operator,
51377
51709
  product: aiUserFetch.product,
51378
51710
  verificationStatus: aiUserFetch.verificationStatus,
51711
+ verificationManifest: aiUserFetch.verificationManifest,
51379
51712
  pathNormalized,
51380
51713
  status: event.status,
51381
51714
  hits: 1,
@@ -53042,6 +53375,38 @@ function writeTrafficEventBatch(opts) {
53042
53375
  updatedAt: opts.receivedAt
53043
53376
  }
53044
53377
  }).run();
53378
+ if (bucket.verificationManifest) {
53379
+ tx.insert(crawlerVerificationManifestsHourly).values({
53380
+ projectId: opts.projectId,
53381
+ sourceId: opts.sourceId,
53382
+ tsHour: bucket.tsHour,
53383
+ botId: bucket.botId,
53384
+ verificationStatus: bucket.verificationStatus,
53385
+ pathNormalized: bucket.pathNormalized,
53386
+ status,
53387
+ manifestId: bucket.verificationManifest.id,
53388
+ manifestJson: bucket.verificationManifest,
53389
+ hits: bucket.hits,
53390
+ createdAt: opts.receivedAt,
53391
+ updatedAt: opts.receivedAt
53392
+ }).onConflictDoUpdate({
53393
+ target: [
53394
+ crawlerVerificationManifestsHourly.projectId,
53395
+ crawlerVerificationManifestsHourly.sourceId,
53396
+ crawlerVerificationManifestsHourly.tsHour,
53397
+ crawlerVerificationManifestsHourly.botId,
53398
+ crawlerVerificationManifestsHourly.verificationStatus,
53399
+ crawlerVerificationManifestsHourly.pathNormalized,
53400
+ crawlerVerificationManifestsHourly.status,
53401
+ crawlerVerificationManifestsHourly.manifestId
53402
+ ],
53403
+ set: {
53404
+ hits: sql18`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
53405
+ manifestJson: bucket.verificationManifest,
53406
+ updatedAt: opts.receivedAt
53407
+ }
53408
+ }).run();
53409
+ }
53045
53410
  crawlerBucketRows += 1;
53046
53411
  }
53047
53412
  for (const bucket of report.aiUserFetchEventsHourly) {
@@ -53075,6 +53440,38 @@ function writeTrafficEventBatch(opts) {
53075
53440
  updatedAt: opts.receivedAt
53076
53441
  }
53077
53442
  }).run();
53443
+ if (bucket.verificationManifest) {
53444
+ tx.insert(aiUserFetchVerificationManifestsHourly).values({
53445
+ projectId: opts.projectId,
53446
+ sourceId: opts.sourceId,
53447
+ tsHour: bucket.tsHour,
53448
+ botId: bucket.botId,
53449
+ verificationStatus: bucket.verificationStatus,
53450
+ pathNormalized: bucket.pathNormalized,
53451
+ status,
53452
+ manifestId: bucket.verificationManifest.id,
53453
+ manifestJson: bucket.verificationManifest,
53454
+ hits: bucket.hits,
53455
+ createdAt: opts.receivedAt,
53456
+ updatedAt: opts.receivedAt
53457
+ }).onConflictDoUpdate({
53458
+ target: [
53459
+ aiUserFetchVerificationManifestsHourly.projectId,
53460
+ aiUserFetchVerificationManifestsHourly.sourceId,
53461
+ aiUserFetchVerificationManifestsHourly.tsHour,
53462
+ aiUserFetchVerificationManifestsHourly.botId,
53463
+ aiUserFetchVerificationManifestsHourly.verificationStatus,
53464
+ aiUserFetchVerificationManifestsHourly.pathNormalized,
53465
+ aiUserFetchVerificationManifestsHourly.status,
53466
+ aiUserFetchVerificationManifestsHourly.manifestId
53467
+ ],
53468
+ set: {
53469
+ hits: sql18`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
53470
+ manifestJson: bucket.verificationManifest,
53471
+ updatedAt: opts.receivedAt
53472
+ }
53473
+ }).run();
53474
+ }
53078
53475
  aiUserFetchBucketRows += 1;
53079
53476
  }
53080
53477
  for (const bucket of report.aiReferralEventsHourly) {
@@ -53393,6 +53790,117 @@ function trafficSeriesPoint(pointsByBucket, bucket) {
53393
53790
  pointsByBucket.set(bucket, created);
53394
53791
  return created;
53395
53792
  }
53793
+ function verificationEventKey(row) {
53794
+ return JSON.stringify([
53795
+ row.sourceId,
53796
+ row.tsHour,
53797
+ row.botId,
53798
+ row.verificationStatus,
53799
+ row.pathNormalized,
53800
+ row.status
53801
+ ]);
53802
+ }
53803
+ function verificationSelectedRowsJson(projectId, rows) {
53804
+ return JSON.stringify(rows.map((row) => [
53805
+ projectId,
53806
+ row.sourceId,
53807
+ row.tsHour,
53808
+ row.botId,
53809
+ row.verificationStatus,
53810
+ row.pathNormalized,
53811
+ row.status
53812
+ ]));
53813
+ }
53814
+ function buildVerificationProvenance(events, rows) {
53815
+ const manifestsByEvent = /* @__PURE__ */ new Map();
53816
+ for (const row of rows) {
53817
+ const key = verificationEventKey(row);
53818
+ const manifests = manifestsByEvent.get(key);
53819
+ const usage = { manifestId: row.manifestId, manifest: row.manifest, hits: row.hits };
53820
+ if (manifests) manifests.push(usage);
53821
+ else manifestsByEvent.set(key, [usage]);
53822
+ }
53823
+ const provenanceByEvent = /* @__PURE__ */ new Map();
53824
+ for (const event of events) {
53825
+ const key = verificationEventKey(event);
53826
+ const verificationManifests = manifestsByEvent.get(key) ?? [];
53827
+ verificationManifests.sort((a, b) => a.manifestId.localeCompare(b.manifestId));
53828
+ const attributedHits = verificationManifests.reduce((sum, manifest) => sum + manifest.hits, 0);
53829
+ provenanceByEvent.set(key, {
53830
+ verificationManifests,
53831
+ verificationUnattributedHits: Math.max(0, event.hits - attributedHits)
53832
+ });
53833
+ }
53834
+ return provenanceByEvent;
53835
+ }
53836
+ function crawlerVerificationProvenance(db, projectId, events) {
53837
+ if (events.length === 0) return /* @__PURE__ */ new Map();
53838
+ const selectedRows = verificationSelectedRowsJson(projectId, events);
53839
+ const rows = db.select({
53840
+ sourceId: crawlerVerificationManifestsHourly.sourceId,
53841
+ tsHour: crawlerVerificationManifestsHourly.tsHour,
53842
+ botId: crawlerVerificationManifestsHourly.botId,
53843
+ verificationStatus: crawlerVerificationManifestsHourly.verificationStatus,
53844
+ pathNormalized: crawlerVerificationManifestsHourly.pathNormalized,
53845
+ status: crawlerVerificationManifestsHourly.status,
53846
+ manifestId: crawlerVerificationManifestsHourly.manifestId,
53847
+ manifest: crawlerVerificationManifestsHourly.manifestJson,
53848
+ hits: crawlerVerificationManifestsHourly.hits
53849
+ }).from(crawlerVerificationManifestsHourly).where(sql19`(
53850
+ ${crawlerVerificationManifestsHourly.projectId},
53851
+ ${crawlerVerificationManifestsHourly.sourceId},
53852
+ ${crawlerVerificationManifestsHourly.tsHour},
53853
+ ${crawlerVerificationManifestsHourly.botId},
53854
+ ${crawlerVerificationManifestsHourly.verificationStatus},
53855
+ ${crawlerVerificationManifestsHourly.pathNormalized},
53856
+ ${crawlerVerificationManifestsHourly.status}
53857
+ ) IN (
53858
+ SELECT
53859
+ json_extract(value, '$[0]'),
53860
+ json_extract(value, '$[1]'),
53861
+ json_extract(value, '$[2]'),
53862
+ json_extract(value, '$[3]'),
53863
+ json_extract(value, '$[4]'),
53864
+ json_extract(value, '$[5]'),
53865
+ json_extract(value, '$[6]')
53866
+ FROM json_each(${selectedRows})
53867
+ )`).orderBy(crawlerVerificationManifestsHourly.manifestId).all();
53868
+ return buildVerificationProvenance(events, rows);
53869
+ }
53870
+ function aiUserFetchVerificationProvenance(db, projectId, events) {
53871
+ if (events.length === 0) return /* @__PURE__ */ new Map();
53872
+ const selectedRows = verificationSelectedRowsJson(projectId, events);
53873
+ const rows = db.select({
53874
+ sourceId: aiUserFetchVerificationManifestsHourly.sourceId,
53875
+ tsHour: aiUserFetchVerificationManifestsHourly.tsHour,
53876
+ botId: aiUserFetchVerificationManifestsHourly.botId,
53877
+ verificationStatus: aiUserFetchVerificationManifestsHourly.verificationStatus,
53878
+ pathNormalized: aiUserFetchVerificationManifestsHourly.pathNormalized,
53879
+ status: aiUserFetchVerificationManifestsHourly.status,
53880
+ manifestId: aiUserFetchVerificationManifestsHourly.manifestId,
53881
+ manifest: aiUserFetchVerificationManifestsHourly.manifestJson,
53882
+ hits: aiUserFetchVerificationManifestsHourly.hits
53883
+ }).from(aiUserFetchVerificationManifestsHourly).where(sql19`(
53884
+ ${aiUserFetchVerificationManifestsHourly.projectId},
53885
+ ${aiUserFetchVerificationManifestsHourly.sourceId},
53886
+ ${aiUserFetchVerificationManifestsHourly.tsHour},
53887
+ ${aiUserFetchVerificationManifestsHourly.botId},
53888
+ ${aiUserFetchVerificationManifestsHourly.verificationStatus},
53889
+ ${aiUserFetchVerificationManifestsHourly.pathNormalized},
53890
+ ${aiUserFetchVerificationManifestsHourly.status}
53891
+ ) IN (
53892
+ SELECT
53893
+ json_extract(value, '$[0]'),
53894
+ json_extract(value, '$[1]'),
53895
+ json_extract(value, '$[2]'),
53896
+ json_extract(value, '$[3]'),
53897
+ json_extract(value, '$[4]'),
53898
+ json_extract(value, '$[5]'),
53899
+ json_extract(value, '$[6]')
53900
+ FROM json_each(${selectedRows})
53901
+ )`).orderBy(aiUserFetchVerificationManifestsHourly.manifestId).all();
53902
+ return buildVerificationProvenance(events, rows);
53903
+ }
53396
53904
  function parseSourceConfig(row) {
53397
53905
  return row.configJson;
53398
53906
  }
@@ -53570,6 +54078,7 @@ async function runBackfillTask(options) {
53570
54078
  )
53571
54079
  ).run();
53572
54080
  for (const bucket of report.crawlerEventsHourly) {
54081
+ const status = bucket.status ?? 0;
53573
54082
  tx.insert(crawlerEventsHourly).values({
53574
54083
  projectId: project.id,
53575
54084
  sourceId: sourceRow2.id,
@@ -53578,14 +54087,62 @@ async function runBackfillTask(options) {
53578
54087
  operator: bucket.operator,
53579
54088
  verificationStatus: bucket.verificationStatus,
53580
54089
  pathNormalized: bucket.pathNormalized,
53581
- status: bucket.status ?? 0,
54090
+ status,
53582
54091
  hits: bucket.hits,
53583
54092
  sampledUserAgent: bucket.sampledUserAgent,
53584
54093
  createdAt: finishedAt,
53585
54094
  updatedAt: finishedAt
54095
+ }).onConflictDoUpdate({
54096
+ target: [
54097
+ crawlerEventsHourly.projectId,
54098
+ crawlerEventsHourly.sourceId,
54099
+ crawlerEventsHourly.tsHour,
54100
+ crawlerEventsHourly.botId,
54101
+ crawlerEventsHourly.verificationStatus,
54102
+ crawlerEventsHourly.pathNormalized,
54103
+ crawlerEventsHourly.status
54104
+ ],
54105
+ set: {
54106
+ hits: sql19`${crawlerEventsHourly.hits} + ${bucket.hits}`,
54107
+ sampledUserAgent: bucket.sampledUserAgent,
54108
+ updatedAt: finishedAt
54109
+ }
53586
54110
  }).run();
54111
+ if (bucket.verificationManifest) {
54112
+ tx.insert(crawlerVerificationManifestsHourly).values({
54113
+ projectId: project.id,
54114
+ sourceId: sourceRow2.id,
54115
+ tsHour: bucket.tsHour,
54116
+ botId: bucket.botId,
54117
+ verificationStatus: bucket.verificationStatus,
54118
+ pathNormalized: bucket.pathNormalized,
54119
+ status,
54120
+ manifestId: bucket.verificationManifest.id,
54121
+ manifestJson: bucket.verificationManifest,
54122
+ hits: bucket.hits,
54123
+ createdAt: finishedAt,
54124
+ updatedAt: finishedAt
54125
+ }).onConflictDoUpdate({
54126
+ target: [
54127
+ crawlerVerificationManifestsHourly.projectId,
54128
+ crawlerVerificationManifestsHourly.sourceId,
54129
+ crawlerVerificationManifestsHourly.tsHour,
54130
+ crawlerVerificationManifestsHourly.botId,
54131
+ crawlerVerificationManifestsHourly.verificationStatus,
54132
+ crawlerVerificationManifestsHourly.pathNormalized,
54133
+ crawlerVerificationManifestsHourly.status,
54134
+ crawlerVerificationManifestsHourly.manifestId
54135
+ ],
54136
+ set: {
54137
+ hits: sql19`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
54138
+ manifestJson: bucket.verificationManifest,
54139
+ updatedAt: finishedAt
54140
+ }
54141
+ }).run();
54142
+ }
53587
54143
  }
53588
54144
  for (const bucket of report.aiUserFetchEventsHourly) {
54145
+ const status = bucket.status ?? 0;
53589
54146
  tx.insert(aiUserFetchEventsHourly).values({
53590
54147
  projectId: project.id,
53591
54148
  sourceId: sourceRow2.id,
@@ -53594,12 +54151,59 @@ async function runBackfillTask(options) {
53594
54151
  operator: bucket.operator,
53595
54152
  verificationStatus: bucket.verificationStatus,
53596
54153
  pathNormalized: bucket.pathNormalized,
53597
- status: bucket.status ?? 0,
54154
+ status,
53598
54155
  hits: bucket.hits,
53599
54156
  sampledUserAgent: bucket.sampledUserAgent,
53600
54157
  createdAt: finishedAt,
53601
54158
  updatedAt: finishedAt
54159
+ }).onConflictDoUpdate({
54160
+ target: [
54161
+ aiUserFetchEventsHourly.projectId,
54162
+ aiUserFetchEventsHourly.sourceId,
54163
+ aiUserFetchEventsHourly.tsHour,
54164
+ aiUserFetchEventsHourly.botId,
54165
+ aiUserFetchEventsHourly.verificationStatus,
54166
+ aiUserFetchEventsHourly.pathNormalized,
54167
+ aiUserFetchEventsHourly.status
54168
+ ],
54169
+ set: {
54170
+ hits: sql19`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
54171
+ sampledUserAgent: bucket.sampledUserAgent,
54172
+ updatedAt: finishedAt
54173
+ }
53602
54174
  }).run();
54175
+ if (bucket.verificationManifest) {
54176
+ tx.insert(aiUserFetchVerificationManifestsHourly).values({
54177
+ projectId: project.id,
54178
+ sourceId: sourceRow2.id,
54179
+ tsHour: bucket.tsHour,
54180
+ botId: bucket.botId,
54181
+ verificationStatus: bucket.verificationStatus,
54182
+ pathNormalized: bucket.pathNormalized,
54183
+ status,
54184
+ manifestId: bucket.verificationManifest.id,
54185
+ manifestJson: bucket.verificationManifest,
54186
+ hits: bucket.hits,
54187
+ createdAt: finishedAt,
54188
+ updatedAt: finishedAt
54189
+ }).onConflictDoUpdate({
54190
+ target: [
54191
+ aiUserFetchVerificationManifestsHourly.projectId,
54192
+ aiUserFetchVerificationManifestsHourly.sourceId,
54193
+ aiUserFetchVerificationManifestsHourly.tsHour,
54194
+ aiUserFetchVerificationManifestsHourly.botId,
54195
+ aiUserFetchVerificationManifestsHourly.verificationStatus,
54196
+ aiUserFetchVerificationManifestsHourly.pathNormalized,
54197
+ aiUserFetchVerificationManifestsHourly.status,
54198
+ aiUserFetchVerificationManifestsHourly.manifestId
54199
+ ],
54200
+ set: {
54201
+ hits: sql19`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
54202
+ manifestJson: bucket.verificationManifest,
54203
+ updatedAt: finishedAt
54204
+ }
54205
+ }).run();
54206
+ }
53603
54207
  }
53604
54208
  for (const bucket of report.aiReferralEventsHourly) {
53605
54209
  tx.insert(aiReferralEventsHourly).values({
@@ -55082,6 +55686,38 @@ async function trafficRoutes(app, opts) {
55082
55686
  updatedAt: finishedAt
55083
55687
  }
55084
55688
  }).run();
55689
+ if (bucket.verificationManifest) {
55690
+ tx.insert(crawlerVerificationManifestsHourly).values({
55691
+ projectId: project.id,
55692
+ sourceId: sourceRow2.id,
55693
+ tsHour: bucket.tsHour,
55694
+ botId: bucket.botId,
55695
+ verificationStatus: bucket.verificationStatus,
55696
+ pathNormalized: bucket.pathNormalized,
55697
+ status,
55698
+ manifestId: bucket.verificationManifest.id,
55699
+ manifestJson: bucket.verificationManifest,
55700
+ hits: bucket.hits,
55701
+ createdAt: finishedAt,
55702
+ updatedAt: finishedAt
55703
+ }).onConflictDoUpdate({
55704
+ target: [
55705
+ crawlerVerificationManifestsHourly.projectId,
55706
+ crawlerVerificationManifestsHourly.sourceId,
55707
+ crawlerVerificationManifestsHourly.tsHour,
55708
+ crawlerVerificationManifestsHourly.botId,
55709
+ crawlerVerificationManifestsHourly.verificationStatus,
55710
+ crawlerVerificationManifestsHourly.pathNormalized,
55711
+ crawlerVerificationManifestsHourly.status,
55712
+ crawlerVerificationManifestsHourly.manifestId
55713
+ ],
55714
+ set: {
55715
+ hits: sql19`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
55716
+ manifestJson: bucket.verificationManifest,
55717
+ updatedAt: finishedAt
55718
+ }
55719
+ }).run();
55720
+ }
55085
55721
  crawlerBucketRows += 1;
55086
55722
  }
55087
55723
  for (const bucket of report.aiUserFetchEventsHourly) {
@@ -55115,6 +55751,38 @@ async function trafficRoutes(app, opts) {
55115
55751
  updatedAt: finishedAt
55116
55752
  }
55117
55753
  }).run();
55754
+ if (bucket.verificationManifest) {
55755
+ tx.insert(aiUserFetchVerificationManifestsHourly).values({
55756
+ projectId: project.id,
55757
+ sourceId: sourceRow2.id,
55758
+ tsHour: bucket.tsHour,
55759
+ botId: bucket.botId,
55760
+ verificationStatus: bucket.verificationStatus,
55761
+ pathNormalized: bucket.pathNormalized,
55762
+ status,
55763
+ manifestId: bucket.verificationManifest.id,
55764
+ manifestJson: bucket.verificationManifest,
55765
+ hits: bucket.hits,
55766
+ createdAt: finishedAt,
55767
+ updatedAt: finishedAt
55768
+ }).onConflictDoUpdate({
55769
+ target: [
55770
+ aiUserFetchVerificationManifestsHourly.projectId,
55771
+ aiUserFetchVerificationManifestsHourly.sourceId,
55772
+ aiUserFetchVerificationManifestsHourly.tsHour,
55773
+ aiUserFetchVerificationManifestsHourly.botId,
55774
+ aiUserFetchVerificationManifestsHourly.verificationStatus,
55775
+ aiUserFetchVerificationManifestsHourly.pathNormalized,
55776
+ aiUserFetchVerificationManifestsHourly.status,
55777
+ aiUserFetchVerificationManifestsHourly.manifestId
55778
+ ],
55779
+ set: {
55780
+ hits: sql19`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
55781
+ manifestJson: bucket.verificationManifest,
55782
+ updatedAt: finishedAt
55783
+ }
55784
+ }).run();
55785
+ }
55118
55786
  aiUserFetchBucketRows += 1;
55119
55787
  }
55120
55788
  for (const bucket of report.aiReferralEventsHourly) {
@@ -55691,6 +56359,8 @@ async function trafficRoutes(app, opts) {
55691
56359
  botId: r.botId,
55692
56360
  operator: r.operator,
55693
56361
  verificationStatus: r.verificationStatus,
56362
+ verificationManifests: [],
56363
+ verificationUnattributedHits: r.hits,
55694
56364
  pathNormalized: r.pathNormalized,
55695
56365
  pathClass: classifyTrafficPath(r.pathNormalized),
55696
56366
  status: r.status,
@@ -55729,6 +56399,8 @@ async function trafficRoutes(app, opts) {
55729
56399
  botId: r.botId,
55730
56400
  operator: r.operator,
55731
56401
  verificationStatus: r.verificationStatus,
56402
+ verificationManifests: [],
56403
+ verificationUnattributedHits: r.hits,
55732
56404
  pathNormalized: r.pathNormalized,
55733
56405
  status: r.status,
55734
56406
  hits: r.hits
@@ -55785,6 +56457,25 @@ async function trafficRoutes(app, opts) {
55785
56457
  }
55786
56458
  events.sort((a, b) => a.tsHour < b.tsHour ? 1 : a.tsHour > b.tsHour ? -1 : 0);
55787
56459
  const trimmed = events.slice(0, limit);
56460
+ const trimmedCrawlerEvents = trimmed.filter(
56461
+ (event) => event.kind === TrafficEventKinds.crawler
56462
+ );
56463
+ const trimmedAiUserFetchEvents = trimmed.filter(
56464
+ (event) => event.kind === TrafficEventKinds["ai-user-fetch"]
56465
+ );
56466
+ const crawlerProvenance = crawlerVerificationProvenance(app.db, project.id, trimmedCrawlerEvents);
56467
+ const aiUserFetchProvenance = aiUserFetchVerificationProvenance(app.db, project.id, trimmedAiUserFetchEvents);
56468
+ const eventsWithProvenance = trimmed.map((event) => {
56469
+ if (event.kind === TrafficEventKinds.crawler) {
56470
+ const provenance = crawlerProvenance.get(verificationEventKey(event));
56471
+ return provenance ? { ...event, ...provenance } : event;
56472
+ }
56473
+ if (event.kind === TrafficEventKinds["ai-user-fetch"]) {
56474
+ const provenance = aiUserFetchProvenance.get(verificationEventKey(event));
56475
+ return provenance ? { ...event, ...provenance } : event;
56476
+ }
56477
+ return event;
56478
+ });
55788
56479
  const response = {
55789
56480
  windowStart: sinceIso,
55790
56481
  windowEnd: untilIso,
@@ -55808,7 +56499,7 @@ async function trafficRoutes(app, opts) {
55808
56499
  returned: trimmed.length,
55809
56500
  truncated: trimmed.length < totalEventRows
55810
56501
  },
55811
- events: trimmed
56502
+ events: eventsWithProvenance
55812
56503
  };
55813
56504
  return response;
55814
56505
  });