@ainyc/canonry 4.163.1 → 4.164.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. package/README.md +120 -66
  2. package/assets/assets/{AuditHistoryPanel-CcoUJRWK.js → AuditHistoryPanel-m4hLLnEl.js} +1 -1
  3. package/assets/assets/{BacklinksPage-CPJcgZMH.js → BacklinksPage-DWl04vdm.js} +1 -1
  4. package/assets/assets/{HistoryPage-CA5hKRuu.js → HistoryPage-BTb0MBB4.js} +1 -1
  5. package/assets/assets/{MeasurementPropertyPage-DxrhxQ0q.js → MeasurementPropertyPage-C8c4huVf.js} +1 -1
  6. package/assets/assets/{ProjectPage-CHhf6m-z.js → ProjectPage-CEbscgtB.js} +1 -1
  7. package/assets/assets/{RunRow-D_8fID7_.js → RunRow-CDzkoy89.js} +1 -1
  8. package/assets/assets/{RunsPage-vSxIzGSk.js → RunsPage-CzLCTfIA.js} +1 -1
  9. package/assets/assets/{SettingsPage-CooTu-Qq.js → SettingsPage-D8Vfdd2n.js} +1 -1
  10. package/assets/assets/{SiteHealthSection-B84ag_7p.js → SiteHealthSection-BGmBOTo0.js} +3 -3
  11. package/assets/assets/{TrafficPage-CXLxgcDy.js → TrafficPage-BcmemZJB.js} +1 -1
  12. package/assets/assets/{TrafficSourceDetailPage-DH47U88b.js → TrafficSourceDetailPage-B9Dz1vcn.js} +1 -1
  13. package/assets/assets/{extract-error-message-DclWcEAn.js → extract-error-message-B5tyCXha.js} +1 -1
  14. package/assets/assets/{index-DH8-WPHn.js → index-CEjDqng5.js} +19 -19
  15. package/assets/assets/{react-sigma_core.esm.min-DXADa_zN.js → react-sigma_core.esm.min-z4XxmpWJ.js} +1 -1
  16. package/assets/index.html +1 -1
  17. package/dist/{chunk-INXI6PO7.js → chunk-76DCR646.js} +1 -1
  18. package/dist/{chunk-ESH3LPYQ.js → chunk-DCCEKIJ7.js} +28 -0
  19. package/dist/{chunk-G5UZTODF.js → chunk-R6QOYKRW.js} +805 -105
  20. package/dist/{chunk-MRHYSEZB.js → chunk-XYZMJA3L.js} +66 -62
  21. package/dist/cli.js +23 -6
  22. package/dist/index.js +4 -4
  23. package/dist/{intelligence-service-AFAFGLCM.js → intelligence-service-5UVHKSHR.js} +2 -2
  24. package/dist/mcp.js +2 -2
  25. package/package.json +10 -10
@@ -581,7 +581,7 @@ import {
581
581
  wordpressSchemaDeployResultDtoSchema,
582
582
  wordpressSchemaStatusResultDtoSchema,
583
583
  wordpressStatusDtoSchema
584
- } from "./chunk-ESH3LPYQ.js";
584
+ } from "./chunk-DCCEKIJ7.js";
585
585
 
586
586
  // src/intelligence-service.ts
587
587
  import { eq as eq59, desc as desc26, asc as asc11, and as and47, ne as ne7, or as or13, inArray as inArray21, gte as gte14, lte as lte12 } from "drizzle-orm";
@@ -608,6 +608,7 @@ __export(schema_exports, {
608
608
  agentToolEvents: () => agentToolEvents,
609
609
  aiReferralEventsHourly: () => aiReferralEventsHourly,
610
610
  aiUserFetchEventsHourly: () => aiUserFetchEventsHourly,
611
+ aiUserFetchVerificationManifestsHourly: () => aiUserFetchVerificationManifestsHourly,
611
612
  apiKeys: () => apiKeys,
612
613
  auditLog: () => auditLog,
613
614
  backlinkDomains: () => backlinkDomains,
@@ -620,6 +621,7 @@ __export(schema_exports, {
620
621
  competitors: () => competitors,
621
622
  contentTargetDismissals: () => contentTargetDismissals,
622
623
  crawlerEventsHourly: () => crawlerEventsHourly,
624
+ crawlerVerificationManifestsHourly: () => crawlerVerificationManifestsHourly,
623
625
  discoveryProbes: () => discoveryProbes,
624
626
  discoverySessions: () => discoverySessions,
625
627
  doctorHealthState: () => doctorHealthState,
@@ -2242,6 +2244,55 @@ var crawlerEventsHourly = sqliteTable("crawler_events_hourly", {
2242
2244
  index("idx_crawler_hourly_project_ts").on(table.projectId, table.tsHour),
2243
2245
  index("idx_crawler_hourly_path").on(table.projectId, table.pathNormalized)
2244
2246
  ]);
2247
+ var crawlerVerificationManifestsHourly = sqliteTable("crawler_verification_manifests_hourly", {
2248
+ projectId: text("project_id").notNull(),
2249
+ sourceId: text("source_id").notNull(),
2250
+ tsHour: text("ts_hour").notNull(),
2251
+ botId: text("bot_id").notNull(),
2252
+ verificationStatus: text("verification_status").notNull(),
2253
+ pathNormalized: text("path_normalized").notNull(),
2254
+ status: integer("status").notNull(),
2255
+ manifestId: text("manifest_id").notNull(),
2256
+ manifestJson: text("manifest_json", { mode: "json" }).$type(),
2257
+ hits: integer("hits").notNull().default(0),
2258
+ createdAt: text("created_at").notNull(),
2259
+ updatedAt: text("updated_at").notNull()
2260
+ }, (table) => [
2261
+ primaryKey({
2262
+ columns: [
2263
+ table.projectId,
2264
+ table.sourceId,
2265
+ table.tsHour,
2266
+ table.botId,
2267
+ table.verificationStatus,
2268
+ table.pathNormalized,
2269
+ table.status,
2270
+ table.manifestId
2271
+ ]
2272
+ }),
2273
+ foreignKey({
2274
+ name: "crawler_verification_manifests_parent_fk",
2275
+ columns: [
2276
+ table.projectId,
2277
+ table.sourceId,
2278
+ table.tsHour,
2279
+ table.botId,
2280
+ table.verificationStatus,
2281
+ table.pathNormalized,
2282
+ table.status
2283
+ ],
2284
+ foreignColumns: [
2285
+ crawlerEventsHourly.projectId,
2286
+ crawlerEventsHourly.sourceId,
2287
+ crawlerEventsHourly.tsHour,
2288
+ crawlerEventsHourly.botId,
2289
+ crawlerEventsHourly.verificationStatus,
2290
+ crawlerEventsHourly.pathNormalized,
2291
+ crawlerEventsHourly.status
2292
+ ]
2293
+ }).onDelete("cascade"),
2294
+ index("idx_crawler_verification_manifests_project_ts").on(table.projectId, table.tsHour)
2295
+ ]);
2245
2296
  var aiUserFetchEventsHourly = sqliteTable("ai_user_fetch_events_hourly", {
2246
2297
  projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
2247
2298
  sourceId: text("source_id").notNull().references(() => trafficSources.id, { onDelete: "cascade" }),
@@ -2270,6 +2321,55 @@ var aiUserFetchEventsHourly = sqliteTable("ai_user_fetch_events_hourly", {
2270
2321
  index("idx_ai_user_fetch_hourly_project_ts").on(table.projectId, table.tsHour),
2271
2322
  index("idx_ai_user_fetch_hourly_path").on(table.projectId, table.pathNormalized)
2272
2323
  ]);
2324
+ var aiUserFetchVerificationManifestsHourly = sqliteTable("ai_user_fetch_verification_manifests_hourly", {
2325
+ projectId: text("project_id").notNull(),
2326
+ sourceId: text("source_id").notNull(),
2327
+ tsHour: text("ts_hour").notNull(),
2328
+ botId: text("bot_id").notNull(),
2329
+ verificationStatus: text("verification_status").notNull(),
2330
+ pathNormalized: text("path_normalized").notNull(),
2331
+ status: integer("status").notNull(),
2332
+ manifestId: text("manifest_id").notNull(),
2333
+ manifestJson: text("manifest_json", { mode: "json" }).$type(),
2334
+ hits: integer("hits").notNull().default(0),
2335
+ createdAt: text("created_at").notNull(),
2336
+ updatedAt: text("updated_at").notNull()
2337
+ }, (table) => [
2338
+ primaryKey({
2339
+ columns: [
2340
+ table.projectId,
2341
+ table.sourceId,
2342
+ table.tsHour,
2343
+ table.botId,
2344
+ table.verificationStatus,
2345
+ table.pathNormalized,
2346
+ table.status,
2347
+ table.manifestId
2348
+ ]
2349
+ }),
2350
+ foreignKey({
2351
+ name: "ai_user_fetch_verification_manifests_parent_fk",
2352
+ columns: [
2353
+ table.projectId,
2354
+ table.sourceId,
2355
+ table.tsHour,
2356
+ table.botId,
2357
+ table.verificationStatus,
2358
+ table.pathNormalized,
2359
+ table.status
2360
+ ],
2361
+ foreignColumns: [
2362
+ aiUserFetchEventsHourly.projectId,
2363
+ aiUserFetchEventsHourly.sourceId,
2364
+ aiUserFetchEventsHourly.tsHour,
2365
+ aiUserFetchEventsHourly.botId,
2366
+ aiUserFetchEventsHourly.verificationStatus,
2367
+ aiUserFetchEventsHourly.pathNormalized,
2368
+ aiUserFetchEventsHourly.status
2369
+ ]
2370
+ }).onDelete("cascade"),
2371
+ index("idx_ai_user_fetch_verification_manifests_project_ts").on(table.projectId, table.tsHour)
2372
+ ]);
2273
2373
  var aiReferralEventsHourly = sqliteTable("ai_referral_events_hourly", {
2274
2374
  projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
2275
2375
  sourceId: text("source_id").notNull().references(() => trafficSources.id, { onDelete: "cascade" }),
@@ -6264,6 +6364,71 @@ var MIGRATION_VERSIONS = [
6264
6364
  `ALTER TABLE site_crawl_edges ADD COLUMN placement_unknown_occurrences INTEGER`,
6265
6365
  `ALTER TABLE site_crawl_snapshots ADD COLUMN link_placement_ruleset_version TEXT`
6266
6366
  ]
6367
+ },
6368
+ {
6369
+ version: 139,
6370
+ name: "traffic-verification-manifest-provenance",
6371
+ // Verification is evidence produced by one exact range manifest. Store
6372
+ // that provenance in additive sidecars: the existing rollup keys stay
6373
+ // unchanged so an older binary can still upsert them after a rollback.
6374
+ // Historical rows deliberately receive no sidecar row because the old
6375
+ // schema never recorded which bundle classified them.
6376
+ statements: [
6377
+ `CREATE TABLE IF NOT EXISTS crawler_verification_manifests_hourly (
6378
+ project_id TEXT NOT NULL,
6379
+ source_id TEXT NOT NULL,
6380
+ ts_hour TEXT NOT NULL,
6381
+ bot_id TEXT NOT NULL,
6382
+ verification_status TEXT NOT NULL,
6383
+ path_normalized TEXT NOT NULL,
6384
+ status INTEGER NOT NULL,
6385
+ manifest_id TEXT NOT NULL,
6386
+ manifest_json TEXT,
6387
+ hits INTEGER NOT NULL DEFAULT 0,
6388
+ created_at TEXT NOT NULL,
6389
+ updated_at TEXT NOT NULL,
6390
+ PRIMARY KEY (
6391
+ project_id, source_id, ts_hour, bot_id, verification_status,
6392
+ path_normalized, status, manifest_id
6393
+ ),
6394
+ FOREIGN KEY (
6395
+ project_id, source_id, ts_hour, bot_id, verification_status,
6396
+ path_normalized, status
6397
+ ) REFERENCES crawler_events_hourly (
6398
+ project_id, source_id, ts_hour, bot_id, verification_status,
6399
+ path_normalized, status
6400
+ ) ON DELETE CASCADE
6401
+ )`,
6402
+ `CREATE INDEX IF NOT EXISTS idx_crawler_verification_manifests_project_ts
6403
+ ON crawler_verification_manifests_hourly(project_id, ts_hour)`,
6404
+ `CREATE TABLE IF NOT EXISTS ai_user_fetch_verification_manifests_hourly (
6405
+ project_id TEXT NOT NULL,
6406
+ source_id TEXT NOT NULL,
6407
+ ts_hour TEXT NOT NULL,
6408
+ bot_id TEXT NOT NULL,
6409
+ verification_status TEXT NOT NULL,
6410
+ path_normalized TEXT NOT NULL,
6411
+ status INTEGER NOT NULL,
6412
+ manifest_id TEXT NOT NULL,
6413
+ manifest_json TEXT,
6414
+ hits INTEGER NOT NULL DEFAULT 0,
6415
+ created_at TEXT NOT NULL,
6416
+ updated_at TEXT NOT NULL,
6417
+ PRIMARY KEY (
6418
+ project_id, source_id, ts_hour, bot_id, verification_status,
6419
+ path_normalized, status, manifest_id
6420
+ ),
6421
+ FOREIGN KEY (
6422
+ project_id, source_id, ts_hour, bot_id, verification_status,
6423
+ path_normalized, status
6424
+ ) REFERENCES ai_user_fetch_events_hourly (
6425
+ project_id, source_id, ts_hour, bot_id, verification_status,
6426
+ path_normalized, status
6427
+ ) ON DELETE CASCADE
6428
+ )`,
6429
+ `CREATE INDEX IF NOT EXISTS idx_ai_user_fetch_verification_manifests_project_ts
6430
+ ON ai_user_fetch_verification_manifests_hourly(project_id, ts_hour)`
6431
+ ]
6267
6432
  }
6268
6433
  ];
6269
6434
  function addRunsMeasurementPlanVersionForeignKey(tx) {
@@ -32820,6 +32985,7 @@ var routeCatalog = [
32820
32985
  method: "post",
32821
32986
  path: "/api/v1/projects/{name}/ga/sync",
32822
32987
  summary: "Sync GA4 traffic and AI referral data",
32988
+ description: "Syncs a window of GA4 history. `days` is bounded to GA4's supported sync range (1-90); a request outside it is clamped rather than rejected. The response reports the window ACTUALLY written as `days`, the unbounded request as `requestedDays`, and sets `clamped` when the two differ.",
32823
32989
  tags: ["ga4"],
32824
32990
  parameters: [nameParameter],
32825
32991
  requestBody: {
@@ -32829,7 +32995,7 @@ var routeCatalog = [
32829
32995
  schema: {
32830
32996
  type: "object",
32831
32997
  properties: {
32832
- days: integerSchema
32998
+ days: { ...integerSchema, description: "Days of history to sync. Clamped to 1-90; check `clamped` in the response to detect truncation. Defaults to 30." }
32833
32999
  }
32834
33000
  }
32835
33001
  }
@@ -35952,6 +36118,11 @@ var GA4_SCOPE = "https://www.googleapis.com/auth/analytics.readonly";
35952
36118
  var GOOGLE_TOKEN_URL2 = "https://oauth2.googleapis.com/token";
35953
36119
  var GA4_DEFAULT_SYNC_DAYS = 30;
35954
36120
  var GA4_MAX_SYNC_DAYS = 90;
36121
+ function resolveGa4SyncDays(days) {
36122
+ const requestedDays = days != null && Number.isFinite(days) ? Math.trunc(days) : GA4_DEFAULT_SYNC_DAYS;
36123
+ const effectiveDays = Math.min(Math.max(1, requestedDays), GA4_MAX_SYNC_DAYS);
36124
+ return { requestedDays, effectiveDays, clamped: effectiveDays !== requestedDays };
36125
+ }
35955
36126
  var GA4_REQUEST_TIMEOUT_MS = 3e4;
35956
36127
  var GA4_MAX_PAGES = 50;
35957
36128
  var GA4_MAX_CONCURRENT_REQUESTS = 4;
@@ -36270,7 +36441,7 @@ function formatDate2(d) {
36270
36441
  return d.toISOString().slice(0, 10);
36271
36442
  }
36272
36443
  function measurementDateRange(days) {
36273
- const syncDays = Math.min(Math.max(1, days ?? GA4_DEFAULT_SYNC_DAYS), GA4_MAX_SYNC_DAYS);
36444
+ const { effectiveDays: syncDays } = resolveGa4SyncDays(days);
36274
36445
  const endDate = /* @__PURE__ */ new Date();
36275
36446
  const startDate = /* @__PURE__ */ new Date();
36276
36447
  startDate.setUTCDate(startDate.getUTCDate() - syncDays);
@@ -36378,7 +36549,7 @@ var AI_REFERRAL_SOURCE_FILTERS = [
36378
36549
  async function fetchTrafficByLandingPage(accessToken, propertyId, days) {
36379
36550
  validateAccessToken2(accessToken);
36380
36551
  validatePropertyId(propertyId);
36381
- const syncDays = Math.min(Math.max(1, days ?? GA4_DEFAULT_SYNC_DAYS), GA4_MAX_SYNC_DAYS);
36552
+ const { effectiveDays: syncDays } = resolveGa4SyncDays(days);
36382
36553
  const endDate = /* @__PURE__ */ new Date();
36383
36554
  const startDate = /* @__PURE__ */ new Date();
36384
36555
  startDate.setDate(startDate.getDate() - syncDays);
@@ -36510,7 +36681,7 @@ async function verifyConnectionWithToken(accessToken, propertyId) {
36510
36681
  async function fetchAggregateSummary(accessToken, propertyId, days) {
36511
36682
  validateAccessToken2(accessToken);
36512
36683
  validatePropertyId(propertyId);
36513
- const syncDays = Math.min(Math.max(1, days ?? GA4_DEFAULT_SYNC_DAYS), GA4_MAX_SYNC_DAYS);
36684
+ const { effectiveDays: syncDays } = resolveGa4SyncDays(days);
36514
36685
  const endDate = /* @__PURE__ */ new Date();
36515
36686
  const startDate = /* @__PURE__ */ new Date();
36516
36687
  startDate.setDate(startDate.getDate() - syncDays);
@@ -36560,7 +36731,7 @@ function parseOptionalCountMetric(value) {
36560
36731
  async function fetchDailyTotals(accessToken, propertyId, days) {
36561
36732
  validateAccessToken2(accessToken);
36562
36733
  validatePropertyId(propertyId);
36563
- const syncDays = Math.min(Math.max(1, days ?? GA4_DEFAULT_SYNC_DAYS), GA4_MAX_SYNC_DAYS);
36734
+ const { effectiveDays: syncDays } = resolveGa4SyncDays(days);
36564
36735
  const endDate = /* @__PURE__ */ new Date();
36565
36736
  const startDate = /* @__PURE__ */ new Date();
36566
36737
  startDate.setDate(startDate.getDate() - syncDays);
@@ -36653,7 +36824,7 @@ async function fetchWindowSummary(accessToken, propertyId, windowKey) {
36653
36824
  async function fetchAiReferrals(accessToken, propertyId, days) {
36654
36825
  validateAccessToken2(accessToken);
36655
36826
  validatePropertyId(propertyId);
36656
- const syncDays = Math.min(Math.max(1, days ?? GA4_DEFAULT_SYNC_DAYS), GA4_MAX_SYNC_DAYS);
36827
+ const { effectiveDays: syncDays } = resolveGa4SyncDays(days);
36657
36828
  const endDate = /* @__PURE__ */ new Date();
36658
36829
  const startDate = /* @__PURE__ */ new Date();
36659
36830
  startDate.setDate(startDate.getDate() - syncDays);
@@ -36746,7 +36917,7 @@ var SOCIAL_CHANNEL_GROUPS = ["Organic Social", "Paid Social"];
36746
36917
  async function fetchSocialReferrals(accessToken, propertyId, days) {
36747
36918
  validateAccessToken2(accessToken);
36748
36919
  validatePropertyId(propertyId);
36749
- const syncDays = Math.min(Math.max(1, days ?? GA4_DEFAULT_SYNC_DAYS), GA4_MAX_SYNC_DAYS);
36920
+ const { effectiveDays: syncDays } = resolveGa4SyncDays(days);
36750
36921
  const endDate = /* @__PURE__ */ new Date();
36751
36922
  const startDate = /* @__PURE__ */ new Date();
36752
36923
  startDate.setDate(startDate.getDate() - syncDays);
@@ -45044,7 +45215,7 @@ async function ga4Routes(app, opts) {
45044
45215
  });
45045
45216
  app.post("/projects/:name/ga/sync", async (request, _reply) => {
45046
45217
  const project = resolveProject(app.db, request.params.name);
45047
- const days = request.body?.days ?? 30;
45218
+ const { requestedDays, effectiveDays: days, clamped } = resolveGa4SyncDays(request.body?.days);
45048
45219
  const only = request.body?.only;
45049
45220
  const validOnlyValues = ["traffic", "ai", "social"];
45050
45221
  if (only !== void 0 && !validOnlyValues.includes(only)) {
@@ -45055,8 +45226,8 @@ async function ga4Routes(app, opts) {
45055
45226
  const syncAi = !only || only === "ai";
45056
45227
  const syncSocial = !only || only === "social";
45057
45228
  const measurementState = app.db.select().from(gaMeasurementSyncStates).where(eq44(gaMeasurementSyncStates.projectId, project.id)).get();
45058
- const acquisitionDays = measurementState?.acquisitionSyncedAt != null ? Math.min(Math.max(1, days), 90) : 90;
45059
- const leadDays = measurementState?.leadSyncedAt != null ? Math.min(Math.max(1, days), 90) : 90;
45229
+ const acquisitionDays = measurementState?.acquisitionSyncedAt != null ? days : GA4_MAX_SYNC_DAYS;
45230
+ const leadDays = measurementState?.leadSyncedAt != null ? days : GA4_MAX_SYNC_DAYS;
45060
45231
  const leadEventNames = project.measurement.leadEventNames;
45061
45232
  const startedAt = (/* @__PURE__ */ new Date()).toISOString();
45062
45233
  const runId = crypto35.randomUUID();
@@ -45319,6 +45490,7 @@ async function ga4Routes(app, opts) {
45319
45490
  aiReferralCount: aiReferrals.length,
45320
45491
  socialReferralCount: socialReferrals.length,
45321
45492
  days,
45493
+ ...clamped ? { requestedDays, clamped } : {},
45322
45494
  totalUsers: summary.totalUsers,
45323
45495
  ...only ? { only } : {}
45324
45496
  });
@@ -45328,6 +45500,8 @@ async function ga4Routes(app, opts) {
45328
45500
  aiReferralCount: aiReferrals.length,
45329
45501
  socialReferralCount: socialReferrals.length,
45330
45502
  days,
45503
+ requestedDays,
45504
+ clamped,
45331
45505
  syncedAt: now,
45332
45506
  measurement: {
45333
45507
  acquisition: acquisitionResult,
@@ -48518,13 +48692,78 @@ async function listCloudRunTrafficEvents(accessToken, options) {
48518
48692
 
48519
48693
  // ../integration-traffic/src/ip-ranges/anthropic.json
48520
48694
  var anthropic_default = {
48521
- _source: "ARIN RDAP \u2014 every network registered to Anthropic, PBC (handle AP-2440)",
48522
- _query: "https://rdap.arin.net/registry/entity/AP-2440",
48523
- _notes: "Anthropic does not publish a clean machine-readable IP-range JSON like Google or OpenAI do. These prefixes are taken straight from ARIN's authoritative allocation records (NOT bgp.tools' AS-based view, which can include misleading announcements \u2014 e.g. AS60808 is registered to Anthropic but announces 209.249.57.0/24 which actually belongs to Mitel Networks). 216.73.216.0/22 (AWS-ANTHROPIC) is the load-bearing prefix \u2014 empirical Cloud Run logs show 100% of real ClaudeBot traffic comes from there; the other two are Anthropic's own allocations for non-crawler infra (kept since the verification gate still requires a Claude-* UA match). Refresh by re-running the ARIN RDAP query above and replacing the prefix list.",
48695
+ _source: "https://claude.com/crawling/bots.json",
48696
+ creationTime: "2026-08-13T20:38:01Z",
48524
48697
  prefixes: [
48525
- { ipv4Prefix: "216.73.216.0/22" },
48526
- { ipv4Prefix: "160.79.104.0/21" },
48527
- { ipv6Prefix: "2607:6bc0::/32" }
48698
+ {
48699
+ ipv4Prefix: "216.73.216.0/22"
48700
+ },
48701
+ {
48702
+ ipv4Prefix: "34.162.230.222/32"
48703
+ },
48704
+ {
48705
+ ipv4Prefix: "34.162.244.71/32"
48706
+ },
48707
+ {
48708
+ ipv4Prefix: "34.162.191.81/32"
48709
+ },
48710
+ {
48711
+ ipv4Prefix: "34.150.241.79/32"
48712
+ },
48713
+ {
48714
+ ipv4Prefix: "34.85.172.162/32"
48715
+ },
48716
+ {
48717
+ ipv4Prefix: "35.245.175.129/32"
48718
+ },
48719
+ {
48720
+ ipv4Prefix: "34.182.222.37/32"
48721
+ },
48722
+ {
48723
+ ipv4Prefix: "34.186.108.163/32"
48724
+ },
48725
+ {
48726
+ ipv4Prefix: "35.245.89.239/32"
48727
+ },
48728
+ {
48729
+ ipv4Prefix: "34.182.161.143/32"
48730
+ },
48731
+ {
48732
+ ipv4Prefix: "34.182.218.27/32"
48733
+ },
48734
+ {
48735
+ ipv4Prefix: "34.182.220.85/32"
48736
+ },
48737
+ {
48738
+ ipv4Prefix: "34.11.34.31/32"
48739
+ },
48740
+ {
48741
+ ipv4Prefix: "34.182.140.95/32"
48742
+ },
48743
+ {
48744
+ ipv4Prefix: "136.107.176.208/32"
48745
+ },
48746
+ {
48747
+ ipv4Prefix: "34.182.226.151/32"
48748
+ },
48749
+ {
48750
+ ipv4Prefix: "34.182.226.221/32"
48751
+ },
48752
+ {
48753
+ ipv4Prefix: "35.221.29.174/32"
48754
+ },
48755
+ {
48756
+ ipv4Prefix: "34.182.225.167/32"
48757
+ },
48758
+ {
48759
+ ipv4Prefix: "40.124.101.48/28"
48760
+ },
48761
+ {
48762
+ ipv4Prefix: "20.102.46.224/28"
48763
+ },
48764
+ {
48765
+ ipv4Prefix: "20.64.57.208/28"
48766
+ }
48528
48767
  ]
48529
48768
  };
48530
48769
 
@@ -50623,6 +50862,141 @@ var perplexitybot_default = {
50623
50862
  ]
50624
50863
  };
50625
50864
 
50865
+ // ../integration-traffic/src/ip-range-manifest.ts
50866
+ import { createHash as createHash4 } from "crypto";
50867
+ function isRecord(value) {
50868
+ return typeof value === "object" && value !== null && !Array.isArray(value);
50869
+ }
50870
+ function stableJson2(value, seen = /* @__PURE__ */ new Set()) {
50871
+ if (value === null) return "null";
50872
+ if (typeof value === "string" || typeof value === "boolean") return JSON.stringify(value);
50873
+ if (typeof value === "number") {
50874
+ return Number.isFinite(value) ? JSON.stringify(value) : JSON.stringify(String(value));
50875
+ }
50876
+ if (typeof value !== "object") return JSON.stringify(`[${typeof value}]`);
50877
+ if (seen.has(value)) return '"[circular]"';
50878
+ seen.add(value);
50879
+ const result = Array.isArray(value) ? `[${value.map((entry) => stableJson2(entry, seen)).join(",")}]` : `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableJson2(value[key], seen)}`).join(",")}}`;
50880
+ seen.delete(value);
50881
+ return result;
50882
+ }
50883
+ function parseIp(ip) {
50884
+ if (typeof ip !== "string" || ip.length === 0) return null;
50885
+ const mappedMatch = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/i.exec(ip);
50886
+ if (mappedMatch) return parseIp(mappedMatch[1]);
50887
+ if (ip.includes(":")) {
50888
+ const sides = ip.split("::");
50889
+ if (sides.length > 2) return null;
50890
+ const left = sides[0].length > 0 ? sides[0].split(":") : [];
50891
+ const right = sides.length === 2 && sides[1].length > 0 ? sides[1].split(":") : [];
50892
+ const groupCount = left.length + right.length;
50893
+ if (sides.length === 1 && groupCount !== 8) return null;
50894
+ if (sides.length === 2 && groupCount >= 8) return null;
50895
+ const fill = 8 - groupCount;
50896
+ const groups = [...left, ...new Array(fill).fill("0"), ...right];
50897
+ let addr2 = 0n;
50898
+ for (const group of groups) {
50899
+ if (!/^[0-9a-f]{1,4}$/i.test(group)) return null;
50900
+ addr2 = addr2 << 16n | BigInt(Number.parseInt(group, 16));
50901
+ }
50902
+ return { version: 6, addr: addr2 };
50903
+ }
50904
+ const octets = ip.split(".");
50905
+ if (octets.length !== 4) return null;
50906
+ let addr = 0n;
50907
+ for (const octet of octets) {
50908
+ if (!/^(?:0|[1-9]\d{0,2})$/.test(octet)) return null;
50909
+ const value = Number(octet);
50910
+ if (value > 255) return null;
50911
+ addr = addr << 8n | BigInt(value);
50912
+ }
50913
+ return { version: 4, addr };
50914
+ }
50915
+ function parseCidr(cidr) {
50916
+ if (typeof cidr !== "string") return null;
50917
+ const slash = cidr.indexOf("/");
50918
+ if (slash <= 0 || slash !== cidr.lastIndexOf("/") || slash === cidr.length - 1) return null;
50919
+ const ipPart = cidr.slice(0, slash);
50920
+ const prefixStr = cidr.slice(slash + 1);
50921
+ if (!/^(?:0|[1-9]\d{0,2})$/.test(prefixStr)) return null;
50922
+ const parsed = parseIp(ipPart);
50923
+ if (!parsed) return null;
50924
+ const prefix = Number(prefixStr);
50925
+ const totalBits = parsed.version === 4 ? 32 : 128;
50926
+ if (prefix > totalBits) return null;
50927
+ const allOnes = (1n << BigInt(totalBits)) - 1n;
50928
+ const mask = allOnes >> BigInt(totalBits - prefix) << BigInt(totalBits - prefix);
50929
+ return {
50930
+ version: parsed.version,
50931
+ network: parsed.addr & mask,
50932
+ mask
50933
+ };
50934
+ }
50935
+ function canonicalPrefixKey(entry) {
50936
+ if (isRecord(entry)) {
50937
+ const hasIpv4 = Object.hasOwn(entry, "ipv4Prefix");
50938
+ const hasIpv6 = Object.hasOwn(entry, "ipv6Prefix");
50939
+ if (hasIpv4 !== hasIpv6) {
50940
+ const value = entry[hasIpv4 ? "ipv4Prefix" : "ipv6Prefix"];
50941
+ if (typeof value === "string") {
50942
+ const parsed = parseCidr(value);
50943
+ const expectedVersion = hasIpv4 ? 4 : 6;
50944
+ if (parsed?.version === expectedVersion) {
50945
+ const hexWidth = parsed.version === 4 ? 8 : 32;
50946
+ const network = parsed.network.toString(16).padStart(hexWidth, "0");
50947
+ const mask = parsed.mask.toString(16).padStart(hexWidth, "0");
50948
+ return `cidr:${parsed.version}:${network}:${mask}`;
50949
+ }
50950
+ }
50951
+ }
50952
+ }
50953
+ return `invalid:${stableJson2(entry)}`;
50954
+ }
50955
+ function ipRangeManifestContentHash(value) {
50956
+ const prefixes = isRecord(value) ? value.prefixes : void 0;
50957
+ const canonical = Array.isArray(prefixes) ? prefixes.map(canonicalPrefixKey).sort() : [`invalid-prefixes:${stableJson2(prefixes)}`];
50958
+ return createHash4("sha256").update(JSON.stringify(canonical)).digest("hex");
50959
+ }
50960
+ function validateIpRangeManifestPayload(value) {
50961
+ if (!isRecord(value)) return { ok: false, error: "response must be a JSON object" };
50962
+ const creationTime = value.creationTime;
50963
+ if (typeof creationTime !== "string" || creationTime.trim().length === 0) {
50964
+ return { ok: false, error: "response missing `creationTime` version" };
50965
+ }
50966
+ if (!Array.isArray(value.prefixes)) {
50967
+ return { ok: false, error: "response missing `prefixes` array" };
50968
+ }
50969
+ const prefixes = [];
50970
+ for (const [index2, entry] of value.prefixes.entries()) {
50971
+ if (!isRecord(entry)) {
50972
+ return { ok: false, error: `prefixes[${index2}] must be an object` };
50973
+ }
50974
+ const hasIpv4 = Object.hasOwn(entry, "ipv4Prefix");
50975
+ const hasIpv6 = Object.hasOwn(entry, "ipv6Prefix");
50976
+ if (hasIpv4 === hasIpv6) {
50977
+ return { ok: false, error: `prefixes[${index2}] must contain exactly one IP prefix` };
50978
+ }
50979
+ const field = hasIpv4 ? "ipv4Prefix" : "ipv6Prefix";
50980
+ const cidr = entry[field];
50981
+ if (typeof cidr !== "string") {
50982
+ return { ok: false, error: `prefixes[${index2}].${field} must be a string` };
50983
+ }
50984
+ const parsed = parseCidr(cidr);
50985
+ const expectedVersion = hasIpv4 ? 4 : 6;
50986
+ if (!parsed || parsed.version !== expectedVersion) {
50987
+ return { ok: false, error: `prefixes[${index2}].${field} is not a valid IPv${expectedVersion} CIDR` };
50988
+ }
50989
+ prefixes.push(hasIpv4 ? { ipv4Prefix: cidr } : { ipv6Prefix: cidr });
50990
+ }
50991
+ return {
50992
+ ok: true,
50993
+ value: {
50994
+ creationTime: creationTime.trim(),
50995
+ prefixes
50996
+ }
50997
+ };
50998
+ }
50999
+
50626
51000
  // ../integration-traffic/src/ip-verify.ts
50627
51001
  var RULE_ID_TO_RANGES = {
50628
51002
  // OpenAI — three separate published lists (training crawler vs
@@ -50656,94 +51030,56 @@ var RULE_ID_TO_RANGES = {
50656
51030
  "perplexity-bot": perplexitybot_default,
50657
51031
  // src: https://www.perplexity.ai/perplexity-user.json
50658
51032
  "perplexity-user": perplexity_user_default,
50659
- // Anthropic — no machine-readable JSON published. The bundled
50660
- // anthropic.json is the set of networks registered to Anthropic,
50661
- // PBC at ARIN (the authoritative allocation record). Maintained by
50662
- // hand; refresh by re-querying the ARIN entity below. The crawler
50663
- // block is AWS-ANTHROPIC 216.73.216.0/22 — empirical Cloud Run
50664
- // logs show all real ClaudeBot traffic comes from there. The same
50665
- // raw set is shared across every Claude-* UA the classifier emits:
50666
- // both the training crawler and the per-user fetcher map here.
50667
- // src: https://rdap.arin.net/registry/entity/AP-2440
51033
+ // Anthropic publishes one shared crawler-origin manifest. It confirms
51034
+ // Anthropic origin but does not attribute a prefix to ClaudeBot,
51035
+ // Claude-SearchBot, or Claude-User individually, so both classifier rules
51036
+ // map to the same file and the UA remains the product discriminator.
51037
+ // src: https://claude.com/crawling/bots.json
50668
51038
  "anthropic-claudebot": anthropic_default,
50669
51039
  "claude-user": anthropic_default
50670
51040
  };
51041
+ function manifestMetadata(raw) {
51042
+ if (typeof raw !== "object" || raw === null || Array.isArray(raw)) return null;
51043
+ const record = raw;
51044
+ const source = typeof record._source === "string" ? record._source.trim() : "";
51045
+ const version = typeof record.creationTime === "string" ? record.creationTime.trim() : "";
51046
+ if (!source || !version) return null;
51047
+ const contentHash2 = ipRangeManifestContentHash(raw);
51048
+ return { id: `${source}#${version}#sha256:${contentHash2}`, source, version };
51049
+ }
51050
+ function verificationDataFromRaw(raw) {
51051
+ const validation = validateIpRangeManifestPayload(raw);
51052
+ const ranges = [];
51053
+ if (validation.ok) {
51054
+ for (const entry of validation.value.prefixes) {
51055
+ const parsed = parseCidr(entry.ipv4Prefix ?? entry.ipv6Prefix);
51056
+ if (parsed) ranges.push(parsed);
51057
+ }
51058
+ }
51059
+ return { ranges, manifest: manifestMetadata(raw) };
51060
+ }
51061
+ function decideIpVerification(ip, data) {
51062
+ if (!ip || data.ranges.length === 0) return { verified: false, manifest: null };
51063
+ const parsed = parseIp(ip);
51064
+ if (!parsed) return { verified: false, manifest: null };
51065
+ for (const cidr of data.ranges) {
51066
+ if (parsed.version !== cidr.version) continue;
51067
+ if ((parsed.addr & cidr.mask) === cidr.network) {
51068
+ return { verified: true, manifest: data.manifest };
51069
+ }
51070
+ }
51071
+ return { verified: false, manifest: data.manifest };
51072
+ }
50671
51073
  var CACHE = (() => {
50672
51074
  const cache = /* @__PURE__ */ new Map();
50673
51075
  for (const [ruleId, raw] of Object.entries(RULE_ID_TO_RANGES)) {
50674
- const parsed = [];
50675
- for (const entry of raw.prefixes) {
50676
- const cidr = entry.ipv4Prefix ?? entry.ipv6Prefix;
50677
- if (!cidr) continue;
50678
- const p = parseCidr(cidr);
50679
- if (p) parsed.push(p);
50680
- }
50681
- cache.set(ruleId, parsed);
51076
+ cache.set(ruleId, verificationDataFromRaw(raw));
50682
51077
  }
50683
51078
  return cache;
50684
51079
  })();
50685
- function parseIp(ip) {
50686
- if (!ip) return null;
50687
- const mappedMatch = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/i.exec(ip);
50688
- if (mappedMatch) return parseIp(mappedMatch[1]);
50689
- if (ip.includes(":")) {
50690
- const sides = ip.split("::");
50691
- if (sides.length > 2) return null;
50692
- const left = sides[0].length > 0 ? sides[0].split(":") : [];
50693
- const right = sides.length === 2 && sides[1].length > 0 ? sides[1].split(":") : [];
50694
- const groupCount = left.length + right.length;
50695
- if (groupCount > 8) return null;
50696
- if (sides.length === 1 && groupCount !== 8) return null;
50697
- const fill = 8 - groupCount;
50698
- const groups = [...left, ...new Array(fill).fill("0"), ...right];
50699
- let addr2 = 0n;
50700
- for (const g of groups) {
50701
- if (g.length === 0 || g.length > 4) return null;
50702
- const n = Number.parseInt(g, 16);
50703
- if (!Number.isFinite(n) || n < 0 || n > 65535) return null;
50704
- addr2 = addr2 << 16n | BigInt(n);
50705
- }
50706
- return { version: 6, addr: addr2 };
50707
- }
50708
- const octets = ip.split(".");
50709
- if (octets.length !== 4) return null;
50710
- let addr = 0n;
50711
- for (const o of octets) {
50712
- if (o.length === 0 || o.length > 3) return null;
50713
- const n = Number.parseInt(o, 10);
50714
- if (!Number.isInteger(n) || n < 0 || n > 255) return null;
50715
- addr = addr << 8n | BigInt(n);
50716
- }
50717
- return { version: 4, addr };
50718
- }
50719
- function parseCidr(cidr) {
50720
- const [ipPart, prefixStr] = cidr.split("/");
50721
- if (!ipPart || !prefixStr) return null;
50722
- const prefix = Number.parseInt(prefixStr, 10);
50723
- if (!Number.isInteger(prefix)) return null;
50724
- const parsed = parseIp(ipPart);
50725
- if (!parsed) return null;
50726
- const totalBits = parsed.version === 4 ? 32 : 128;
50727
- if (prefix < 0 || prefix > totalBits) return null;
50728
- const allOnes = (1n << BigInt(totalBits)) - 1n;
50729
- const mask = allOnes >> BigInt(totalBits - prefix) << BigInt(totalBits - prefix);
50730
- return {
50731
- version: parsed.version,
50732
- network: parsed.addr & mask,
50733
- mask
50734
- };
50735
- }
50736
- function verifyIpForRule(ip, ruleId) {
50737
- if (!ip) return false;
50738
- const ranges = CACHE.get(ruleId);
50739
- if (!ranges || ranges.length === 0) return false;
50740
- const parsed = parseIp(ip);
50741
- if (!parsed) return false;
50742
- for (const cidr of ranges) {
50743
- if (parsed.version !== cidr.version) continue;
50744
- if ((parsed.addr & cidr.mask) === cidr.network) return true;
50745
- }
50746
- return false;
51080
+ function verifyIpForRuleDecision(ip, ruleId) {
51081
+ const data = CACHE.get(ruleId);
51082
+ return data ? decideIpVerification(ip, data) : { verified: false, manifest: null };
50747
51083
  }
50748
51084
 
50749
51085
  // ../integration-traffic/src/rules.ts
@@ -51113,13 +51449,14 @@ function classifyCrawler(event) {
51113
51449
  for (const rule of DEFAULT_AI_CRAWLER_RULES) {
51114
51450
  if (rule.purpose === USER_FETCH_PURPOSE) continue;
51115
51451
  if (rule.userAgentPatterns.some((pattern) => pattern.test(userAgent))) {
51116
- const verified = verifyIpForRule(event.remoteIp, rule.id);
51452
+ const decision = verifyIpForRuleDecision(event.remoteIp, rule.id);
51117
51453
  return {
51118
51454
  botId: rule.id,
51119
51455
  operator: rule.operator,
51120
51456
  product: rule.product,
51121
51457
  purpose: rule.purpose,
51122
- verificationStatus: verified ? "verified" : "claimed_unverified",
51458
+ verificationStatus: decision.verified ? "verified" : "claimed_unverified",
51459
+ verificationManifest: decision.manifest,
51123
51460
  matchedUserAgent: userAgent
51124
51461
  };
51125
51462
  }
@@ -51132,12 +51469,13 @@ function classifyAiUserFetch(event) {
51132
51469
  for (const rule of DEFAULT_AI_CRAWLER_RULES) {
51133
51470
  if (rule.purpose !== USER_FETCH_PURPOSE) continue;
51134
51471
  if (rule.userAgentPatterns.some((pattern) => pattern.test(userAgent))) {
51135
- const verified = verifyIpForRule(event.remoteIp, rule.id);
51472
+ const decision = verifyIpForRuleDecision(event.remoteIp, rule.id);
51136
51473
  return {
51137
51474
  botId: rule.id,
51138
51475
  operator: rule.operator,
51139
51476
  product: rule.product,
51140
- verificationStatus: verified ? "verified" : "claimed_unverified",
51477
+ verificationStatus: decision.verified ? "verified" : "claimed_unverified",
51478
+ verificationManifest: decision.manifest,
51141
51479
  matchedUserAgent: userAgent
51142
51480
  };
51143
51481
  }
@@ -51286,10 +51624,10 @@ function mergeReferralSession(current, next) {
51286
51624
  return { ...winner, trafficClass: paid ? AiReferralTrafficClasses.paid : winner.trafficClass };
51287
51625
  }
51288
51626
  function sortCrawlerBuckets(a, b) {
51289
- return a.tsHour.localeCompare(b.tsHour) || a.botId.localeCompare(b.botId) || a.pathNormalized.localeCompare(b.pathNormalized) || String(a.status).localeCompare(String(b.status));
51627
+ return a.tsHour.localeCompare(b.tsHour) || a.botId.localeCompare(b.botId) || a.verificationStatus.localeCompare(b.verificationStatus) || (a.verificationManifest?.id ?? "").localeCompare(b.verificationManifest?.id ?? "") || a.pathNormalized.localeCompare(b.pathNormalized) || String(a.status).localeCompare(String(b.status));
51290
51628
  }
51291
51629
  function sortAiUserFetchBuckets(a, b) {
51292
- return a.tsHour.localeCompare(b.tsHour) || a.botId.localeCompare(b.botId) || a.pathNormalized.localeCompare(b.pathNormalized) || String(a.status).localeCompare(String(b.status));
51630
+ return a.tsHour.localeCompare(b.tsHour) || a.botId.localeCompare(b.botId) || a.verificationStatus.localeCompare(b.verificationStatus) || (a.verificationManifest?.id ?? "").localeCompare(b.verificationManifest?.id ?? "") || a.pathNormalized.localeCompare(b.pathNormalized) || String(a.status).localeCompare(String(b.status));
51293
51631
  }
51294
51632
  function sortReferralBuckets(a, b) {
51295
51633
  return a.tsHour.localeCompare(b.tsHour) || a.product.localeCompare(b.product) || a.sourceDomain.localeCompare(b.sourceDomain) || a.landingPathNormalized.localeCompare(b.landingPathNormalized) || String(a.status).localeCompare(String(b.status));
@@ -51332,6 +51670,7 @@ function buildTrafficProbeReport(events, options = {}) {
51332
51670
  tsHour,
51333
51671
  crawler.botId,
51334
51672
  crawler.verificationStatus,
51673
+ crawler.verificationManifest?.id ?? "no-manifest",
51335
51674
  pathNormalized,
51336
51675
  event.status ?? "null"
51337
51676
  ].join(" ");
@@ -51345,6 +51684,7 @@ function buildTrafficProbeReport(events, options = {}) {
51345
51684
  operator: crawler.operator,
51346
51685
  product: crawler.product,
51347
51686
  verificationStatus: crawler.verificationStatus,
51687
+ verificationManifest: crawler.verificationManifest,
51348
51688
  pathNormalized,
51349
51689
  status: event.status,
51350
51690
  hits: 1,
@@ -51363,6 +51703,7 @@ function buildTrafficProbeReport(events, options = {}) {
51363
51703
  tsHour,
51364
51704
  aiUserFetch.botId,
51365
51705
  aiUserFetch.verificationStatus,
51706
+ aiUserFetch.verificationManifest?.id ?? "no-manifest",
51366
51707
  pathNormalized,
51367
51708
  event.status ?? "null"
51368
51709
  ].join(" ");
@@ -51376,6 +51717,7 @@ function buildTrafficProbeReport(events, options = {}) {
51376
51717
  operator: aiUserFetch.operator,
51377
51718
  product: aiUserFetch.product,
51378
51719
  verificationStatus: aiUserFetch.verificationStatus,
51720
+ verificationManifest: aiUserFetch.verificationManifest,
51379
51721
  pathNormalized,
51380
51722
  status: event.status,
51381
51723
  hits: 1,
@@ -53042,6 +53384,38 @@ function writeTrafficEventBatch(opts) {
53042
53384
  updatedAt: opts.receivedAt
53043
53385
  }
53044
53386
  }).run();
53387
+ if (bucket.verificationManifest) {
53388
+ tx.insert(crawlerVerificationManifestsHourly).values({
53389
+ projectId: opts.projectId,
53390
+ sourceId: opts.sourceId,
53391
+ tsHour: bucket.tsHour,
53392
+ botId: bucket.botId,
53393
+ verificationStatus: bucket.verificationStatus,
53394
+ pathNormalized: bucket.pathNormalized,
53395
+ status,
53396
+ manifestId: bucket.verificationManifest.id,
53397
+ manifestJson: bucket.verificationManifest,
53398
+ hits: bucket.hits,
53399
+ createdAt: opts.receivedAt,
53400
+ updatedAt: opts.receivedAt
53401
+ }).onConflictDoUpdate({
53402
+ target: [
53403
+ crawlerVerificationManifestsHourly.projectId,
53404
+ crawlerVerificationManifestsHourly.sourceId,
53405
+ crawlerVerificationManifestsHourly.tsHour,
53406
+ crawlerVerificationManifestsHourly.botId,
53407
+ crawlerVerificationManifestsHourly.verificationStatus,
53408
+ crawlerVerificationManifestsHourly.pathNormalized,
53409
+ crawlerVerificationManifestsHourly.status,
53410
+ crawlerVerificationManifestsHourly.manifestId
53411
+ ],
53412
+ set: {
53413
+ hits: sql18`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
53414
+ manifestJson: bucket.verificationManifest,
53415
+ updatedAt: opts.receivedAt
53416
+ }
53417
+ }).run();
53418
+ }
53045
53419
  crawlerBucketRows += 1;
53046
53420
  }
53047
53421
  for (const bucket of report.aiUserFetchEventsHourly) {
@@ -53075,6 +53449,38 @@ function writeTrafficEventBatch(opts) {
53075
53449
  updatedAt: opts.receivedAt
53076
53450
  }
53077
53451
  }).run();
53452
+ if (bucket.verificationManifest) {
53453
+ tx.insert(aiUserFetchVerificationManifestsHourly).values({
53454
+ projectId: opts.projectId,
53455
+ sourceId: opts.sourceId,
53456
+ tsHour: bucket.tsHour,
53457
+ botId: bucket.botId,
53458
+ verificationStatus: bucket.verificationStatus,
53459
+ pathNormalized: bucket.pathNormalized,
53460
+ status,
53461
+ manifestId: bucket.verificationManifest.id,
53462
+ manifestJson: bucket.verificationManifest,
53463
+ hits: bucket.hits,
53464
+ createdAt: opts.receivedAt,
53465
+ updatedAt: opts.receivedAt
53466
+ }).onConflictDoUpdate({
53467
+ target: [
53468
+ aiUserFetchVerificationManifestsHourly.projectId,
53469
+ aiUserFetchVerificationManifestsHourly.sourceId,
53470
+ aiUserFetchVerificationManifestsHourly.tsHour,
53471
+ aiUserFetchVerificationManifestsHourly.botId,
53472
+ aiUserFetchVerificationManifestsHourly.verificationStatus,
53473
+ aiUserFetchVerificationManifestsHourly.pathNormalized,
53474
+ aiUserFetchVerificationManifestsHourly.status,
53475
+ aiUserFetchVerificationManifestsHourly.manifestId
53476
+ ],
53477
+ set: {
53478
+ hits: sql18`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
53479
+ manifestJson: bucket.verificationManifest,
53480
+ updatedAt: opts.receivedAt
53481
+ }
53482
+ }).run();
53483
+ }
53078
53484
  aiUserFetchBucketRows += 1;
53079
53485
  }
53080
53486
  for (const bucket of report.aiReferralEventsHourly) {
@@ -53393,6 +53799,117 @@ function trafficSeriesPoint(pointsByBucket, bucket) {
53393
53799
  pointsByBucket.set(bucket, created);
53394
53800
  return created;
53395
53801
  }
53802
+ function verificationEventKey(row) {
53803
+ return JSON.stringify([
53804
+ row.sourceId,
53805
+ row.tsHour,
53806
+ row.botId,
53807
+ row.verificationStatus,
53808
+ row.pathNormalized,
53809
+ row.status
53810
+ ]);
53811
+ }
53812
+ function verificationSelectedRowsJson(projectId, rows) {
53813
+ return JSON.stringify(rows.map((row) => [
53814
+ projectId,
53815
+ row.sourceId,
53816
+ row.tsHour,
53817
+ row.botId,
53818
+ row.verificationStatus,
53819
+ row.pathNormalized,
53820
+ row.status
53821
+ ]));
53822
+ }
53823
+ function buildVerificationProvenance(events, rows) {
53824
+ const manifestsByEvent = /* @__PURE__ */ new Map();
53825
+ for (const row of rows) {
53826
+ const key = verificationEventKey(row);
53827
+ const manifests = manifestsByEvent.get(key);
53828
+ const usage = { manifestId: row.manifestId, manifest: row.manifest, hits: row.hits };
53829
+ if (manifests) manifests.push(usage);
53830
+ else manifestsByEvent.set(key, [usage]);
53831
+ }
53832
+ const provenanceByEvent = /* @__PURE__ */ new Map();
53833
+ for (const event of events) {
53834
+ const key = verificationEventKey(event);
53835
+ const verificationManifests = manifestsByEvent.get(key) ?? [];
53836
+ verificationManifests.sort((a, b) => a.manifestId.localeCompare(b.manifestId));
53837
+ const attributedHits = verificationManifests.reduce((sum, manifest) => sum + manifest.hits, 0);
53838
+ provenanceByEvent.set(key, {
53839
+ verificationManifests,
53840
+ verificationUnattributedHits: Math.max(0, event.hits - attributedHits)
53841
+ });
53842
+ }
53843
+ return provenanceByEvent;
53844
+ }
53845
+ function crawlerVerificationProvenance(db, projectId, events) {
53846
+ if (events.length === 0) return /* @__PURE__ */ new Map();
53847
+ const selectedRows = verificationSelectedRowsJson(projectId, events);
53848
+ const rows = db.select({
53849
+ sourceId: crawlerVerificationManifestsHourly.sourceId,
53850
+ tsHour: crawlerVerificationManifestsHourly.tsHour,
53851
+ botId: crawlerVerificationManifestsHourly.botId,
53852
+ verificationStatus: crawlerVerificationManifestsHourly.verificationStatus,
53853
+ pathNormalized: crawlerVerificationManifestsHourly.pathNormalized,
53854
+ status: crawlerVerificationManifestsHourly.status,
53855
+ manifestId: crawlerVerificationManifestsHourly.manifestId,
53856
+ manifest: crawlerVerificationManifestsHourly.manifestJson,
53857
+ hits: crawlerVerificationManifestsHourly.hits
53858
+ }).from(crawlerVerificationManifestsHourly).where(sql19`(
53859
+ ${crawlerVerificationManifestsHourly.projectId},
53860
+ ${crawlerVerificationManifestsHourly.sourceId},
53861
+ ${crawlerVerificationManifestsHourly.tsHour},
53862
+ ${crawlerVerificationManifestsHourly.botId},
53863
+ ${crawlerVerificationManifestsHourly.verificationStatus},
53864
+ ${crawlerVerificationManifestsHourly.pathNormalized},
53865
+ ${crawlerVerificationManifestsHourly.status}
53866
+ ) IN (
53867
+ SELECT
53868
+ json_extract(value, '$[0]'),
53869
+ json_extract(value, '$[1]'),
53870
+ json_extract(value, '$[2]'),
53871
+ json_extract(value, '$[3]'),
53872
+ json_extract(value, '$[4]'),
53873
+ json_extract(value, '$[5]'),
53874
+ json_extract(value, '$[6]')
53875
+ FROM json_each(${selectedRows})
53876
+ )`).orderBy(crawlerVerificationManifestsHourly.manifestId).all();
53877
+ return buildVerificationProvenance(events, rows);
53878
+ }
53879
+ function aiUserFetchVerificationProvenance(db, projectId, events) {
53880
+ if (events.length === 0) return /* @__PURE__ */ new Map();
53881
+ const selectedRows = verificationSelectedRowsJson(projectId, events);
53882
+ const rows = db.select({
53883
+ sourceId: aiUserFetchVerificationManifestsHourly.sourceId,
53884
+ tsHour: aiUserFetchVerificationManifestsHourly.tsHour,
53885
+ botId: aiUserFetchVerificationManifestsHourly.botId,
53886
+ verificationStatus: aiUserFetchVerificationManifestsHourly.verificationStatus,
53887
+ pathNormalized: aiUserFetchVerificationManifestsHourly.pathNormalized,
53888
+ status: aiUserFetchVerificationManifestsHourly.status,
53889
+ manifestId: aiUserFetchVerificationManifestsHourly.manifestId,
53890
+ manifest: aiUserFetchVerificationManifestsHourly.manifestJson,
53891
+ hits: aiUserFetchVerificationManifestsHourly.hits
53892
+ }).from(aiUserFetchVerificationManifestsHourly).where(sql19`(
53893
+ ${aiUserFetchVerificationManifestsHourly.projectId},
53894
+ ${aiUserFetchVerificationManifestsHourly.sourceId},
53895
+ ${aiUserFetchVerificationManifestsHourly.tsHour},
53896
+ ${aiUserFetchVerificationManifestsHourly.botId},
53897
+ ${aiUserFetchVerificationManifestsHourly.verificationStatus},
53898
+ ${aiUserFetchVerificationManifestsHourly.pathNormalized},
53899
+ ${aiUserFetchVerificationManifestsHourly.status}
53900
+ ) IN (
53901
+ SELECT
53902
+ json_extract(value, '$[0]'),
53903
+ json_extract(value, '$[1]'),
53904
+ json_extract(value, '$[2]'),
53905
+ json_extract(value, '$[3]'),
53906
+ json_extract(value, '$[4]'),
53907
+ json_extract(value, '$[5]'),
53908
+ json_extract(value, '$[6]')
53909
+ FROM json_each(${selectedRows})
53910
+ )`).orderBy(aiUserFetchVerificationManifestsHourly.manifestId).all();
53911
+ return buildVerificationProvenance(events, rows);
53912
+ }
53396
53913
  function parseSourceConfig(row) {
53397
53914
  return row.configJson;
53398
53915
  }
@@ -53570,6 +54087,7 @@ async function runBackfillTask(options) {
53570
54087
  )
53571
54088
  ).run();
53572
54089
  for (const bucket of report.crawlerEventsHourly) {
54090
+ const status = bucket.status ?? 0;
53573
54091
  tx.insert(crawlerEventsHourly).values({
53574
54092
  projectId: project.id,
53575
54093
  sourceId: sourceRow2.id,
@@ -53578,14 +54096,62 @@ async function runBackfillTask(options) {
53578
54096
  operator: bucket.operator,
53579
54097
  verificationStatus: bucket.verificationStatus,
53580
54098
  pathNormalized: bucket.pathNormalized,
53581
- status: bucket.status ?? 0,
54099
+ status,
53582
54100
  hits: bucket.hits,
53583
54101
  sampledUserAgent: bucket.sampledUserAgent,
53584
54102
  createdAt: finishedAt,
53585
54103
  updatedAt: finishedAt
54104
+ }).onConflictDoUpdate({
54105
+ target: [
54106
+ crawlerEventsHourly.projectId,
54107
+ crawlerEventsHourly.sourceId,
54108
+ crawlerEventsHourly.tsHour,
54109
+ crawlerEventsHourly.botId,
54110
+ crawlerEventsHourly.verificationStatus,
54111
+ crawlerEventsHourly.pathNormalized,
54112
+ crawlerEventsHourly.status
54113
+ ],
54114
+ set: {
54115
+ hits: sql19`${crawlerEventsHourly.hits} + ${bucket.hits}`,
54116
+ sampledUserAgent: bucket.sampledUserAgent,
54117
+ updatedAt: finishedAt
54118
+ }
53586
54119
  }).run();
54120
+ if (bucket.verificationManifest) {
54121
+ tx.insert(crawlerVerificationManifestsHourly).values({
54122
+ projectId: project.id,
54123
+ sourceId: sourceRow2.id,
54124
+ tsHour: bucket.tsHour,
54125
+ botId: bucket.botId,
54126
+ verificationStatus: bucket.verificationStatus,
54127
+ pathNormalized: bucket.pathNormalized,
54128
+ status,
54129
+ manifestId: bucket.verificationManifest.id,
54130
+ manifestJson: bucket.verificationManifest,
54131
+ hits: bucket.hits,
54132
+ createdAt: finishedAt,
54133
+ updatedAt: finishedAt
54134
+ }).onConflictDoUpdate({
54135
+ target: [
54136
+ crawlerVerificationManifestsHourly.projectId,
54137
+ crawlerVerificationManifestsHourly.sourceId,
54138
+ crawlerVerificationManifestsHourly.tsHour,
54139
+ crawlerVerificationManifestsHourly.botId,
54140
+ crawlerVerificationManifestsHourly.verificationStatus,
54141
+ crawlerVerificationManifestsHourly.pathNormalized,
54142
+ crawlerVerificationManifestsHourly.status,
54143
+ crawlerVerificationManifestsHourly.manifestId
54144
+ ],
54145
+ set: {
54146
+ hits: sql19`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
54147
+ manifestJson: bucket.verificationManifest,
54148
+ updatedAt: finishedAt
54149
+ }
54150
+ }).run();
54151
+ }
53587
54152
  }
53588
54153
  for (const bucket of report.aiUserFetchEventsHourly) {
54154
+ const status = bucket.status ?? 0;
53589
54155
  tx.insert(aiUserFetchEventsHourly).values({
53590
54156
  projectId: project.id,
53591
54157
  sourceId: sourceRow2.id,
@@ -53594,12 +54160,59 @@ async function runBackfillTask(options) {
53594
54160
  operator: bucket.operator,
53595
54161
  verificationStatus: bucket.verificationStatus,
53596
54162
  pathNormalized: bucket.pathNormalized,
53597
- status: bucket.status ?? 0,
54163
+ status,
53598
54164
  hits: bucket.hits,
53599
54165
  sampledUserAgent: bucket.sampledUserAgent,
53600
54166
  createdAt: finishedAt,
53601
54167
  updatedAt: finishedAt
54168
+ }).onConflictDoUpdate({
54169
+ target: [
54170
+ aiUserFetchEventsHourly.projectId,
54171
+ aiUserFetchEventsHourly.sourceId,
54172
+ aiUserFetchEventsHourly.tsHour,
54173
+ aiUserFetchEventsHourly.botId,
54174
+ aiUserFetchEventsHourly.verificationStatus,
54175
+ aiUserFetchEventsHourly.pathNormalized,
54176
+ aiUserFetchEventsHourly.status
54177
+ ],
54178
+ set: {
54179
+ hits: sql19`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
54180
+ sampledUserAgent: bucket.sampledUserAgent,
54181
+ updatedAt: finishedAt
54182
+ }
53602
54183
  }).run();
54184
+ if (bucket.verificationManifest) {
54185
+ tx.insert(aiUserFetchVerificationManifestsHourly).values({
54186
+ projectId: project.id,
54187
+ sourceId: sourceRow2.id,
54188
+ tsHour: bucket.tsHour,
54189
+ botId: bucket.botId,
54190
+ verificationStatus: bucket.verificationStatus,
54191
+ pathNormalized: bucket.pathNormalized,
54192
+ status,
54193
+ manifestId: bucket.verificationManifest.id,
54194
+ manifestJson: bucket.verificationManifest,
54195
+ hits: bucket.hits,
54196
+ createdAt: finishedAt,
54197
+ updatedAt: finishedAt
54198
+ }).onConflictDoUpdate({
54199
+ target: [
54200
+ aiUserFetchVerificationManifestsHourly.projectId,
54201
+ aiUserFetchVerificationManifestsHourly.sourceId,
54202
+ aiUserFetchVerificationManifestsHourly.tsHour,
54203
+ aiUserFetchVerificationManifestsHourly.botId,
54204
+ aiUserFetchVerificationManifestsHourly.verificationStatus,
54205
+ aiUserFetchVerificationManifestsHourly.pathNormalized,
54206
+ aiUserFetchVerificationManifestsHourly.status,
54207
+ aiUserFetchVerificationManifestsHourly.manifestId
54208
+ ],
54209
+ set: {
54210
+ hits: sql19`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
54211
+ manifestJson: bucket.verificationManifest,
54212
+ updatedAt: finishedAt
54213
+ }
54214
+ }).run();
54215
+ }
53603
54216
  }
53604
54217
  for (const bucket of report.aiReferralEventsHourly) {
53605
54218
  tx.insert(aiReferralEventsHourly).values({
@@ -55082,6 +55695,38 @@ async function trafficRoutes(app, opts) {
55082
55695
  updatedAt: finishedAt
55083
55696
  }
55084
55697
  }).run();
55698
+ if (bucket.verificationManifest) {
55699
+ tx.insert(crawlerVerificationManifestsHourly).values({
55700
+ projectId: project.id,
55701
+ sourceId: sourceRow2.id,
55702
+ tsHour: bucket.tsHour,
55703
+ botId: bucket.botId,
55704
+ verificationStatus: bucket.verificationStatus,
55705
+ pathNormalized: bucket.pathNormalized,
55706
+ status,
55707
+ manifestId: bucket.verificationManifest.id,
55708
+ manifestJson: bucket.verificationManifest,
55709
+ hits: bucket.hits,
55710
+ createdAt: finishedAt,
55711
+ updatedAt: finishedAt
55712
+ }).onConflictDoUpdate({
55713
+ target: [
55714
+ crawlerVerificationManifestsHourly.projectId,
55715
+ crawlerVerificationManifestsHourly.sourceId,
55716
+ crawlerVerificationManifestsHourly.tsHour,
55717
+ crawlerVerificationManifestsHourly.botId,
55718
+ crawlerVerificationManifestsHourly.verificationStatus,
55719
+ crawlerVerificationManifestsHourly.pathNormalized,
55720
+ crawlerVerificationManifestsHourly.status,
55721
+ crawlerVerificationManifestsHourly.manifestId
55722
+ ],
55723
+ set: {
55724
+ hits: sql19`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
55725
+ manifestJson: bucket.verificationManifest,
55726
+ updatedAt: finishedAt
55727
+ }
55728
+ }).run();
55729
+ }
55085
55730
  crawlerBucketRows += 1;
55086
55731
  }
55087
55732
  for (const bucket of report.aiUserFetchEventsHourly) {
@@ -55115,6 +55760,38 @@ async function trafficRoutes(app, opts) {
55115
55760
  updatedAt: finishedAt
55116
55761
  }
55117
55762
  }).run();
55763
+ if (bucket.verificationManifest) {
55764
+ tx.insert(aiUserFetchVerificationManifestsHourly).values({
55765
+ projectId: project.id,
55766
+ sourceId: sourceRow2.id,
55767
+ tsHour: bucket.tsHour,
55768
+ botId: bucket.botId,
55769
+ verificationStatus: bucket.verificationStatus,
55770
+ pathNormalized: bucket.pathNormalized,
55771
+ status,
55772
+ manifestId: bucket.verificationManifest.id,
55773
+ manifestJson: bucket.verificationManifest,
55774
+ hits: bucket.hits,
55775
+ createdAt: finishedAt,
55776
+ updatedAt: finishedAt
55777
+ }).onConflictDoUpdate({
55778
+ target: [
55779
+ aiUserFetchVerificationManifestsHourly.projectId,
55780
+ aiUserFetchVerificationManifestsHourly.sourceId,
55781
+ aiUserFetchVerificationManifestsHourly.tsHour,
55782
+ aiUserFetchVerificationManifestsHourly.botId,
55783
+ aiUserFetchVerificationManifestsHourly.verificationStatus,
55784
+ aiUserFetchVerificationManifestsHourly.pathNormalized,
55785
+ aiUserFetchVerificationManifestsHourly.status,
55786
+ aiUserFetchVerificationManifestsHourly.manifestId
55787
+ ],
55788
+ set: {
55789
+ hits: sql19`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
55790
+ manifestJson: bucket.verificationManifest,
55791
+ updatedAt: finishedAt
55792
+ }
55793
+ }).run();
55794
+ }
55118
55795
  aiUserFetchBucketRows += 1;
55119
55796
  }
55120
55797
  for (const bucket of report.aiReferralEventsHourly) {
@@ -55691,6 +56368,8 @@ async function trafficRoutes(app, opts) {
55691
56368
  botId: r.botId,
55692
56369
  operator: r.operator,
55693
56370
  verificationStatus: r.verificationStatus,
56371
+ verificationManifests: [],
56372
+ verificationUnattributedHits: r.hits,
55694
56373
  pathNormalized: r.pathNormalized,
55695
56374
  pathClass: classifyTrafficPath(r.pathNormalized),
55696
56375
  status: r.status,
@@ -55729,6 +56408,8 @@ async function trafficRoutes(app, opts) {
55729
56408
  botId: r.botId,
55730
56409
  operator: r.operator,
55731
56410
  verificationStatus: r.verificationStatus,
56411
+ verificationManifests: [],
56412
+ verificationUnattributedHits: r.hits,
55732
56413
  pathNormalized: r.pathNormalized,
55733
56414
  status: r.status,
55734
56415
  hits: r.hits
@@ -55785,6 +56466,25 @@ async function trafficRoutes(app, opts) {
55785
56466
  }
55786
56467
  events.sort((a, b) => a.tsHour < b.tsHour ? 1 : a.tsHour > b.tsHour ? -1 : 0);
55787
56468
  const trimmed = events.slice(0, limit);
56469
+ const trimmedCrawlerEvents = trimmed.filter(
56470
+ (event) => event.kind === TrafficEventKinds.crawler
56471
+ );
56472
+ const trimmedAiUserFetchEvents = trimmed.filter(
56473
+ (event) => event.kind === TrafficEventKinds["ai-user-fetch"]
56474
+ );
56475
+ const crawlerProvenance = crawlerVerificationProvenance(app.db, project.id, trimmedCrawlerEvents);
56476
+ const aiUserFetchProvenance = aiUserFetchVerificationProvenance(app.db, project.id, trimmedAiUserFetchEvents);
56477
+ const eventsWithProvenance = trimmed.map((event) => {
56478
+ if (event.kind === TrafficEventKinds.crawler) {
56479
+ const provenance = crawlerProvenance.get(verificationEventKey(event));
56480
+ return provenance ? { ...event, ...provenance } : event;
56481
+ }
56482
+ if (event.kind === TrafficEventKinds["ai-user-fetch"]) {
56483
+ const provenance = aiUserFetchProvenance.get(verificationEventKey(event));
56484
+ return provenance ? { ...event, ...provenance } : event;
56485
+ }
56486
+ return event;
56487
+ });
55788
56488
  const response = {
55789
56489
  windowStart: sinceIso,
55790
56490
  windowEnd: untilIso,
@@ -55808,7 +56508,7 @@ async function trafficRoutes(app, opts) {
55808
56508
  returned: trimmed.length,
55809
56509
  truncated: trimmed.length < totalEventRows
55810
56510
  },
55811
- events: trimmed
56511
+ events: eventsWithProvenance
55812
56512
  };
55813
56513
  return response;
55814
56514
  });