@canonry/canonry 7.83.2 → 7.85.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. package/assets/agent-workspace/skills/aero/references/ai-traffic-tracking.md +2 -1
  2. package/assets/agent-workspace/skills/canonry/references/ai-traffic-tracking.md +2 -1
  3. package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +4 -0
  4. package/assets/agent-workspace/skills/canonry/references/server-side-traffic.md +45 -8
  5. package/assets/assets/{AgentSetupPage-B9NOQP8a.js → AgentSetupPage-NdqmQkYa.js} +1 -1
  6. package/assets/assets/{AuditHistoryPanel-Fvh4WFjq.js → AuditHistoryPanel-CuqdFCb9.js} +1 -1
  7. package/assets/assets/{BacklinksPage-D1UHFLoV.js → BacklinksPage-CVjZiG1i.js} +1 -1
  8. package/assets/assets/{HistoryPage-CBcacDgP.js → HistoryPage-DRvdgemW.js} +1 -1
  9. package/assets/assets/{MeasurementPropertyPage-ZuZpurwr.js → MeasurementPropertyPage-Bc5EpJFV.js} +1 -1
  10. package/assets/assets/{ProjectPage-BoqDyoGi.js → ProjectPage-CGGBSiK7.js} +1 -1
  11. package/assets/assets/{RunsPage-tlCk_gOw.js → RunsPage-B6_o81G3.js} +1 -1
  12. package/assets/assets/{SettingsPage-7mRTFLFQ.js → SettingsPage-DjmVapHO.js} +1 -1
  13. package/assets/assets/{SiteHealthSection-l_A_AmY_.js → SiteHealthSection-BPPB0D9D.js} +2 -2
  14. package/assets/assets/{ToneBadge-CXyUIWzB.js → ToneBadge-DwUBSAjL.js} +1 -1
  15. package/assets/assets/{TrafficPage-DQbBT3TP.js → TrafficPage-BvbjxOw3.js} +1 -1
  16. package/assets/assets/{TrafficSourceDetailPage-D3DwE6Qd.js → TrafficSourceDetailPage-BGJwxS_Q.js} +1 -1
  17. package/assets/assets/{account-context-Bhgg1lEH.js → account-context-D7M4W4wP.js} +1 -1
  18. package/assets/assets/agent-trust-copy-WXXeDvDs.js +1 -0
  19. package/assets/assets/{card-B16Dvxau.js → card-DULVRmzI.js} +1 -1
  20. package/assets/assets/{extract-error-message-B5_v_-U8.js → extract-error-message-B9-6WgLm.js} +1 -1
  21. package/assets/assets/{format-helpers-Bs9rrO74.js → format-helpers-Cgq0VzXz.js} +1 -1
  22. package/assets/assets/{index-3aBRkQpI.js → index-A-4JN4Pv.js} +4 -4
  23. package/assets/assets/{query-client-D-tt1sn3.js → query-client-1Q6EwPba.js} +1 -1
  24. package/assets/assets/{react-query-DQr9PoNm.js → react-query-D0OiMBt3.js} +1 -1
  25. package/assets/assets/{react-sigma_core.esm.min-BfZVKh7v.js → react-sigma_core.esm.min-CR09uPhZ.js} +1 -1
  26. package/assets/assets/{service-Cmt36bhL.js → service-Bu-7icpb.js} +1 -1
  27. package/assets/index.html +7 -7
  28. package/dist/{ads-BSVYKQ2L.js → ads-ETZIGLL5.js} +3 -3
  29. package/dist/{agent-L7VUHOHS.js → agent-WG2FUC7C.js} +7 -7
  30. package/dist/{agent-ask-A4HQCNLV.js → agent-ask-775XBFOH.js} +3 -3
  31. package/dist/{agent-conversations-L3TJJKPJ.js → agent-conversations-AYZ3LQJQ.js} +3 -3
  32. package/dist/{agent-journeys-FEXRAZYN.js → agent-journeys-GWNZG73C.js} +3 -3
  33. package/dist/{agent-memory-LBTLOHUA.js → agent-memory-E5MRRS5P.js} +3 -3
  34. package/dist/{agent-providers-X7V2IHRK.js → agent-providers-DEQZWO2M.js} +3 -3
  35. package/dist/{agent-transcript-YSP2HKUZ.js → agent-transcript-4AKVVHQC.js} +3 -3
  36. package/dist/{analytics-FBPHES6M.js → analytics-HSLTWKDA.js} +3 -3
  37. package/dist/{apply-PBCXZC5F.js → apply-4GD7VGG7.js} +3 -3
  38. package/dist/{backfill-PZBLCRPI.js → backfill-LNE2BW34.js} +10 -9
  39. package/dist/{backlinks-QMLTF2VF.js → backlinks-TGE3DWA2.js} +3 -3
  40. package/dist/{bing-H65WGTXU.js → bing-56GNRC55.js} +3 -3
  41. package/dist/{bootstrap-JAZNMAKP.js → bootstrap-42XCWZTS.js} +10 -10
  42. package/dist/{cdp-4DAMAHTB.js → cdp-UODIMKE3.js} +7 -7
  43. package/dist/{chunk-UGUON7IK.js → chunk-2O5XBKEO.js} +7 -7
  44. package/dist/{chunk-IQTMHY23.js → chunk-2R6WI237.js} +17643 -17454
  45. package/dist/{chunk-OMNS42ZQ.js → chunk-2X6UAPAC.js} +523 -4
  46. package/dist/{chunk-SEAXXCU6.js → chunk-4KK3OTHE.js} +1 -1
  47. package/dist/{chunk-X7U7XYJQ.js → chunk-5NEZO3SC.js} +3 -3
  48. package/dist/{chunk-SVMTN7MI.js → chunk-6SGHSSWQ.js} +2 -2
  49. package/dist/{chunk-NCM6PX64.js → chunk-AICFUZ6X.js} +1 -1
  50. package/dist/{chunk-QAJDHHC5.js → chunk-BQTXXXUG.js} +1 -1
  51. package/dist/{chunk-V52KSHH7.js → chunk-C2H5TPKD.js} +338 -338
  52. package/dist/{chunk-IJQJSNOZ.js → chunk-D45AMUYA.js} +4 -4
  53. package/dist/{chunk-WA4FD5Y6.js → chunk-DWK6NIJF.js} +1 -1
  54. package/dist/{chunk-CBAYQHNX.js → chunk-GR7VVRNN.js} +1 -1
  55. package/dist/{chunk-SKH3N6WI.js → chunk-IOW23XNK.js} +3 -3
  56. package/dist/{chunk-5AGYH5PX.js → chunk-JVO5GQZH.js} +4287 -3934
  57. package/dist/{chunk-3GXAL3F7.js → chunk-KBLMQDUZ.js} +37 -171
  58. package/dist/{chunk-PHVIMUAQ.js → chunk-KNKOTLVW.js} +6 -2
  59. package/dist/{chunk-3UFQI4KL.js → chunk-L5R5S57T.js} +75 -1
  60. package/dist/{chunk-32U5LFIT.js → chunk-NY252PIY.js} +3 -3
  61. package/dist/{chunk-W5ZBMBBZ.js → chunk-OHDSI5XS.js} +1 -1
  62. package/dist/{chunk-KCFIQJOL.js → chunk-OS5BVALI.js} +2 -2
  63. package/dist/{chunk-AA6P5NK3.js → chunk-PQWZCAVS.js} +1 -1
  64. package/dist/{chunk-WDEDEEWA.js → chunk-PX2KVGOJ.js} +1 -1
  65. package/dist/{chunk-HIC62JTS.js → chunk-Q7354YI3.js} +3 -3
  66. package/dist/{chunk-KFMU6CLU.js → chunk-QEATYB7C.js} +1 -1
  67. package/dist/{chunk-Y7WHB65J.js → chunk-RCAB7M6A.js} +2 -2
  68. package/dist/{chunk-LZGVQSZL.js → chunk-SS33RWKG.js} +1 -1
  69. package/dist/{chunk-YCRPOY2Y.js → chunk-TPUBPXBN.js} +2 -2
  70. package/dist/{chunk-QHUCHZA3.js → chunk-TVZMFCX2.js} +6 -6
  71. package/dist/{chunk-4D2QHIRS.js → chunk-UMEQ7EYN.js} +4 -4
  72. package/dist/{chunk-IPPNN5KE.js → chunk-UMV4HYHD.js} +2 -2
  73. package/dist/{chunk-7Q5BSCKN.js → chunk-VFSHZHZS.js} +2 -2
  74. package/dist/{chunk-YJOGWS7F.js → chunk-X7XKMARG.js} +2 -2
  75. package/dist/{chunk-KVSCEZBR.js → chunk-YDC2V5FQ.js} +30 -25
  76. package/dist/{chunk-JCWXEVYW.js → chunk-Z53CKLDS.js} +3 -3
  77. package/dist/{chunk-P76XOEWL.js → chunk-ZXMDIWW6.js} +4 -4
  78. package/dist/{citations-MNLMO33V.js → citations-RXBPR5MX.js} +3 -3
  79. package/dist/{cli-catalog-IKF4A7ZS.js → cli-catalog-QVCJWPCI.js} +3 -3
  80. package/dist/{cli-commands-KNMPQFQO.js → cli-commands-ZXR4Z2RW.js} +7 -7
  81. package/dist/cli.js +13 -13
  82. package/dist/{competitor-5DY5PKNU.js → competitor-MDNNHRSD.js} +3 -3
  83. package/dist/{content-XNK2YY4S.js → content-D3JZZQ64.js} +3 -3
  84. package/dist/{dashboard-SMI6A6OF.js → dashboard-5S2A46GM.js} +6 -6
  85. package/dist/{demo-DIP5JBEI.js → demo-DQHWRIKR.js} +1 -1
  86. package/dist/{demo-server-EWOBWWV4.js → demo-server-FY2XBSD2.js} +12 -12
  87. package/dist/{discover-D4H4ALIE.js → discover-NVNWHOTZ.js} +3 -3
  88. package/dist/{discover-eval-2CID2IBT.js → discover-eval-CJB6GYUS.js} +3 -3
  89. package/dist/{doctor-OWRQNDWF.js → doctor-DLSKNC4W.js} +3 -3
  90. package/dist/{evidence-SQX4IMQM.js → evidence-QNJULIXM.js} +3 -3
  91. package/dist/{export-cmd-73PYMZZ5.js → export-cmd-Z2YG2WLK.js} +3 -3
  92. package/dist/{ga-S6AKEVAL.js → ga-RSNHBP2B.js} +3 -3
  93. package/dist/{gbp-57J3AXEY.js → gbp-ZEV62LSZ.js} +3 -3
  94. package/dist/{google-PPU5MJSV.js → google-RZHEIKW5.js} +4 -4
  95. package/dist/{google-marketing-W4KZ74HX.js → google-marketing-DYQXPTD3.js} +1 -1
  96. package/dist/{health-cmd-QYP4MS3E.js → health-cmd-6ATPYUYG.js} +3 -3
  97. package/dist/{history-MFHBHYH7.js → history-KLX6XRY4.js} +3 -3
  98. package/dist/index.d.ts +337 -47
  99. package/dist/index.js +25 -25
  100. package/dist/{init-UZPL4H55.js → init-5BALV6SU.js} +12 -12
  101. package/dist/{insights-IIMMXGRO.js → insights-ARCEFFK5.js} +3 -3
  102. package/dist/intelligence-service-YJFZ6IBI.js +16 -0
  103. package/dist/{journeys-DXXRNANM.js → journeys-WVRQRZQJ.js} +3 -3
  104. package/dist/{keys-O3T52MLX.js → keys-5KAJFRZZ.js} +3 -3
  105. package/dist/{keyword-MS23BR7O.js → keyword-PTUH3TFD.js} +3 -3
  106. package/dist/{logs-SKR5OLTY.js → logs-NM7ACXGQ.js} +3 -3
  107. package/dist/{mcp-NFC2PBGD.js → mcp-ALRQXYL3.js} +4 -4
  108. package/dist/mcp.js +9 -9
  109. package/dist/{measurement-plan-Z73LMQYB.js → measurement-plan-T52JHK2J.js} +5 -5
  110. package/dist/{notify-2RUWK2AR.js → notify-MJBMRDXD.js} +3 -3
  111. package/dist/{organic-evidence-BGAOEVMZ.js → organic-evidence-ZUCQFWD5.js} +3 -3
  112. package/dist/{overview-DONWVYZT.js → overview-74IHPU2A.js} +4 -4
  113. package/dist/{project-FFSPWR6R.js → project-DKQOPMH6.js} +3 -3
  114. package/dist/{query-AXEUWIW3.js → query-YDJ5T46L.js} +3 -3
  115. package/dist/{research-YURAJRCV.js → research-5DU6DFZW.js} +3 -3
  116. package/dist/{results-export-ECSJKZHZ.js → results-export-EDVVXKVW.js} +3 -3
  117. package/dist/{run-27EXPBE4.js → run-HRHGSIYE.js} +4 -4
  118. package/dist/{schedule-NY55SZZF.js → schedule-GZVKQGMP.js} +3 -3
  119. package/dist/{search-RXCBX3ST.js → search-M37GRKVC.js} +3 -3
  120. package/dist/{serve-KGZW5WPA.js → serve-DW6PPZPJ.js} +26 -26
  121. package/dist/{session-BY2FSRYU.js → session-H427H2SK.js} +13 -13
  122. package/dist/{settings-IPC5SSDU.js → settings-WGS7GP7C.js} +3 -3
  123. package/dist/{skills-Y3VNCWQM.js → skills-CDDUPMAU.js} +2 -2
  124. package/dist/{snapshot-TPDSMADU.js → snapshot-5MAK7PWE.js} +3 -3
  125. package/dist/{src-IDX2F2SR.js → src-FEV4KYXN.js} +8 -2
  126. package/dist/{status-YSQJBNM7.js → status-EHFOYTXQ.js} +4 -4
  127. package/dist/{technical-aeo-NMUQVIFG.js → technical-aeo-KVY4RQWT.js} +3 -3
  128. package/dist/{telemetry-OA6W7Q54.js → telemetry-7XRRIHVC.js} +4 -4
  129. package/dist/{tools-5YRH5NEI.js → tools-EHONHGB6.js} +11 -11
  130. package/dist/{traffic-CE47IFFI.js → traffic-TBR6GOWN.js} +5 -5
  131. package/dist/{users-NMWBTCDI.js → users-YQS4X47Q.js} +3 -3
  132. package/dist/{visibility-stats-DE64FVOJ.js → visibility-stats-JXIOSJXC.js} +3 -3
  133. package/dist/{wordpress-AJWIJSM7.js → wordpress-HVOLEPQY.js} +3 -3
  134. package/package.json +1 -1
  135. package/assets/assets/agent-trust-copy-CKZsKhc0.js +0 -1
  136. package/dist/intelligence-service-AMXDCTS2.js +0 -16
@@ -1,9 +1,47 @@
1
1
  import {
2
- parseJsonColumn
3
- } from "./chunk-3UFQI4KL.js";
2
+ createLogger
3
+ } from "./chunk-6SGHSSWQ.js";
4
+ import {
5
+ addLikelyFakeAgentHits,
6
+ correctBackgroundCallJourneys,
7
+ mergeSplitJourneyVisitors,
8
+ writeUnrecognizedAgentBuckets
9
+ } from "./chunk-JVO5GQZH.js";
10
+ import {
11
+ AGENT_TRUST_RULES_VERSION,
12
+ BROWSER_SHAPED_AGENT_MARKERS,
13
+ DEFAULT_AI_CRAWLER_RULES,
14
+ GENERIC_HTTP_CLIENTS,
15
+ JOURNEY_RULES_VERSION,
16
+ SCANNER_BURST_WINDOW_MS,
17
+ SELF_TRAFFIC_USER_AGENT_PATTERNS,
18
+ UNLISTED_AGENT_SKIPPED_EXTENSIONS,
19
+ UNRECOGNIZED_AGENT_NAMING_VERSION,
20
+ classifyAiUserFetch,
21
+ classifyCrawler,
22
+ isScannerPath,
23
+ isSelfTraffic,
24
+ normalizeTrafficPathPattern,
25
+ unrecognizedAgentName
26
+ } from "./chunk-2R6WI237.js";
27
+ import {
28
+ RawEventSampleEventTypes,
29
+ agentJourneyCaptureDays,
30
+ agentJourneyEvents,
31
+ aiUserFetchEventsHourly,
32
+ aiUserFetchVerificationManifestsHourly,
33
+ crawlerEventsHourly,
34
+ migrationsTable,
35
+ parseJsonColumn,
36
+ rawEventSamples,
37
+ trafficReclassificationState,
38
+ unrecognizedAgentEventsHourly
39
+ } from "./chunk-L5R5S57T.js";
4
40
  import {
5
41
  AI_ENGINE_DOMAINS,
6
42
  AI_ENGINE_SELF_DOMAINS,
43
+ AgentTrustLevels,
44
+ AgentTrustReasons,
7
45
  AnchoredSpanSources,
8
46
  OPENROUTER_MODEL_PATTERN,
9
47
  OPENROUTER_ROUTE_FAMILY,
@@ -12,7 +50,12 @@ import {
12
50
  ProviderNames,
13
51
  RetrievalContracts,
14
52
  RetrievalStatuses,
53
+ TrafficEventConfidences,
54
+ TrafficEvidenceKinds,
55
+ TrafficSourceTypes,
56
+ UNRECORDED_TRAFFIC_PATH,
15
57
  VERTEX_AI_SEARCH_PROXY_DOMAIN,
58
+ VerificationStatuses,
16
59
  asRecord,
17
60
  describeError,
18
61
  hostMatchesAnyDomain,
@@ -26,7 +69,7 @@ import {
26
69
  retryAfterDelayMs,
27
70
  usageCount,
28
71
  withRetry
29
- } from "./chunk-PHVIMUAQ.js";
72
+ } from "./chunk-KNKOTLVW.js";
30
73
 
31
74
  // ../provider-openai/src/normalize.ts
32
75
  import OpenAI from "openai";
@@ -2895,6 +2938,480 @@ function extractStoredAnswerAnchors(provider, rawResponse) {
2895
2938
  }
2896
2939
  }
2897
2940
 
2941
+ // src/traffic-reclassify.ts
2942
+ import { createHash } from "crypto";
2943
+ import { setImmediate as yieldToEventLoop } from "timers/promises";
2944
+ import { and, eq, gte, inArray, lte, ne, sql } from "drizzle-orm";
2945
+ var log = createLogger("TrafficReclassify");
2946
+ var STATE_ID = "default";
2947
+ var BATCH_SIZE = 200;
2948
+ var UNRECOGNIZED_TABLE_MIGRATION = 201;
2949
+ var TRAFFIC_RECLASSIFY_START_DELAY_MS = 3e4;
2950
+ function trafficClassificationFingerprint() {
2951
+ const pattern = (value) => `/${value.source}/${value.flags}`;
2952
+ return createHash("sha256").update(JSON.stringify({
2953
+ rules: DEFAULT_AI_CRAWLER_RULES.map((rule) => ({
2954
+ id: rule.id,
2955
+ operator: rule.operator,
2956
+ product: rule.product,
2957
+ purpose: rule.purpose,
2958
+ patterns: rule.userAgentPatterns.map(pattern)
2959
+ })),
2960
+ self: SELF_TRAFFIC_USER_AGENT_PATTERNS.map(pattern),
2961
+ naming: UNRECOGNIZED_AGENT_NAMING_VERSION,
2962
+ trust: AGENT_TRUST_RULES_VERSION,
2963
+ journeys: JOURNEY_RULES_VERSION,
2964
+ genericClients: GENERIC_HTTP_CLIENTS,
2965
+ browserMarkers: BROWSER_SHAPED_AGENT_MARKERS,
2966
+ skippedExtensions: UNLISTED_AGENT_SKIPPED_EXTENSIONS
2967
+ })).digest("hex");
2968
+ }
2969
+ function probeRequest(userAgent, path, observedAt) {
2970
+ return {
2971
+ sourceType: TrafficSourceTypes["cloud-run"],
2972
+ evidenceKind: TrafficEvidenceKinds["raw-request"],
2973
+ confidence: TrafficEventConfidences.observed,
2974
+ eventId: "reclassify",
2975
+ observedAt,
2976
+ method: "GET",
2977
+ requestUrl: `https://placeholder${path.startsWith("/") ? path : "/"}`,
2978
+ host: "placeholder",
2979
+ path,
2980
+ queryString: null,
2981
+ status: 200,
2982
+ userAgent,
2983
+ // Stored traffic keeps no client address, so no replay can verify an IP.
2984
+ remoteIp: null,
2985
+ referer: null,
2986
+ latencyMs: null,
2987
+ requestSizeBytes: null,
2988
+ responseSizeBytes: null,
2989
+ providerResource: { type: "cloud_run_revision", labels: {} },
2990
+ providerLabels: {}
2991
+ };
2992
+ }
2993
+ function classifyStored(userAgent, path, observedAt) {
2994
+ const request2 = probeRequest(userAgent, path, observedAt);
2995
+ if (isSelfTraffic(request2)) return { kind: "self" };
2996
+ const fetch2 = classifyAiUserFetch(request2);
2997
+ if (fetch2) return { kind: "user-fetch", fetch: fetch2 };
2998
+ const crawler = classifyCrawler(request2);
2999
+ if (crawler) return { kind: "crawler", crawler };
3000
+ return { kind: "unknown" };
3001
+ }
3002
+ function sqliteUtcToIso(value) {
3003
+ return /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)?$/.test(value) ? `${value.replace(" ", "T")}Z` : value;
3004
+ }
3005
+ function hourOf(iso) {
3006
+ const hour = new Date(iso);
3007
+ hour.setUTCMinutes(0, 0, 0);
3008
+ return hour.toISOString();
3009
+ }
3010
+ function unrecognizedCountingStart(db) {
3011
+ const row = db.select({ appliedAt: migrationsTable.appliedAt }).from(migrationsTable).where(eq(migrationsTable.version, UNRECOGNIZED_TABLE_MIGRATION)).get();
3012
+ return row ? hourOf(sqliteUtcToIso(row.appliedAt)) : null;
3013
+ }
3014
+ function addHourlyHits(tx, opts) {
3015
+ const { projectId, sourceId, tsHour, verdict, pathNormalized, status, hits, sampledUserAgent, now } = opts;
3016
+ if (verdict.kind === "user-fetch" && verdict.fetch.trust === AgentTrustLevels.likely_fake) {
3017
+ addLikelyFakeAgentHits(tx, {
3018
+ projectId,
3019
+ sourceId,
3020
+ buckets: [{ tsHour, botId: verdict.fetch.botId, operator: verdict.fetch.operator, reason: verdict.fetch.trustReasons[0], hits }],
3021
+ now
3022
+ });
3023
+ return;
3024
+ }
3025
+ const table = verdict.kind === "user-fetch" ? aiUserFetchEventsHourly : crawlerEventsHourly;
3026
+ const classified = verdict.kind === "user-fetch" ? verdict.fetch : verdict.crawler;
3027
+ tx.insert(table).values({
3028
+ projectId,
3029
+ sourceId,
3030
+ tsHour,
3031
+ botId: classified.botId,
3032
+ operator: classified.operator,
3033
+ verificationStatus: classified.verificationStatus,
3034
+ pathNormalized,
3035
+ status,
3036
+ hits,
3037
+ sampledUserAgent,
3038
+ createdAt: now,
3039
+ updatedAt: now
3040
+ }).onConflictDoUpdate({
3041
+ target: [table.projectId, table.sourceId, table.tsHour, table.botId, table.verificationStatus, table.pathNormalized, table.status],
3042
+ set: { hits: sql`${table.hits} + ${hits}`, updatedAt: now }
3043
+ }).run();
3044
+ }
3045
+ function botIdOf(verdict) {
3046
+ return verdict.kind === "user-fetch" ? verdict.fetch.botId : verdict.crawler.botId;
3047
+ }
3048
+ async function reclassifyUnrecognizedRows(db, opts) {
3049
+ const result = { examined: 0, moved: 0, movedHits: 0, renamed: 0, removed: 0, byBot: {} };
3050
+ const table = unrecognizedAgentEventsHourly;
3051
+ const verdicts = /* @__PURE__ */ new Map();
3052
+ const lastStartingRowid = Number(db.select({ max: sql`max(rowid)` }).from(table).get()?.max ?? 0);
3053
+ let lastRowid = 0;
3054
+ for (; ; ) {
3055
+ if (opts.shouldStop()) break;
3056
+ const keys = db.select({ rowid: sql`rowid`, projectId: table.projectId, sourceId: table.sourceId, tsHour: table.tsHour, agentName: table.agentName }).from(table).where(and(
3057
+ sql`rowid > ${lastRowid}`,
3058
+ sql`rowid <= ${lastStartingRowid}`,
3059
+ opts.projectIds ? inArray(table.projectId, [...opts.projectIds]) : void 0
3060
+ )).orderBy(sql`rowid`).limit(BATCH_SIZE).all();
3061
+ if (keys.length === 0) break;
3062
+ lastRowid = keys[keys.length - 1].rowid;
3063
+ result.examined += keys.length;
3064
+ db.transaction((tx) => {
3065
+ for (const key of keys) {
3066
+ const match = and(
3067
+ eq(table.projectId, key.projectId),
3068
+ eq(table.sourceId, key.sourceId),
3069
+ eq(table.tsHour, key.tsHour),
3070
+ eq(table.agentName, key.agentName)
3071
+ );
3072
+ const row = tx.select().from(table).where(match).get();
3073
+ if (!row?.sampledUserAgent) continue;
3074
+ const userAgent = row.sampledUserAgent;
3075
+ let verdict = verdicts.get(userAgent);
3076
+ if (!verdict) {
3077
+ verdict = classifyStored(userAgent, "/", row.tsHour);
3078
+ verdicts.set(userAgent, verdict);
3079
+ }
3080
+ if (verdict.kind === "self") {
3081
+ result.removed++;
3082
+ if (!opts.dryRun) tx.delete(table).where(match).run();
3083
+ continue;
3084
+ }
3085
+ if (verdict.kind === "user-fetch" || verdict.kind === "crawler") {
3086
+ result.moved++;
3087
+ result.movedHits += row.hits;
3088
+ const botId = botIdOf(verdict);
3089
+ result.byBot[botId] = (result.byBot[botId] ?? 0) + row.hits;
3090
+ if (opts.dryRun) continue;
3091
+ addHourlyHits(tx, {
3092
+ projectId: row.projectId,
3093
+ sourceId: row.sourceId,
3094
+ tsHour: row.tsHour,
3095
+ verdict,
3096
+ pathNormalized: UNRECORDED_TRAFFIC_PATH,
3097
+ status: 0,
3098
+ hits: row.hits,
3099
+ sampledUserAgent: userAgent,
3100
+ now: opts.now
3101
+ });
3102
+ tx.delete(table).where(match).run();
3103
+ continue;
3104
+ }
3105
+ const name = unrecognizedAgentName({ method: "GET", path: row.sampledPath, userAgent });
3106
+ if (name === row.agentName) continue;
3107
+ if (name === null) {
3108
+ result.removed++;
3109
+ if (!opts.dryRun) tx.delete(table).where(match).run();
3110
+ continue;
3111
+ }
3112
+ result.renamed++;
3113
+ if (opts.dryRun) continue;
3114
+ tx.delete(table).where(match).run();
3115
+ writeUnrecognizedAgentBuckets(tx, {
3116
+ projectId: row.projectId,
3117
+ sourceId: row.sourceId,
3118
+ buckets: [{ tsHour: row.tsHour, agentName: name, hits: row.hits, sampledUserAgent: userAgent, sampledPath: row.sampledPath ?? "" }],
3119
+ now: opts.now
3120
+ });
3121
+ }
3122
+ }, { behavior: "immediate" });
3123
+ await yieldToEventLoop();
3124
+ }
3125
+ return result;
3126
+ }
3127
+ async function reclassifyUnknownSamples(db, opts) {
3128
+ const result = { examined: 0, reclassified: 0, counted: 0, removed: 0, byBot: {} };
3129
+ const countingStart = unrecognizedCountingStart(db);
3130
+ const lastStartingRowid = Number(db.select({ max: sql`max(rowid)` }).from(rawEventSamples).get()?.max ?? 0);
3131
+ let lastRowid = 0;
3132
+ for (; ; ) {
3133
+ if (opts.shouldStop()) break;
3134
+ const samples = db.select({
3135
+ rowid: sql`rowid`,
3136
+ id: rawEventSamples.id,
3137
+ projectId: rawEventSamples.projectId,
3138
+ sourceId: rawEventSamples.sourceId,
3139
+ ts: rawEventSamples.ts,
3140
+ userAgent: rawEventSamples.userAgent,
3141
+ pathNormalized: rawEventSamples.pathNormalized,
3142
+ status: rawEventSamples.status
3143
+ }).from(rawEventSamples).where(and(
3144
+ eq(rawEventSamples.eventType, RawEventSampleEventTypes.unknown),
3145
+ sql`rowid > ${lastRowid}`,
3146
+ sql`rowid <= ${lastStartingRowid}`,
3147
+ opts.projectIds ? inArray(rawEventSamples.projectId, [...opts.projectIds]) : void 0
3148
+ )).orderBy(sql`rowid`).limit(BATCH_SIZE).all();
3149
+ if (samples.length === 0) break;
3150
+ lastRowid = samples[samples.length - 1].rowid;
3151
+ result.examined += samples.length;
3152
+ db.transaction((tx) => {
3153
+ for (const sample of samples) {
3154
+ if (!sample.userAgent) continue;
3155
+ const verdict = classifyStored(sample.userAgent, sample.pathNormalized, sample.ts);
3156
+ if (verdict.kind === "self") {
3157
+ result.removed++;
3158
+ if (!opts.dryRun) tx.delete(rawEventSamples).where(eq(rawEventSamples.id, sample.id)).run();
3159
+ continue;
3160
+ }
3161
+ if (verdict.kind === "unknown") continue;
3162
+ result.reclassified++;
3163
+ const botId = botIdOf(verdict);
3164
+ result.byBot[botId] = (result.byBot[botId] ?? 0) + 1;
3165
+ const tsHour = hourOf(sample.ts);
3166
+ const countSample = countingStart === null || tsHour < countingStart || unrecognizedAgentName({ method: "GET", path: sample.pathNormalized, userAgent: sample.userAgent }) === null;
3167
+ if (countSample) result.counted++;
3168
+ if (opts.dryRun) continue;
3169
+ tx.update(rawEventSamples).set({
3170
+ eventType: verdict.kind === "user-fetch" ? RawEventSampleEventTypes.aiUserFetch : RawEventSampleEventTypes.crawler,
3171
+ classifierDetailsJson: {
3172
+ crawler: verdict.kind === "crawler" ? { ...verdict.crawler, verificationManifest: null } : null,
3173
+ aiUserFetch: verdict.kind === "user-fetch" ? { ...verdict.fetch, verificationManifest: null } : null,
3174
+ aiReferral: null
3175
+ }
3176
+ }).where(eq(rawEventSamples.id, sample.id)).run();
3177
+ if (!countSample) continue;
3178
+ addHourlyHits(tx, {
3179
+ projectId: sample.projectId,
3180
+ sourceId: sample.sourceId,
3181
+ tsHour,
3182
+ verdict,
3183
+ pathNormalized: sample.pathNormalized,
3184
+ status: sample.status ?? 200,
3185
+ hits: 1,
3186
+ sampledUserAgent: sample.userAgent,
3187
+ now: opts.now
3188
+ });
3189
+ }
3190
+ }, { behavior: "immediate" });
3191
+ await yieldToEventLoop();
3192
+ }
3193
+ return result;
3194
+ }
3195
+ function hourlyAgentRow(key) {
3196
+ const table = aiUserFetchEventsHourly;
3197
+ return and(
3198
+ eq(table.projectId, key.projectId),
3199
+ eq(table.sourceId, key.sourceId),
3200
+ eq(table.tsHour, key.tsHour),
3201
+ eq(table.botId, key.botId),
3202
+ eq(table.verificationStatus, key.verificationStatus),
3203
+ eq(table.pathNormalized, key.pathNormalized),
3204
+ eq(table.status, key.status)
3205
+ );
3206
+ }
3207
+ async function reassessAgentHits(db, opts) {
3208
+ const result = { examined: 0, moved: 0, movedHits: 0 };
3209
+ const table = aiUserFetchEventsHourly;
3210
+ const lastStartingRowid = Number(db.select({ max: sql`max(rowid)` }).from(table).get()?.max ?? 0);
3211
+ let lastRowid = 0;
3212
+ for (; ; ) {
3213
+ if (opts.shouldStop()) break;
3214
+ const rows = db.select({
3215
+ rowid: sql`rowid`,
3216
+ projectId: table.projectId,
3217
+ sourceId: table.sourceId,
3218
+ tsHour: table.tsHour,
3219
+ botId: table.botId,
3220
+ operator: table.operator,
3221
+ verificationStatus: table.verificationStatus,
3222
+ pathNormalized: table.pathNormalized,
3223
+ status: table.status
3224
+ }).from(table).where(and(
3225
+ sql`rowid > ${lastRowid}`,
3226
+ sql`rowid <= ${lastStartingRowid}`,
3227
+ ne(table.verificationStatus, VerificationStatuses.verified),
3228
+ opts.projectIds ? inArray(table.projectId, [...opts.projectIds]) : void 0
3229
+ )).orderBy(sql`rowid`).limit(BATCH_SIZE).all();
3230
+ if (rows.length === 0) break;
3231
+ lastRowid = rows[rows.length - 1].rowid;
3232
+ result.examined += rows.length;
3233
+ const probes = rows.filter((row) => isScannerPath(row.pathNormalized));
3234
+ if (probes.length > 0) {
3235
+ db.transaction((tx) => {
3236
+ for (const row of probes) {
3237
+ const current = tx.select({ hits: table.hits }).from(table).where(hourlyAgentRow(row)).get();
3238
+ if (!current) continue;
3239
+ result.moved++;
3240
+ result.movedHits += current.hits;
3241
+ if (opts.dryRun) continue;
3242
+ addLikelyFakeAgentHits(tx, {
3243
+ projectId: row.projectId,
3244
+ sourceId: row.sourceId,
3245
+ buckets: [{ tsHour: row.tsHour, botId: row.botId, operator: row.operator, reason: AgentTrustReasons["scanner-path"], hits: current.hits }],
3246
+ now: opts.now
3247
+ });
3248
+ tx.delete(table).where(hourlyAgentRow(row)).run();
3249
+ }
3250
+ }, { behavior: "immediate" });
3251
+ }
3252
+ await yieldToEventLoop();
3253
+ }
3254
+ return result;
3255
+ }
3256
+ function moveAgentHitToBurst(tx, row, now) {
3257
+ const key = {
3258
+ projectId: row.projectId,
3259
+ sourceId: row.sourceId,
3260
+ tsHour: hourOf(row.ts),
3261
+ botId: row.agentId,
3262
+ verificationStatus: row.verificationStatus,
3263
+ pathNormalized: normalizeTrafficPathPattern(row.path),
3264
+ status: row.status ?? 0
3265
+ };
3266
+ const table = aiUserFetchEventsHourly;
3267
+ const taken = tx.update(table).set({ hits: sql`${table.hits} - 1`, updatedAt: now }).where(and(hourlyAgentRow(key), sql`${table.hits} > 0`)).run();
3268
+ if (taken.changes === 0) return;
3269
+ const manifests = aiUserFetchVerificationManifestsHourly;
3270
+ if (row.verificationManifest) {
3271
+ const manifestRow = and(
3272
+ eq(manifests.projectId, key.projectId),
3273
+ eq(manifests.sourceId, key.sourceId),
3274
+ eq(manifests.tsHour, key.tsHour),
3275
+ eq(manifests.botId, key.botId),
3276
+ eq(manifests.verificationStatus, key.verificationStatus),
3277
+ eq(manifests.pathNormalized, key.pathNormalized),
3278
+ eq(manifests.status, key.status),
3279
+ eq(manifests.manifestId, row.verificationManifest.id)
3280
+ );
3281
+ tx.update(manifests).set({ hits: sql`${manifests.hits} - 1`, updatedAt: now }).where(and(manifestRow, sql`${manifests.hits} > 0`)).run();
3282
+ tx.delete(manifests).where(and(manifestRow, sql`${manifests.hits} <= 0`)).run();
3283
+ }
3284
+ tx.delete(table).where(and(hourlyAgentRow(key), sql`${table.hits} <= 0`)).run();
3285
+ addLikelyFakeAgentHits(tx, {
3286
+ projectId: row.projectId,
3287
+ sourceId: row.sourceId,
3288
+ buckets: [{ tsHour: key.tsHour, botId: row.agentId, operator: row.operator, reason: AgentTrustReasons["scanner-burst"], hits: 1 }],
3289
+ now
3290
+ });
3291
+ }
3292
+ async function reassessAgentRequests(db, opts) {
3293
+ const result = { examined: 0, probes: 0, removed: 0 };
3294
+ const table = agentJourneyEvents;
3295
+ const clients = /* @__PURE__ */ new Map();
3296
+ const lastStartingRowid = Number(db.select({ max: sql`max(rowid)` }).from(table).get()?.max ?? 0);
3297
+ let lastRowid = 0;
3298
+ for (; ; ) {
3299
+ if (opts.shouldStop()) return result;
3300
+ const rows = db.select({ rowid: sql`rowid`, projectId: table.projectId, sourceId: table.sourceId, correlationKey: table.correlationKey, ts: table.ts, path: table.path }).from(table).where(and(
3301
+ sql`rowid > ${lastRowid}`,
3302
+ sql`rowid <= ${lastStartingRowid}`,
3303
+ ne(table.trust, AgentTrustLevels.verified),
3304
+ opts.projectIds ? inArray(table.projectId, [...opts.projectIds]) : void 0
3305
+ )).orderBy(sql`rowid`).limit(BATCH_SIZE).all();
3306
+ if (rows.length === 0) break;
3307
+ lastRowid = rows[rows.length - 1].rowid;
3308
+ result.examined += rows.length;
3309
+ for (const row of rows) {
3310
+ if (!isScannerPath(row.path)) continue;
3311
+ result.probes++;
3312
+ const id = JSON.stringify([row.projectId, row.sourceId, row.correlationKey]);
3313
+ const client = clients.get(id) ?? { projectId: row.projectId, sourceId: row.sourceId, correlationKey: row.correlationKey, probes: [] };
3314
+ client.probes.push(Date.parse(row.ts));
3315
+ clients.set(id, client);
3316
+ }
3317
+ await yieldToEventLoop();
3318
+ }
3319
+ for (const client of clients.values()) {
3320
+ if (opts.shouldStop()) break;
3321
+ const probes = client.probes.sort((a, b) => a - b);
3322
+ const from = new Date(probes[0] - SCANNER_BURST_WINDOW_MS).toISOString();
3323
+ const to = new Date(probes.at(-1) + SCANNER_BURST_WINDOW_MS).toISOString();
3324
+ db.transaction((tx) => {
3325
+ const rows = tx.select().from(table).where(and(
3326
+ eq(table.projectId, client.projectId),
3327
+ eq(table.sourceId, client.sourceId),
3328
+ eq(table.correlationKey, client.correlationKey),
3329
+ gte(table.ts, from),
3330
+ lte(table.ts, to),
3331
+ ne(table.trust, AgentTrustLevels.verified)
3332
+ )).all();
3333
+ for (const row of rows) {
3334
+ const probe = isScannerPath(row.path);
3335
+ const at = Date.parse(row.ts);
3336
+ if (!probe && !probes.some((t) => Math.abs(at - t) <= SCANNER_BURST_WINDOW_MS)) continue;
3337
+ result.removed++;
3338
+ if (opts.dryRun) continue;
3339
+ tx.delete(table).where(eq(table.id, row.id)).run();
3340
+ tx.update(agentJourneyCaptureDays).set({ storedEvents: sql`max(${agentJourneyCaptureDays.storedEvents} - 1, 0)` }).where(and(
3341
+ eq(agentJourneyCaptureDays.projectId, row.projectId),
3342
+ eq(agentJourneyCaptureDays.sourceId, row.sourceId),
3343
+ eq(agentJourneyCaptureDays.day, row.ts.slice(0, 10))
3344
+ )).run();
3345
+ if (!probe) moveAgentHitToBurst(tx, row, opts.now);
3346
+ }
3347
+ }, { behavior: "immediate" });
3348
+ await yieldToEventLoop();
3349
+ }
3350
+ return result;
3351
+ }
3352
+ async function reclassifyStoredTraffic(db, opts = {}) {
3353
+ const shared = {
3354
+ projectIds: opts.projectIds,
3355
+ dryRun: opts.dryRun === true,
3356
+ now: opts.now ?? (/* @__PURE__ */ new Date()).toISOString(),
3357
+ shouldStop: opts.shouldStop ?? (() => false)
3358
+ };
3359
+ const samples = await reclassifyUnknownSamples(db, shared);
3360
+ const unrecognized = shared.shouldStop() ? { examined: 0, moved: 0, movedHits: 0, renamed: 0, removed: 0, byBot: {} } : await reclassifyUnrecognizedRows(db, shared);
3361
+ const agentHits = shared.shouldStop() ? { examined: 0, moved: 0, movedHits: 0 } : await reassessAgentHits(db, shared);
3362
+ const agentRequests = shared.shouldStop() ? { examined: 0, probes: 0, removed: 0 } : await reassessAgentRequests(db, shared);
3363
+ const journeys = shared.shouldStop() ? { visitors: 0, visitorsRemoved: 0, visitsRemoved: 0, visitsRelanded: 0 } : correctBackgroundCallJourneys(db, shared);
3364
+ const journeyMerges = shared.shouldStop() ? { visitorsMerged: 0, visitsJoined: 0 } : mergeSplitJourneyVisitors(db, shared);
3365
+ return { dryRun: shared.dryRun, samples, unrecognized, agentHits, agentRequests, journeys, journeyMerges };
3366
+ }
3367
+ async function reclassifyStoredTrafficIfRulesChanged(db, opts = {}) {
3368
+ const fingerprint = trafficClassificationFingerprint();
3369
+ const state = db.select().from(trafficReclassificationState).where(eq(trafficReclassificationState.id, STATE_ID)).get();
3370
+ if (state?.fingerprint === fingerprint) return { ran: false };
3371
+ const now = opts.now ?? (() => (/* @__PURE__ */ new Date()).toISOString());
3372
+ const shouldStop = opts.shouldStop ?? (() => false);
3373
+ const result = await reclassifyStoredTraffic(db, { now: now(), shouldStop });
3374
+ if (shouldStop()) return { ran: false };
3375
+ const completedAt = now();
3376
+ db.insert(trafficReclassificationState).values({ id: STATE_ID, fingerprint, completedAt, resultJson: { ...result } }).onConflictDoUpdate({
3377
+ target: trafficReclassificationState.id,
3378
+ set: { fingerprint, completedAt, resultJson: { ...result } }
3379
+ }).run();
3380
+ return { ran: true, result };
3381
+ }
3382
+ function startTrafficReclassification(opts) {
3383
+ let stopped = false;
3384
+ const timer = setTimeout(() => {
3385
+ reclassifyStoredTrafficIfRulesChanged(opts.db, { shouldStop: () => stopped }).then((outcome) => {
3386
+ if (!outcome.ran || !outcome.result) return;
3387
+ const { samples, unrecognized, agentHits, agentRequests, journeys, journeyMerges } = outcome.result;
3388
+ log.info("traffic-reclassify.completed", {
3389
+ samplesReclassified: samples.reclassified,
3390
+ samplesCounted: samples.counted,
3391
+ samplesRemoved: samples.removed,
3392
+ unrecognizedMoved: unrecognized.moved,
3393
+ unrecognizedMovedHits: unrecognized.movedHits,
3394
+ unrecognizedRenamed: unrecognized.renamed,
3395
+ unrecognizedRemoved: unrecognized.removed,
3396
+ agentHitsMoved: agentHits.movedHits,
3397
+ agentRequestsRemoved: agentRequests.removed,
3398
+ journeyVisitorsRemoved: journeys.visitorsRemoved,
3399
+ journeyVisitsRemoved: journeys.visitsRemoved,
3400
+ journeyVisitsRelanded: journeys.visitsRelanded,
3401
+ journeyVisitorsMerged: journeyMerges.visitorsMerged,
3402
+ journeyVisitsJoined: journeyMerges.visitsJoined
3403
+ });
3404
+ }).catch((err) => {
3405
+ if (!stopped) log.error("traffic-reclassify.failed", { error: describeError(err) });
3406
+ });
3407
+ }, opts.delayMs ?? TRAFFIC_RECLASSIFY_START_DELAY_MS);
3408
+ timer.unref();
3409
+ return () => {
3410
+ stopped = true;
3411
+ clearTimeout(timer);
3412
+ };
3413
+ }
3414
+
2898
3415
  export {
2899
3416
  OPENAI_RETRIEVAL_CONTRACT,
2900
3417
  reparseStoredResult,
@@ -2911,5 +3428,7 @@ export {
2911
3428
  museAdapter,
2912
3429
  openrouterAdapter,
2913
3430
  resolveStoredApiResponse,
2914
- extractStoredAnswerAnchors
3431
+ extractStoredAnswerAnchors,
3432
+ reclassifyStoredTraffic,
3433
+ startTrafficReclassification
2915
3434
  };
@@ -5,7 +5,7 @@ import {
5
5
  AppError,
6
6
  parseWindowIn,
7
7
  resolveDateRange
8
- } from "./chunk-PHVIMUAQ.js";
8
+ } from "./chunk-KNKOTLVW.js";
9
9
 
10
10
  // src/cli-access.ts
11
11
  var CLI_ACCESS_BY_ID = Object.freeze({
@@ -3,7 +3,7 @@ import {
3
3
  resolveCliAccess,
4
4
  resolveCliAccessForInvocation,
5
5
  windowOptions
6
- } from "./chunk-SEAXXCU6.js";
6
+ } from "./chunk-4KK3OTHE.js";
7
7
  import {
8
8
  emitJsonl
9
9
  } from "./chunk-Z7IUAJOF.js";
@@ -14,7 +14,7 @@ import {
14
14
  } from "./chunk-HGMYEDAQ.js";
15
15
  import {
16
16
  OperationAccesses
17
- } from "./chunk-PHVIMUAQ.js";
17
+ } from "./chunk-KNKOTLVW.js";
18
18
 
19
19
  // src/cli-catalog.ts
20
20
  import { createRequire } from "module";
@@ -111,7 +111,7 @@ async function dispatchRegisteredCommand(args, fallbackFormat, specs) {
111
111
  const accessValue = accessFilterFromArgs(argsWithoutHelp);
112
112
  validateAccessFilter(accessValue);
113
113
  if (fallbackFormat !== "text") {
114
- const { printCliCatalog: printCliCatalog2 } = await import("./cli-catalog-IKF4A7ZS.js");
114
+ const { printCliCatalog: printCliCatalog2 } = await import("./cli-catalog-QVCJWPCI.js");
115
115
  const group2 = argsWithoutHelp.find((arg) => !arg.startsWith("--"));
116
116
  printCliCatalog2(specs, fallbackFormat, group2, accessValue);
117
117
  return true;
@@ -1,11 +1,11 @@
1
1
  import {
2
2
  getRequestContext
3
- } from "./chunk-WDEDEEWA.js";
3
+ } from "./chunk-PX2KVGOJ.js";
4
4
  import {
5
5
  diagnosticIdentity,
6
6
  redactLogString,
7
7
  redactLogValue
8
- } from "./chunk-PHVIMUAQ.js";
8
+ } from "./chunk-KNKOTLVW.js";
9
9
 
10
10
  // ../api-routes/src/runtime-logger.ts
11
11
  import { format } from "util";
@@ -28,7 +28,7 @@ import {
28
28
  unknownUsageObservation,
29
29
  usdToMicros,
30
30
  withRetry
31
- } from "./chunk-PHVIMUAQ.js";
31
+ } from "./chunk-KNKOTLVW.js";
32
32
 
33
33
  // ../provider-answer-services/src/core/http.ts
34
34
  var MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  cloudflareQueueNameSchema
3
- } from "./chunk-PHVIMUAQ.js";
3
+ } from "./chunk-KNKOTLVW.js";
4
4
 
5
5
  // src/cloudflare-traffic-config.ts
6
6
  function unsupportedDeliveryMode(deliveryMode) {