@canonry/canonry 7.83.2 → 7.85.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agent-workspace/skills/aero/references/ai-traffic-tracking.md +2 -1
- package/assets/agent-workspace/skills/canonry/references/ai-traffic-tracking.md +2 -1
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +4 -0
- package/assets/agent-workspace/skills/canonry/references/server-side-traffic.md +45 -8
- package/assets/assets/{AgentSetupPage-B9NOQP8a.js → AgentSetupPage-NdqmQkYa.js} +1 -1
- package/assets/assets/{AuditHistoryPanel-Fvh4WFjq.js → AuditHistoryPanel-CuqdFCb9.js} +1 -1
- package/assets/assets/{BacklinksPage-D1UHFLoV.js → BacklinksPage-CVjZiG1i.js} +1 -1
- package/assets/assets/{HistoryPage-CBcacDgP.js → HistoryPage-DRvdgemW.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-ZuZpurwr.js → MeasurementPropertyPage-Bc5EpJFV.js} +1 -1
- package/assets/assets/{ProjectPage-BoqDyoGi.js → ProjectPage-CGGBSiK7.js} +1 -1
- package/assets/assets/{RunsPage-tlCk_gOw.js → RunsPage-B6_o81G3.js} +1 -1
- package/assets/assets/{SettingsPage-7mRTFLFQ.js → SettingsPage-DjmVapHO.js} +1 -1
- package/assets/assets/{SiteHealthSection-l_A_AmY_.js → SiteHealthSection-BPPB0D9D.js} +2 -2
- package/assets/assets/{ToneBadge-CXyUIWzB.js → ToneBadge-DwUBSAjL.js} +1 -1
- package/assets/assets/{TrafficPage-DQbBT3TP.js → TrafficPage-BvbjxOw3.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-D3DwE6Qd.js → TrafficSourceDetailPage-BGJwxS_Q.js} +1 -1
- package/assets/assets/{account-context-Bhgg1lEH.js → account-context-D7M4W4wP.js} +1 -1
- package/assets/assets/agent-trust-copy-WXXeDvDs.js +1 -0
- package/assets/assets/{card-B16Dvxau.js → card-DULVRmzI.js} +1 -1
- package/assets/assets/{extract-error-message-B5_v_-U8.js → extract-error-message-B9-6WgLm.js} +1 -1
- package/assets/assets/{format-helpers-Bs9rrO74.js → format-helpers-Cgq0VzXz.js} +1 -1
- package/assets/assets/{index-3aBRkQpI.js → index-A-4JN4Pv.js} +4 -4
- package/assets/assets/{query-client-D-tt1sn3.js → query-client-1Q6EwPba.js} +1 -1
- package/assets/assets/{react-query-DQr9PoNm.js → react-query-D0OiMBt3.js} +1 -1
- package/assets/assets/{react-sigma_core.esm.min-BfZVKh7v.js → react-sigma_core.esm.min-CR09uPhZ.js} +1 -1
- package/assets/assets/{service-Cmt36bhL.js → service-Bu-7icpb.js} +1 -1
- package/assets/index.html +7 -7
- package/dist/{ads-BSVYKQ2L.js → ads-ETZIGLL5.js} +3 -3
- package/dist/{agent-L7VUHOHS.js → agent-WG2FUC7C.js} +7 -7
- package/dist/{agent-ask-A4HQCNLV.js → agent-ask-775XBFOH.js} +3 -3
- package/dist/{agent-conversations-L3TJJKPJ.js → agent-conversations-AYZ3LQJQ.js} +3 -3
- package/dist/{agent-journeys-FEXRAZYN.js → agent-journeys-GWNZG73C.js} +3 -3
- package/dist/{agent-memory-LBTLOHUA.js → agent-memory-E5MRRS5P.js} +3 -3
- package/dist/{agent-providers-X7V2IHRK.js → agent-providers-DEQZWO2M.js} +3 -3
- package/dist/{agent-transcript-YSP2HKUZ.js → agent-transcript-4AKVVHQC.js} +3 -3
- package/dist/{analytics-FBPHES6M.js → analytics-HSLTWKDA.js} +3 -3
- package/dist/{apply-PBCXZC5F.js → apply-4GD7VGG7.js} +3 -3
- package/dist/{backfill-PZBLCRPI.js → backfill-LNE2BW34.js} +10 -9
- package/dist/{backlinks-QMLTF2VF.js → backlinks-TGE3DWA2.js} +3 -3
- package/dist/{bing-H65WGTXU.js → bing-56GNRC55.js} +3 -3
- package/dist/{bootstrap-JAZNMAKP.js → bootstrap-42XCWZTS.js} +10 -10
- package/dist/{cdp-4DAMAHTB.js → cdp-UODIMKE3.js} +7 -7
- package/dist/{chunk-UGUON7IK.js → chunk-2O5XBKEO.js} +7 -7
- package/dist/{chunk-IQTMHY23.js → chunk-2R6WI237.js} +17643 -17454
- package/dist/{chunk-OMNS42ZQ.js → chunk-2X6UAPAC.js} +523 -4
- package/dist/{chunk-SEAXXCU6.js → chunk-4KK3OTHE.js} +1 -1
- package/dist/{chunk-X7U7XYJQ.js → chunk-5NEZO3SC.js} +3 -3
- package/dist/{chunk-SVMTN7MI.js → chunk-6SGHSSWQ.js} +2 -2
- package/dist/{chunk-NCM6PX64.js → chunk-AICFUZ6X.js} +1 -1
- package/dist/{chunk-QAJDHHC5.js → chunk-BQTXXXUG.js} +1 -1
- package/dist/{chunk-V52KSHH7.js → chunk-C2H5TPKD.js} +338 -338
- package/dist/{chunk-IJQJSNOZ.js → chunk-D45AMUYA.js} +4 -4
- package/dist/{chunk-WA4FD5Y6.js → chunk-DWK6NIJF.js} +1 -1
- package/dist/{chunk-CBAYQHNX.js → chunk-GR7VVRNN.js} +1 -1
- package/dist/{chunk-SKH3N6WI.js → chunk-IOW23XNK.js} +3 -3
- package/dist/{chunk-5AGYH5PX.js → chunk-JVO5GQZH.js} +4287 -3934
- package/dist/{chunk-3GXAL3F7.js → chunk-KBLMQDUZ.js} +37 -171
- package/dist/{chunk-PHVIMUAQ.js → chunk-KNKOTLVW.js} +6 -2
- package/dist/{chunk-3UFQI4KL.js → chunk-L5R5S57T.js} +75 -1
- package/dist/{chunk-32U5LFIT.js → chunk-NY252PIY.js} +3 -3
- package/dist/{chunk-W5ZBMBBZ.js → chunk-OHDSI5XS.js} +1 -1
- package/dist/{chunk-KCFIQJOL.js → chunk-OS5BVALI.js} +2 -2
- package/dist/{chunk-AA6P5NK3.js → chunk-PQWZCAVS.js} +1 -1
- package/dist/{chunk-WDEDEEWA.js → chunk-PX2KVGOJ.js} +1 -1
- package/dist/{chunk-HIC62JTS.js → chunk-Q7354YI3.js} +3 -3
- package/dist/{chunk-KFMU6CLU.js → chunk-QEATYB7C.js} +1 -1
- package/dist/{chunk-Y7WHB65J.js → chunk-RCAB7M6A.js} +2 -2
- package/dist/{chunk-LZGVQSZL.js → chunk-SS33RWKG.js} +1 -1
- package/dist/{chunk-YCRPOY2Y.js → chunk-TPUBPXBN.js} +2 -2
- package/dist/{chunk-QHUCHZA3.js → chunk-TVZMFCX2.js} +6 -6
- package/dist/{chunk-4D2QHIRS.js → chunk-UMEQ7EYN.js} +4 -4
- package/dist/{chunk-IPPNN5KE.js → chunk-UMV4HYHD.js} +2 -2
- package/dist/{chunk-7Q5BSCKN.js → chunk-VFSHZHZS.js} +2 -2
- package/dist/{chunk-YJOGWS7F.js → chunk-X7XKMARG.js} +2 -2
- package/dist/{chunk-KVSCEZBR.js → chunk-YDC2V5FQ.js} +30 -25
- package/dist/{chunk-JCWXEVYW.js → chunk-Z53CKLDS.js} +3 -3
- package/dist/{chunk-P76XOEWL.js → chunk-ZXMDIWW6.js} +4 -4
- package/dist/{citations-MNLMO33V.js → citations-RXBPR5MX.js} +3 -3
- package/dist/{cli-catalog-IKF4A7ZS.js → cli-catalog-QVCJWPCI.js} +3 -3
- package/dist/{cli-commands-KNMPQFQO.js → cli-commands-ZXR4Z2RW.js} +7 -7
- package/dist/cli.js +13 -13
- package/dist/{competitor-5DY5PKNU.js → competitor-MDNNHRSD.js} +3 -3
- package/dist/{content-XNK2YY4S.js → content-D3JZZQ64.js} +3 -3
- package/dist/{dashboard-SMI6A6OF.js → dashboard-5S2A46GM.js} +6 -6
- package/dist/{demo-DIP5JBEI.js → demo-DQHWRIKR.js} +1 -1
- package/dist/{demo-server-EWOBWWV4.js → demo-server-FY2XBSD2.js} +12 -12
- package/dist/{discover-D4H4ALIE.js → discover-NVNWHOTZ.js} +3 -3
- package/dist/{discover-eval-2CID2IBT.js → discover-eval-CJB6GYUS.js} +3 -3
- package/dist/{doctor-OWRQNDWF.js → doctor-DLSKNC4W.js} +3 -3
- package/dist/{evidence-SQX4IMQM.js → evidence-QNJULIXM.js} +3 -3
- package/dist/{export-cmd-73PYMZZ5.js → export-cmd-Z2YG2WLK.js} +3 -3
- package/dist/{ga-S6AKEVAL.js → ga-RSNHBP2B.js} +3 -3
- package/dist/{gbp-57J3AXEY.js → gbp-ZEV62LSZ.js} +3 -3
- package/dist/{google-PPU5MJSV.js → google-RZHEIKW5.js} +4 -4
- package/dist/{google-marketing-W4KZ74HX.js → google-marketing-DYQXPTD3.js} +1 -1
- package/dist/{health-cmd-QYP4MS3E.js → health-cmd-6ATPYUYG.js} +3 -3
- package/dist/{history-MFHBHYH7.js → history-KLX6XRY4.js} +3 -3
- package/dist/index.d.ts +337 -47
- package/dist/index.js +25 -25
- package/dist/{init-UZPL4H55.js → init-5BALV6SU.js} +12 -12
- package/dist/{insights-IIMMXGRO.js → insights-ARCEFFK5.js} +3 -3
- package/dist/intelligence-service-YJFZ6IBI.js +16 -0
- package/dist/{journeys-DXXRNANM.js → journeys-WVRQRZQJ.js} +3 -3
- package/dist/{keys-O3T52MLX.js → keys-5KAJFRZZ.js} +3 -3
- package/dist/{keyword-MS23BR7O.js → keyword-PTUH3TFD.js} +3 -3
- package/dist/{logs-SKR5OLTY.js → logs-NM7ACXGQ.js} +3 -3
- package/dist/{mcp-NFC2PBGD.js → mcp-ALRQXYL3.js} +4 -4
- package/dist/mcp.js +9 -9
- package/dist/{measurement-plan-Z73LMQYB.js → measurement-plan-T52JHK2J.js} +5 -5
- package/dist/{notify-2RUWK2AR.js → notify-MJBMRDXD.js} +3 -3
- package/dist/{organic-evidence-BGAOEVMZ.js → organic-evidence-ZUCQFWD5.js} +3 -3
- package/dist/{overview-DONWVYZT.js → overview-74IHPU2A.js} +4 -4
- package/dist/{project-FFSPWR6R.js → project-DKQOPMH6.js} +3 -3
- package/dist/{query-AXEUWIW3.js → query-YDJ5T46L.js} +3 -3
- package/dist/{research-YURAJRCV.js → research-5DU6DFZW.js} +3 -3
- package/dist/{results-export-ECSJKZHZ.js → results-export-EDVVXKVW.js} +3 -3
- package/dist/{run-27EXPBE4.js → run-HRHGSIYE.js} +4 -4
- package/dist/{schedule-NY55SZZF.js → schedule-GZVKQGMP.js} +3 -3
- package/dist/{search-RXCBX3ST.js → search-M37GRKVC.js} +3 -3
- package/dist/{serve-KGZW5WPA.js → serve-DW6PPZPJ.js} +26 -26
- package/dist/{session-BY2FSRYU.js → session-H427H2SK.js} +13 -13
- package/dist/{settings-IPC5SSDU.js → settings-WGS7GP7C.js} +3 -3
- package/dist/{skills-Y3VNCWQM.js → skills-CDDUPMAU.js} +2 -2
- package/dist/{snapshot-TPDSMADU.js → snapshot-5MAK7PWE.js} +3 -3
- package/dist/{src-IDX2F2SR.js → src-FEV4KYXN.js} +8 -2
- package/dist/{status-YSQJBNM7.js → status-EHFOYTXQ.js} +4 -4
- package/dist/{technical-aeo-NMUQVIFG.js → technical-aeo-KVY4RQWT.js} +3 -3
- package/dist/{telemetry-OA6W7Q54.js → telemetry-7XRRIHVC.js} +4 -4
- package/dist/{tools-5YRH5NEI.js → tools-EHONHGB6.js} +11 -11
- package/dist/{traffic-CE47IFFI.js → traffic-TBR6GOWN.js} +5 -5
- package/dist/{users-NMWBTCDI.js → users-YQS4X47Q.js} +3 -3
- package/dist/{visibility-stats-DE64FVOJ.js → visibility-stats-JXIOSJXC.js} +3 -3
- package/dist/{wordpress-AJWIJSM7.js → wordpress-HVOLEPQY.js} +3 -3
- package/package.json +1 -1
- package/assets/assets/agent-trust-copy-CKZsKhc0.js +0 -1
- package/dist/intelligence-service-AMXDCTS2.js +0 -16
|
@@ -1,9 +1,47 @@
|
|
|
1
1
|
import {
|
|
2
|
-
|
|
3
|
-
} from "./chunk-
|
|
2
|
+
createLogger
|
|
3
|
+
} from "./chunk-6SGHSSWQ.js";
|
|
4
|
+
import {
|
|
5
|
+
addLikelyFakeAgentHits,
|
|
6
|
+
correctBackgroundCallJourneys,
|
|
7
|
+
mergeSplitJourneyVisitors,
|
|
8
|
+
writeUnrecognizedAgentBuckets
|
|
9
|
+
} from "./chunk-JVO5GQZH.js";
|
|
10
|
+
import {
|
|
11
|
+
AGENT_TRUST_RULES_VERSION,
|
|
12
|
+
BROWSER_SHAPED_AGENT_MARKERS,
|
|
13
|
+
DEFAULT_AI_CRAWLER_RULES,
|
|
14
|
+
GENERIC_HTTP_CLIENTS,
|
|
15
|
+
JOURNEY_RULES_VERSION,
|
|
16
|
+
SCANNER_BURST_WINDOW_MS,
|
|
17
|
+
SELF_TRAFFIC_USER_AGENT_PATTERNS,
|
|
18
|
+
UNLISTED_AGENT_SKIPPED_EXTENSIONS,
|
|
19
|
+
UNRECOGNIZED_AGENT_NAMING_VERSION,
|
|
20
|
+
classifyAiUserFetch,
|
|
21
|
+
classifyCrawler,
|
|
22
|
+
isScannerPath,
|
|
23
|
+
isSelfTraffic,
|
|
24
|
+
normalizeTrafficPathPattern,
|
|
25
|
+
unrecognizedAgentName
|
|
26
|
+
} from "./chunk-2R6WI237.js";
|
|
27
|
+
import {
|
|
28
|
+
RawEventSampleEventTypes,
|
|
29
|
+
agentJourneyCaptureDays,
|
|
30
|
+
agentJourneyEvents,
|
|
31
|
+
aiUserFetchEventsHourly,
|
|
32
|
+
aiUserFetchVerificationManifestsHourly,
|
|
33
|
+
crawlerEventsHourly,
|
|
34
|
+
migrationsTable,
|
|
35
|
+
parseJsonColumn,
|
|
36
|
+
rawEventSamples,
|
|
37
|
+
trafficReclassificationState,
|
|
38
|
+
unrecognizedAgentEventsHourly
|
|
39
|
+
} from "./chunk-L5R5S57T.js";
|
|
4
40
|
import {
|
|
5
41
|
AI_ENGINE_DOMAINS,
|
|
6
42
|
AI_ENGINE_SELF_DOMAINS,
|
|
43
|
+
AgentTrustLevels,
|
|
44
|
+
AgentTrustReasons,
|
|
7
45
|
AnchoredSpanSources,
|
|
8
46
|
OPENROUTER_MODEL_PATTERN,
|
|
9
47
|
OPENROUTER_ROUTE_FAMILY,
|
|
@@ -12,7 +50,12 @@ import {
|
|
|
12
50
|
ProviderNames,
|
|
13
51
|
RetrievalContracts,
|
|
14
52
|
RetrievalStatuses,
|
|
53
|
+
TrafficEventConfidences,
|
|
54
|
+
TrafficEvidenceKinds,
|
|
55
|
+
TrafficSourceTypes,
|
|
56
|
+
UNRECORDED_TRAFFIC_PATH,
|
|
15
57
|
VERTEX_AI_SEARCH_PROXY_DOMAIN,
|
|
58
|
+
VerificationStatuses,
|
|
16
59
|
asRecord,
|
|
17
60
|
describeError,
|
|
18
61
|
hostMatchesAnyDomain,
|
|
@@ -26,7 +69,7 @@ import {
|
|
|
26
69
|
retryAfterDelayMs,
|
|
27
70
|
usageCount,
|
|
28
71
|
withRetry
|
|
29
|
-
} from "./chunk-
|
|
72
|
+
} from "./chunk-KNKOTLVW.js";
|
|
30
73
|
|
|
31
74
|
// ../provider-openai/src/normalize.ts
|
|
32
75
|
import OpenAI from "openai";
|
|
@@ -2895,6 +2938,480 @@ function extractStoredAnswerAnchors(provider, rawResponse) {
|
|
|
2895
2938
|
}
|
|
2896
2939
|
}
|
|
2897
2940
|
|
|
2941
|
+
// src/traffic-reclassify.ts
|
|
2942
|
+
import { createHash } from "crypto";
|
|
2943
|
+
import { setImmediate as yieldToEventLoop } from "timers/promises";
|
|
2944
|
+
import { and, eq, gte, inArray, lte, ne, sql } from "drizzle-orm";
|
|
2945
|
+
var log = createLogger("TrafficReclassify");
|
|
2946
|
+
var STATE_ID = "default";
|
|
2947
|
+
var BATCH_SIZE = 200;
|
|
2948
|
+
var UNRECOGNIZED_TABLE_MIGRATION = 201;
|
|
2949
|
+
var TRAFFIC_RECLASSIFY_START_DELAY_MS = 3e4;
|
|
2950
|
+
function trafficClassificationFingerprint() {
|
|
2951
|
+
const pattern = (value) => `/${value.source}/${value.flags}`;
|
|
2952
|
+
return createHash("sha256").update(JSON.stringify({
|
|
2953
|
+
rules: DEFAULT_AI_CRAWLER_RULES.map((rule) => ({
|
|
2954
|
+
id: rule.id,
|
|
2955
|
+
operator: rule.operator,
|
|
2956
|
+
product: rule.product,
|
|
2957
|
+
purpose: rule.purpose,
|
|
2958
|
+
patterns: rule.userAgentPatterns.map(pattern)
|
|
2959
|
+
})),
|
|
2960
|
+
self: SELF_TRAFFIC_USER_AGENT_PATTERNS.map(pattern),
|
|
2961
|
+
naming: UNRECOGNIZED_AGENT_NAMING_VERSION,
|
|
2962
|
+
trust: AGENT_TRUST_RULES_VERSION,
|
|
2963
|
+
journeys: JOURNEY_RULES_VERSION,
|
|
2964
|
+
genericClients: GENERIC_HTTP_CLIENTS,
|
|
2965
|
+
browserMarkers: BROWSER_SHAPED_AGENT_MARKERS,
|
|
2966
|
+
skippedExtensions: UNLISTED_AGENT_SKIPPED_EXTENSIONS
|
|
2967
|
+
})).digest("hex");
|
|
2968
|
+
}
|
|
2969
|
+
function probeRequest(userAgent, path, observedAt) {
|
|
2970
|
+
return {
|
|
2971
|
+
sourceType: TrafficSourceTypes["cloud-run"],
|
|
2972
|
+
evidenceKind: TrafficEvidenceKinds["raw-request"],
|
|
2973
|
+
confidence: TrafficEventConfidences.observed,
|
|
2974
|
+
eventId: "reclassify",
|
|
2975
|
+
observedAt,
|
|
2976
|
+
method: "GET",
|
|
2977
|
+
requestUrl: `https://placeholder${path.startsWith("/") ? path : "/"}`,
|
|
2978
|
+
host: "placeholder",
|
|
2979
|
+
path,
|
|
2980
|
+
queryString: null,
|
|
2981
|
+
status: 200,
|
|
2982
|
+
userAgent,
|
|
2983
|
+
// Stored traffic keeps no client address, so no replay can verify an IP.
|
|
2984
|
+
remoteIp: null,
|
|
2985
|
+
referer: null,
|
|
2986
|
+
latencyMs: null,
|
|
2987
|
+
requestSizeBytes: null,
|
|
2988
|
+
responseSizeBytes: null,
|
|
2989
|
+
providerResource: { type: "cloud_run_revision", labels: {} },
|
|
2990
|
+
providerLabels: {}
|
|
2991
|
+
};
|
|
2992
|
+
}
|
|
2993
|
+
function classifyStored(userAgent, path, observedAt) {
|
|
2994
|
+
const request2 = probeRequest(userAgent, path, observedAt);
|
|
2995
|
+
if (isSelfTraffic(request2)) return { kind: "self" };
|
|
2996
|
+
const fetch2 = classifyAiUserFetch(request2);
|
|
2997
|
+
if (fetch2) return { kind: "user-fetch", fetch: fetch2 };
|
|
2998
|
+
const crawler = classifyCrawler(request2);
|
|
2999
|
+
if (crawler) return { kind: "crawler", crawler };
|
|
3000
|
+
return { kind: "unknown" };
|
|
3001
|
+
}
|
|
3002
|
+
function sqliteUtcToIso(value) {
|
|
3003
|
+
return /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)?$/.test(value) ? `${value.replace(" ", "T")}Z` : value;
|
|
3004
|
+
}
|
|
3005
|
+
function hourOf(iso) {
|
|
3006
|
+
const hour = new Date(iso);
|
|
3007
|
+
hour.setUTCMinutes(0, 0, 0);
|
|
3008
|
+
return hour.toISOString();
|
|
3009
|
+
}
|
|
3010
|
+
function unrecognizedCountingStart(db) {
|
|
3011
|
+
const row = db.select({ appliedAt: migrationsTable.appliedAt }).from(migrationsTable).where(eq(migrationsTable.version, UNRECOGNIZED_TABLE_MIGRATION)).get();
|
|
3012
|
+
return row ? hourOf(sqliteUtcToIso(row.appliedAt)) : null;
|
|
3013
|
+
}
|
|
3014
|
+
function addHourlyHits(tx, opts) {
|
|
3015
|
+
const { projectId, sourceId, tsHour, verdict, pathNormalized, status, hits, sampledUserAgent, now } = opts;
|
|
3016
|
+
if (verdict.kind === "user-fetch" && verdict.fetch.trust === AgentTrustLevels.likely_fake) {
|
|
3017
|
+
addLikelyFakeAgentHits(tx, {
|
|
3018
|
+
projectId,
|
|
3019
|
+
sourceId,
|
|
3020
|
+
buckets: [{ tsHour, botId: verdict.fetch.botId, operator: verdict.fetch.operator, reason: verdict.fetch.trustReasons[0], hits }],
|
|
3021
|
+
now
|
|
3022
|
+
});
|
|
3023
|
+
return;
|
|
3024
|
+
}
|
|
3025
|
+
const table = verdict.kind === "user-fetch" ? aiUserFetchEventsHourly : crawlerEventsHourly;
|
|
3026
|
+
const classified = verdict.kind === "user-fetch" ? verdict.fetch : verdict.crawler;
|
|
3027
|
+
tx.insert(table).values({
|
|
3028
|
+
projectId,
|
|
3029
|
+
sourceId,
|
|
3030
|
+
tsHour,
|
|
3031
|
+
botId: classified.botId,
|
|
3032
|
+
operator: classified.operator,
|
|
3033
|
+
verificationStatus: classified.verificationStatus,
|
|
3034
|
+
pathNormalized,
|
|
3035
|
+
status,
|
|
3036
|
+
hits,
|
|
3037
|
+
sampledUserAgent,
|
|
3038
|
+
createdAt: now,
|
|
3039
|
+
updatedAt: now
|
|
3040
|
+
}).onConflictDoUpdate({
|
|
3041
|
+
target: [table.projectId, table.sourceId, table.tsHour, table.botId, table.verificationStatus, table.pathNormalized, table.status],
|
|
3042
|
+
set: { hits: sql`${table.hits} + ${hits}`, updatedAt: now }
|
|
3043
|
+
}).run();
|
|
3044
|
+
}
|
|
3045
|
+
function botIdOf(verdict) {
|
|
3046
|
+
return verdict.kind === "user-fetch" ? verdict.fetch.botId : verdict.crawler.botId;
|
|
3047
|
+
}
|
|
3048
|
+
async function reclassifyUnrecognizedRows(db, opts) {
|
|
3049
|
+
const result = { examined: 0, moved: 0, movedHits: 0, renamed: 0, removed: 0, byBot: {} };
|
|
3050
|
+
const table = unrecognizedAgentEventsHourly;
|
|
3051
|
+
const verdicts = /* @__PURE__ */ new Map();
|
|
3052
|
+
const lastStartingRowid = Number(db.select({ max: sql`max(rowid)` }).from(table).get()?.max ?? 0);
|
|
3053
|
+
let lastRowid = 0;
|
|
3054
|
+
for (; ; ) {
|
|
3055
|
+
if (opts.shouldStop()) break;
|
|
3056
|
+
const keys = db.select({ rowid: sql`rowid`, projectId: table.projectId, sourceId: table.sourceId, tsHour: table.tsHour, agentName: table.agentName }).from(table).where(and(
|
|
3057
|
+
sql`rowid > ${lastRowid}`,
|
|
3058
|
+
sql`rowid <= ${lastStartingRowid}`,
|
|
3059
|
+
opts.projectIds ? inArray(table.projectId, [...opts.projectIds]) : void 0
|
|
3060
|
+
)).orderBy(sql`rowid`).limit(BATCH_SIZE).all();
|
|
3061
|
+
if (keys.length === 0) break;
|
|
3062
|
+
lastRowid = keys[keys.length - 1].rowid;
|
|
3063
|
+
result.examined += keys.length;
|
|
3064
|
+
db.transaction((tx) => {
|
|
3065
|
+
for (const key of keys) {
|
|
3066
|
+
const match = and(
|
|
3067
|
+
eq(table.projectId, key.projectId),
|
|
3068
|
+
eq(table.sourceId, key.sourceId),
|
|
3069
|
+
eq(table.tsHour, key.tsHour),
|
|
3070
|
+
eq(table.agentName, key.agentName)
|
|
3071
|
+
);
|
|
3072
|
+
const row = tx.select().from(table).where(match).get();
|
|
3073
|
+
if (!row?.sampledUserAgent) continue;
|
|
3074
|
+
const userAgent = row.sampledUserAgent;
|
|
3075
|
+
let verdict = verdicts.get(userAgent);
|
|
3076
|
+
if (!verdict) {
|
|
3077
|
+
verdict = classifyStored(userAgent, "/", row.tsHour);
|
|
3078
|
+
verdicts.set(userAgent, verdict);
|
|
3079
|
+
}
|
|
3080
|
+
if (verdict.kind === "self") {
|
|
3081
|
+
result.removed++;
|
|
3082
|
+
if (!opts.dryRun) tx.delete(table).where(match).run();
|
|
3083
|
+
continue;
|
|
3084
|
+
}
|
|
3085
|
+
if (verdict.kind === "user-fetch" || verdict.kind === "crawler") {
|
|
3086
|
+
result.moved++;
|
|
3087
|
+
result.movedHits += row.hits;
|
|
3088
|
+
const botId = botIdOf(verdict);
|
|
3089
|
+
result.byBot[botId] = (result.byBot[botId] ?? 0) + row.hits;
|
|
3090
|
+
if (opts.dryRun) continue;
|
|
3091
|
+
addHourlyHits(tx, {
|
|
3092
|
+
projectId: row.projectId,
|
|
3093
|
+
sourceId: row.sourceId,
|
|
3094
|
+
tsHour: row.tsHour,
|
|
3095
|
+
verdict,
|
|
3096
|
+
pathNormalized: UNRECORDED_TRAFFIC_PATH,
|
|
3097
|
+
status: 0,
|
|
3098
|
+
hits: row.hits,
|
|
3099
|
+
sampledUserAgent: userAgent,
|
|
3100
|
+
now: opts.now
|
|
3101
|
+
});
|
|
3102
|
+
tx.delete(table).where(match).run();
|
|
3103
|
+
continue;
|
|
3104
|
+
}
|
|
3105
|
+
const name = unrecognizedAgentName({ method: "GET", path: row.sampledPath, userAgent });
|
|
3106
|
+
if (name === row.agentName) continue;
|
|
3107
|
+
if (name === null) {
|
|
3108
|
+
result.removed++;
|
|
3109
|
+
if (!opts.dryRun) tx.delete(table).where(match).run();
|
|
3110
|
+
continue;
|
|
3111
|
+
}
|
|
3112
|
+
result.renamed++;
|
|
3113
|
+
if (opts.dryRun) continue;
|
|
3114
|
+
tx.delete(table).where(match).run();
|
|
3115
|
+
writeUnrecognizedAgentBuckets(tx, {
|
|
3116
|
+
projectId: row.projectId,
|
|
3117
|
+
sourceId: row.sourceId,
|
|
3118
|
+
buckets: [{ tsHour: row.tsHour, agentName: name, hits: row.hits, sampledUserAgent: userAgent, sampledPath: row.sampledPath ?? "" }],
|
|
3119
|
+
now: opts.now
|
|
3120
|
+
});
|
|
3121
|
+
}
|
|
3122
|
+
}, { behavior: "immediate" });
|
|
3123
|
+
await yieldToEventLoop();
|
|
3124
|
+
}
|
|
3125
|
+
return result;
|
|
3126
|
+
}
|
|
3127
|
+
async function reclassifyUnknownSamples(db, opts) {
|
|
3128
|
+
const result = { examined: 0, reclassified: 0, counted: 0, removed: 0, byBot: {} };
|
|
3129
|
+
const countingStart = unrecognizedCountingStart(db);
|
|
3130
|
+
const lastStartingRowid = Number(db.select({ max: sql`max(rowid)` }).from(rawEventSamples).get()?.max ?? 0);
|
|
3131
|
+
let lastRowid = 0;
|
|
3132
|
+
for (; ; ) {
|
|
3133
|
+
if (opts.shouldStop()) break;
|
|
3134
|
+
const samples = db.select({
|
|
3135
|
+
rowid: sql`rowid`,
|
|
3136
|
+
id: rawEventSamples.id,
|
|
3137
|
+
projectId: rawEventSamples.projectId,
|
|
3138
|
+
sourceId: rawEventSamples.sourceId,
|
|
3139
|
+
ts: rawEventSamples.ts,
|
|
3140
|
+
userAgent: rawEventSamples.userAgent,
|
|
3141
|
+
pathNormalized: rawEventSamples.pathNormalized,
|
|
3142
|
+
status: rawEventSamples.status
|
|
3143
|
+
}).from(rawEventSamples).where(and(
|
|
3144
|
+
eq(rawEventSamples.eventType, RawEventSampleEventTypes.unknown),
|
|
3145
|
+
sql`rowid > ${lastRowid}`,
|
|
3146
|
+
sql`rowid <= ${lastStartingRowid}`,
|
|
3147
|
+
opts.projectIds ? inArray(rawEventSamples.projectId, [...opts.projectIds]) : void 0
|
|
3148
|
+
)).orderBy(sql`rowid`).limit(BATCH_SIZE).all();
|
|
3149
|
+
if (samples.length === 0) break;
|
|
3150
|
+
lastRowid = samples[samples.length - 1].rowid;
|
|
3151
|
+
result.examined += samples.length;
|
|
3152
|
+
db.transaction((tx) => {
|
|
3153
|
+
for (const sample of samples) {
|
|
3154
|
+
if (!sample.userAgent) continue;
|
|
3155
|
+
const verdict = classifyStored(sample.userAgent, sample.pathNormalized, sample.ts);
|
|
3156
|
+
if (verdict.kind === "self") {
|
|
3157
|
+
result.removed++;
|
|
3158
|
+
if (!opts.dryRun) tx.delete(rawEventSamples).where(eq(rawEventSamples.id, sample.id)).run();
|
|
3159
|
+
continue;
|
|
3160
|
+
}
|
|
3161
|
+
if (verdict.kind === "unknown") continue;
|
|
3162
|
+
result.reclassified++;
|
|
3163
|
+
const botId = botIdOf(verdict);
|
|
3164
|
+
result.byBot[botId] = (result.byBot[botId] ?? 0) + 1;
|
|
3165
|
+
const tsHour = hourOf(sample.ts);
|
|
3166
|
+
const countSample = countingStart === null || tsHour < countingStart || unrecognizedAgentName({ method: "GET", path: sample.pathNormalized, userAgent: sample.userAgent }) === null;
|
|
3167
|
+
if (countSample) result.counted++;
|
|
3168
|
+
if (opts.dryRun) continue;
|
|
3169
|
+
tx.update(rawEventSamples).set({
|
|
3170
|
+
eventType: verdict.kind === "user-fetch" ? RawEventSampleEventTypes.aiUserFetch : RawEventSampleEventTypes.crawler,
|
|
3171
|
+
classifierDetailsJson: {
|
|
3172
|
+
crawler: verdict.kind === "crawler" ? { ...verdict.crawler, verificationManifest: null } : null,
|
|
3173
|
+
aiUserFetch: verdict.kind === "user-fetch" ? { ...verdict.fetch, verificationManifest: null } : null,
|
|
3174
|
+
aiReferral: null
|
|
3175
|
+
}
|
|
3176
|
+
}).where(eq(rawEventSamples.id, sample.id)).run();
|
|
3177
|
+
if (!countSample) continue;
|
|
3178
|
+
addHourlyHits(tx, {
|
|
3179
|
+
projectId: sample.projectId,
|
|
3180
|
+
sourceId: sample.sourceId,
|
|
3181
|
+
tsHour,
|
|
3182
|
+
verdict,
|
|
3183
|
+
pathNormalized: sample.pathNormalized,
|
|
3184
|
+
status: sample.status ?? 200,
|
|
3185
|
+
hits: 1,
|
|
3186
|
+
sampledUserAgent: sample.userAgent,
|
|
3187
|
+
now: opts.now
|
|
3188
|
+
});
|
|
3189
|
+
}
|
|
3190
|
+
}, { behavior: "immediate" });
|
|
3191
|
+
await yieldToEventLoop();
|
|
3192
|
+
}
|
|
3193
|
+
return result;
|
|
3194
|
+
}
|
|
3195
|
+
function hourlyAgentRow(key) {
|
|
3196
|
+
const table = aiUserFetchEventsHourly;
|
|
3197
|
+
return and(
|
|
3198
|
+
eq(table.projectId, key.projectId),
|
|
3199
|
+
eq(table.sourceId, key.sourceId),
|
|
3200
|
+
eq(table.tsHour, key.tsHour),
|
|
3201
|
+
eq(table.botId, key.botId),
|
|
3202
|
+
eq(table.verificationStatus, key.verificationStatus),
|
|
3203
|
+
eq(table.pathNormalized, key.pathNormalized),
|
|
3204
|
+
eq(table.status, key.status)
|
|
3205
|
+
);
|
|
3206
|
+
}
|
|
3207
|
+
async function reassessAgentHits(db, opts) {
|
|
3208
|
+
const result = { examined: 0, moved: 0, movedHits: 0 };
|
|
3209
|
+
const table = aiUserFetchEventsHourly;
|
|
3210
|
+
const lastStartingRowid = Number(db.select({ max: sql`max(rowid)` }).from(table).get()?.max ?? 0);
|
|
3211
|
+
let lastRowid = 0;
|
|
3212
|
+
for (; ; ) {
|
|
3213
|
+
if (opts.shouldStop()) break;
|
|
3214
|
+
const rows = db.select({
|
|
3215
|
+
rowid: sql`rowid`,
|
|
3216
|
+
projectId: table.projectId,
|
|
3217
|
+
sourceId: table.sourceId,
|
|
3218
|
+
tsHour: table.tsHour,
|
|
3219
|
+
botId: table.botId,
|
|
3220
|
+
operator: table.operator,
|
|
3221
|
+
verificationStatus: table.verificationStatus,
|
|
3222
|
+
pathNormalized: table.pathNormalized,
|
|
3223
|
+
status: table.status
|
|
3224
|
+
}).from(table).where(and(
|
|
3225
|
+
sql`rowid > ${lastRowid}`,
|
|
3226
|
+
sql`rowid <= ${lastStartingRowid}`,
|
|
3227
|
+
ne(table.verificationStatus, VerificationStatuses.verified),
|
|
3228
|
+
opts.projectIds ? inArray(table.projectId, [...opts.projectIds]) : void 0
|
|
3229
|
+
)).orderBy(sql`rowid`).limit(BATCH_SIZE).all();
|
|
3230
|
+
if (rows.length === 0) break;
|
|
3231
|
+
lastRowid = rows[rows.length - 1].rowid;
|
|
3232
|
+
result.examined += rows.length;
|
|
3233
|
+
const probes = rows.filter((row) => isScannerPath(row.pathNormalized));
|
|
3234
|
+
if (probes.length > 0) {
|
|
3235
|
+
db.transaction((tx) => {
|
|
3236
|
+
for (const row of probes) {
|
|
3237
|
+
const current = tx.select({ hits: table.hits }).from(table).where(hourlyAgentRow(row)).get();
|
|
3238
|
+
if (!current) continue;
|
|
3239
|
+
result.moved++;
|
|
3240
|
+
result.movedHits += current.hits;
|
|
3241
|
+
if (opts.dryRun) continue;
|
|
3242
|
+
addLikelyFakeAgentHits(tx, {
|
|
3243
|
+
projectId: row.projectId,
|
|
3244
|
+
sourceId: row.sourceId,
|
|
3245
|
+
buckets: [{ tsHour: row.tsHour, botId: row.botId, operator: row.operator, reason: AgentTrustReasons["scanner-path"], hits: current.hits }],
|
|
3246
|
+
now: opts.now
|
|
3247
|
+
});
|
|
3248
|
+
tx.delete(table).where(hourlyAgentRow(row)).run();
|
|
3249
|
+
}
|
|
3250
|
+
}, { behavior: "immediate" });
|
|
3251
|
+
}
|
|
3252
|
+
await yieldToEventLoop();
|
|
3253
|
+
}
|
|
3254
|
+
return result;
|
|
3255
|
+
}
|
|
3256
|
+
function moveAgentHitToBurst(tx, row, now) {
|
|
3257
|
+
const key = {
|
|
3258
|
+
projectId: row.projectId,
|
|
3259
|
+
sourceId: row.sourceId,
|
|
3260
|
+
tsHour: hourOf(row.ts),
|
|
3261
|
+
botId: row.agentId,
|
|
3262
|
+
verificationStatus: row.verificationStatus,
|
|
3263
|
+
pathNormalized: normalizeTrafficPathPattern(row.path),
|
|
3264
|
+
status: row.status ?? 0
|
|
3265
|
+
};
|
|
3266
|
+
const table = aiUserFetchEventsHourly;
|
|
3267
|
+
const taken = tx.update(table).set({ hits: sql`${table.hits} - 1`, updatedAt: now }).where(and(hourlyAgentRow(key), sql`${table.hits} > 0`)).run();
|
|
3268
|
+
if (taken.changes === 0) return;
|
|
3269
|
+
const manifests = aiUserFetchVerificationManifestsHourly;
|
|
3270
|
+
if (row.verificationManifest) {
|
|
3271
|
+
const manifestRow = and(
|
|
3272
|
+
eq(manifests.projectId, key.projectId),
|
|
3273
|
+
eq(manifests.sourceId, key.sourceId),
|
|
3274
|
+
eq(manifests.tsHour, key.tsHour),
|
|
3275
|
+
eq(manifests.botId, key.botId),
|
|
3276
|
+
eq(manifests.verificationStatus, key.verificationStatus),
|
|
3277
|
+
eq(manifests.pathNormalized, key.pathNormalized),
|
|
3278
|
+
eq(manifests.status, key.status),
|
|
3279
|
+
eq(manifests.manifestId, row.verificationManifest.id)
|
|
3280
|
+
);
|
|
3281
|
+
tx.update(manifests).set({ hits: sql`${manifests.hits} - 1`, updatedAt: now }).where(and(manifestRow, sql`${manifests.hits} > 0`)).run();
|
|
3282
|
+
tx.delete(manifests).where(and(manifestRow, sql`${manifests.hits} <= 0`)).run();
|
|
3283
|
+
}
|
|
3284
|
+
tx.delete(table).where(and(hourlyAgentRow(key), sql`${table.hits} <= 0`)).run();
|
|
3285
|
+
addLikelyFakeAgentHits(tx, {
|
|
3286
|
+
projectId: row.projectId,
|
|
3287
|
+
sourceId: row.sourceId,
|
|
3288
|
+
buckets: [{ tsHour: key.tsHour, botId: row.agentId, operator: row.operator, reason: AgentTrustReasons["scanner-burst"], hits: 1 }],
|
|
3289
|
+
now
|
|
3290
|
+
});
|
|
3291
|
+
}
|
|
3292
|
+
async function reassessAgentRequests(db, opts) {
|
|
3293
|
+
const result = { examined: 0, probes: 0, removed: 0 };
|
|
3294
|
+
const table = agentJourneyEvents;
|
|
3295
|
+
const clients = /* @__PURE__ */ new Map();
|
|
3296
|
+
const lastStartingRowid = Number(db.select({ max: sql`max(rowid)` }).from(table).get()?.max ?? 0);
|
|
3297
|
+
let lastRowid = 0;
|
|
3298
|
+
for (; ; ) {
|
|
3299
|
+
if (opts.shouldStop()) return result;
|
|
3300
|
+
const rows = db.select({ rowid: sql`rowid`, projectId: table.projectId, sourceId: table.sourceId, correlationKey: table.correlationKey, ts: table.ts, path: table.path }).from(table).where(and(
|
|
3301
|
+
sql`rowid > ${lastRowid}`,
|
|
3302
|
+
sql`rowid <= ${lastStartingRowid}`,
|
|
3303
|
+
ne(table.trust, AgentTrustLevels.verified),
|
|
3304
|
+
opts.projectIds ? inArray(table.projectId, [...opts.projectIds]) : void 0
|
|
3305
|
+
)).orderBy(sql`rowid`).limit(BATCH_SIZE).all();
|
|
3306
|
+
if (rows.length === 0) break;
|
|
3307
|
+
lastRowid = rows[rows.length - 1].rowid;
|
|
3308
|
+
result.examined += rows.length;
|
|
3309
|
+
for (const row of rows) {
|
|
3310
|
+
if (!isScannerPath(row.path)) continue;
|
|
3311
|
+
result.probes++;
|
|
3312
|
+
const id = JSON.stringify([row.projectId, row.sourceId, row.correlationKey]);
|
|
3313
|
+
const client = clients.get(id) ?? { projectId: row.projectId, sourceId: row.sourceId, correlationKey: row.correlationKey, probes: [] };
|
|
3314
|
+
client.probes.push(Date.parse(row.ts));
|
|
3315
|
+
clients.set(id, client);
|
|
3316
|
+
}
|
|
3317
|
+
await yieldToEventLoop();
|
|
3318
|
+
}
|
|
3319
|
+
for (const client of clients.values()) {
|
|
3320
|
+
if (opts.shouldStop()) break;
|
|
3321
|
+
const probes = client.probes.sort((a, b) => a - b);
|
|
3322
|
+
const from = new Date(probes[0] - SCANNER_BURST_WINDOW_MS).toISOString();
|
|
3323
|
+
const to = new Date(probes.at(-1) + SCANNER_BURST_WINDOW_MS).toISOString();
|
|
3324
|
+
db.transaction((tx) => {
|
|
3325
|
+
const rows = tx.select().from(table).where(and(
|
|
3326
|
+
eq(table.projectId, client.projectId),
|
|
3327
|
+
eq(table.sourceId, client.sourceId),
|
|
3328
|
+
eq(table.correlationKey, client.correlationKey),
|
|
3329
|
+
gte(table.ts, from),
|
|
3330
|
+
lte(table.ts, to),
|
|
3331
|
+
ne(table.trust, AgentTrustLevels.verified)
|
|
3332
|
+
)).all();
|
|
3333
|
+
for (const row of rows) {
|
|
3334
|
+
const probe = isScannerPath(row.path);
|
|
3335
|
+
const at = Date.parse(row.ts);
|
|
3336
|
+
if (!probe && !probes.some((t) => Math.abs(at - t) <= SCANNER_BURST_WINDOW_MS)) continue;
|
|
3337
|
+
result.removed++;
|
|
3338
|
+
if (opts.dryRun) continue;
|
|
3339
|
+
tx.delete(table).where(eq(table.id, row.id)).run();
|
|
3340
|
+
tx.update(agentJourneyCaptureDays).set({ storedEvents: sql`max(${agentJourneyCaptureDays.storedEvents} - 1, 0)` }).where(and(
|
|
3341
|
+
eq(agentJourneyCaptureDays.projectId, row.projectId),
|
|
3342
|
+
eq(agentJourneyCaptureDays.sourceId, row.sourceId),
|
|
3343
|
+
eq(agentJourneyCaptureDays.day, row.ts.slice(0, 10))
|
|
3344
|
+
)).run();
|
|
3345
|
+
if (!probe) moveAgentHitToBurst(tx, row, opts.now);
|
|
3346
|
+
}
|
|
3347
|
+
}, { behavior: "immediate" });
|
|
3348
|
+
await yieldToEventLoop();
|
|
3349
|
+
}
|
|
3350
|
+
return result;
|
|
3351
|
+
}
|
|
3352
|
+
async function reclassifyStoredTraffic(db, opts = {}) {
|
|
3353
|
+
const shared = {
|
|
3354
|
+
projectIds: opts.projectIds,
|
|
3355
|
+
dryRun: opts.dryRun === true,
|
|
3356
|
+
now: opts.now ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
3357
|
+
shouldStop: opts.shouldStop ?? (() => false)
|
|
3358
|
+
};
|
|
3359
|
+
const samples = await reclassifyUnknownSamples(db, shared);
|
|
3360
|
+
const unrecognized = shared.shouldStop() ? { examined: 0, moved: 0, movedHits: 0, renamed: 0, removed: 0, byBot: {} } : await reclassifyUnrecognizedRows(db, shared);
|
|
3361
|
+
const agentHits = shared.shouldStop() ? { examined: 0, moved: 0, movedHits: 0 } : await reassessAgentHits(db, shared);
|
|
3362
|
+
const agentRequests = shared.shouldStop() ? { examined: 0, probes: 0, removed: 0 } : await reassessAgentRequests(db, shared);
|
|
3363
|
+
const journeys = shared.shouldStop() ? { visitors: 0, visitorsRemoved: 0, visitsRemoved: 0, visitsRelanded: 0 } : correctBackgroundCallJourneys(db, shared);
|
|
3364
|
+
const journeyMerges = shared.shouldStop() ? { visitorsMerged: 0, visitsJoined: 0 } : mergeSplitJourneyVisitors(db, shared);
|
|
3365
|
+
return { dryRun: shared.dryRun, samples, unrecognized, agentHits, agentRequests, journeys, journeyMerges };
|
|
3366
|
+
}
|
|
3367
|
+
async function reclassifyStoredTrafficIfRulesChanged(db, opts = {}) {
|
|
3368
|
+
const fingerprint = trafficClassificationFingerprint();
|
|
3369
|
+
const state = db.select().from(trafficReclassificationState).where(eq(trafficReclassificationState.id, STATE_ID)).get();
|
|
3370
|
+
if (state?.fingerprint === fingerprint) return { ran: false };
|
|
3371
|
+
const now = opts.now ?? (() => (/* @__PURE__ */ new Date()).toISOString());
|
|
3372
|
+
const shouldStop = opts.shouldStop ?? (() => false);
|
|
3373
|
+
const result = await reclassifyStoredTraffic(db, { now: now(), shouldStop });
|
|
3374
|
+
if (shouldStop()) return { ran: false };
|
|
3375
|
+
const completedAt = now();
|
|
3376
|
+
db.insert(trafficReclassificationState).values({ id: STATE_ID, fingerprint, completedAt, resultJson: { ...result } }).onConflictDoUpdate({
|
|
3377
|
+
target: trafficReclassificationState.id,
|
|
3378
|
+
set: { fingerprint, completedAt, resultJson: { ...result } }
|
|
3379
|
+
}).run();
|
|
3380
|
+
return { ran: true, result };
|
|
3381
|
+
}
|
|
3382
|
+
function startTrafficReclassification(opts) {
|
|
3383
|
+
let stopped = false;
|
|
3384
|
+
const timer = setTimeout(() => {
|
|
3385
|
+
reclassifyStoredTrafficIfRulesChanged(opts.db, { shouldStop: () => stopped }).then((outcome) => {
|
|
3386
|
+
if (!outcome.ran || !outcome.result) return;
|
|
3387
|
+
const { samples, unrecognized, agentHits, agentRequests, journeys, journeyMerges } = outcome.result;
|
|
3388
|
+
log.info("traffic-reclassify.completed", {
|
|
3389
|
+
samplesReclassified: samples.reclassified,
|
|
3390
|
+
samplesCounted: samples.counted,
|
|
3391
|
+
samplesRemoved: samples.removed,
|
|
3392
|
+
unrecognizedMoved: unrecognized.moved,
|
|
3393
|
+
unrecognizedMovedHits: unrecognized.movedHits,
|
|
3394
|
+
unrecognizedRenamed: unrecognized.renamed,
|
|
3395
|
+
unrecognizedRemoved: unrecognized.removed,
|
|
3396
|
+
agentHitsMoved: agentHits.movedHits,
|
|
3397
|
+
agentRequestsRemoved: agentRequests.removed,
|
|
3398
|
+
journeyVisitorsRemoved: journeys.visitorsRemoved,
|
|
3399
|
+
journeyVisitsRemoved: journeys.visitsRemoved,
|
|
3400
|
+
journeyVisitsRelanded: journeys.visitsRelanded,
|
|
3401
|
+
journeyVisitorsMerged: journeyMerges.visitorsMerged,
|
|
3402
|
+
journeyVisitsJoined: journeyMerges.visitsJoined
|
|
3403
|
+
});
|
|
3404
|
+
}).catch((err) => {
|
|
3405
|
+
if (!stopped) log.error("traffic-reclassify.failed", { error: describeError(err) });
|
|
3406
|
+
});
|
|
3407
|
+
}, opts.delayMs ?? TRAFFIC_RECLASSIFY_START_DELAY_MS);
|
|
3408
|
+
timer.unref();
|
|
3409
|
+
return () => {
|
|
3410
|
+
stopped = true;
|
|
3411
|
+
clearTimeout(timer);
|
|
3412
|
+
};
|
|
3413
|
+
}
|
|
3414
|
+
|
|
2898
3415
|
export {
|
|
2899
3416
|
OPENAI_RETRIEVAL_CONTRACT,
|
|
2900
3417
|
reparseStoredResult,
|
|
@@ -2911,5 +3428,7 @@ export {
|
|
|
2911
3428
|
museAdapter,
|
|
2912
3429
|
openrouterAdapter,
|
|
2913
3430
|
resolveStoredApiResponse,
|
|
2914
|
-
extractStoredAnswerAnchors
|
|
3431
|
+
extractStoredAnswerAnchors,
|
|
3432
|
+
reclassifyStoredTraffic,
|
|
3433
|
+
startTrafficReclassification
|
|
2915
3434
|
};
|
|
@@ -3,7 +3,7 @@ import {
|
|
|
3
3
|
resolveCliAccess,
|
|
4
4
|
resolveCliAccessForInvocation,
|
|
5
5
|
windowOptions
|
|
6
|
-
} from "./chunk-
|
|
6
|
+
} from "./chunk-4KK3OTHE.js";
|
|
7
7
|
import {
|
|
8
8
|
emitJsonl
|
|
9
9
|
} from "./chunk-Z7IUAJOF.js";
|
|
@@ -14,7 +14,7 @@ import {
|
|
|
14
14
|
} from "./chunk-HGMYEDAQ.js";
|
|
15
15
|
import {
|
|
16
16
|
OperationAccesses
|
|
17
|
-
} from "./chunk-
|
|
17
|
+
} from "./chunk-KNKOTLVW.js";
|
|
18
18
|
|
|
19
19
|
// src/cli-catalog.ts
|
|
20
20
|
import { createRequire } from "module";
|
|
@@ -111,7 +111,7 @@ async function dispatchRegisteredCommand(args, fallbackFormat, specs) {
|
|
|
111
111
|
const accessValue = accessFilterFromArgs(argsWithoutHelp);
|
|
112
112
|
validateAccessFilter(accessValue);
|
|
113
113
|
if (fallbackFormat !== "text") {
|
|
114
|
-
const { printCliCatalog: printCliCatalog2 } = await import("./cli-catalog-
|
|
114
|
+
const { printCliCatalog: printCliCatalog2 } = await import("./cli-catalog-QVCJWPCI.js");
|
|
115
115
|
const group2 = argsWithoutHelp.find((arg) => !arg.startsWith("--"));
|
|
116
116
|
printCliCatalog2(specs, fallbackFormat, group2, accessValue);
|
|
117
117
|
return true;
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import {
|
|
2
2
|
getRequestContext
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-PX2KVGOJ.js";
|
|
4
4
|
import {
|
|
5
5
|
diagnosticIdentity,
|
|
6
6
|
redactLogString,
|
|
7
7
|
redactLogValue
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-KNKOTLVW.js";
|
|
9
9
|
|
|
10
10
|
// ../api-routes/src/runtime-logger.ts
|
|
11
11
|
import { format } from "util";
|