@ainyc/canonry 4.163.1 → 4.164.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +120 -66
- package/assets/assets/{AuditHistoryPanel-CcoUJRWK.js → AuditHistoryPanel-m4hLLnEl.js} +1 -1
- package/assets/assets/{BacklinksPage-CPJcgZMH.js → BacklinksPage-DWl04vdm.js} +1 -1
- package/assets/assets/{HistoryPage-CA5hKRuu.js → HistoryPage-BTb0MBB4.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-DxrhxQ0q.js → MeasurementPropertyPage-C8c4huVf.js} +1 -1
- package/assets/assets/{ProjectPage-CHhf6m-z.js → ProjectPage-CEbscgtB.js} +1 -1
- package/assets/assets/{RunRow-D_8fID7_.js → RunRow-CDzkoy89.js} +1 -1
- package/assets/assets/{RunsPage-vSxIzGSk.js → RunsPage-CzLCTfIA.js} +1 -1
- package/assets/assets/{SettingsPage-CooTu-Qq.js → SettingsPage-D8Vfdd2n.js} +1 -1
- package/assets/assets/{SiteHealthSection-B84ag_7p.js → SiteHealthSection-BGmBOTo0.js} +3 -3
- package/assets/assets/{TrafficPage-CXLxgcDy.js → TrafficPage-BcmemZJB.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-DH47U88b.js → TrafficSourceDetailPage-B9Dz1vcn.js} +1 -1
- package/assets/assets/{extract-error-message-DclWcEAn.js → extract-error-message-B5tyCXha.js} +1 -1
- package/assets/assets/{index-DH8-WPHn.js → index-CEjDqng5.js} +19 -19
- package/assets/assets/{react-sigma_core.esm.min-DXADa_zN.js → react-sigma_core.esm.min-z4XxmpWJ.js} +1 -1
- package/assets/index.html +1 -1
- package/dist/{chunk-INXI6PO7.js → chunk-76DCR646.js} +1 -1
- package/dist/{chunk-ESH3LPYQ.js → chunk-DCCEKIJ7.js} +28 -0
- package/dist/{chunk-G5UZTODF.js → chunk-R6QOYKRW.js} +805 -105
- package/dist/{chunk-MRHYSEZB.js → chunk-XYZMJA3L.js} +66 -62
- package/dist/cli.js +23 -6
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-AFAFGLCM.js → intelligence-service-5UVHKSHR.js} +2 -2
- package/dist/mcp.js +2 -2
- package/package.json +10 -10
|
@@ -581,7 +581,7 @@ import {
|
|
|
581
581
|
wordpressSchemaDeployResultDtoSchema,
|
|
582
582
|
wordpressSchemaStatusResultDtoSchema,
|
|
583
583
|
wordpressStatusDtoSchema
|
|
584
|
-
} from "./chunk-
|
|
584
|
+
} from "./chunk-DCCEKIJ7.js";
|
|
585
585
|
|
|
586
586
|
// src/intelligence-service.ts
|
|
587
587
|
import { eq as eq59, desc as desc26, asc as asc11, and as and47, ne as ne7, or as or13, inArray as inArray21, gte as gte14, lte as lte12 } from "drizzle-orm";
|
|
@@ -608,6 +608,7 @@ __export(schema_exports, {
|
|
|
608
608
|
agentToolEvents: () => agentToolEvents,
|
|
609
609
|
aiReferralEventsHourly: () => aiReferralEventsHourly,
|
|
610
610
|
aiUserFetchEventsHourly: () => aiUserFetchEventsHourly,
|
|
611
|
+
aiUserFetchVerificationManifestsHourly: () => aiUserFetchVerificationManifestsHourly,
|
|
611
612
|
apiKeys: () => apiKeys,
|
|
612
613
|
auditLog: () => auditLog,
|
|
613
614
|
backlinkDomains: () => backlinkDomains,
|
|
@@ -620,6 +621,7 @@ __export(schema_exports, {
|
|
|
620
621
|
competitors: () => competitors,
|
|
621
622
|
contentTargetDismissals: () => contentTargetDismissals,
|
|
622
623
|
crawlerEventsHourly: () => crawlerEventsHourly,
|
|
624
|
+
crawlerVerificationManifestsHourly: () => crawlerVerificationManifestsHourly,
|
|
623
625
|
discoveryProbes: () => discoveryProbes,
|
|
624
626
|
discoverySessions: () => discoverySessions,
|
|
625
627
|
doctorHealthState: () => doctorHealthState,
|
|
@@ -2242,6 +2244,55 @@ var crawlerEventsHourly = sqliteTable("crawler_events_hourly", {
|
|
|
2242
2244
|
index("idx_crawler_hourly_project_ts").on(table.projectId, table.tsHour),
|
|
2243
2245
|
index("idx_crawler_hourly_path").on(table.projectId, table.pathNormalized)
|
|
2244
2246
|
]);
|
|
2247
|
+
var crawlerVerificationManifestsHourly = sqliteTable("crawler_verification_manifests_hourly", {
|
|
2248
|
+
projectId: text("project_id").notNull(),
|
|
2249
|
+
sourceId: text("source_id").notNull(),
|
|
2250
|
+
tsHour: text("ts_hour").notNull(),
|
|
2251
|
+
botId: text("bot_id").notNull(),
|
|
2252
|
+
verificationStatus: text("verification_status").notNull(),
|
|
2253
|
+
pathNormalized: text("path_normalized").notNull(),
|
|
2254
|
+
status: integer("status").notNull(),
|
|
2255
|
+
manifestId: text("manifest_id").notNull(),
|
|
2256
|
+
manifestJson: text("manifest_json", { mode: "json" }).$type(),
|
|
2257
|
+
hits: integer("hits").notNull().default(0),
|
|
2258
|
+
createdAt: text("created_at").notNull(),
|
|
2259
|
+
updatedAt: text("updated_at").notNull()
|
|
2260
|
+
}, (table) => [
|
|
2261
|
+
primaryKey({
|
|
2262
|
+
columns: [
|
|
2263
|
+
table.projectId,
|
|
2264
|
+
table.sourceId,
|
|
2265
|
+
table.tsHour,
|
|
2266
|
+
table.botId,
|
|
2267
|
+
table.verificationStatus,
|
|
2268
|
+
table.pathNormalized,
|
|
2269
|
+
table.status,
|
|
2270
|
+
table.manifestId
|
|
2271
|
+
]
|
|
2272
|
+
}),
|
|
2273
|
+
foreignKey({
|
|
2274
|
+
name: "crawler_verification_manifests_parent_fk",
|
|
2275
|
+
columns: [
|
|
2276
|
+
table.projectId,
|
|
2277
|
+
table.sourceId,
|
|
2278
|
+
table.tsHour,
|
|
2279
|
+
table.botId,
|
|
2280
|
+
table.verificationStatus,
|
|
2281
|
+
table.pathNormalized,
|
|
2282
|
+
table.status
|
|
2283
|
+
],
|
|
2284
|
+
foreignColumns: [
|
|
2285
|
+
crawlerEventsHourly.projectId,
|
|
2286
|
+
crawlerEventsHourly.sourceId,
|
|
2287
|
+
crawlerEventsHourly.tsHour,
|
|
2288
|
+
crawlerEventsHourly.botId,
|
|
2289
|
+
crawlerEventsHourly.verificationStatus,
|
|
2290
|
+
crawlerEventsHourly.pathNormalized,
|
|
2291
|
+
crawlerEventsHourly.status
|
|
2292
|
+
]
|
|
2293
|
+
}).onDelete("cascade"),
|
|
2294
|
+
index("idx_crawler_verification_manifests_project_ts").on(table.projectId, table.tsHour)
|
|
2295
|
+
]);
|
|
2245
2296
|
var aiUserFetchEventsHourly = sqliteTable("ai_user_fetch_events_hourly", {
|
|
2246
2297
|
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
2247
2298
|
sourceId: text("source_id").notNull().references(() => trafficSources.id, { onDelete: "cascade" }),
|
|
@@ -2270,6 +2321,55 @@ var aiUserFetchEventsHourly = sqliteTable("ai_user_fetch_events_hourly", {
|
|
|
2270
2321
|
index("idx_ai_user_fetch_hourly_project_ts").on(table.projectId, table.tsHour),
|
|
2271
2322
|
index("idx_ai_user_fetch_hourly_path").on(table.projectId, table.pathNormalized)
|
|
2272
2323
|
]);
|
|
2324
|
+
var aiUserFetchVerificationManifestsHourly = sqliteTable("ai_user_fetch_verification_manifests_hourly", {
|
|
2325
|
+
projectId: text("project_id").notNull(),
|
|
2326
|
+
sourceId: text("source_id").notNull(),
|
|
2327
|
+
tsHour: text("ts_hour").notNull(),
|
|
2328
|
+
botId: text("bot_id").notNull(),
|
|
2329
|
+
verificationStatus: text("verification_status").notNull(),
|
|
2330
|
+
pathNormalized: text("path_normalized").notNull(),
|
|
2331
|
+
status: integer("status").notNull(),
|
|
2332
|
+
manifestId: text("manifest_id").notNull(),
|
|
2333
|
+
manifestJson: text("manifest_json", { mode: "json" }).$type(),
|
|
2334
|
+
hits: integer("hits").notNull().default(0),
|
|
2335
|
+
createdAt: text("created_at").notNull(),
|
|
2336
|
+
updatedAt: text("updated_at").notNull()
|
|
2337
|
+
}, (table) => [
|
|
2338
|
+
primaryKey({
|
|
2339
|
+
columns: [
|
|
2340
|
+
table.projectId,
|
|
2341
|
+
table.sourceId,
|
|
2342
|
+
table.tsHour,
|
|
2343
|
+
table.botId,
|
|
2344
|
+
table.verificationStatus,
|
|
2345
|
+
table.pathNormalized,
|
|
2346
|
+
table.status,
|
|
2347
|
+
table.manifestId
|
|
2348
|
+
]
|
|
2349
|
+
}),
|
|
2350
|
+
foreignKey({
|
|
2351
|
+
name: "ai_user_fetch_verification_manifests_parent_fk",
|
|
2352
|
+
columns: [
|
|
2353
|
+
table.projectId,
|
|
2354
|
+
table.sourceId,
|
|
2355
|
+
table.tsHour,
|
|
2356
|
+
table.botId,
|
|
2357
|
+
table.verificationStatus,
|
|
2358
|
+
table.pathNormalized,
|
|
2359
|
+
table.status
|
|
2360
|
+
],
|
|
2361
|
+
foreignColumns: [
|
|
2362
|
+
aiUserFetchEventsHourly.projectId,
|
|
2363
|
+
aiUserFetchEventsHourly.sourceId,
|
|
2364
|
+
aiUserFetchEventsHourly.tsHour,
|
|
2365
|
+
aiUserFetchEventsHourly.botId,
|
|
2366
|
+
aiUserFetchEventsHourly.verificationStatus,
|
|
2367
|
+
aiUserFetchEventsHourly.pathNormalized,
|
|
2368
|
+
aiUserFetchEventsHourly.status
|
|
2369
|
+
]
|
|
2370
|
+
}).onDelete("cascade"),
|
|
2371
|
+
index("idx_ai_user_fetch_verification_manifests_project_ts").on(table.projectId, table.tsHour)
|
|
2372
|
+
]);
|
|
2273
2373
|
var aiReferralEventsHourly = sqliteTable("ai_referral_events_hourly", {
|
|
2274
2374
|
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
2275
2375
|
sourceId: text("source_id").notNull().references(() => trafficSources.id, { onDelete: "cascade" }),
|
|
@@ -6264,6 +6364,71 @@ var MIGRATION_VERSIONS = [
|
|
|
6264
6364
|
`ALTER TABLE site_crawl_edges ADD COLUMN placement_unknown_occurrences INTEGER`,
|
|
6265
6365
|
`ALTER TABLE site_crawl_snapshots ADD COLUMN link_placement_ruleset_version TEXT`
|
|
6266
6366
|
]
|
|
6367
|
+
},
|
|
6368
|
+
{
|
|
6369
|
+
version: 139,
|
|
6370
|
+
name: "traffic-verification-manifest-provenance",
|
|
6371
|
+
// Verification is evidence produced by one exact range manifest. Store
|
|
6372
|
+
// that provenance in additive sidecars: the existing rollup keys stay
|
|
6373
|
+
// unchanged so an older binary can still upsert them after a rollback.
|
|
6374
|
+
// Historical rows deliberately receive no sidecar row because the old
|
|
6375
|
+
// schema never recorded which bundle classified them.
|
|
6376
|
+
statements: [
|
|
6377
|
+
`CREATE TABLE IF NOT EXISTS crawler_verification_manifests_hourly (
|
|
6378
|
+
project_id TEXT NOT NULL,
|
|
6379
|
+
source_id TEXT NOT NULL,
|
|
6380
|
+
ts_hour TEXT NOT NULL,
|
|
6381
|
+
bot_id TEXT NOT NULL,
|
|
6382
|
+
verification_status TEXT NOT NULL,
|
|
6383
|
+
path_normalized TEXT NOT NULL,
|
|
6384
|
+
status INTEGER NOT NULL,
|
|
6385
|
+
manifest_id TEXT NOT NULL,
|
|
6386
|
+
manifest_json TEXT,
|
|
6387
|
+
hits INTEGER NOT NULL DEFAULT 0,
|
|
6388
|
+
created_at TEXT NOT NULL,
|
|
6389
|
+
updated_at TEXT NOT NULL,
|
|
6390
|
+
PRIMARY KEY (
|
|
6391
|
+
project_id, source_id, ts_hour, bot_id, verification_status,
|
|
6392
|
+
path_normalized, status, manifest_id
|
|
6393
|
+
),
|
|
6394
|
+
FOREIGN KEY (
|
|
6395
|
+
project_id, source_id, ts_hour, bot_id, verification_status,
|
|
6396
|
+
path_normalized, status
|
|
6397
|
+
) REFERENCES crawler_events_hourly (
|
|
6398
|
+
project_id, source_id, ts_hour, bot_id, verification_status,
|
|
6399
|
+
path_normalized, status
|
|
6400
|
+
) ON DELETE CASCADE
|
|
6401
|
+
)`,
|
|
6402
|
+
`CREATE INDEX IF NOT EXISTS idx_crawler_verification_manifests_project_ts
|
|
6403
|
+
ON crawler_verification_manifests_hourly(project_id, ts_hour)`,
|
|
6404
|
+
`CREATE TABLE IF NOT EXISTS ai_user_fetch_verification_manifests_hourly (
|
|
6405
|
+
project_id TEXT NOT NULL,
|
|
6406
|
+
source_id TEXT NOT NULL,
|
|
6407
|
+
ts_hour TEXT NOT NULL,
|
|
6408
|
+
bot_id TEXT NOT NULL,
|
|
6409
|
+
verification_status TEXT NOT NULL,
|
|
6410
|
+
path_normalized TEXT NOT NULL,
|
|
6411
|
+
status INTEGER NOT NULL,
|
|
6412
|
+
manifest_id TEXT NOT NULL,
|
|
6413
|
+
manifest_json TEXT,
|
|
6414
|
+
hits INTEGER NOT NULL DEFAULT 0,
|
|
6415
|
+
created_at TEXT NOT NULL,
|
|
6416
|
+
updated_at TEXT NOT NULL,
|
|
6417
|
+
PRIMARY KEY (
|
|
6418
|
+
project_id, source_id, ts_hour, bot_id, verification_status,
|
|
6419
|
+
path_normalized, status, manifest_id
|
|
6420
|
+
),
|
|
6421
|
+
FOREIGN KEY (
|
|
6422
|
+
project_id, source_id, ts_hour, bot_id, verification_status,
|
|
6423
|
+
path_normalized, status
|
|
6424
|
+
) REFERENCES ai_user_fetch_events_hourly (
|
|
6425
|
+
project_id, source_id, ts_hour, bot_id, verification_status,
|
|
6426
|
+
path_normalized, status
|
|
6427
|
+
) ON DELETE CASCADE
|
|
6428
|
+
)`,
|
|
6429
|
+
`CREATE INDEX IF NOT EXISTS idx_ai_user_fetch_verification_manifests_project_ts
|
|
6430
|
+
ON ai_user_fetch_verification_manifests_hourly(project_id, ts_hour)`
|
|
6431
|
+
]
|
|
6267
6432
|
}
|
|
6268
6433
|
];
|
|
6269
6434
|
function addRunsMeasurementPlanVersionForeignKey(tx) {
|
|
@@ -32820,6 +32985,7 @@ var routeCatalog = [
|
|
|
32820
32985
|
method: "post",
|
|
32821
32986
|
path: "/api/v1/projects/{name}/ga/sync",
|
|
32822
32987
|
summary: "Sync GA4 traffic and AI referral data",
|
|
32988
|
+
description: "Syncs a window of GA4 history. `days` is bounded to GA4's supported sync range (1-90); a request outside it is clamped rather than rejected. The response reports the window ACTUALLY written as `days`, the unbounded request as `requestedDays`, and sets `clamped` when the two differ.",
|
|
32823
32989
|
tags: ["ga4"],
|
|
32824
32990
|
parameters: [nameParameter],
|
|
32825
32991
|
requestBody: {
|
|
@@ -32829,7 +32995,7 @@ var routeCatalog = [
|
|
|
32829
32995
|
schema: {
|
|
32830
32996
|
type: "object",
|
|
32831
32997
|
properties: {
|
|
32832
|
-
days: integerSchema
|
|
32998
|
+
days: { ...integerSchema, description: "Days of history to sync. Clamped to 1-90; check `clamped` in the response to detect truncation. Defaults to 30." }
|
|
32833
32999
|
}
|
|
32834
33000
|
}
|
|
32835
33001
|
}
|
|
@@ -35952,6 +36118,11 @@ var GA4_SCOPE = "https://www.googleapis.com/auth/analytics.readonly";
|
|
|
35952
36118
|
var GOOGLE_TOKEN_URL2 = "https://oauth2.googleapis.com/token";
|
|
35953
36119
|
var GA4_DEFAULT_SYNC_DAYS = 30;
|
|
35954
36120
|
var GA4_MAX_SYNC_DAYS = 90;
|
|
36121
|
+
function resolveGa4SyncDays(days) {
|
|
36122
|
+
const requestedDays = days != null && Number.isFinite(days) ? Math.trunc(days) : GA4_DEFAULT_SYNC_DAYS;
|
|
36123
|
+
const effectiveDays = Math.min(Math.max(1, requestedDays), GA4_MAX_SYNC_DAYS);
|
|
36124
|
+
return { requestedDays, effectiveDays, clamped: effectiveDays !== requestedDays };
|
|
36125
|
+
}
|
|
35955
36126
|
var GA4_REQUEST_TIMEOUT_MS = 3e4;
|
|
35956
36127
|
var GA4_MAX_PAGES = 50;
|
|
35957
36128
|
var GA4_MAX_CONCURRENT_REQUESTS = 4;
|
|
@@ -36270,7 +36441,7 @@ function formatDate2(d) {
|
|
|
36270
36441
|
return d.toISOString().slice(0, 10);
|
|
36271
36442
|
}
|
|
36272
36443
|
function measurementDateRange(days) {
|
|
36273
|
-
const syncDays =
|
|
36444
|
+
const { effectiveDays: syncDays } = resolveGa4SyncDays(days);
|
|
36274
36445
|
const endDate = /* @__PURE__ */ new Date();
|
|
36275
36446
|
const startDate = /* @__PURE__ */ new Date();
|
|
36276
36447
|
startDate.setUTCDate(startDate.getUTCDate() - syncDays);
|
|
@@ -36378,7 +36549,7 @@ var AI_REFERRAL_SOURCE_FILTERS = [
|
|
|
36378
36549
|
async function fetchTrafficByLandingPage(accessToken, propertyId, days) {
|
|
36379
36550
|
validateAccessToken2(accessToken);
|
|
36380
36551
|
validatePropertyId(propertyId);
|
|
36381
|
-
const syncDays =
|
|
36552
|
+
const { effectiveDays: syncDays } = resolveGa4SyncDays(days);
|
|
36382
36553
|
const endDate = /* @__PURE__ */ new Date();
|
|
36383
36554
|
const startDate = /* @__PURE__ */ new Date();
|
|
36384
36555
|
startDate.setDate(startDate.getDate() - syncDays);
|
|
@@ -36510,7 +36681,7 @@ async function verifyConnectionWithToken(accessToken, propertyId) {
|
|
|
36510
36681
|
async function fetchAggregateSummary(accessToken, propertyId, days) {
|
|
36511
36682
|
validateAccessToken2(accessToken);
|
|
36512
36683
|
validatePropertyId(propertyId);
|
|
36513
|
-
const syncDays =
|
|
36684
|
+
const { effectiveDays: syncDays } = resolveGa4SyncDays(days);
|
|
36514
36685
|
const endDate = /* @__PURE__ */ new Date();
|
|
36515
36686
|
const startDate = /* @__PURE__ */ new Date();
|
|
36516
36687
|
startDate.setDate(startDate.getDate() - syncDays);
|
|
@@ -36560,7 +36731,7 @@ function parseOptionalCountMetric(value) {
|
|
|
36560
36731
|
async function fetchDailyTotals(accessToken, propertyId, days) {
|
|
36561
36732
|
validateAccessToken2(accessToken);
|
|
36562
36733
|
validatePropertyId(propertyId);
|
|
36563
|
-
const syncDays =
|
|
36734
|
+
const { effectiveDays: syncDays } = resolveGa4SyncDays(days);
|
|
36564
36735
|
const endDate = /* @__PURE__ */ new Date();
|
|
36565
36736
|
const startDate = /* @__PURE__ */ new Date();
|
|
36566
36737
|
startDate.setDate(startDate.getDate() - syncDays);
|
|
@@ -36653,7 +36824,7 @@ async function fetchWindowSummary(accessToken, propertyId, windowKey) {
|
|
|
36653
36824
|
async function fetchAiReferrals(accessToken, propertyId, days) {
|
|
36654
36825
|
validateAccessToken2(accessToken);
|
|
36655
36826
|
validatePropertyId(propertyId);
|
|
36656
|
-
const syncDays =
|
|
36827
|
+
const { effectiveDays: syncDays } = resolveGa4SyncDays(days);
|
|
36657
36828
|
const endDate = /* @__PURE__ */ new Date();
|
|
36658
36829
|
const startDate = /* @__PURE__ */ new Date();
|
|
36659
36830
|
startDate.setDate(startDate.getDate() - syncDays);
|
|
@@ -36746,7 +36917,7 @@ var SOCIAL_CHANNEL_GROUPS = ["Organic Social", "Paid Social"];
|
|
|
36746
36917
|
async function fetchSocialReferrals(accessToken, propertyId, days) {
|
|
36747
36918
|
validateAccessToken2(accessToken);
|
|
36748
36919
|
validatePropertyId(propertyId);
|
|
36749
|
-
const syncDays =
|
|
36920
|
+
const { effectiveDays: syncDays } = resolveGa4SyncDays(days);
|
|
36750
36921
|
const endDate = /* @__PURE__ */ new Date();
|
|
36751
36922
|
const startDate = /* @__PURE__ */ new Date();
|
|
36752
36923
|
startDate.setDate(startDate.getDate() - syncDays);
|
|
@@ -45044,7 +45215,7 @@ async function ga4Routes(app, opts) {
|
|
|
45044
45215
|
});
|
|
45045
45216
|
app.post("/projects/:name/ga/sync", async (request, _reply) => {
|
|
45046
45217
|
const project = resolveProject(app.db, request.params.name);
|
|
45047
|
-
const days = request.body?.days
|
|
45218
|
+
const { requestedDays, effectiveDays: days, clamped } = resolveGa4SyncDays(request.body?.days);
|
|
45048
45219
|
const only = request.body?.only;
|
|
45049
45220
|
const validOnlyValues = ["traffic", "ai", "social"];
|
|
45050
45221
|
if (only !== void 0 && !validOnlyValues.includes(only)) {
|
|
@@ -45055,8 +45226,8 @@ async function ga4Routes(app, opts) {
|
|
|
45055
45226
|
const syncAi = !only || only === "ai";
|
|
45056
45227
|
const syncSocial = !only || only === "social";
|
|
45057
45228
|
const measurementState = app.db.select().from(gaMeasurementSyncStates).where(eq44(gaMeasurementSyncStates.projectId, project.id)).get();
|
|
45058
|
-
const acquisitionDays = measurementState?.acquisitionSyncedAt != null ?
|
|
45059
|
-
const leadDays = measurementState?.leadSyncedAt != null ?
|
|
45229
|
+
const acquisitionDays = measurementState?.acquisitionSyncedAt != null ? days : GA4_MAX_SYNC_DAYS;
|
|
45230
|
+
const leadDays = measurementState?.leadSyncedAt != null ? days : GA4_MAX_SYNC_DAYS;
|
|
45060
45231
|
const leadEventNames = project.measurement.leadEventNames;
|
|
45061
45232
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
45062
45233
|
const runId = crypto35.randomUUID();
|
|
@@ -45319,6 +45490,7 @@ async function ga4Routes(app, opts) {
|
|
|
45319
45490
|
aiReferralCount: aiReferrals.length,
|
|
45320
45491
|
socialReferralCount: socialReferrals.length,
|
|
45321
45492
|
days,
|
|
45493
|
+
...clamped ? { requestedDays, clamped } : {},
|
|
45322
45494
|
totalUsers: summary.totalUsers,
|
|
45323
45495
|
...only ? { only } : {}
|
|
45324
45496
|
});
|
|
@@ -45328,6 +45500,8 @@ async function ga4Routes(app, opts) {
|
|
|
45328
45500
|
aiReferralCount: aiReferrals.length,
|
|
45329
45501
|
socialReferralCount: socialReferrals.length,
|
|
45330
45502
|
days,
|
|
45503
|
+
requestedDays,
|
|
45504
|
+
clamped,
|
|
45331
45505
|
syncedAt: now,
|
|
45332
45506
|
measurement: {
|
|
45333
45507
|
acquisition: acquisitionResult,
|
|
@@ -48518,13 +48692,78 @@ async function listCloudRunTrafficEvents(accessToken, options) {
|
|
|
48518
48692
|
|
|
48519
48693
|
// ../integration-traffic/src/ip-ranges/anthropic.json
|
|
48520
48694
|
var anthropic_default = {
|
|
48521
|
-
_source: "
|
|
48522
|
-
|
|
48523
|
-
_notes: "Anthropic does not publish a clean machine-readable IP-range JSON like Google or OpenAI do. These prefixes are taken straight from ARIN's authoritative allocation records (NOT bgp.tools' AS-based view, which can include misleading announcements \u2014 e.g. AS60808 is registered to Anthropic but announces 209.249.57.0/24 which actually belongs to Mitel Networks). 216.73.216.0/22 (AWS-ANTHROPIC) is the load-bearing prefix \u2014 empirical Cloud Run logs show 100% of real ClaudeBot traffic comes from there; the other two are Anthropic's own allocations for non-crawler infra (kept since the verification gate still requires a Claude-* UA match). Refresh by re-running the ARIN RDAP query above and replacing the prefix list.",
|
|
48695
|
+
_source: "https://claude.com/crawling/bots.json",
|
|
48696
|
+
creationTime: "2026-08-13T20:38:01Z",
|
|
48524
48697
|
prefixes: [
|
|
48525
|
-
{
|
|
48526
|
-
|
|
48527
|
-
|
|
48698
|
+
{
|
|
48699
|
+
ipv4Prefix: "216.73.216.0/22"
|
|
48700
|
+
},
|
|
48701
|
+
{
|
|
48702
|
+
ipv4Prefix: "34.162.230.222/32"
|
|
48703
|
+
},
|
|
48704
|
+
{
|
|
48705
|
+
ipv4Prefix: "34.162.244.71/32"
|
|
48706
|
+
},
|
|
48707
|
+
{
|
|
48708
|
+
ipv4Prefix: "34.162.191.81/32"
|
|
48709
|
+
},
|
|
48710
|
+
{
|
|
48711
|
+
ipv4Prefix: "34.150.241.79/32"
|
|
48712
|
+
},
|
|
48713
|
+
{
|
|
48714
|
+
ipv4Prefix: "34.85.172.162/32"
|
|
48715
|
+
},
|
|
48716
|
+
{
|
|
48717
|
+
ipv4Prefix: "35.245.175.129/32"
|
|
48718
|
+
},
|
|
48719
|
+
{
|
|
48720
|
+
ipv4Prefix: "34.182.222.37/32"
|
|
48721
|
+
},
|
|
48722
|
+
{
|
|
48723
|
+
ipv4Prefix: "34.186.108.163/32"
|
|
48724
|
+
},
|
|
48725
|
+
{
|
|
48726
|
+
ipv4Prefix: "35.245.89.239/32"
|
|
48727
|
+
},
|
|
48728
|
+
{
|
|
48729
|
+
ipv4Prefix: "34.182.161.143/32"
|
|
48730
|
+
},
|
|
48731
|
+
{
|
|
48732
|
+
ipv4Prefix: "34.182.218.27/32"
|
|
48733
|
+
},
|
|
48734
|
+
{
|
|
48735
|
+
ipv4Prefix: "34.182.220.85/32"
|
|
48736
|
+
},
|
|
48737
|
+
{
|
|
48738
|
+
ipv4Prefix: "34.11.34.31/32"
|
|
48739
|
+
},
|
|
48740
|
+
{
|
|
48741
|
+
ipv4Prefix: "34.182.140.95/32"
|
|
48742
|
+
},
|
|
48743
|
+
{
|
|
48744
|
+
ipv4Prefix: "136.107.176.208/32"
|
|
48745
|
+
},
|
|
48746
|
+
{
|
|
48747
|
+
ipv4Prefix: "34.182.226.151/32"
|
|
48748
|
+
},
|
|
48749
|
+
{
|
|
48750
|
+
ipv4Prefix: "34.182.226.221/32"
|
|
48751
|
+
},
|
|
48752
|
+
{
|
|
48753
|
+
ipv4Prefix: "35.221.29.174/32"
|
|
48754
|
+
},
|
|
48755
|
+
{
|
|
48756
|
+
ipv4Prefix: "34.182.225.167/32"
|
|
48757
|
+
},
|
|
48758
|
+
{
|
|
48759
|
+
ipv4Prefix: "40.124.101.48/28"
|
|
48760
|
+
},
|
|
48761
|
+
{
|
|
48762
|
+
ipv4Prefix: "20.102.46.224/28"
|
|
48763
|
+
},
|
|
48764
|
+
{
|
|
48765
|
+
ipv4Prefix: "20.64.57.208/28"
|
|
48766
|
+
}
|
|
48528
48767
|
]
|
|
48529
48768
|
};
|
|
48530
48769
|
|
|
@@ -50623,6 +50862,141 @@ var perplexitybot_default = {
|
|
|
50623
50862
|
]
|
|
50624
50863
|
};
|
|
50625
50864
|
|
|
50865
|
+
// ../integration-traffic/src/ip-range-manifest.ts
|
|
50866
|
+
import { createHash as createHash4 } from "crypto";
|
|
50867
|
+
function isRecord(value) {
|
|
50868
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
50869
|
+
}
|
|
50870
|
+
function stableJson2(value, seen = /* @__PURE__ */ new Set()) {
|
|
50871
|
+
if (value === null) return "null";
|
|
50872
|
+
if (typeof value === "string" || typeof value === "boolean") return JSON.stringify(value);
|
|
50873
|
+
if (typeof value === "number") {
|
|
50874
|
+
return Number.isFinite(value) ? JSON.stringify(value) : JSON.stringify(String(value));
|
|
50875
|
+
}
|
|
50876
|
+
if (typeof value !== "object") return JSON.stringify(`[${typeof value}]`);
|
|
50877
|
+
if (seen.has(value)) return '"[circular]"';
|
|
50878
|
+
seen.add(value);
|
|
50879
|
+
const result = Array.isArray(value) ? `[${value.map((entry) => stableJson2(entry, seen)).join(",")}]` : `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableJson2(value[key], seen)}`).join(",")}}`;
|
|
50880
|
+
seen.delete(value);
|
|
50881
|
+
return result;
|
|
50882
|
+
}
|
|
50883
|
+
function parseIp(ip) {
|
|
50884
|
+
if (typeof ip !== "string" || ip.length === 0) return null;
|
|
50885
|
+
const mappedMatch = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/i.exec(ip);
|
|
50886
|
+
if (mappedMatch) return parseIp(mappedMatch[1]);
|
|
50887
|
+
if (ip.includes(":")) {
|
|
50888
|
+
const sides = ip.split("::");
|
|
50889
|
+
if (sides.length > 2) return null;
|
|
50890
|
+
const left = sides[0].length > 0 ? sides[0].split(":") : [];
|
|
50891
|
+
const right = sides.length === 2 && sides[1].length > 0 ? sides[1].split(":") : [];
|
|
50892
|
+
const groupCount = left.length + right.length;
|
|
50893
|
+
if (sides.length === 1 && groupCount !== 8) return null;
|
|
50894
|
+
if (sides.length === 2 && groupCount >= 8) return null;
|
|
50895
|
+
const fill = 8 - groupCount;
|
|
50896
|
+
const groups = [...left, ...new Array(fill).fill("0"), ...right];
|
|
50897
|
+
let addr2 = 0n;
|
|
50898
|
+
for (const group of groups) {
|
|
50899
|
+
if (!/^[0-9a-f]{1,4}$/i.test(group)) return null;
|
|
50900
|
+
addr2 = addr2 << 16n | BigInt(Number.parseInt(group, 16));
|
|
50901
|
+
}
|
|
50902
|
+
return { version: 6, addr: addr2 };
|
|
50903
|
+
}
|
|
50904
|
+
const octets = ip.split(".");
|
|
50905
|
+
if (octets.length !== 4) return null;
|
|
50906
|
+
let addr = 0n;
|
|
50907
|
+
for (const octet of octets) {
|
|
50908
|
+
if (!/^(?:0|[1-9]\d{0,2})$/.test(octet)) return null;
|
|
50909
|
+
const value = Number(octet);
|
|
50910
|
+
if (value > 255) return null;
|
|
50911
|
+
addr = addr << 8n | BigInt(value);
|
|
50912
|
+
}
|
|
50913
|
+
return { version: 4, addr };
|
|
50914
|
+
}
|
|
50915
|
+
function parseCidr(cidr) {
|
|
50916
|
+
if (typeof cidr !== "string") return null;
|
|
50917
|
+
const slash = cidr.indexOf("/");
|
|
50918
|
+
if (slash <= 0 || slash !== cidr.lastIndexOf("/") || slash === cidr.length - 1) return null;
|
|
50919
|
+
const ipPart = cidr.slice(0, slash);
|
|
50920
|
+
const prefixStr = cidr.slice(slash + 1);
|
|
50921
|
+
if (!/^(?:0|[1-9]\d{0,2})$/.test(prefixStr)) return null;
|
|
50922
|
+
const parsed = parseIp(ipPart);
|
|
50923
|
+
if (!parsed) return null;
|
|
50924
|
+
const prefix = Number(prefixStr);
|
|
50925
|
+
const totalBits = parsed.version === 4 ? 32 : 128;
|
|
50926
|
+
if (prefix > totalBits) return null;
|
|
50927
|
+
const allOnes = (1n << BigInt(totalBits)) - 1n;
|
|
50928
|
+
const mask = allOnes >> BigInt(totalBits - prefix) << BigInt(totalBits - prefix);
|
|
50929
|
+
return {
|
|
50930
|
+
version: parsed.version,
|
|
50931
|
+
network: parsed.addr & mask,
|
|
50932
|
+
mask
|
|
50933
|
+
};
|
|
50934
|
+
}
|
|
50935
|
+
function canonicalPrefixKey(entry) {
|
|
50936
|
+
if (isRecord(entry)) {
|
|
50937
|
+
const hasIpv4 = Object.hasOwn(entry, "ipv4Prefix");
|
|
50938
|
+
const hasIpv6 = Object.hasOwn(entry, "ipv6Prefix");
|
|
50939
|
+
if (hasIpv4 !== hasIpv6) {
|
|
50940
|
+
const value = entry[hasIpv4 ? "ipv4Prefix" : "ipv6Prefix"];
|
|
50941
|
+
if (typeof value === "string") {
|
|
50942
|
+
const parsed = parseCidr(value);
|
|
50943
|
+
const expectedVersion = hasIpv4 ? 4 : 6;
|
|
50944
|
+
if (parsed?.version === expectedVersion) {
|
|
50945
|
+
const hexWidth = parsed.version === 4 ? 8 : 32;
|
|
50946
|
+
const network = parsed.network.toString(16).padStart(hexWidth, "0");
|
|
50947
|
+
const mask = parsed.mask.toString(16).padStart(hexWidth, "0");
|
|
50948
|
+
return `cidr:${parsed.version}:${network}:${mask}`;
|
|
50949
|
+
}
|
|
50950
|
+
}
|
|
50951
|
+
}
|
|
50952
|
+
}
|
|
50953
|
+
return `invalid:${stableJson2(entry)}`;
|
|
50954
|
+
}
|
|
50955
|
+
function ipRangeManifestContentHash(value) {
|
|
50956
|
+
const prefixes = isRecord(value) ? value.prefixes : void 0;
|
|
50957
|
+
const canonical = Array.isArray(prefixes) ? prefixes.map(canonicalPrefixKey).sort() : [`invalid-prefixes:${stableJson2(prefixes)}`];
|
|
50958
|
+
return createHash4("sha256").update(JSON.stringify(canonical)).digest("hex");
|
|
50959
|
+
}
|
|
50960
|
+
function validateIpRangeManifestPayload(value) {
|
|
50961
|
+
if (!isRecord(value)) return { ok: false, error: "response must be a JSON object" };
|
|
50962
|
+
const creationTime = value.creationTime;
|
|
50963
|
+
if (typeof creationTime !== "string" || creationTime.trim().length === 0) {
|
|
50964
|
+
return { ok: false, error: "response missing `creationTime` version" };
|
|
50965
|
+
}
|
|
50966
|
+
if (!Array.isArray(value.prefixes)) {
|
|
50967
|
+
return { ok: false, error: "response missing `prefixes` array" };
|
|
50968
|
+
}
|
|
50969
|
+
const prefixes = [];
|
|
50970
|
+
for (const [index2, entry] of value.prefixes.entries()) {
|
|
50971
|
+
if (!isRecord(entry)) {
|
|
50972
|
+
return { ok: false, error: `prefixes[${index2}] must be an object` };
|
|
50973
|
+
}
|
|
50974
|
+
const hasIpv4 = Object.hasOwn(entry, "ipv4Prefix");
|
|
50975
|
+
const hasIpv6 = Object.hasOwn(entry, "ipv6Prefix");
|
|
50976
|
+
if (hasIpv4 === hasIpv6) {
|
|
50977
|
+
return { ok: false, error: `prefixes[${index2}] must contain exactly one IP prefix` };
|
|
50978
|
+
}
|
|
50979
|
+
const field = hasIpv4 ? "ipv4Prefix" : "ipv6Prefix";
|
|
50980
|
+
const cidr = entry[field];
|
|
50981
|
+
if (typeof cidr !== "string") {
|
|
50982
|
+
return { ok: false, error: `prefixes[${index2}].${field} must be a string` };
|
|
50983
|
+
}
|
|
50984
|
+
const parsed = parseCidr(cidr);
|
|
50985
|
+
const expectedVersion = hasIpv4 ? 4 : 6;
|
|
50986
|
+
if (!parsed || parsed.version !== expectedVersion) {
|
|
50987
|
+
return { ok: false, error: `prefixes[${index2}].${field} is not a valid IPv${expectedVersion} CIDR` };
|
|
50988
|
+
}
|
|
50989
|
+
prefixes.push(hasIpv4 ? { ipv4Prefix: cidr } : { ipv6Prefix: cidr });
|
|
50990
|
+
}
|
|
50991
|
+
return {
|
|
50992
|
+
ok: true,
|
|
50993
|
+
value: {
|
|
50994
|
+
creationTime: creationTime.trim(),
|
|
50995
|
+
prefixes
|
|
50996
|
+
}
|
|
50997
|
+
};
|
|
50998
|
+
}
|
|
50999
|
+
|
|
50626
51000
|
// ../integration-traffic/src/ip-verify.ts
|
|
50627
51001
|
var RULE_ID_TO_RANGES = {
|
|
50628
51002
|
// OpenAI — three separate published lists (training crawler vs
|
|
@@ -50656,94 +51030,56 @@ var RULE_ID_TO_RANGES = {
|
|
|
50656
51030
|
"perplexity-bot": perplexitybot_default,
|
|
50657
51031
|
// src: https://www.perplexity.ai/perplexity-user.json
|
|
50658
51032
|
"perplexity-user": perplexity_user_default,
|
|
50659
|
-
// Anthropic
|
|
50660
|
-
//
|
|
50661
|
-
//
|
|
50662
|
-
//
|
|
50663
|
-
//
|
|
50664
|
-
// logs show all real ClaudeBot traffic comes from there. The same
|
|
50665
|
-
// raw set is shared across every Claude-* UA the classifier emits:
|
|
50666
|
-
// both the training crawler and the per-user fetcher map here.
|
|
50667
|
-
// src: https://rdap.arin.net/registry/entity/AP-2440
|
|
51033
|
+
// Anthropic publishes one shared crawler-origin manifest. It confirms
|
|
51034
|
+
// Anthropic origin but does not attribute a prefix to ClaudeBot,
|
|
51035
|
+
// Claude-SearchBot, or Claude-User individually, so both classifier rules
|
|
51036
|
+
// map to the same file and the UA remains the product discriminator.
|
|
51037
|
+
// src: https://claude.com/crawling/bots.json
|
|
50668
51038
|
"anthropic-claudebot": anthropic_default,
|
|
50669
51039
|
"claude-user": anthropic_default
|
|
50670
51040
|
};
|
|
51041
|
+
function manifestMetadata(raw) {
|
|
51042
|
+
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) return null;
|
|
51043
|
+
const record = raw;
|
|
51044
|
+
const source = typeof record._source === "string" ? record._source.trim() : "";
|
|
51045
|
+
const version = typeof record.creationTime === "string" ? record.creationTime.trim() : "";
|
|
51046
|
+
if (!source || !version) return null;
|
|
51047
|
+
const contentHash2 = ipRangeManifestContentHash(raw);
|
|
51048
|
+
return { id: `${source}#${version}#sha256:${contentHash2}`, source, version };
|
|
51049
|
+
}
|
|
51050
|
+
function verificationDataFromRaw(raw) {
|
|
51051
|
+
const validation = validateIpRangeManifestPayload(raw);
|
|
51052
|
+
const ranges = [];
|
|
51053
|
+
if (validation.ok) {
|
|
51054
|
+
for (const entry of validation.value.prefixes) {
|
|
51055
|
+
const parsed = parseCidr(entry.ipv4Prefix ?? entry.ipv6Prefix);
|
|
51056
|
+
if (parsed) ranges.push(parsed);
|
|
51057
|
+
}
|
|
51058
|
+
}
|
|
51059
|
+
return { ranges, manifest: manifestMetadata(raw) };
|
|
51060
|
+
}
|
|
51061
|
+
function decideIpVerification(ip, data) {
|
|
51062
|
+
if (!ip || data.ranges.length === 0) return { verified: false, manifest: null };
|
|
51063
|
+
const parsed = parseIp(ip);
|
|
51064
|
+
if (!parsed) return { verified: false, manifest: null };
|
|
51065
|
+
for (const cidr of data.ranges) {
|
|
51066
|
+
if (parsed.version !== cidr.version) continue;
|
|
51067
|
+
if ((parsed.addr & cidr.mask) === cidr.network) {
|
|
51068
|
+
return { verified: true, manifest: data.manifest };
|
|
51069
|
+
}
|
|
51070
|
+
}
|
|
51071
|
+
return { verified: false, manifest: data.manifest };
|
|
51072
|
+
}
|
|
50671
51073
|
var CACHE = (() => {
|
|
50672
51074
|
const cache = /* @__PURE__ */ new Map();
|
|
50673
51075
|
for (const [ruleId, raw] of Object.entries(RULE_ID_TO_RANGES)) {
|
|
50674
|
-
|
|
50675
|
-
for (const entry of raw.prefixes) {
|
|
50676
|
-
const cidr = entry.ipv4Prefix ?? entry.ipv6Prefix;
|
|
50677
|
-
if (!cidr) continue;
|
|
50678
|
-
const p = parseCidr(cidr);
|
|
50679
|
-
if (p) parsed.push(p);
|
|
50680
|
-
}
|
|
50681
|
-
cache.set(ruleId, parsed);
|
|
51076
|
+
cache.set(ruleId, verificationDataFromRaw(raw));
|
|
50682
51077
|
}
|
|
50683
51078
|
return cache;
|
|
50684
51079
|
})();
|
|
50685
|
-
function
|
|
50686
|
-
|
|
50687
|
-
|
|
50688
|
-
if (mappedMatch) return parseIp(mappedMatch[1]);
|
|
50689
|
-
if (ip.includes(":")) {
|
|
50690
|
-
const sides = ip.split("::");
|
|
50691
|
-
if (sides.length > 2) return null;
|
|
50692
|
-
const left = sides[0].length > 0 ? sides[0].split(":") : [];
|
|
50693
|
-
const right = sides.length === 2 && sides[1].length > 0 ? sides[1].split(":") : [];
|
|
50694
|
-
const groupCount = left.length + right.length;
|
|
50695
|
-
if (groupCount > 8) return null;
|
|
50696
|
-
if (sides.length === 1 && groupCount !== 8) return null;
|
|
50697
|
-
const fill = 8 - groupCount;
|
|
50698
|
-
const groups = [...left, ...new Array(fill).fill("0"), ...right];
|
|
50699
|
-
let addr2 = 0n;
|
|
50700
|
-
for (const g of groups) {
|
|
50701
|
-
if (g.length === 0 || g.length > 4) return null;
|
|
50702
|
-
const n = Number.parseInt(g, 16);
|
|
50703
|
-
if (!Number.isFinite(n) || n < 0 || n > 65535) return null;
|
|
50704
|
-
addr2 = addr2 << 16n | BigInt(n);
|
|
50705
|
-
}
|
|
50706
|
-
return { version: 6, addr: addr2 };
|
|
50707
|
-
}
|
|
50708
|
-
const octets = ip.split(".");
|
|
50709
|
-
if (octets.length !== 4) return null;
|
|
50710
|
-
let addr = 0n;
|
|
50711
|
-
for (const o of octets) {
|
|
50712
|
-
if (o.length === 0 || o.length > 3) return null;
|
|
50713
|
-
const n = Number.parseInt(o, 10);
|
|
50714
|
-
if (!Number.isInteger(n) || n < 0 || n > 255) return null;
|
|
50715
|
-
addr = addr << 8n | BigInt(n);
|
|
50716
|
-
}
|
|
50717
|
-
return { version: 4, addr };
|
|
50718
|
-
}
|
|
50719
|
-
function parseCidr(cidr) {
|
|
50720
|
-
const [ipPart, prefixStr] = cidr.split("/");
|
|
50721
|
-
if (!ipPart || !prefixStr) return null;
|
|
50722
|
-
const prefix = Number.parseInt(prefixStr, 10);
|
|
50723
|
-
if (!Number.isInteger(prefix)) return null;
|
|
50724
|
-
const parsed = parseIp(ipPart);
|
|
50725
|
-
if (!parsed) return null;
|
|
50726
|
-
const totalBits = parsed.version === 4 ? 32 : 128;
|
|
50727
|
-
if (prefix < 0 || prefix > totalBits) return null;
|
|
50728
|
-
const allOnes = (1n << BigInt(totalBits)) - 1n;
|
|
50729
|
-
const mask = allOnes >> BigInt(totalBits - prefix) << BigInt(totalBits - prefix);
|
|
50730
|
-
return {
|
|
50731
|
-
version: parsed.version,
|
|
50732
|
-
network: parsed.addr & mask,
|
|
50733
|
-
mask
|
|
50734
|
-
};
|
|
50735
|
-
}
|
|
50736
|
-
function verifyIpForRule(ip, ruleId) {
|
|
50737
|
-
if (!ip) return false;
|
|
50738
|
-
const ranges = CACHE.get(ruleId);
|
|
50739
|
-
if (!ranges || ranges.length === 0) return false;
|
|
50740
|
-
const parsed = parseIp(ip);
|
|
50741
|
-
if (!parsed) return false;
|
|
50742
|
-
for (const cidr of ranges) {
|
|
50743
|
-
if (parsed.version !== cidr.version) continue;
|
|
50744
|
-
if ((parsed.addr & cidr.mask) === cidr.network) return true;
|
|
50745
|
-
}
|
|
50746
|
-
return false;
|
|
51080
|
+
function verifyIpForRuleDecision(ip, ruleId) {
|
|
51081
|
+
const data = CACHE.get(ruleId);
|
|
51082
|
+
return data ? decideIpVerification(ip, data) : { verified: false, manifest: null };
|
|
50747
51083
|
}
|
|
50748
51084
|
|
|
50749
51085
|
// ../integration-traffic/src/rules.ts
|
|
@@ -51113,13 +51449,14 @@ function classifyCrawler(event) {
|
|
|
51113
51449
|
for (const rule of DEFAULT_AI_CRAWLER_RULES) {
|
|
51114
51450
|
if (rule.purpose === USER_FETCH_PURPOSE) continue;
|
|
51115
51451
|
if (rule.userAgentPatterns.some((pattern) => pattern.test(userAgent))) {
|
|
51116
|
-
const
|
|
51452
|
+
const decision = verifyIpForRuleDecision(event.remoteIp, rule.id);
|
|
51117
51453
|
return {
|
|
51118
51454
|
botId: rule.id,
|
|
51119
51455
|
operator: rule.operator,
|
|
51120
51456
|
product: rule.product,
|
|
51121
51457
|
purpose: rule.purpose,
|
|
51122
|
-
verificationStatus: verified ? "verified" : "claimed_unverified",
|
|
51458
|
+
verificationStatus: decision.verified ? "verified" : "claimed_unverified",
|
|
51459
|
+
verificationManifest: decision.manifest,
|
|
51123
51460
|
matchedUserAgent: userAgent
|
|
51124
51461
|
};
|
|
51125
51462
|
}
|
|
@@ -51132,12 +51469,13 @@ function classifyAiUserFetch(event) {
|
|
|
51132
51469
|
for (const rule of DEFAULT_AI_CRAWLER_RULES) {
|
|
51133
51470
|
if (rule.purpose !== USER_FETCH_PURPOSE) continue;
|
|
51134
51471
|
if (rule.userAgentPatterns.some((pattern) => pattern.test(userAgent))) {
|
|
51135
|
-
const
|
|
51472
|
+
const decision = verifyIpForRuleDecision(event.remoteIp, rule.id);
|
|
51136
51473
|
return {
|
|
51137
51474
|
botId: rule.id,
|
|
51138
51475
|
operator: rule.operator,
|
|
51139
51476
|
product: rule.product,
|
|
51140
|
-
verificationStatus: verified ? "verified" : "claimed_unverified",
|
|
51477
|
+
verificationStatus: decision.verified ? "verified" : "claimed_unverified",
|
|
51478
|
+
verificationManifest: decision.manifest,
|
|
51141
51479
|
matchedUserAgent: userAgent
|
|
51142
51480
|
};
|
|
51143
51481
|
}
|
|
@@ -51286,10 +51624,10 @@ function mergeReferralSession(current, next) {
|
|
|
51286
51624
|
return { ...winner, trafficClass: paid ? AiReferralTrafficClasses.paid : winner.trafficClass };
|
|
51287
51625
|
}
|
|
51288
51626
|
function sortCrawlerBuckets(a, b) {
|
|
51289
|
-
return a.tsHour.localeCompare(b.tsHour) || a.botId.localeCompare(b.botId) || a.pathNormalized.localeCompare(b.pathNormalized) || String(a.status).localeCompare(String(b.status));
|
|
51627
|
+
return a.tsHour.localeCompare(b.tsHour) || a.botId.localeCompare(b.botId) || a.verificationStatus.localeCompare(b.verificationStatus) || (a.verificationManifest?.id ?? "").localeCompare(b.verificationManifest?.id ?? "") || a.pathNormalized.localeCompare(b.pathNormalized) || String(a.status).localeCompare(String(b.status));
|
|
51290
51628
|
}
|
|
51291
51629
|
function sortAiUserFetchBuckets(a, b) {
|
|
51292
|
-
return a.tsHour.localeCompare(b.tsHour) || a.botId.localeCompare(b.botId) || a.pathNormalized.localeCompare(b.pathNormalized) || String(a.status).localeCompare(String(b.status));
|
|
51630
|
+
return a.tsHour.localeCompare(b.tsHour) || a.botId.localeCompare(b.botId) || a.verificationStatus.localeCompare(b.verificationStatus) || (a.verificationManifest?.id ?? "").localeCompare(b.verificationManifest?.id ?? "") || a.pathNormalized.localeCompare(b.pathNormalized) || String(a.status).localeCompare(String(b.status));
|
|
51293
51631
|
}
|
|
51294
51632
|
function sortReferralBuckets(a, b) {
|
|
51295
51633
|
return a.tsHour.localeCompare(b.tsHour) || a.product.localeCompare(b.product) || a.sourceDomain.localeCompare(b.sourceDomain) || a.landingPathNormalized.localeCompare(b.landingPathNormalized) || String(a.status).localeCompare(String(b.status));
|
|
@@ -51332,6 +51670,7 @@ function buildTrafficProbeReport(events, options = {}) {
|
|
|
51332
51670
|
tsHour,
|
|
51333
51671
|
crawler.botId,
|
|
51334
51672
|
crawler.verificationStatus,
|
|
51673
|
+
crawler.verificationManifest?.id ?? "no-manifest",
|
|
51335
51674
|
pathNormalized,
|
|
51336
51675
|
event.status ?? "null"
|
|
51337
51676
|
].join(" ");
|
|
@@ -51345,6 +51684,7 @@ function buildTrafficProbeReport(events, options = {}) {
|
|
|
51345
51684
|
operator: crawler.operator,
|
|
51346
51685
|
product: crawler.product,
|
|
51347
51686
|
verificationStatus: crawler.verificationStatus,
|
|
51687
|
+
verificationManifest: crawler.verificationManifest,
|
|
51348
51688
|
pathNormalized,
|
|
51349
51689
|
status: event.status,
|
|
51350
51690
|
hits: 1,
|
|
@@ -51363,6 +51703,7 @@ function buildTrafficProbeReport(events, options = {}) {
|
|
|
51363
51703
|
tsHour,
|
|
51364
51704
|
aiUserFetch.botId,
|
|
51365
51705
|
aiUserFetch.verificationStatus,
|
|
51706
|
+
aiUserFetch.verificationManifest?.id ?? "no-manifest",
|
|
51366
51707
|
pathNormalized,
|
|
51367
51708
|
event.status ?? "null"
|
|
51368
51709
|
].join(" ");
|
|
@@ -51376,6 +51717,7 @@ function buildTrafficProbeReport(events, options = {}) {
|
|
|
51376
51717
|
operator: aiUserFetch.operator,
|
|
51377
51718
|
product: aiUserFetch.product,
|
|
51378
51719
|
verificationStatus: aiUserFetch.verificationStatus,
|
|
51720
|
+
verificationManifest: aiUserFetch.verificationManifest,
|
|
51379
51721
|
pathNormalized,
|
|
51380
51722
|
status: event.status,
|
|
51381
51723
|
hits: 1,
|
|
@@ -53042,6 +53384,38 @@ function writeTrafficEventBatch(opts) {
|
|
|
53042
53384
|
updatedAt: opts.receivedAt
|
|
53043
53385
|
}
|
|
53044
53386
|
}).run();
|
|
53387
|
+
if (bucket.verificationManifest) {
|
|
53388
|
+
tx.insert(crawlerVerificationManifestsHourly).values({
|
|
53389
|
+
projectId: opts.projectId,
|
|
53390
|
+
sourceId: opts.sourceId,
|
|
53391
|
+
tsHour: bucket.tsHour,
|
|
53392
|
+
botId: bucket.botId,
|
|
53393
|
+
verificationStatus: bucket.verificationStatus,
|
|
53394
|
+
pathNormalized: bucket.pathNormalized,
|
|
53395
|
+
status,
|
|
53396
|
+
manifestId: bucket.verificationManifest.id,
|
|
53397
|
+
manifestJson: bucket.verificationManifest,
|
|
53398
|
+
hits: bucket.hits,
|
|
53399
|
+
createdAt: opts.receivedAt,
|
|
53400
|
+
updatedAt: opts.receivedAt
|
|
53401
|
+
}).onConflictDoUpdate({
|
|
53402
|
+
target: [
|
|
53403
|
+
crawlerVerificationManifestsHourly.projectId,
|
|
53404
|
+
crawlerVerificationManifestsHourly.sourceId,
|
|
53405
|
+
crawlerVerificationManifestsHourly.tsHour,
|
|
53406
|
+
crawlerVerificationManifestsHourly.botId,
|
|
53407
|
+
crawlerVerificationManifestsHourly.verificationStatus,
|
|
53408
|
+
crawlerVerificationManifestsHourly.pathNormalized,
|
|
53409
|
+
crawlerVerificationManifestsHourly.status,
|
|
53410
|
+
crawlerVerificationManifestsHourly.manifestId
|
|
53411
|
+
],
|
|
53412
|
+
set: {
|
|
53413
|
+
hits: sql18`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
53414
|
+
manifestJson: bucket.verificationManifest,
|
|
53415
|
+
updatedAt: opts.receivedAt
|
|
53416
|
+
}
|
|
53417
|
+
}).run();
|
|
53418
|
+
}
|
|
53045
53419
|
crawlerBucketRows += 1;
|
|
53046
53420
|
}
|
|
53047
53421
|
for (const bucket of report.aiUserFetchEventsHourly) {
|
|
@@ -53075,6 +53449,38 @@ function writeTrafficEventBatch(opts) {
|
|
|
53075
53449
|
updatedAt: opts.receivedAt
|
|
53076
53450
|
}
|
|
53077
53451
|
}).run();
|
|
53452
|
+
if (bucket.verificationManifest) {
|
|
53453
|
+
tx.insert(aiUserFetchVerificationManifestsHourly).values({
|
|
53454
|
+
projectId: opts.projectId,
|
|
53455
|
+
sourceId: opts.sourceId,
|
|
53456
|
+
tsHour: bucket.tsHour,
|
|
53457
|
+
botId: bucket.botId,
|
|
53458
|
+
verificationStatus: bucket.verificationStatus,
|
|
53459
|
+
pathNormalized: bucket.pathNormalized,
|
|
53460
|
+
status,
|
|
53461
|
+
manifestId: bucket.verificationManifest.id,
|
|
53462
|
+
manifestJson: bucket.verificationManifest,
|
|
53463
|
+
hits: bucket.hits,
|
|
53464
|
+
createdAt: opts.receivedAt,
|
|
53465
|
+
updatedAt: opts.receivedAt
|
|
53466
|
+
}).onConflictDoUpdate({
|
|
53467
|
+
target: [
|
|
53468
|
+
aiUserFetchVerificationManifestsHourly.projectId,
|
|
53469
|
+
aiUserFetchVerificationManifestsHourly.sourceId,
|
|
53470
|
+
aiUserFetchVerificationManifestsHourly.tsHour,
|
|
53471
|
+
aiUserFetchVerificationManifestsHourly.botId,
|
|
53472
|
+
aiUserFetchVerificationManifestsHourly.verificationStatus,
|
|
53473
|
+
aiUserFetchVerificationManifestsHourly.pathNormalized,
|
|
53474
|
+
aiUserFetchVerificationManifestsHourly.status,
|
|
53475
|
+
aiUserFetchVerificationManifestsHourly.manifestId
|
|
53476
|
+
],
|
|
53477
|
+
set: {
|
|
53478
|
+
hits: sql18`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
53479
|
+
manifestJson: bucket.verificationManifest,
|
|
53480
|
+
updatedAt: opts.receivedAt
|
|
53481
|
+
}
|
|
53482
|
+
}).run();
|
|
53483
|
+
}
|
|
53078
53484
|
aiUserFetchBucketRows += 1;
|
|
53079
53485
|
}
|
|
53080
53486
|
for (const bucket of report.aiReferralEventsHourly) {
|
|
@@ -53393,6 +53799,117 @@ function trafficSeriesPoint(pointsByBucket, bucket) {
|
|
|
53393
53799
|
pointsByBucket.set(bucket, created);
|
|
53394
53800
|
return created;
|
|
53395
53801
|
}
|
|
53802
|
+
function verificationEventKey(row) {
|
|
53803
|
+
return JSON.stringify([
|
|
53804
|
+
row.sourceId,
|
|
53805
|
+
row.tsHour,
|
|
53806
|
+
row.botId,
|
|
53807
|
+
row.verificationStatus,
|
|
53808
|
+
row.pathNormalized,
|
|
53809
|
+
row.status
|
|
53810
|
+
]);
|
|
53811
|
+
}
|
|
53812
|
+
function verificationSelectedRowsJson(projectId, rows) {
|
|
53813
|
+
return JSON.stringify(rows.map((row) => [
|
|
53814
|
+
projectId,
|
|
53815
|
+
row.sourceId,
|
|
53816
|
+
row.tsHour,
|
|
53817
|
+
row.botId,
|
|
53818
|
+
row.verificationStatus,
|
|
53819
|
+
row.pathNormalized,
|
|
53820
|
+
row.status
|
|
53821
|
+
]));
|
|
53822
|
+
}
|
|
53823
|
+
function buildVerificationProvenance(events, rows) {
|
|
53824
|
+
const manifestsByEvent = /* @__PURE__ */ new Map();
|
|
53825
|
+
for (const row of rows) {
|
|
53826
|
+
const key = verificationEventKey(row);
|
|
53827
|
+
const manifests = manifestsByEvent.get(key);
|
|
53828
|
+
const usage = { manifestId: row.manifestId, manifest: row.manifest, hits: row.hits };
|
|
53829
|
+
if (manifests) manifests.push(usage);
|
|
53830
|
+
else manifestsByEvent.set(key, [usage]);
|
|
53831
|
+
}
|
|
53832
|
+
const provenanceByEvent = /* @__PURE__ */ new Map();
|
|
53833
|
+
for (const event of events) {
|
|
53834
|
+
const key = verificationEventKey(event);
|
|
53835
|
+
const verificationManifests = manifestsByEvent.get(key) ?? [];
|
|
53836
|
+
verificationManifests.sort((a, b) => a.manifestId.localeCompare(b.manifestId));
|
|
53837
|
+
const attributedHits = verificationManifests.reduce((sum, manifest) => sum + manifest.hits, 0);
|
|
53838
|
+
provenanceByEvent.set(key, {
|
|
53839
|
+
verificationManifests,
|
|
53840
|
+
verificationUnattributedHits: Math.max(0, event.hits - attributedHits)
|
|
53841
|
+
});
|
|
53842
|
+
}
|
|
53843
|
+
return provenanceByEvent;
|
|
53844
|
+
}
|
|
53845
|
+
function crawlerVerificationProvenance(db, projectId, events) {
|
|
53846
|
+
if (events.length === 0) return /* @__PURE__ */ new Map();
|
|
53847
|
+
const selectedRows = verificationSelectedRowsJson(projectId, events);
|
|
53848
|
+
const rows = db.select({
|
|
53849
|
+
sourceId: crawlerVerificationManifestsHourly.sourceId,
|
|
53850
|
+
tsHour: crawlerVerificationManifestsHourly.tsHour,
|
|
53851
|
+
botId: crawlerVerificationManifestsHourly.botId,
|
|
53852
|
+
verificationStatus: crawlerVerificationManifestsHourly.verificationStatus,
|
|
53853
|
+
pathNormalized: crawlerVerificationManifestsHourly.pathNormalized,
|
|
53854
|
+
status: crawlerVerificationManifestsHourly.status,
|
|
53855
|
+
manifestId: crawlerVerificationManifestsHourly.manifestId,
|
|
53856
|
+
manifest: crawlerVerificationManifestsHourly.manifestJson,
|
|
53857
|
+
hits: crawlerVerificationManifestsHourly.hits
|
|
53858
|
+
}).from(crawlerVerificationManifestsHourly).where(sql19`(
|
|
53859
|
+
${crawlerVerificationManifestsHourly.projectId},
|
|
53860
|
+
${crawlerVerificationManifestsHourly.sourceId},
|
|
53861
|
+
${crawlerVerificationManifestsHourly.tsHour},
|
|
53862
|
+
${crawlerVerificationManifestsHourly.botId},
|
|
53863
|
+
${crawlerVerificationManifestsHourly.verificationStatus},
|
|
53864
|
+
${crawlerVerificationManifestsHourly.pathNormalized},
|
|
53865
|
+
${crawlerVerificationManifestsHourly.status}
|
|
53866
|
+
) IN (
|
|
53867
|
+
SELECT
|
|
53868
|
+
json_extract(value, '$[0]'),
|
|
53869
|
+
json_extract(value, '$[1]'),
|
|
53870
|
+
json_extract(value, '$[2]'),
|
|
53871
|
+
json_extract(value, '$[3]'),
|
|
53872
|
+
json_extract(value, '$[4]'),
|
|
53873
|
+
json_extract(value, '$[5]'),
|
|
53874
|
+
json_extract(value, '$[6]')
|
|
53875
|
+
FROM json_each(${selectedRows})
|
|
53876
|
+
)`).orderBy(crawlerVerificationManifestsHourly.manifestId).all();
|
|
53877
|
+
return buildVerificationProvenance(events, rows);
|
|
53878
|
+
}
|
|
53879
|
+
function aiUserFetchVerificationProvenance(db, projectId, events) {
|
|
53880
|
+
if (events.length === 0) return /* @__PURE__ */ new Map();
|
|
53881
|
+
const selectedRows = verificationSelectedRowsJson(projectId, events);
|
|
53882
|
+
const rows = db.select({
|
|
53883
|
+
sourceId: aiUserFetchVerificationManifestsHourly.sourceId,
|
|
53884
|
+
tsHour: aiUserFetchVerificationManifestsHourly.tsHour,
|
|
53885
|
+
botId: aiUserFetchVerificationManifestsHourly.botId,
|
|
53886
|
+
verificationStatus: aiUserFetchVerificationManifestsHourly.verificationStatus,
|
|
53887
|
+
pathNormalized: aiUserFetchVerificationManifestsHourly.pathNormalized,
|
|
53888
|
+
status: aiUserFetchVerificationManifestsHourly.status,
|
|
53889
|
+
manifestId: aiUserFetchVerificationManifestsHourly.manifestId,
|
|
53890
|
+
manifest: aiUserFetchVerificationManifestsHourly.manifestJson,
|
|
53891
|
+
hits: aiUserFetchVerificationManifestsHourly.hits
|
|
53892
|
+
}).from(aiUserFetchVerificationManifestsHourly).where(sql19`(
|
|
53893
|
+
${aiUserFetchVerificationManifestsHourly.projectId},
|
|
53894
|
+
${aiUserFetchVerificationManifestsHourly.sourceId},
|
|
53895
|
+
${aiUserFetchVerificationManifestsHourly.tsHour},
|
|
53896
|
+
${aiUserFetchVerificationManifestsHourly.botId},
|
|
53897
|
+
${aiUserFetchVerificationManifestsHourly.verificationStatus},
|
|
53898
|
+
${aiUserFetchVerificationManifestsHourly.pathNormalized},
|
|
53899
|
+
${aiUserFetchVerificationManifestsHourly.status}
|
|
53900
|
+
) IN (
|
|
53901
|
+
SELECT
|
|
53902
|
+
json_extract(value, '$[0]'),
|
|
53903
|
+
json_extract(value, '$[1]'),
|
|
53904
|
+
json_extract(value, '$[2]'),
|
|
53905
|
+
json_extract(value, '$[3]'),
|
|
53906
|
+
json_extract(value, '$[4]'),
|
|
53907
|
+
json_extract(value, '$[5]'),
|
|
53908
|
+
json_extract(value, '$[6]')
|
|
53909
|
+
FROM json_each(${selectedRows})
|
|
53910
|
+
)`).orderBy(aiUserFetchVerificationManifestsHourly.manifestId).all();
|
|
53911
|
+
return buildVerificationProvenance(events, rows);
|
|
53912
|
+
}
|
|
53396
53913
|
function parseSourceConfig(row) {
|
|
53397
53914
|
return row.configJson;
|
|
53398
53915
|
}
|
|
@@ -53570,6 +54087,7 @@ async function runBackfillTask(options) {
|
|
|
53570
54087
|
)
|
|
53571
54088
|
).run();
|
|
53572
54089
|
for (const bucket of report.crawlerEventsHourly) {
|
|
54090
|
+
const status = bucket.status ?? 0;
|
|
53573
54091
|
tx.insert(crawlerEventsHourly).values({
|
|
53574
54092
|
projectId: project.id,
|
|
53575
54093
|
sourceId: sourceRow2.id,
|
|
@@ -53578,14 +54096,62 @@ async function runBackfillTask(options) {
|
|
|
53578
54096
|
operator: bucket.operator,
|
|
53579
54097
|
verificationStatus: bucket.verificationStatus,
|
|
53580
54098
|
pathNormalized: bucket.pathNormalized,
|
|
53581
|
-
status
|
|
54099
|
+
status,
|
|
53582
54100
|
hits: bucket.hits,
|
|
53583
54101
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
53584
54102
|
createdAt: finishedAt,
|
|
53585
54103
|
updatedAt: finishedAt
|
|
54104
|
+
}).onConflictDoUpdate({
|
|
54105
|
+
target: [
|
|
54106
|
+
crawlerEventsHourly.projectId,
|
|
54107
|
+
crawlerEventsHourly.sourceId,
|
|
54108
|
+
crawlerEventsHourly.tsHour,
|
|
54109
|
+
crawlerEventsHourly.botId,
|
|
54110
|
+
crawlerEventsHourly.verificationStatus,
|
|
54111
|
+
crawlerEventsHourly.pathNormalized,
|
|
54112
|
+
crawlerEventsHourly.status
|
|
54113
|
+
],
|
|
54114
|
+
set: {
|
|
54115
|
+
hits: sql19`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
54116
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
54117
|
+
updatedAt: finishedAt
|
|
54118
|
+
}
|
|
53586
54119
|
}).run();
|
|
54120
|
+
if (bucket.verificationManifest) {
|
|
54121
|
+
tx.insert(crawlerVerificationManifestsHourly).values({
|
|
54122
|
+
projectId: project.id,
|
|
54123
|
+
sourceId: sourceRow2.id,
|
|
54124
|
+
tsHour: bucket.tsHour,
|
|
54125
|
+
botId: bucket.botId,
|
|
54126
|
+
verificationStatus: bucket.verificationStatus,
|
|
54127
|
+
pathNormalized: bucket.pathNormalized,
|
|
54128
|
+
status,
|
|
54129
|
+
manifestId: bucket.verificationManifest.id,
|
|
54130
|
+
manifestJson: bucket.verificationManifest,
|
|
54131
|
+
hits: bucket.hits,
|
|
54132
|
+
createdAt: finishedAt,
|
|
54133
|
+
updatedAt: finishedAt
|
|
54134
|
+
}).onConflictDoUpdate({
|
|
54135
|
+
target: [
|
|
54136
|
+
crawlerVerificationManifestsHourly.projectId,
|
|
54137
|
+
crawlerVerificationManifestsHourly.sourceId,
|
|
54138
|
+
crawlerVerificationManifestsHourly.tsHour,
|
|
54139
|
+
crawlerVerificationManifestsHourly.botId,
|
|
54140
|
+
crawlerVerificationManifestsHourly.verificationStatus,
|
|
54141
|
+
crawlerVerificationManifestsHourly.pathNormalized,
|
|
54142
|
+
crawlerVerificationManifestsHourly.status,
|
|
54143
|
+
crawlerVerificationManifestsHourly.manifestId
|
|
54144
|
+
],
|
|
54145
|
+
set: {
|
|
54146
|
+
hits: sql19`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
54147
|
+
manifestJson: bucket.verificationManifest,
|
|
54148
|
+
updatedAt: finishedAt
|
|
54149
|
+
}
|
|
54150
|
+
}).run();
|
|
54151
|
+
}
|
|
53587
54152
|
}
|
|
53588
54153
|
for (const bucket of report.aiUserFetchEventsHourly) {
|
|
54154
|
+
const status = bucket.status ?? 0;
|
|
53589
54155
|
tx.insert(aiUserFetchEventsHourly).values({
|
|
53590
54156
|
projectId: project.id,
|
|
53591
54157
|
sourceId: sourceRow2.id,
|
|
@@ -53594,12 +54160,59 @@ async function runBackfillTask(options) {
|
|
|
53594
54160
|
operator: bucket.operator,
|
|
53595
54161
|
verificationStatus: bucket.verificationStatus,
|
|
53596
54162
|
pathNormalized: bucket.pathNormalized,
|
|
53597
|
-
status
|
|
54163
|
+
status,
|
|
53598
54164
|
hits: bucket.hits,
|
|
53599
54165
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
53600
54166
|
createdAt: finishedAt,
|
|
53601
54167
|
updatedAt: finishedAt
|
|
54168
|
+
}).onConflictDoUpdate({
|
|
54169
|
+
target: [
|
|
54170
|
+
aiUserFetchEventsHourly.projectId,
|
|
54171
|
+
aiUserFetchEventsHourly.sourceId,
|
|
54172
|
+
aiUserFetchEventsHourly.tsHour,
|
|
54173
|
+
aiUserFetchEventsHourly.botId,
|
|
54174
|
+
aiUserFetchEventsHourly.verificationStatus,
|
|
54175
|
+
aiUserFetchEventsHourly.pathNormalized,
|
|
54176
|
+
aiUserFetchEventsHourly.status
|
|
54177
|
+
],
|
|
54178
|
+
set: {
|
|
54179
|
+
hits: sql19`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
54180
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
54181
|
+
updatedAt: finishedAt
|
|
54182
|
+
}
|
|
53602
54183
|
}).run();
|
|
54184
|
+
if (bucket.verificationManifest) {
|
|
54185
|
+
tx.insert(aiUserFetchVerificationManifestsHourly).values({
|
|
54186
|
+
projectId: project.id,
|
|
54187
|
+
sourceId: sourceRow2.id,
|
|
54188
|
+
tsHour: bucket.tsHour,
|
|
54189
|
+
botId: bucket.botId,
|
|
54190
|
+
verificationStatus: bucket.verificationStatus,
|
|
54191
|
+
pathNormalized: bucket.pathNormalized,
|
|
54192
|
+
status,
|
|
54193
|
+
manifestId: bucket.verificationManifest.id,
|
|
54194
|
+
manifestJson: bucket.verificationManifest,
|
|
54195
|
+
hits: bucket.hits,
|
|
54196
|
+
createdAt: finishedAt,
|
|
54197
|
+
updatedAt: finishedAt
|
|
54198
|
+
}).onConflictDoUpdate({
|
|
54199
|
+
target: [
|
|
54200
|
+
aiUserFetchVerificationManifestsHourly.projectId,
|
|
54201
|
+
aiUserFetchVerificationManifestsHourly.sourceId,
|
|
54202
|
+
aiUserFetchVerificationManifestsHourly.tsHour,
|
|
54203
|
+
aiUserFetchVerificationManifestsHourly.botId,
|
|
54204
|
+
aiUserFetchVerificationManifestsHourly.verificationStatus,
|
|
54205
|
+
aiUserFetchVerificationManifestsHourly.pathNormalized,
|
|
54206
|
+
aiUserFetchVerificationManifestsHourly.status,
|
|
54207
|
+
aiUserFetchVerificationManifestsHourly.manifestId
|
|
54208
|
+
],
|
|
54209
|
+
set: {
|
|
54210
|
+
hits: sql19`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
54211
|
+
manifestJson: bucket.verificationManifest,
|
|
54212
|
+
updatedAt: finishedAt
|
|
54213
|
+
}
|
|
54214
|
+
}).run();
|
|
54215
|
+
}
|
|
53603
54216
|
}
|
|
53604
54217
|
for (const bucket of report.aiReferralEventsHourly) {
|
|
53605
54218
|
tx.insert(aiReferralEventsHourly).values({
|
|
@@ -55082,6 +55695,38 @@ async function trafficRoutes(app, opts) {
|
|
|
55082
55695
|
updatedAt: finishedAt
|
|
55083
55696
|
}
|
|
55084
55697
|
}).run();
|
|
55698
|
+
if (bucket.verificationManifest) {
|
|
55699
|
+
tx.insert(crawlerVerificationManifestsHourly).values({
|
|
55700
|
+
projectId: project.id,
|
|
55701
|
+
sourceId: sourceRow2.id,
|
|
55702
|
+
tsHour: bucket.tsHour,
|
|
55703
|
+
botId: bucket.botId,
|
|
55704
|
+
verificationStatus: bucket.verificationStatus,
|
|
55705
|
+
pathNormalized: bucket.pathNormalized,
|
|
55706
|
+
status,
|
|
55707
|
+
manifestId: bucket.verificationManifest.id,
|
|
55708
|
+
manifestJson: bucket.verificationManifest,
|
|
55709
|
+
hits: bucket.hits,
|
|
55710
|
+
createdAt: finishedAt,
|
|
55711
|
+
updatedAt: finishedAt
|
|
55712
|
+
}).onConflictDoUpdate({
|
|
55713
|
+
target: [
|
|
55714
|
+
crawlerVerificationManifestsHourly.projectId,
|
|
55715
|
+
crawlerVerificationManifestsHourly.sourceId,
|
|
55716
|
+
crawlerVerificationManifestsHourly.tsHour,
|
|
55717
|
+
crawlerVerificationManifestsHourly.botId,
|
|
55718
|
+
crawlerVerificationManifestsHourly.verificationStatus,
|
|
55719
|
+
crawlerVerificationManifestsHourly.pathNormalized,
|
|
55720
|
+
crawlerVerificationManifestsHourly.status,
|
|
55721
|
+
crawlerVerificationManifestsHourly.manifestId
|
|
55722
|
+
],
|
|
55723
|
+
set: {
|
|
55724
|
+
hits: sql19`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
55725
|
+
manifestJson: bucket.verificationManifest,
|
|
55726
|
+
updatedAt: finishedAt
|
|
55727
|
+
}
|
|
55728
|
+
}).run();
|
|
55729
|
+
}
|
|
55085
55730
|
crawlerBucketRows += 1;
|
|
55086
55731
|
}
|
|
55087
55732
|
for (const bucket of report.aiUserFetchEventsHourly) {
|
|
@@ -55115,6 +55760,38 @@ async function trafficRoutes(app, opts) {
|
|
|
55115
55760
|
updatedAt: finishedAt
|
|
55116
55761
|
}
|
|
55117
55762
|
}).run();
|
|
55763
|
+
if (bucket.verificationManifest) {
|
|
55764
|
+
tx.insert(aiUserFetchVerificationManifestsHourly).values({
|
|
55765
|
+
projectId: project.id,
|
|
55766
|
+
sourceId: sourceRow2.id,
|
|
55767
|
+
tsHour: bucket.tsHour,
|
|
55768
|
+
botId: bucket.botId,
|
|
55769
|
+
verificationStatus: bucket.verificationStatus,
|
|
55770
|
+
pathNormalized: bucket.pathNormalized,
|
|
55771
|
+
status,
|
|
55772
|
+
manifestId: bucket.verificationManifest.id,
|
|
55773
|
+
manifestJson: bucket.verificationManifest,
|
|
55774
|
+
hits: bucket.hits,
|
|
55775
|
+
createdAt: finishedAt,
|
|
55776
|
+
updatedAt: finishedAt
|
|
55777
|
+
}).onConflictDoUpdate({
|
|
55778
|
+
target: [
|
|
55779
|
+
aiUserFetchVerificationManifestsHourly.projectId,
|
|
55780
|
+
aiUserFetchVerificationManifestsHourly.sourceId,
|
|
55781
|
+
aiUserFetchVerificationManifestsHourly.tsHour,
|
|
55782
|
+
aiUserFetchVerificationManifestsHourly.botId,
|
|
55783
|
+
aiUserFetchVerificationManifestsHourly.verificationStatus,
|
|
55784
|
+
aiUserFetchVerificationManifestsHourly.pathNormalized,
|
|
55785
|
+
aiUserFetchVerificationManifestsHourly.status,
|
|
55786
|
+
aiUserFetchVerificationManifestsHourly.manifestId
|
|
55787
|
+
],
|
|
55788
|
+
set: {
|
|
55789
|
+
hits: sql19`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
55790
|
+
manifestJson: bucket.verificationManifest,
|
|
55791
|
+
updatedAt: finishedAt
|
|
55792
|
+
}
|
|
55793
|
+
}).run();
|
|
55794
|
+
}
|
|
55118
55795
|
aiUserFetchBucketRows += 1;
|
|
55119
55796
|
}
|
|
55120
55797
|
for (const bucket of report.aiReferralEventsHourly) {
|
|
@@ -55691,6 +56368,8 @@ async function trafficRoutes(app, opts) {
|
|
|
55691
56368
|
botId: r.botId,
|
|
55692
56369
|
operator: r.operator,
|
|
55693
56370
|
verificationStatus: r.verificationStatus,
|
|
56371
|
+
verificationManifests: [],
|
|
56372
|
+
verificationUnattributedHits: r.hits,
|
|
55694
56373
|
pathNormalized: r.pathNormalized,
|
|
55695
56374
|
pathClass: classifyTrafficPath(r.pathNormalized),
|
|
55696
56375
|
status: r.status,
|
|
@@ -55729,6 +56408,8 @@ async function trafficRoutes(app, opts) {
|
|
|
55729
56408
|
botId: r.botId,
|
|
55730
56409
|
operator: r.operator,
|
|
55731
56410
|
verificationStatus: r.verificationStatus,
|
|
56411
|
+
verificationManifests: [],
|
|
56412
|
+
verificationUnattributedHits: r.hits,
|
|
55732
56413
|
pathNormalized: r.pathNormalized,
|
|
55733
56414
|
status: r.status,
|
|
55734
56415
|
hits: r.hits
|
|
@@ -55785,6 +56466,25 @@ async function trafficRoutes(app, opts) {
|
|
|
55785
56466
|
}
|
|
55786
56467
|
events.sort((a, b) => a.tsHour < b.tsHour ? 1 : a.tsHour > b.tsHour ? -1 : 0);
|
|
55787
56468
|
const trimmed = events.slice(0, limit);
|
|
56469
|
+
const trimmedCrawlerEvents = trimmed.filter(
|
|
56470
|
+
(event) => event.kind === TrafficEventKinds.crawler
|
|
56471
|
+
);
|
|
56472
|
+
const trimmedAiUserFetchEvents = trimmed.filter(
|
|
56473
|
+
(event) => event.kind === TrafficEventKinds["ai-user-fetch"]
|
|
56474
|
+
);
|
|
56475
|
+
const crawlerProvenance = crawlerVerificationProvenance(app.db, project.id, trimmedCrawlerEvents);
|
|
56476
|
+
const aiUserFetchProvenance = aiUserFetchVerificationProvenance(app.db, project.id, trimmedAiUserFetchEvents);
|
|
56477
|
+
const eventsWithProvenance = trimmed.map((event) => {
|
|
56478
|
+
if (event.kind === TrafficEventKinds.crawler) {
|
|
56479
|
+
const provenance = crawlerProvenance.get(verificationEventKey(event));
|
|
56480
|
+
return provenance ? { ...event, ...provenance } : event;
|
|
56481
|
+
}
|
|
56482
|
+
if (event.kind === TrafficEventKinds["ai-user-fetch"]) {
|
|
56483
|
+
const provenance = aiUserFetchProvenance.get(verificationEventKey(event));
|
|
56484
|
+
return provenance ? { ...event, ...provenance } : event;
|
|
56485
|
+
}
|
|
56486
|
+
return event;
|
|
56487
|
+
});
|
|
55788
56488
|
const response = {
|
|
55789
56489
|
windowStart: sinceIso,
|
|
55790
56490
|
windowEnd: untilIso,
|
|
@@ -55808,7 +56508,7 @@ async function trafficRoutes(app, opts) {
|
|
|
55808
56508
|
returned: trimmed.length,
|
|
55809
56509
|
truncated: trimmed.length < totalEventRows
|
|
55810
56510
|
},
|
|
55811
|
-
events:
|
|
56511
|
+
events: eventsWithProvenance
|
|
55812
56512
|
};
|
|
55813
56513
|
return response;
|
|
55814
56514
|
});
|