@canonry/canonry 4.155.0 → 4.157.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/assets/agent-workspace/skills/aero/references/orchestration.md +1 -1
- package/assets/agent-workspace/skills/canonry/SKILL.md +9 -4
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +14 -2
- package/assets/agent-workspace/skills/canonry/references/server-side-traffic.md +454 -21
- package/assets/assets/{AuditHistoryPanel-BlbMBxOY.js → AuditHistoryPanel-D12lHs6n.js} +1 -1
- package/assets/assets/{BacklinksPage-DA9EaV9q.js → BacklinksPage-D2ScFYru.js} +1 -1
- package/assets/assets/{HistoryPage-BTrIHjMQ.js → HistoryPage-jZ_I4rwX.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-CN6o5xFg.js → MeasurementPropertyPage-BIdQjXJT.js} +1 -1
- package/assets/assets/ProjectPage-CLvLOOtj.js +15 -0
- package/assets/assets/{RunRow-ch_OlCPs.js → RunRow-Q2X0NR74.js} +1 -1
- package/assets/assets/{RunsPage-T89Y_yS8.js → RunsPage-Cw7YVsbJ.js} +1 -1
- package/assets/assets/{SettingsPage-DnmEq3F9.js → SettingsPage-bFl2EXOD.js} +1 -1
- package/assets/assets/{TrafficPage-CW1UhaJt.js → TrafficPage-DJjhk7Jw.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-r4Ji3wJm.js → TrafficSourceDetailPage-CGKwIu9A.js} +1 -1
- package/assets/assets/{extract-error-message-a8EErBoZ.js → extract-error-message-BHm98tXH.js} +1 -1
- package/assets/assets/index-BxegEwic.css +1 -0
- package/assets/assets/index-VO6nKq5b.js +83 -0
- package/assets/assets/react-sigma_core.esm.min-EzswMBkr.js +312 -0
- package/assets/assets/vendor-lucide-BtYdosYk.js +1 -0
- package/assets/assets/{vendor-markdown-CpDJch4G.js → vendor-markdown-D274_wsI.js} +1 -1
- package/assets/assets/{vendor-radix-DnzzbllM.js → vendor-radix-BxMXSwbC.js} +1 -1
- package/assets/assets/{vendor-recharts-33ACLxsD.js → vendor-recharts-C4ktw_aE.js} +1 -1
- package/assets/assets/{vendor-tanstack-BjS7iY3g.js → vendor-tanstack-pFxw0eGP.js} +1 -1
- package/assets/index.html +7 -7
- package/dist/{chunk-YSESZNVT.js → chunk-AEIKAWOW.js} +795 -239
- package/dist/{chunk-OSHRDV36.js → chunk-KMTRO56O.js} +281 -1
- package/dist/{chunk-VOL2BSM5.js → chunk-MEBL4EJO.js} +3030 -338
- package/dist/{chunk-VIKPFI6F.js → chunk-TRRVBTU5.js} +399 -1
- package/dist/cli.js +1087 -140
- package/dist/index.d.ts +32 -0
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-G4NV3X36.js → intelligence-service-YUUPSJRC.js} +2 -2
- package/dist/mcp.js +4 -4
- package/package.json +13 -10
- package/assets/assets/ProjectPage-CsDbgp_0.js +0 -12
- package/assets/assets/index-BKq6k5Of.js +0 -83
- package/assets/assets/index-DyL54s8x.css +0 -1
- package/assets/assets/vendor-lucide-ByU7vBHt.js +0 -1
|
@@ -35,6 +35,7 @@ import {
|
|
|
35
35
|
CheckScopes,
|
|
36
36
|
CheckStatuses,
|
|
37
37
|
CitationStates,
|
|
38
|
+
CloudflareTrafficDeliveryModes,
|
|
38
39
|
ContentActions,
|
|
39
40
|
DEFAULT_DISCOVERY_PROMOTE_BUCKETS,
|
|
40
41
|
DEFAULT_DISCOVERY_PROMOTE_COMPETITOR_TYPES,
|
|
@@ -72,9 +73,23 @@ import {
|
|
|
72
73
|
RunKinds,
|
|
73
74
|
RunStatuses,
|
|
74
75
|
RunTriggers,
|
|
76
|
+
SITE_CRAWL_GRAPH_DEFAULT_MAX_EDGES,
|
|
77
|
+
SITE_CRAWL_GRAPH_DEFAULT_MAX_NODES,
|
|
78
|
+
SITE_CRAWL_GRAPH_MAX_EDGES,
|
|
79
|
+
SITE_CRAWL_GRAPH_MAX_NODES,
|
|
80
|
+
SITE_HEALTH_CHANGES_DEFAULT_LIMIT,
|
|
81
|
+
SITE_HEALTH_CHANGES_MAX_LIMIT,
|
|
82
|
+
SITE_HEALTH_PATH_DEFAULT_MAX_DEPTH,
|
|
83
|
+
SITE_HEALTH_PATH_MAX_DEPTH,
|
|
84
|
+
SITE_HEALTH_PATH_MAX_VISITED_NODES,
|
|
85
|
+
SITE_HEALTH_SUBGRAPH_DEFAULT_MAX_EDGES,
|
|
86
|
+
SITE_HEALTH_SUBGRAPH_DEFAULT_MAX_NODES,
|
|
87
|
+
SITE_HEALTH_SUBGRAPH_MAX_EDGES,
|
|
88
|
+
SITE_HEALTH_SUBGRAPH_MAX_NODES,
|
|
75
89
|
SKILL_MANIFEST_FILENAME,
|
|
76
90
|
SchedulableRunKinds,
|
|
77
91
|
SiteAuditTrendDirections,
|
|
92
|
+
SiteCrawlFetchedStates,
|
|
78
93
|
TrafficEventConfidences,
|
|
79
94
|
TrafficEventKinds,
|
|
80
95
|
TrafficEvidenceKinds,
|
|
@@ -183,6 +198,9 @@ import {
|
|
|
183
198
|
classifySkillFile,
|
|
184
199
|
classifySurfaceFromCategory,
|
|
185
200
|
classifyTrafficPath,
|
|
201
|
+
cloudflareTrafficSourceConfigSchema,
|
|
202
|
+
cloudflareWorkerIngestRequestSchema,
|
|
203
|
+
cloudflareWorkerIngestResponseSchema,
|
|
186
204
|
clusterByCosine,
|
|
187
205
|
coerceSkillManifest,
|
|
188
206
|
compactDateToIso,
|
|
@@ -209,6 +227,7 @@ import {
|
|
|
209
227
|
deltaPercent,
|
|
210
228
|
deltaTone,
|
|
211
229
|
deriveCitedUrlCandidates,
|
|
230
|
+
deriveSiteHealthState,
|
|
212
231
|
deriveWinnabilityClass,
|
|
213
232
|
determineAnswerMentioned,
|
|
214
233
|
discoveryBucketSchema,
|
|
@@ -228,6 +247,7 @@ import {
|
|
|
228
247
|
escapeLikePattern,
|
|
229
248
|
evaluateModelPointerExposure,
|
|
230
249
|
extractAnswerMentions,
|
|
250
|
+
factorStatusFromScore,
|
|
231
251
|
filterBrandedSeedCandidates,
|
|
232
252
|
filterCapturedCitedUrls,
|
|
233
253
|
findDuplicateLocationLabels,
|
|
@@ -469,18 +489,26 @@ import {
|
|
|
469
489
|
segmentCrawlerHits,
|
|
470
490
|
serializeRunError,
|
|
471
491
|
settingsDtoSchema,
|
|
492
|
+
siteAuditPageFactorSchema,
|
|
472
493
|
siteAuditPagesResponseSchema,
|
|
473
494
|
siteAuditRequestIdentity,
|
|
474
495
|
siteAuditRunRequestSchema,
|
|
475
496
|
siteAuditRunResponseSchema,
|
|
476
497
|
siteAuditScoreSchema,
|
|
477
498
|
siteAuditTrendResponseSchema,
|
|
499
|
+
siteCrawlAuditFactorSchema,
|
|
500
|
+
siteCrawlCriticalDefectSchema,
|
|
478
501
|
siteCrawlDeadLinksResponseSchema,
|
|
502
|
+
siteCrawlGraphResponseSchema,
|
|
479
503
|
siteCrawlInternalLinksResponseSchema,
|
|
480
504
|
siteCrawlNeighborsResponseSchema,
|
|
505
|
+
siteCrawlPageAuditSchema,
|
|
481
506
|
siteCrawlPagesResponseSchema,
|
|
482
507
|
siteCrawlStructureResponseSchema,
|
|
483
508
|
siteCrawlSummarySchema,
|
|
509
|
+
siteHealthChangesResponseSchema,
|
|
510
|
+
siteHealthPathResponseSchema,
|
|
511
|
+
siteHealthSubgraphResponseSchema,
|
|
484
512
|
snapshotDiffResponseSchema,
|
|
485
513
|
snapshotListResponseSchema,
|
|
486
514
|
snapshotReportSchema,
|
|
@@ -496,6 +524,8 @@ import {
|
|
|
496
524
|
textContainsBrandAlias,
|
|
497
525
|
textContainsDomain,
|
|
498
526
|
trafficBackfillResponseSchema,
|
|
527
|
+
trafficConnectCloudflareRequestSchema,
|
|
528
|
+
trafficConnectCloudflareResponseSchema,
|
|
499
529
|
trafficConnectVercelRequestSchema,
|
|
500
530
|
trafficConnectWordpressRequestSchema,
|
|
501
531
|
trafficEventsResponseSchema,
|
|
@@ -529,10 +559,10 @@ import {
|
|
|
529
559
|
wordpressSchemaDeployResultDtoSchema,
|
|
530
560
|
wordpressSchemaStatusResultDtoSchema,
|
|
531
561
|
wordpressStatusDtoSchema
|
|
532
|
-
} from "./chunk-
|
|
562
|
+
} from "./chunk-TRRVBTU5.js";
|
|
533
563
|
|
|
534
564
|
// src/intelligence-service.ts
|
|
535
|
-
import { eq as
|
|
565
|
+
import { eq as eq58, desc as desc26, asc as asc11, and as and46, ne as ne7, or as or12, inArray as inArray20, gte as gte14, lte as lte11 } from "drizzle-orm";
|
|
536
566
|
|
|
537
567
|
// ../db/src/client.ts
|
|
538
568
|
import { mkdirSync } from "fs";
|
|
@@ -627,9 +657,13 @@ __export(schema_exports, {
|
|
|
627
657
|
siteCrawlEdges: () => siteCrawlEdges,
|
|
628
658
|
siteCrawlEventReceipts: () => siteCrawlEventReceipts,
|
|
629
659
|
siteCrawlFindings: () => siteCrawlFindings,
|
|
660
|
+
siteCrawlGraphEdges: () => siteCrawlGraphEdges,
|
|
661
|
+
siteCrawlGraphLayouts: () => siteCrawlGraphLayouts,
|
|
662
|
+
siteCrawlGraphNodes: () => siteCrawlGraphNodes,
|
|
630
663
|
siteCrawlPages: () => siteCrawlPages,
|
|
631
664
|
siteCrawlRunRequests: () => siteCrawlRunRequests,
|
|
632
665
|
siteCrawlSnapshots: () => siteCrawlSnapshots,
|
|
666
|
+
trafficEventReceipts: () => trafficEventReceipts,
|
|
633
667
|
trafficSources: () => trafficSources,
|
|
634
668
|
usageCounters: () => usageCounters,
|
|
635
669
|
userSessions: () => userSessions,
|
|
@@ -1251,6 +1285,8 @@ var siteCrawlSnapshots = sqliteTable("site_crawl_snapshots", {
|
|
|
1251
1285
|
runId: text("run_id").notNull(),
|
|
1252
1286
|
attemptId: text("attempt_id"),
|
|
1253
1287
|
rootUrl: text("root_url").notNull(),
|
|
1288
|
+
/** Original requested crawl root; null for snapshots written before v128. */
|
|
1289
|
+
requestedRootUrl: text("requested_root_url"),
|
|
1254
1290
|
crawlSchemaVersion: text("crawl_schema_version").notNull().default("1.0"),
|
|
1255
1291
|
engineVersion: text("engine_version").notNull().default(""),
|
|
1256
1292
|
normalizationVersion: text("normalization_version").notNull().default(""),
|
|
@@ -1361,12 +1397,116 @@ var siteCrawlEdges = sqliteTable("site_crawl_edges", {
|
|
|
1361
1397
|
index("idx_site_crawl_edges_inbound").on(table.projectId, table.runId, table.attemptId, table.targetNodeKey, table.edgeKey),
|
|
1362
1398
|
index("idx_site_crawl_edges_source_url").on(table.projectId, table.runId, table.attemptId, table.sourceUrl),
|
|
1363
1399
|
index("idx_site_crawl_edges_target_url").on(table.projectId, table.runId, table.attemptId, table.targetUrl),
|
|
1400
|
+
// Publish-time graph sampling filters this exact scope and consumes the
|
|
1401
|
+
// highest-occurrence internal anchors first. The mixed-direction suffix
|
|
1402
|
+
// avoids a million-row temporary ORDER BY at the maximum crawl budget.
|
|
1403
|
+
index("idx_site_crawl_edges_graph_sample").on(
|
|
1404
|
+
table.projectId,
|
|
1405
|
+
table.runId,
|
|
1406
|
+
table.attemptId,
|
|
1407
|
+
table.internal,
|
|
1408
|
+
table.relation,
|
|
1409
|
+
sql`${table.occurrences} desc`,
|
|
1410
|
+
table.edgeKey
|
|
1411
|
+
),
|
|
1364
1412
|
foreignKey({
|
|
1365
1413
|
name: "site_crawl_edges_attempt_fk",
|
|
1366
1414
|
columns: [table.projectId, table.runId, table.attemptId],
|
|
1367
1415
|
foreignColumns: [siteCrawlAttempts.projectId, siteCrawlAttempts.runId, siteCrawlAttempts.id]
|
|
1368
1416
|
}).onDelete("cascade")
|
|
1369
1417
|
]);
|
|
1418
|
+
var siteCrawlGraphLayouts = sqliteTable("site_crawl_graph_layouts", {
|
|
1419
|
+
id: text("id").primaryKey(),
|
|
1420
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1421
|
+
runId: text("run_id").notNull(),
|
|
1422
|
+
attemptId: text("attempt_id").notNull(),
|
|
1423
|
+
state: text("state").notNull(),
|
|
1424
|
+
layoutVersion: text("layout_version"),
|
|
1425
|
+
failureCode: text("failure_code"),
|
|
1426
|
+
totalNodes: integer("total_nodes").notNull().default(0),
|
|
1427
|
+
totalEdges: integer("total_edges").notNull().default(0),
|
|
1428
|
+
nodeCount: integer("node_count").notNull().default(0),
|
|
1429
|
+
edgeCount: integer("edge_count").notNull().default(0),
|
|
1430
|
+
createdAt: text("created_at").notNull(),
|
|
1431
|
+
updatedAt: text("updated_at").notNull()
|
|
1432
|
+
}, (table) => [
|
|
1433
|
+
uniqueIndex("idx_site_crawl_graph_layouts_attempt").on(table.projectId, table.runId, table.attemptId),
|
|
1434
|
+
index("idx_site_crawl_graph_layouts_run").on(table.projectId, table.runId),
|
|
1435
|
+
check("site_crawl_graph_layouts_state_check", sql`${table.state} in ('ready', 'unavailable')`),
|
|
1436
|
+
foreignKey({
|
|
1437
|
+
name: "site_crawl_graph_layouts_attempt_fk",
|
|
1438
|
+
columns: [table.projectId, table.runId, table.attemptId],
|
|
1439
|
+
foreignColumns: [siteCrawlAttempts.projectId, siteCrawlAttempts.runId, siteCrawlAttempts.id]
|
|
1440
|
+
}).onDelete("cascade")
|
|
1441
|
+
]);
|
|
1442
|
+
var siteCrawlGraphNodes = sqliteTable("site_crawl_graph_nodes", {
|
|
1443
|
+
id: text("id").primaryKey(),
|
|
1444
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1445
|
+
runId: text("run_id").notNull(),
|
|
1446
|
+
attemptId: text("attempt_id").notNull(),
|
|
1447
|
+
nodeKey: text("node_key").notNull(),
|
|
1448
|
+
sampleRank: integer("sample_rank").notNull(),
|
|
1449
|
+
x: real("x").notNull(),
|
|
1450
|
+
y: real("y").notNull(),
|
|
1451
|
+
createdAt: text("created_at").notNull()
|
|
1452
|
+
}, (table) => [
|
|
1453
|
+
uniqueIndex("idx_site_crawl_graph_nodes_attempt_node").on(table.projectId, table.runId, table.attemptId, table.nodeKey),
|
|
1454
|
+
uniqueIndex("idx_site_crawl_graph_nodes_attempt_rank").on(table.projectId, table.runId, table.attemptId, table.sampleRank),
|
|
1455
|
+
index("idx_site_crawl_graph_nodes_read").on(table.projectId, table.runId, table.attemptId, table.sampleRank),
|
|
1456
|
+
check("site_crawl_graph_nodes_rank_check", sql`${table.sampleRank} >= 0`),
|
|
1457
|
+
foreignKey({
|
|
1458
|
+
name: "site_crawl_graph_nodes_layout_fk",
|
|
1459
|
+
columns: [table.projectId, table.runId, table.attemptId],
|
|
1460
|
+
foreignColumns: [siteCrawlGraphLayouts.projectId, siteCrawlGraphLayouts.runId, siteCrawlGraphLayouts.attemptId]
|
|
1461
|
+
}).onDelete("cascade"),
|
|
1462
|
+
foreignKey({
|
|
1463
|
+
name: "site_crawl_graph_nodes_page_fk",
|
|
1464
|
+
columns: [table.projectId, table.runId, table.attemptId, table.nodeKey],
|
|
1465
|
+
foreignColumns: [siteCrawlPages.projectId, siteCrawlPages.runId, siteCrawlPages.attemptId, siteCrawlPages.nodeKey]
|
|
1466
|
+
}).onDelete("cascade")
|
|
1467
|
+
]);
|
|
1468
|
+
var siteCrawlGraphEdges = sqliteTable("site_crawl_graph_edges", {
|
|
1469
|
+
id: text("id").primaryKey(),
|
|
1470
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1471
|
+
runId: text("run_id").notNull(),
|
|
1472
|
+
attemptId: text("attempt_id").notNull(),
|
|
1473
|
+
edgeKey: text("edge_key").notNull(),
|
|
1474
|
+
sampleRank: integer("sample_rank").notNull(),
|
|
1475
|
+
sourceNodeKey: text("source_node_key").notNull(),
|
|
1476
|
+
targetNodeKey: text("target_node_key").notNull(),
|
|
1477
|
+
followable: integer("followable", { mode: "boolean" }).notNull(),
|
|
1478
|
+
occurrences: integer("occurrences").notNull(),
|
|
1479
|
+
createdAt: text("created_at").notNull()
|
|
1480
|
+
}, (table) => [
|
|
1481
|
+
uniqueIndex("idx_site_crawl_graph_edges_attempt_edge").on(table.projectId, table.runId, table.attemptId, table.edgeKey),
|
|
1482
|
+
uniqueIndex("idx_site_crawl_graph_edges_attempt_rank").on(table.projectId, table.runId, table.attemptId, table.sampleRank),
|
|
1483
|
+
index("idx_site_crawl_graph_edges_read").on(table.projectId, table.runId, table.attemptId, table.sampleRank),
|
|
1484
|
+
// These FKs otherwise make a layout cascade scan graph edges once per node.
|
|
1485
|
+
index("idx_site_crawl_graph_edges_source_node").on(table.projectId, table.runId, table.attemptId, table.sourceNodeKey),
|
|
1486
|
+
index("idx_site_crawl_graph_edges_target_node").on(table.projectId, table.runId, table.attemptId, table.targetNodeKey),
|
|
1487
|
+
check("site_crawl_graph_edges_rank_check", sql`${table.sampleRank} >= 0`),
|
|
1488
|
+
check("site_crawl_graph_edges_occurrences_check", sql`${table.occurrences} > 0`),
|
|
1489
|
+
foreignKey({
|
|
1490
|
+
name: "site_crawl_graph_edges_layout_fk",
|
|
1491
|
+
columns: [table.projectId, table.runId, table.attemptId],
|
|
1492
|
+
foreignColumns: [siteCrawlGraphLayouts.projectId, siteCrawlGraphLayouts.runId, siteCrawlGraphLayouts.attemptId]
|
|
1493
|
+
}).onDelete("cascade"),
|
|
1494
|
+
foreignKey({
|
|
1495
|
+
name: "site_crawl_graph_edges_source_node_fk",
|
|
1496
|
+
columns: [table.projectId, table.runId, table.attemptId, table.sourceNodeKey],
|
|
1497
|
+
foreignColumns: [siteCrawlGraphNodes.projectId, siteCrawlGraphNodes.runId, siteCrawlGraphNodes.attemptId, siteCrawlGraphNodes.nodeKey]
|
|
1498
|
+
}).onDelete("cascade"),
|
|
1499
|
+
foreignKey({
|
|
1500
|
+
name: "site_crawl_graph_edges_target_node_fk",
|
|
1501
|
+
columns: [table.projectId, table.runId, table.attemptId, table.targetNodeKey],
|
|
1502
|
+
foreignColumns: [siteCrawlGraphNodes.projectId, siteCrawlGraphNodes.runId, siteCrawlGraphNodes.attemptId, siteCrawlGraphNodes.nodeKey]
|
|
1503
|
+
}).onDelete("cascade"),
|
|
1504
|
+
foreignKey({
|
|
1505
|
+
name: "site_crawl_graph_edges_canonical_edge_fk",
|
|
1506
|
+
columns: [table.projectId, table.runId, table.attemptId, table.edgeKey],
|
|
1507
|
+
foreignColumns: [siteCrawlEdges.projectId, siteCrawlEdges.runId, siteCrawlEdges.attemptId, siteCrawlEdges.edgeKey]
|
|
1508
|
+
}).onDelete("cascade")
|
|
1509
|
+
]);
|
|
1370
1510
|
var siteCrawlFindings = sqliteTable("site_crawl_findings", {
|
|
1371
1511
|
id: text("id").primaryKey(),
|
|
1372
1512
|
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
@@ -1915,12 +2055,31 @@ var trafficSources = sqliteTable("traffic_sources", {
|
|
|
1915
2055
|
lastEventIds: text("last_event_ids", { mode: "json" }).$type(),
|
|
1916
2056
|
archivedAt: text("archived_at"),
|
|
1917
2057
|
configJson: text("config_json", { mode: "json" }).$type().notNull().default({}),
|
|
2058
|
+
// sha256 hex of the per-source bearer token issued at connect time. Only
|
|
2059
|
+
// populated for push-receive source types (currently `cloudflare`); pull
|
|
2060
|
+
// adapters leave this NULL. The cleartext bearer + HMAC secret never live
|
|
2061
|
+
// in the DB — they go to `~/.canonry/config.yaml` under the per-type
|
|
2062
|
+
// connections block.
|
|
2063
|
+
ingestTokenHash: text("ingest_token_hash"),
|
|
2064
|
+
// Semver reported by the most recent forwarded event. Drives the
|
|
2065
|
+
// `traffic.source.worker-version` doctor check. NULL until the first
|
|
2066
|
+
// event arrives or for source types that don't forward versioned events.
|
|
2067
|
+
lastWorkerVersion: text("last_worker_version"),
|
|
1918
2068
|
createdAt: text("created_at").notNull(),
|
|
1919
2069
|
updatedAt: text("updated_at").notNull()
|
|
1920
2070
|
}, (table) => [
|
|
1921
2071
|
index("idx_traffic_sources_project").on(table.projectId),
|
|
1922
2072
|
index("idx_traffic_sources_project_status").on(table.projectId, table.status)
|
|
1923
2073
|
]);
|
|
2074
|
+
var trafficEventReceipts = sqliteTable("traffic_event_receipts", {
|
|
2075
|
+
sourceId: text("source_id").notNull().references(() => trafficSources.id, { onDelete: "cascade" }),
|
|
2076
|
+
eventId: text("event_id").notNull(),
|
|
2077
|
+
receivedAt: text("received_at").notNull(),
|
|
2078
|
+
expiresAt: text("expires_at").notNull()
|
|
2079
|
+
}, (table) => [
|
|
2080
|
+
primaryKey({ columns: [table.sourceId, table.eventId] }),
|
|
2081
|
+
index("idx_traffic_event_receipts_expires").on(table.sourceId, table.expiresAt)
|
|
2082
|
+
]);
|
|
1924
2083
|
var crawlerEventsHourly = sqliteTable("crawler_events_hourly", {
|
|
1925
2084
|
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1926
2085
|
sourceId: text("source_id").notNull().references(() => trafficSources.id, { onDelete: "cascade" }),
|
|
@@ -5526,6 +5685,126 @@ var MIGRATION_VERSIONS = [
|
|
|
5526
5685
|
`CREATE INDEX IF NOT EXISTS idx_site_crawl_receipts_project_run
|
|
5527
5686
|
ON site_crawl_event_receipts(project_id, run_id, attempt_id)`
|
|
5528
5687
|
]
|
|
5688
|
+
},
|
|
5689
|
+
{
|
|
5690
|
+
// Graph coordinates are optional derived snapshot data. Separate tables
|
|
5691
|
+
// preserve the canonical crawl-page shape and make a missing row the
|
|
5692
|
+
// truthful compatibility signal for snapshots published before v127.
|
|
5693
|
+
version: 127,
|
|
5694
|
+
name: "site-crawl-persisted-graph-layout",
|
|
5695
|
+
statements: [
|
|
5696
|
+
`CREATE TABLE IF NOT EXISTS site_crawl_graph_layouts (
|
|
5697
|
+
id TEXT PRIMARY KEY,
|
|
5698
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
5699
|
+
run_id TEXT NOT NULL,
|
|
5700
|
+
attempt_id TEXT NOT NULL,
|
|
5701
|
+
state TEXT NOT NULL CHECK (state IN ('ready', 'unavailable')),
|
|
5702
|
+
layout_version TEXT,
|
|
5703
|
+
failure_code TEXT,
|
|
5704
|
+
total_nodes INTEGER NOT NULL DEFAULT 0,
|
|
5705
|
+
total_edges INTEGER NOT NULL DEFAULT 0,
|
|
5706
|
+
node_count INTEGER NOT NULL DEFAULT 0,
|
|
5707
|
+
edge_count INTEGER NOT NULL DEFAULT 0,
|
|
5708
|
+
created_at TEXT NOT NULL,
|
|
5709
|
+
updated_at TEXT NOT NULL,
|
|
5710
|
+
FOREIGN KEY (project_id, run_id, attempt_id)
|
|
5711
|
+
REFERENCES site_crawl_attempts(project_id, run_id, id) ON DELETE CASCADE
|
|
5712
|
+
)`,
|
|
5713
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_graph_layouts_attempt
|
|
5714
|
+
ON site_crawl_graph_layouts(project_id, run_id, attempt_id)`,
|
|
5715
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_graph_layouts_run
|
|
5716
|
+
ON site_crawl_graph_layouts(project_id, run_id)`,
|
|
5717
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_edges_graph_sample
|
|
5718
|
+
ON site_crawl_edges(project_id, run_id, attempt_id, internal, relation, occurrences DESC, edge_key ASC)`,
|
|
5719
|
+
`CREATE TABLE IF NOT EXISTS site_crawl_graph_nodes (
|
|
5720
|
+
id TEXT PRIMARY KEY,
|
|
5721
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
5722
|
+
run_id TEXT NOT NULL,
|
|
5723
|
+
attempt_id TEXT NOT NULL,
|
|
5724
|
+
node_key TEXT NOT NULL,
|
|
5725
|
+
sample_rank INTEGER NOT NULL CHECK (sample_rank >= 0),
|
|
5726
|
+
x REAL NOT NULL,
|
|
5727
|
+
y REAL NOT NULL,
|
|
5728
|
+
created_at TEXT NOT NULL,
|
|
5729
|
+
FOREIGN KEY (project_id, run_id, attempt_id)
|
|
5730
|
+
REFERENCES site_crawl_graph_layouts(project_id, run_id, attempt_id) ON DELETE CASCADE,
|
|
5731
|
+
FOREIGN KEY (project_id, run_id, attempt_id, node_key)
|
|
5732
|
+
REFERENCES site_crawl_pages(project_id, run_id, attempt_id, node_key) ON DELETE CASCADE
|
|
5733
|
+
)`,
|
|
5734
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_graph_nodes_attempt_node
|
|
5735
|
+
ON site_crawl_graph_nodes(project_id, run_id, attempt_id, node_key)`,
|
|
5736
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_graph_nodes_attempt_rank
|
|
5737
|
+
ON site_crawl_graph_nodes(project_id, run_id, attempt_id, sample_rank)`,
|
|
5738
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_graph_nodes_read
|
|
5739
|
+
ON site_crawl_graph_nodes(project_id, run_id, attempt_id, sample_rank)`,
|
|
5740
|
+
`CREATE TABLE IF NOT EXISTS site_crawl_graph_edges (
|
|
5741
|
+
id TEXT PRIMARY KEY,
|
|
5742
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
5743
|
+
run_id TEXT NOT NULL,
|
|
5744
|
+
attempt_id TEXT NOT NULL,
|
|
5745
|
+
edge_key TEXT NOT NULL,
|
|
5746
|
+
sample_rank INTEGER NOT NULL CHECK (sample_rank >= 0),
|
|
5747
|
+
source_node_key TEXT NOT NULL,
|
|
5748
|
+
target_node_key TEXT NOT NULL,
|
|
5749
|
+
followable INTEGER NOT NULL,
|
|
5750
|
+
occurrences INTEGER NOT NULL CHECK (occurrences > 0),
|
|
5751
|
+
created_at TEXT NOT NULL,
|
|
5752
|
+
FOREIGN KEY (project_id, run_id, attempt_id)
|
|
5753
|
+
REFERENCES site_crawl_graph_layouts(project_id, run_id, attempt_id) ON DELETE CASCADE,
|
|
5754
|
+
FOREIGN KEY (project_id, run_id, attempt_id, source_node_key)
|
|
5755
|
+
REFERENCES site_crawl_graph_nodes(project_id, run_id, attempt_id, node_key) ON DELETE CASCADE,
|
|
5756
|
+
FOREIGN KEY (project_id, run_id, attempt_id, target_node_key)
|
|
5757
|
+
REFERENCES site_crawl_graph_nodes(project_id, run_id, attempt_id, node_key) ON DELETE CASCADE,
|
|
5758
|
+
FOREIGN KEY (project_id, run_id, attempt_id, edge_key)
|
|
5759
|
+
REFERENCES site_crawl_edges(project_id, run_id, attempt_id, edge_key) ON DELETE CASCADE
|
|
5760
|
+
)`,
|
|
5761
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_graph_edges_attempt_edge
|
|
5762
|
+
ON site_crawl_graph_edges(project_id, run_id, attempt_id, edge_key)`,
|
|
5763
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_graph_edges_attempt_rank
|
|
5764
|
+
ON site_crawl_graph_edges(project_id, run_id, attempt_id, sample_rank)`,
|
|
5765
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_graph_edges_read
|
|
5766
|
+
ON site_crawl_graph_edges(project_id, run_id, attempt_id, sample_rank)`
|
|
5767
|
+
]
|
|
5768
|
+
},
|
|
5769
|
+
{
|
|
5770
|
+
// Keep the operator-requested root beside the terminal root without
|
|
5771
|
+
// rewriting immutable historical snapshots. The edge indexes make layout
|
|
5772
|
+
// cleanup/cascades linear at the graph's 20k/50k publish cap.
|
|
5773
|
+
version: 128,
|
|
5774
|
+
name: "site-crawl-requested-root-and-graph-edge-cleanup-indexes",
|
|
5775
|
+
statements: [
|
|
5776
|
+
`ALTER TABLE site_crawl_snapshots ADD COLUMN requested_root_url TEXT`,
|
|
5777
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_graph_edges_source_node
|
|
5778
|
+
ON site_crawl_graph_edges(project_id, run_id, attempt_id, source_node_key)`,
|
|
5779
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_graph_edges_target_node
|
|
5780
|
+
ON site_crawl_graph_edges(project_id, run_id, attempt_id, target_node_key)`
|
|
5781
|
+
]
|
|
5782
|
+
},
|
|
5783
|
+
{
|
|
5784
|
+
version: 129,
|
|
5785
|
+
name: "traffic-ingest-foundation",
|
|
5786
|
+
// Push-receive traffic sources (currently only `cloudflare`) need a
|
|
5787
|
+
// per-source bearer for the Worker to authenticate against canonry's
|
|
5788
|
+
// ingest endpoint, plus a place to remember the deployed Worker version.
|
|
5789
|
+
// Durable receipts are transport-neutral: direct push and a future Queue
|
|
5790
|
+
// pull consumer both claim an event in the same transaction as rollups.
|
|
5791
|
+
// Cleartext credentials remain outside the database.
|
|
5792
|
+
//
|
|
5793
|
+
// Idempotent: `ALTER TABLE ADD COLUMN` errors with "duplicate column
|
|
5794
|
+
// name" on retry, which the runner already swallows.
|
|
5795
|
+
statements: [
|
|
5796
|
+
`ALTER TABLE traffic_sources ADD COLUMN ingest_token_hash TEXT`,
|
|
5797
|
+
`ALTER TABLE traffic_sources ADD COLUMN last_worker_version TEXT`,
|
|
5798
|
+
`CREATE TABLE IF NOT EXISTS traffic_event_receipts (
|
|
5799
|
+
source_id TEXT NOT NULL REFERENCES traffic_sources(id) ON DELETE CASCADE,
|
|
5800
|
+
event_id TEXT NOT NULL,
|
|
5801
|
+
received_at TEXT NOT NULL,
|
|
5802
|
+
expires_at TEXT NOT NULL,
|
|
5803
|
+
PRIMARY KEY (source_id, event_id)
|
|
5804
|
+
)`,
|
|
5805
|
+
`CREATE INDEX IF NOT EXISTS idx_traffic_event_receipts_expires
|
|
5806
|
+
ON traffic_event_receipts(source_id, expires_at)`
|
|
5807
|
+
]
|
|
5529
5808
|
}
|
|
5530
5809
|
];
|
|
5531
5810
|
function addRunsMeasurementPlanVersionForeignKey(tx) {
|
|
@@ -7465,7 +7744,7 @@ function buildMentionShare(snapshots, options) {
|
|
|
7465
7744
|
competitorCounts.set(competitor.domain, 0);
|
|
7466
7745
|
competitorAliases.set(
|
|
7467
7746
|
competitor.domain,
|
|
7468
|
-
competitor.brandTokens.filter((
|
|
7747
|
+
competitor.brandTokens.filter((alias2) => brandKeyFromText(alias2).length >= 3)
|
|
7469
7748
|
);
|
|
7470
7749
|
}
|
|
7471
7750
|
for (const snap of snapshots) {
|
|
@@ -8333,6 +8612,7 @@ function shouldSkipAuth(url) {
|
|
|
8333
8612
|
if (url.endsWith("/session") || url.endsWith("/session/setup")) return true;
|
|
8334
8613
|
if (url.endsWith("/auth/session") || url.endsWith("/auth/login") || url.endsWith("/auth/logout")) return true;
|
|
8335
8614
|
if (url.endsWith("/auth/sessions")) return true;
|
|
8615
|
+
if (url.endsWith("/traffic/cloudflare/ingest")) return true;
|
|
8336
8616
|
return false;
|
|
8337
8617
|
}
|
|
8338
8618
|
var USERS_READ_SCOPE = "users.read";
|
|
@@ -8398,7 +8678,7 @@ function isOverviewRead(url) {
|
|
|
8398
8678
|
}
|
|
8399
8679
|
function isTechnicalAeoRead(url) {
|
|
8400
8680
|
const rest = projectRouteRest(url);
|
|
8401
|
-
return rest === "technical-aeo" || rest === "technical-aeo/pages" || rest === "technical-aeo/trend" || rest === "technical-aeo/crawl" || rest === "technical-aeo/crawl/pages" || rest === "technical-aeo/structure" || rest === "technical-aeo/internal-links" || rest === "technical-aeo/internal-links/neighbors" || rest === "technical-aeo/dead-links";
|
|
8681
|
+
return rest === "technical-aeo" || rest === "technical-aeo/pages" || rest === "technical-aeo/trend" || rest === "technical-aeo/crawl" || rest === "technical-aeo/graph" || rest === "technical-aeo/subgraph" || rest === "technical-aeo/path" || rest === "technical-aeo/changes" || rest === "technical-aeo/crawl/pages/audit" || rest === "technical-aeo/crawl/pages" || rest === "technical-aeo/structure" || rest === "technical-aeo/internal-links" || rest === "technical-aeo/internal-links/neighbors" || rest === "technical-aeo/dead-links";
|
|
8402
8682
|
}
|
|
8403
8683
|
function isReportRead(url) {
|
|
8404
8684
|
const rest = projectRouteRest(url);
|
|
@@ -9766,9 +10046,9 @@ function aliasMatchesAt(textWords, aliasWords, start) {
|
|
|
9766
10046
|
}
|
|
9767
10047
|
function containsAnyAlias(answerText, aliases) {
|
|
9768
10048
|
const textWords = words(answerText);
|
|
9769
|
-
const candidates = aliases.map(words).filter((
|
|
10049
|
+
const candidates = aliases.map(words).filter((alias2) => alias2.length > 0);
|
|
9770
10050
|
for (let start = 0; start < textWords.length; start++) {
|
|
9771
|
-
if (candidates.some((
|
|
10051
|
+
if (candidates.some((alias2) => aliasMatchesAt(textWords, alias2, start))) return true;
|
|
9772
10052
|
}
|
|
9773
10053
|
return false;
|
|
9774
10054
|
}
|
|
@@ -9776,9 +10056,9 @@ function mentionedTargets(answerText, targets) {
|
|
|
9776
10056
|
const result = /* @__PURE__ */ new Set();
|
|
9777
10057
|
if (answerText === null) return result;
|
|
9778
10058
|
const textWords = words(answerText);
|
|
9779
|
-
const aliases = targets.flatMap((target) => target.aliases.map((
|
|
10059
|
+
const aliases = targets.flatMap((target) => target.aliases.map((alias2) => ({ targetId: target.id, words: words(alias2) }))).filter((alias2) => alias2.words.length > 0);
|
|
9780
10060
|
for (let start = 0; start < textWords.length; ) {
|
|
9781
|
-
const matches = aliases.filter((
|
|
10061
|
+
const matches = aliases.filter((alias2) => aliasMatchesAt(textWords, alias2.words, start));
|
|
9782
10062
|
if (matches.length === 0) {
|
|
9783
10063
|
start++;
|
|
9784
10064
|
continue;
|
|
@@ -9862,7 +10142,7 @@ function prepareReport(input) {
|
|
|
9862
10142
|
}
|
|
9863
10143
|
const answers = [];
|
|
9864
10144
|
const mentionableIds = new Set(
|
|
9865
|
-
input.targets.filter((target) => target.aliases.some((
|
|
10145
|
+
input.targets.filter((target) => target.aliases.some((alias2) => words(alias2).length > 0)).map((target) => target.id)
|
|
9866
10146
|
);
|
|
9867
10147
|
for (const observation of observationsBySlot.values()) {
|
|
9868
10148
|
const edges = [...edgesByExecution.get(observation.slot.executionId) ?? []].sort((left, right) => compareText(left.id, right.id));
|
|
@@ -9985,7 +10265,7 @@ function targetCoverageRate(targetIds, slots, edges, prepared) {
|
|
|
9985
10265
|
}
|
|
9986
10266
|
function mentionRate(target, slots, prepared) {
|
|
9987
10267
|
const numerator = slots.filter((slot) => prepared.observationsBySlot.get(slot.id)?.mentionedTargetIds.has(target.id)).length;
|
|
9988
|
-
if (target.aliases.every((
|
|
10268
|
+
if (target.aliases.every((alias2) => words(alias2).length === 0)) {
|
|
9989
10269
|
return { numerator: null, denominator: null, rate: null, reason: "aliasless" };
|
|
9990
10270
|
}
|
|
9991
10271
|
if (slots.length === 0) return { numerator: null, denominator: null, rate: null, reason: "no-population" };
|
|
@@ -10013,11 +10293,11 @@ function buildSovForSlots(slots, competitors2, projectBrandNames, projectDomain,
|
|
|
10013
10293
|
let reason = null;
|
|
10014
10294
|
if (selected.length === 0) reason = "no-population";
|
|
10015
10295
|
else if (!status.complete || !status.answerComplete) reason = "incomplete";
|
|
10016
|
-
else if (projectBrandNames.every((
|
|
10296
|
+
else if (projectBrandNames.every((alias2) => words(alias2).length === 0)) reason = "no-project-aliases";
|
|
10017
10297
|
else if (sortedCompetitors.length === 0) reason = "no-competitors";
|
|
10018
10298
|
return rows.map((row) => {
|
|
10019
10299
|
if (reason !== null) return { domain: row.domain, own: row.own, presentIn: null, of: null, reason };
|
|
10020
|
-
if (row.aliases.every((
|
|
10300
|
+
if (row.aliases.every((alias2) => words(alias2).length === 0)) {
|
|
10021
10301
|
return { domain: row.domain, own: row.own, presentIn: null, of: null, reason: "aliasless" };
|
|
10022
10302
|
}
|
|
10023
10303
|
const presentIn = selected.filter((slot) => {
|
|
@@ -10167,7 +10447,7 @@ function mentionedForEdge(observation, edge, mentionableIds) {
|
|
|
10167
10447
|
return observation.mentionedTargetIds.has(edge.targetId);
|
|
10168
10448
|
}
|
|
10169
10449
|
function mentionableTargets(input, targetIds) {
|
|
10170
|
-
return input.targets.filter((target) => targetIds.has(target.id) && target.aliases.some((
|
|
10450
|
+
return input.targets.filter((target) => targetIds.has(target.id) && target.aliases.some((alias2) => words(alias2).length > 0));
|
|
10171
10451
|
}
|
|
10172
10452
|
function scopeMentionRate(input, targetIds, slots, answered, prepared) {
|
|
10173
10453
|
const mentionable = mentionableTargets(input, targetIds);
|
|
@@ -10193,7 +10473,7 @@ function indexedScopeCitationRate(slots, edges, indexes) {
|
|
|
10193
10473
|
return { numerator, denominator: basis.length, rate: numerator / basis.length };
|
|
10194
10474
|
}
|
|
10195
10475
|
function targetMentionRate(targets, slots, answered, indexes) {
|
|
10196
|
-
const mentionable = targets?.filter((target) => target.aliases.some((
|
|
10476
|
+
const mentionable = targets?.filter((target) => target.aliases.some((alias2) => words(alias2).length > 0)) ?? [];
|
|
10197
10477
|
if (mentionable.length === 0) return unavailable("aliasless");
|
|
10198
10478
|
if (slots.length === 0) return unavailable("no-population");
|
|
10199
10479
|
if (answered.length === 0) return unavailable("evidence-incomplete");
|
|
@@ -10209,7 +10489,7 @@ function presenceIn(slots, aliases, prepared) {
|
|
|
10209
10489
|
}
|
|
10210
10490
|
function scopeBrandPresence(input, slots, answered, prepared) {
|
|
10211
10491
|
if (slots.length === 0) return unavailable("no-population");
|
|
10212
|
-
if (input.projectBrandNames.every((
|
|
10492
|
+
if (input.projectBrandNames.every((alias2) => words(alias2).length === 0)) return unavailable("no-project-aliases");
|
|
10213
10493
|
if (answered.length === 0) return unavailable("evidence-incomplete");
|
|
10214
10494
|
const numerator = presenceIn(answered, input.projectBrandNames, prepared);
|
|
10215
10495
|
return { numerator, denominator: answered.length, rate: numerator / answered.length };
|
|
@@ -10508,8 +10788,8 @@ function reportInput(revision, plan, manifest, snapshots, legacy) {
|
|
|
10508
10788
|
label: group.label,
|
|
10509
10789
|
targetIds: group.targetKeys,
|
|
10510
10790
|
competitors: (group.competitors ?? []).map((domain) => {
|
|
10511
|
-
const
|
|
10512
|
-
return { domain, aliases: brandKeyFromText(
|
|
10791
|
+
const alias2 = brandLabelFromDomain(domain);
|
|
10792
|
+
return { domain, aliases: brandKeyFromText(alias2).length >= 4 ? [alias2] : [] };
|
|
10513
10793
|
})
|
|
10514
10794
|
})),
|
|
10515
10795
|
expectedSlots: slots,
|
|
@@ -12029,7 +12309,7 @@ function compileRules(input) {
|
|
|
12029
12309
|
const aliases = (input.rules.aliases ?? []).map(
|
|
12030
12310
|
(rule, index2) => compileRouteRule(rule, `rules.aliases[${index2}]`, ownedHosts)
|
|
12031
12311
|
);
|
|
12032
|
-
if (aliases.some((
|
|
12312
|
+
if (aliases.some((alias2) => alias2.host === primary.host)) {
|
|
12033
12313
|
throw new MeasurementDiscoveryConfigurationError("alias rules must use another owned host");
|
|
12034
12314
|
}
|
|
12035
12315
|
const excludedSlugSuffixes = sortedUnique3(
|
|
@@ -12947,14 +13227,14 @@ function compileMeasurementDraft(authoring, context) {
|
|
|
12947
13227
|
matcherClaims.set(identity, target.stableKey);
|
|
12948
13228
|
matchers.push(matcher);
|
|
12949
13229
|
});
|
|
12950
|
-
target.aliases.forEach((
|
|
12951
|
-
const identity = measurementMentionAliasKey(
|
|
13230
|
+
target.aliases.forEach((alias2, aliasIndex) => {
|
|
13231
|
+
const identity = measurementMentionAliasKey(alias2);
|
|
12952
13232
|
if (!identity) return;
|
|
12953
13233
|
const claimedBy = aliasClaims.get(identity);
|
|
12954
13234
|
if (claimedBy !== void 0 && claimedBy !== target.stableKey) {
|
|
12955
13235
|
sink.fail(
|
|
12956
13236
|
"target-alias-ambiguous",
|
|
12957
|
-
`Target alias normalizes to the same mention identity claimed by Target ${claimedBy}: ${
|
|
13237
|
+
`Target alias normalizes to the same mention identity claimed by Target ${claimedBy}: ${alias2}`,
|
|
12958
13238
|
["targets", targetIndex, "aliases", aliasIndex]
|
|
12959
13239
|
);
|
|
12960
13240
|
return;
|
|
@@ -27123,17 +27403,26 @@ var SCHEMA_TABLE = {
|
|
|
27123
27403
|
SiteAuditScoreDto: siteAuditScoreSchema,
|
|
27124
27404
|
SiteAuditTrendResponseDto: siteAuditTrendResponseSchema,
|
|
27125
27405
|
SiteCrawlDeadLinksResponseDto: siteCrawlDeadLinksResponseSchema,
|
|
27406
|
+
SiteCrawlGraphResponseDto: siteCrawlGraphResponseSchema,
|
|
27126
27407
|
SiteCrawlInternalLinksResponseDto: siteCrawlInternalLinksResponseSchema,
|
|
27127
27408
|
SiteCrawlNeighborsResponseDto: siteCrawlNeighborsResponseSchema,
|
|
27409
|
+
SiteCrawlPageAuditDto: siteCrawlPageAuditSchema,
|
|
27128
27410
|
SiteCrawlPagesResponseDto: siteCrawlPagesResponseSchema,
|
|
27129
27411
|
SiteCrawlStructureResponseDto: siteCrawlStructureResponseSchema,
|
|
27130
27412
|
SiteCrawlSummaryDto: siteCrawlSummarySchema,
|
|
27413
|
+
SiteHealthChangesResponseDto: siteHealthChangesResponseSchema,
|
|
27414
|
+
SiteHealthPathResponseDto: siteHealthPathResponseSchema,
|
|
27415
|
+
SiteHealthSubgraphResponseDto: siteHealthSubgraphResponseSchema,
|
|
27131
27416
|
SnapshotDiffResponse: snapshotDiffResponseSchema,
|
|
27132
27417
|
SnapshotListResponse: snapshotListResponseSchema,
|
|
27133
27418
|
SnapshotReportDto: snapshotReportSchema,
|
|
27134
27419
|
SourceBreakdownDto: sourceBreakdownDtoSchema,
|
|
27135
27420
|
TelemetryEventAcceptedDto: telemetryEventAcceptedDtoSchema,
|
|
27421
|
+
CloudflareWorkerIngestRequest: cloudflareWorkerIngestRequestSchema,
|
|
27422
|
+
CloudflareWorkerIngestResponse: cloudflareWorkerIngestResponseSchema,
|
|
27136
27423
|
TrafficBackfillResponse: trafficBackfillResponseSchema,
|
|
27424
|
+
TrafficConnectCloudflareRequest: trafficConnectCloudflareRequestSchema,
|
|
27425
|
+
TrafficConnectCloudflareResponse: trafficConnectCloudflareResponseSchema,
|
|
27137
27426
|
TrafficEventsResponse: trafficEventsResponseSchema,
|
|
27138
27427
|
TrafficSourceDetailDto: trafficSourceDetailDtoSchema,
|
|
27139
27428
|
TrafficSourceDto: trafficSourceDtoSchema,
|
|
@@ -32701,6 +32990,54 @@ var routeCatalog = [
|
|
|
32701
32990
|
502: errorResponse("Vercel request-logs endpoint probe failed (bad token, wrong project / team id, unreachable host, etc.).")
|
|
32702
32991
|
}
|
|
32703
32992
|
},
|
|
32993
|
+
{
|
|
32994
|
+
method: "post",
|
|
32995
|
+
path: "/api/v1/projects/{name}/traffic/connect/cloudflare",
|
|
32996
|
+
summary: "Connect a Cloudflare Worker traffic source",
|
|
32997
|
+
description: "Creates or updates a Cloudflare `direct-push` traffic source and returns a secret-free ES-module Worker plus Wrangler configuration. Per-source bearer and HMAC credentials remain in Canonry's credential store and are installed as Worker secret bindings; neither secret appears in source, TOML, API, or MCP output. Reconnect is idempotent: it reuses a matching source and credentials, preserves omitted metadata, and emits the current deployment package. Wrangler deploys the Worker without a route. The operator must attach the exact site route manually and set its Request limit failure mode to Fail open. No upstream probe is performed at connect time.",
|
|
32998
|
+
tags: ["traffic"],
|
|
32999
|
+
parameters: [nameParameter],
|
|
33000
|
+
requestBody: {
|
|
33001
|
+
required: false,
|
|
33002
|
+
content: {
|
|
33003
|
+
"application/json": {
|
|
33004
|
+
schema: { $ref: "#/components/schemas/TrafficConnectCloudflareRequest" }
|
|
33005
|
+
}
|
|
33006
|
+
}
|
|
33007
|
+
},
|
|
33008
|
+
responses: {
|
|
33009
|
+
200: jsonResponse("Connect response with generated Worker script.", "TrafficConnectCloudflareResponse"),
|
|
33010
|
+
400: errorResponse("Invalid Cloudflare connection request or credential storage not configured."),
|
|
33011
|
+
404: errorResponse("Project not found.")
|
|
33012
|
+
}
|
|
33013
|
+
},
|
|
33014
|
+
{
|
|
33015
|
+
method: "post",
|
|
33016
|
+
path: "/api/v1/projects/{name}/traffic/cloudflare/ingest",
|
|
33017
|
+
summary: "Ingest a batch of Cloudflare Worker events",
|
|
33018
|
+
description: "Direct-push protocol endpoint. The customer Worker sends selected edge-event batches authenticated by a per-source bearer and HMAC-SHA256 over `${timestamp}.${canonicalJson(payload)}`; object keys are sorted recursively before signing. The global cnry_* bearer does not apply. Headers: `Authorization: Bearer <per-source-token>`, `X-Canonry-Timestamp` (unix seconds), `X-Canonry-Signature` (hex HMAC), `X-Canonry-Source-Id`, and `X-Canonry-Worker-Version`. Authentication completes before project resolution and every auth/path/source mismatch returns the same 401. Accepted event ids are claimed durably in the same transaction as shared crawler, user-fetch, and paid/organic referral rollups.",
|
|
33019
|
+
tags: ["traffic"],
|
|
33020
|
+
auth: false,
|
|
33021
|
+
parameters: [nameParameter],
|
|
33022
|
+
requestBody: {
|
|
33023
|
+
required: true,
|
|
33024
|
+
content: {
|
|
33025
|
+
"application/json": {
|
|
33026
|
+
schema: { $ref: "#/components/schemas/CloudflareWorkerIngestRequest" }
|
|
33027
|
+
}
|
|
33028
|
+
}
|
|
33029
|
+
},
|
|
33030
|
+
responses: {
|
|
33031
|
+
200: jsonResponse(
|
|
33032
|
+
"Ingest acknowledged; droppedEvents includes normalization failures and durable-receipt duplicates.",
|
|
33033
|
+
"CloudflareWorkerIngestResponse"
|
|
33034
|
+
),
|
|
33035
|
+
400: errorResponse("Invalid ingest payload."),
|
|
33036
|
+
401: errorResponse("Authentication failed (bearer, signature, timestamp, or source id mismatch)."),
|
|
33037
|
+
413: errorResponse("The ingest payload exceeds 256 KiB."),
|
|
33038
|
+
429: errorResponse("The per-IP or authenticated-source ingest request budget was exceeded.")
|
|
33039
|
+
}
|
|
33040
|
+
},
|
|
32704
33041
|
{
|
|
32705
33042
|
method: "post",
|
|
32706
33043
|
path: "/api/v1/projects/{name}/traffic/sources/{id}/sync",
|
|
@@ -33062,6 +33399,103 @@ var routeCatalog = [
|
|
|
33062
33399
|
404: errorResponse("Project or crawl-bearing site-audit run not found.")
|
|
33063
33400
|
}
|
|
33064
33401
|
},
|
|
33402
|
+
{
|
|
33403
|
+
method: "get",
|
|
33404
|
+
path: "/api/v1/projects/{name}/technical-aeo/graph",
|
|
33405
|
+
summary: "Get a persisted Site Health graph projection",
|
|
33406
|
+
description: "Returns the deterministic graph projection computed once when the latest complete or selected historical crawl was published. ForceAtlas2 positions and the exact internal-anchor edge sample are persisted, so reads run no layout physics and never rescan the crawl edge table. The response is bounded to 20,000 nodes / 50,000 edges; `layout`, `omittedNodes`, `omittedEdges`, and `sampled` disclose legacy/unavailable layouts and intentional truncation.",
|
|
33407
|
+
tags: ["technical-aeo"],
|
|
33408
|
+
parameters: [
|
|
33409
|
+
nameParameter,
|
|
33410
|
+
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest complete crawl.", schema: stringSchema },
|
|
33411
|
+
{ name: "maxNodes", in: "query", description: "Maximum graph nodes. Defaults to and is capped at 20,000.", schema: { type: "integer", minimum: 1, maximum: 2e4 } },
|
|
33412
|
+
{ name: "maxEdges", in: "query", description: "Maximum graph edges. Defaults to and is capped at 50,000.", schema: { type: "integer", minimum: 1, maximum: 5e4 } }
|
|
33413
|
+
],
|
|
33414
|
+
responses: {
|
|
33415
|
+
200: jsonResponse("Bounded Site Health graph projection returned.", "SiteCrawlGraphResponseDto"),
|
|
33416
|
+
404: errorResponse("Project or crawl-bearing site-audit run not found.")
|
|
33417
|
+
}
|
|
33418
|
+
},
|
|
33419
|
+
{
|
|
33420
|
+
method: "get",
|
|
33421
|
+
path: "/api/v1/projects/{name}/technical-aeo/subgraph",
|
|
33422
|
+
summary: "Get a bounded canonical Site Health neighborhood",
|
|
33423
|
+
description: "Agent-oriented canonical crawl neighborhood around one page. Reads full crawl page/link rows rather than the sampled visualization, excludes layout coordinates, and returns snapshot completeness/termination. When a hard bound is reached, countAccuracy=lower-bound makes totalNodes, totalEdges, omittedNodes, and omittedEdges explicit minimums rather than false exact totals.",
|
|
33424
|
+
tags: ["technical-aeo"],
|
|
33425
|
+
parameters: [
|
|
33426
|
+
nameParameter,
|
|
33427
|
+
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest complete crawl.", schema: stringSchema },
|
|
33428
|
+
{ name: "nodeKey", in: "query", description: "Focus page canonical node key. Omit nodeKey and url to use the crawl root.", schema: stringSchema },
|
|
33429
|
+
{ name: "url", in: "query", description: "Focus page canonical URL. Omit nodeKey and url to use the crawl root.", schema: stringSchema },
|
|
33430
|
+
{ name: "hops", in: "query", description: "Undirected neighborhood radius. Defaults to 1; maximum 3.", schema: { type: "integer", minimum: 0, maximum: 3 } },
|
|
33431
|
+
{ name: "maxNodes", in: "query", description: "Maximum canonical nodes. Defaults to 25; maximum 200.", schema: { type: "integer", minimum: 1, maximum: 200 } },
|
|
33432
|
+
{ name: "maxEdges", in: "query", description: "Maximum canonical edges. Defaults to 50; maximum 500.", schema: { type: "integer", minimum: 1, maximum: 500 } }
|
|
33433
|
+
],
|
|
33434
|
+
responses: {
|
|
33435
|
+
200: jsonResponse("Bounded canonical Site Health neighborhood returned.", "SiteHealthSubgraphResponseDto"),
|
|
33436
|
+
404: errorResponse("Project, crawl-bearing run, or focus page not found.")
|
|
33437
|
+
}
|
|
33438
|
+
},
|
|
33439
|
+
{
|
|
33440
|
+
method: "get",
|
|
33441
|
+
path: "/api/v1/projects/{name}/technical-aeo/path",
|
|
33442
|
+
summary: "Find a directed internal-link path between pages",
|
|
33443
|
+
description: "Returns the shortest directed path over followable internal anchor links. The source defaults to the crawl root; the target must be identified by toNodeKey or toUrl. Exploration is bounded to 5,000 nodes and returns an explicit unreachable or truncated state. Snapshot complete and termination provenance prevents an unreachable result from a selected partial crawl being mistaken for a site-wide claim.",
|
|
33444
|
+
tags: ["technical-aeo"],
|
|
33445
|
+
parameters: [
|
|
33446
|
+
nameParameter,
|
|
33447
|
+
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest complete crawl.", schema: stringSchema },
|
|
33448
|
+
{ name: "fromNodeKey", in: "query", description: "Source canonical node key. Omit both source selectors to start at the crawl root.", schema: stringSchema },
|
|
33449
|
+
{ name: "fromUrl", in: "query", description: "Source canonical URL. Omit both source selectors to start at the crawl root.", schema: stringSchema },
|
|
33450
|
+
{ name: "toNodeKey", in: "query", description: "Target canonical node key. Required when toUrl is omitted.", schema: stringSchema },
|
|
33451
|
+
{ name: "toUrl", in: "query", description: "Target canonical URL. Required when toNodeKey is omitted.", schema: stringSchema },
|
|
33452
|
+
{ name: "maxDepth", in: "query", description: "Maximum directed link depth. Defaults to 12; maximum 24.", schema: { type: "integer", minimum: 1, maximum: 24 } }
|
|
33453
|
+
],
|
|
33454
|
+
responses: {
|
|
33455
|
+
200: jsonResponse("Directed internal-link path result returned.", "SiteHealthPathResponseDto"),
|
|
33456
|
+
400: errorResponse("A target selector is required."),
|
|
33457
|
+
404: errorResponse("Project, crawl-bearing run, source page, or target page not found.")
|
|
33458
|
+
}
|
|
33459
|
+
},
|
|
33460
|
+
{
|
|
33461
|
+
method: "get",
|
|
33462
|
+
path: "/api/v1/projects/{name}/technical-aeo/changes",
|
|
33463
|
+
summary: "Compare canonical Site Health snapshots",
|
|
33464
|
+
description: "Returns exact page and internal-link additions, removals, and semantic field changes between immutable complete crawl snapshots. Omitted toRunId selects the latest complete crawl; omitted fromRunId selects the complete crawl immediately before the target. ForceAtlas2 positions are presentation data and never count as a change. The first page includes exact post-filter summary counts; cursor pages set summary and total to null so they never repeat full snapshot joins or trust caller-carried counts. Results use a snapshot- and filter-bound keyset cursor.",
|
|
33465
|
+
tags: ["technical-aeo"],
|
|
33466
|
+
parameters: [
|
|
33467
|
+
nameParameter,
|
|
33468
|
+
{ name: "fromRunId", in: "query", description: "Baseline complete crawl. Omit for the complete crawl immediately before the target.", schema: stringSchema },
|
|
33469
|
+
{ name: "toRunId", in: "query", description: "Target complete crawl. Omit for the latest complete crawl.", schema: stringSchema },
|
|
33470
|
+
{ name: "scope", in: "query", description: "Restrict records and first-page summary counts to pages, links, or both.", schema: { type: "string", enum: ["all", "pages", "links"], default: "all" } },
|
|
33471
|
+
{ name: "change", in: "query", description: "Restrict records and first-page summary counts to one change kind.", schema: { type: "string", enum: ["all", "added", "removed", "changed"], default: "all" } },
|
|
33472
|
+
{ name: "cursor", in: "query", description: "Opaque keyset cursor bound to both run IDs and the selected filters.", schema: stringSchema },
|
|
33473
|
+
{ name: "limit", in: "query", description: "Maximum change records. Defaults to 25; maximum 100.", schema: { type: "integer", minimum: 1, maximum: 100 } }
|
|
33474
|
+
],
|
|
33475
|
+
responses: {
|
|
33476
|
+
200: jsonResponse("Canonical Site Health snapshot comparison returned.", "SiteHealthChangesResponseDto"),
|
|
33477
|
+
400: errorResponse("Invalid filters, run direction, or cursor context."),
|
|
33478
|
+
404: errorResponse("Project or selected crawl-bearing run not found.")
|
|
33479
|
+
}
|
|
33480
|
+
},
|
|
33481
|
+
{
|
|
33482
|
+
method: "get",
|
|
33483
|
+
path: "/api/v1/projects/{name}/technical-aeo/crawl/pages/audit",
|
|
33484
|
+
summary: "Get exact audit evidence for one persisted crawl page",
|
|
33485
|
+
description: "Returns one page score with the exact weighted-factor findings, recommendations, applicability, and independent critical defects persisted for the selected crawl. Exactly one of nodeKey or url is required. The discriminated state preserves crawl completeness and termination provenance, distinguishes a page that was not audited from one that was not found, and marks legacy score-only rows without fabricating missing evidence. This one-page read keeps verbose evidence out of the bounded interactive graph projection.",
|
|
33486
|
+
tags: ["technical-aeo"],
|
|
33487
|
+
parameters: [
|
|
33488
|
+
nameParameter,
|
|
33489
|
+
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest complete crawl.", schema: stringSchema },
|
|
33490
|
+
{ name: "nodeKey", in: "query", description: "Exact canonical crawl node key. Required when url is omitted.", schema: stringSchema },
|
|
33491
|
+
{ name: "url", in: "query", description: "Exact persisted crawl URL. Required when nodeKey is omitted.", schema: stringSchema }
|
|
33492
|
+
],
|
|
33493
|
+
responses: {
|
|
33494
|
+
200: jsonResponse("Page audit evidence state returned.", "SiteCrawlPageAuditDto"),
|
|
33495
|
+
400: errorResponse("Exactly one page selector is required."),
|
|
33496
|
+
404: errorResponse("Project or explicitly selected crawl-bearing run not found.")
|
|
33497
|
+
}
|
|
33498
|
+
},
|
|
33065
33499
|
{
|
|
33066
33500
|
method: "get",
|
|
33067
33501
|
path: "/api/v1/projects/{name}/technical-aeo/crawl/pages",
|
|
@@ -46482,7 +46916,7 @@ async function queryBacklinks(opts) {
|
|
|
46482
46916
|
const reversed = opts.targets.map(reverseDomain);
|
|
46483
46917
|
const targetList = reversed.map(quote).join(", ");
|
|
46484
46918
|
const limitClause = opts.limitPerTarget ? `QUALIFY row_number() OVER (PARTITION BY t.target_rev_domain ORDER BY v.num_hosts DESC) <= ${Math.floor(opts.limitPerTarget)}` : "";
|
|
46485
|
-
const
|
|
46919
|
+
const sql22 = `
|
|
46486
46920
|
WITH vertices AS (
|
|
46487
46921
|
SELECT * FROM read_csv(
|
|
46488
46922
|
${quote(opts.vertexPath)},
|
|
@@ -46518,7 +46952,7 @@ async function queryBacklinks(opts) {
|
|
|
46518
46952
|
const conn = await instance.connect();
|
|
46519
46953
|
let rows;
|
|
46520
46954
|
try {
|
|
46521
|
-
const reader = await conn.runAndReadAll(
|
|
46955
|
+
const reader = await conn.runAndReadAll(sql22);
|
|
46522
46956
|
rows = reader.getRowObjects();
|
|
46523
46957
|
} finally {
|
|
46524
46958
|
conn.disconnectSync?.();
|
|
@@ -46974,9 +47408,10 @@ async function backlinksRoutes(app, opts) {
|
|
|
46974
47408
|
}
|
|
46975
47409
|
|
|
46976
47410
|
// ../api-routes/src/traffic.ts
|
|
46977
|
-
import
|
|
47411
|
+
import crypto40 from "crypto";
|
|
47412
|
+
import { isIP } from "net";
|
|
46978
47413
|
import { Agent as UndiciAgent } from "undici";
|
|
46979
|
-
import { and as
|
|
47414
|
+
import { and as and39, desc as desc22, eq as eq47, gte as gte11, lte as lte10, sql as sql19 } from "drizzle-orm";
|
|
46980
47415
|
|
|
46981
47416
|
// ../api-routes/src/traffic-limits.ts
|
|
46982
47417
|
var VERCEL_MAX_SYNC_WINDOW_MS = 24 * 60 * 6e4;
|
|
@@ -49760,6 +50195,39 @@ var DEFAULT_AI_CRAWLER_RULES = [
|
|
|
49760
50195
|
userAgentPatterns: [/Amazonbot\//i]
|
|
49761
50196
|
}
|
|
49762
50197
|
];
|
|
50198
|
+
var DEFAULT_AI_CRAWLER_USER_AGENT_SUBSTRINGS = [
|
|
50199
|
+
"GPTBot/",
|
|
50200
|
+
"OAI-SearchBot/",
|
|
50201
|
+
"ChatGPT-User/",
|
|
50202
|
+
"ClaudeBot/",
|
|
50203
|
+
"Claude-Web/",
|
|
50204
|
+
"Claude-SearchBot/",
|
|
50205
|
+
"Claude-User/",
|
|
50206
|
+
"anthropic-ai",
|
|
50207
|
+
"PerplexityBot/",
|
|
50208
|
+
"Google-Extended",
|
|
50209
|
+
"Google-Agent",
|
|
50210
|
+
"Bytespider",
|
|
50211
|
+
"Applebot-Extended",
|
|
50212
|
+
"Applebot/",
|
|
50213
|
+
"meta-externalagent",
|
|
50214
|
+
"CCBot/",
|
|
50215
|
+
"cohere-ai",
|
|
50216
|
+
"Diffbot",
|
|
50217
|
+
"MistralAI-User/",
|
|
50218
|
+
"MistralBot/",
|
|
50219
|
+
"DeepSeekBot",
|
|
50220
|
+
"xAI-Bot/",
|
|
50221
|
+
"Grok-Bot/",
|
|
50222
|
+
"GrokBot/",
|
|
50223
|
+
"Googlebot",
|
|
50224
|
+
"bingbot/",
|
|
50225
|
+
"DuckDuckBot",
|
|
50226
|
+
"YandexBot/",
|
|
50227
|
+
"Baiduspider",
|
|
50228
|
+
"Amazonbot/",
|
|
50229
|
+
"Perplexity-User/"
|
|
50230
|
+
];
|
|
49763
50231
|
var DEFAULT_AI_REFERRER_RULES = [
|
|
49764
50232
|
{ domain: AI_ENGINE_DOMAINS.chatgpt, operator: "OpenAI", product: "ChatGPT" },
|
|
49765
50233
|
{ domain: LEGACY_CHATGPT_DOMAIN, operator: "OpenAI", product: "ChatGPT" },
|
|
@@ -50780,6 +51248,716 @@ async function drainVercelTrafficEvents(options) {
|
|
|
50780
51248
|
return { events, subWindowCount, effectiveStartMs, retentionClamped, truncatedSliceCount, truncatedSliceStartsMs, deadlineSkippedSliceCount: deadlineSkippedSliceStartsMs.length, deadlineSkippedSliceStartsMs, drainedThroughMs: cursorMs, deadlineReached };
|
|
50781
51249
|
}
|
|
50782
51250
|
|
|
51251
|
+
// ../integration-cloudflare-worker/src/canonical-json.ts
|
|
51252
|
+
function canonicalizeCloudflareJson(value) {
|
|
51253
|
+
if (value === null) return "null";
|
|
51254
|
+
if (typeof value === "string" || typeof value === "boolean") {
|
|
51255
|
+
return JSON.stringify(value);
|
|
51256
|
+
}
|
|
51257
|
+
if (typeof value === "number") {
|
|
51258
|
+
if (!Number.isFinite(value)) {
|
|
51259
|
+
throw new TypeError("Cloudflare canonical JSON does not support non-finite numbers");
|
|
51260
|
+
}
|
|
51261
|
+
return JSON.stringify(value);
|
|
51262
|
+
}
|
|
51263
|
+
if (Array.isArray(value)) {
|
|
51264
|
+
return `[${value.map((item2) => canonicalizeCloudflareJson(item2)).join(",")}]`;
|
|
51265
|
+
}
|
|
51266
|
+
if (typeof value === "object") {
|
|
51267
|
+
const record = value;
|
|
51268
|
+
const fields = [];
|
|
51269
|
+
for (const key of Object.keys(record).sort()) {
|
|
51270
|
+
const fieldValue = record[key];
|
|
51271
|
+
if (fieldValue === void 0) continue;
|
|
51272
|
+
fields.push(`${JSON.stringify(key)}:${canonicalizeCloudflareJson(fieldValue)}`);
|
|
51273
|
+
}
|
|
51274
|
+
return `{${fields.join(",")}}`;
|
|
51275
|
+
}
|
|
51276
|
+
throw new TypeError(`Cloudflare canonical JSON does not support ${typeof value}`);
|
|
51277
|
+
}
|
|
51278
|
+
|
|
51279
|
+
// ../integration-cloudflare-worker/src/normalize.ts
|
|
51280
|
+
function emptyToNull2(value) {
|
|
51281
|
+
if (value === null || value.trim() === "") return null;
|
|
51282
|
+
return value;
|
|
51283
|
+
}
|
|
51284
|
+
function maybeLabel(value) {
|
|
51285
|
+
if (value === null) return void 0;
|
|
51286
|
+
if (typeof value === "string" && value === "") return void 0;
|
|
51287
|
+
return String(value);
|
|
51288
|
+
}
|
|
51289
|
+
function buildProviderLabels(cf) {
|
|
51290
|
+
if (!cf) return {};
|
|
51291
|
+
const candidates = {
|
|
51292
|
+
verifiedBot: maybeLabel(cf.verifiedBot),
|
|
51293
|
+
botScore: maybeLabel(cf.botScore),
|
|
51294
|
+
country: maybeLabel(cf.country),
|
|
51295
|
+
asn: maybeLabel(cf.asn),
|
|
51296
|
+
asOrganization: maybeLabel(cf.asOrganization)
|
|
51297
|
+
};
|
|
51298
|
+
return Object.fromEntries(
|
|
51299
|
+
Object.entries(candidates).filter(
|
|
51300
|
+
(entry) => typeof entry[1] === "string"
|
|
51301
|
+
)
|
|
51302
|
+
);
|
|
51303
|
+
}
|
|
51304
|
+
function normalizeCloudflareEdgeEvent(event) {
|
|
51305
|
+
if (!event.path) return null;
|
|
51306
|
+
if (!event.observedAt) return null;
|
|
51307
|
+
if (!event.eventId) return null;
|
|
51308
|
+
const host = emptyToNull2(event.host);
|
|
51309
|
+
const path8 = event.path;
|
|
51310
|
+
const queryString = emptyToNull2(event.queryString);
|
|
51311
|
+
const requestUrl = host ? `https://${host}${path8}${queryString ? `?${queryString}` : ""}` : null;
|
|
51312
|
+
return {
|
|
51313
|
+
sourceType: TrafficSourceTypes.cloudflare,
|
|
51314
|
+
evidenceKind: TrafficEvidenceKinds["raw-request"],
|
|
51315
|
+
confidence: TrafficEventConfidences.observed,
|
|
51316
|
+
eventId: `cloudflare-worker:${event.eventId}`,
|
|
51317
|
+
observedAt: event.observedAt,
|
|
51318
|
+
method: emptyToNull2(event.method),
|
|
51319
|
+
requestUrl,
|
|
51320
|
+
host,
|
|
51321
|
+
path: path8,
|
|
51322
|
+
queryString,
|
|
51323
|
+
status: event.status,
|
|
51324
|
+
userAgent: emptyToNull2(event.userAgent),
|
|
51325
|
+
remoteIp: emptyToNull2(event.remoteIp),
|
|
51326
|
+
referer: emptyToNull2(event.referer),
|
|
51327
|
+
latencyMs: null,
|
|
51328
|
+
requestSizeBytes: null,
|
|
51329
|
+
responseSizeBytes: null,
|
|
51330
|
+
providerResource: {
|
|
51331
|
+
type: "cloudflare_zone",
|
|
51332
|
+
labels: {}
|
|
51333
|
+
},
|
|
51334
|
+
providerLabels: buildProviderLabels(event.cf)
|
|
51335
|
+
};
|
|
51336
|
+
}
|
|
51337
|
+
var normalizeCloudflareWorkerEvent = normalizeCloudflareEdgeEvent;
|
|
51338
|
+
|
|
51339
|
+
// ../integration-cloudflare-worker/src/types.ts
|
|
51340
|
+
var CLOUDFLARE_WORKER_BINDINGS = {
|
|
51341
|
+
deliveryMode: "CANONRY_DELIVERY_MODE",
|
|
51342
|
+
sourceId: "CANONRY_SOURCE_ID",
|
|
51343
|
+
ingestUrl: "CANONRY_INGEST_URL",
|
|
51344
|
+
workerVersion: "CANONRY_WORKER_VERSION",
|
|
51345
|
+
bearerToken: "CANONRY_BEARER_TOKEN",
|
|
51346
|
+
hmacSecret: "CANONRY_HMAC_SECRET"
|
|
51347
|
+
};
|
|
51348
|
+
var CLOUDFLARE_DIRECT_PUSH_SECRET_BINDINGS = [
|
|
51349
|
+
CLOUDFLARE_WORKER_BINDINGS.bearerToken,
|
|
51350
|
+
CLOUDFLARE_WORKER_BINDINGS.hmacSecret
|
|
51351
|
+
];
|
|
51352
|
+
var CLOUDFLARE_WORKER_GENERATED_MARKER = "// canonry traffic Worker \u2014 generated; do not edit by hand";
|
|
51353
|
+
var CLOUDFLARE_WRANGLER_GENERATED_MARKER = "# canonry traffic Worker configuration \u2014 generated; do not edit by hand";
|
|
51354
|
+
|
|
51355
|
+
// ../integration-cloudflare-worker/src/script.ts
|
|
51356
|
+
var DEFAULT_BOT_LIST = {
|
|
51357
|
+
version: "2026-08-09",
|
|
51358
|
+
uaKeywords: [
|
|
51359
|
+
...DEFAULT_AI_CRAWLER_USER_AGENT_SUBSTRINGS,
|
|
51360
|
+
// Preserve a small forward-looking machine-traffic net. The shared
|
|
51361
|
+
// classifier still drops unknown products after delivery.
|
|
51362
|
+
"bot",
|
|
51363
|
+
"crawler",
|
|
51364
|
+
"spider"
|
|
51365
|
+
],
|
|
51366
|
+
refererDomains: DEFAULT_AI_REFERRER_RULES.map((rule) => rule.domain),
|
|
51367
|
+
utmSourceTokens: [
|
|
51368
|
+
.../* @__PURE__ */ new Set([
|
|
51369
|
+
...DEFAULT_AI_REFERRER_RULES.map((rule) => rule.domain),
|
|
51370
|
+
...DEFAULT_AI_REFERRER_RULES.map((rule) => rule.domain.split(".")[0]),
|
|
51371
|
+
// Legacy chat.openai.com referrals classify from their first label.
|
|
51372
|
+
"chat"
|
|
51373
|
+
])
|
|
51374
|
+
]
|
|
51375
|
+
};
|
|
51376
|
+
var DEFAULT_BOT_SCORE_MAX_FORWARD = 30;
|
|
51377
|
+
var WORKER_COMPATIBILITY_DATE = "2026-05-01";
|
|
51378
|
+
function jsString(value) {
|
|
51379
|
+
return JSON.stringify(value);
|
|
51380
|
+
}
|
|
51381
|
+
function jsArray(values) {
|
|
51382
|
+
return `[${values.map((v) => jsString(v)).join(", ")}]`;
|
|
51383
|
+
}
|
|
51384
|
+
function generateWorkerScript(opts) {
|
|
51385
|
+
const botScoreMax = opts.botScoreMaxForward ?? DEFAULT_BOT_SCORE_MAX_FORWARD;
|
|
51386
|
+
const canonicalJsonFunction = canonicalizeCloudflareJson.toString();
|
|
51387
|
+
return `${CLOUDFLARE_WORKER_GENERATED_MARKER}
|
|
51388
|
+
// worker version: ${opts.workerVersion}
|
|
51389
|
+
// bot-list version: ${opts.botList.version}
|
|
51390
|
+
// delivery mode: ${opts.deliveryMode}
|
|
51391
|
+
|
|
51392
|
+
const UA_KEYWORDS = ${jsArray(opts.botList.uaKeywords)}
|
|
51393
|
+
const REFERER_DOMAINS = ${jsArray(opts.botList.refererDomains)}
|
|
51394
|
+
const UTM_SOURCE_TOKENS = ${jsArray(opts.botList.utmSourceTokens)}
|
|
51395
|
+
const BOT_SCORE_MAX_FORWARD = ${String(botScoreMax)}
|
|
51396
|
+
const RETRY_DELAYS_MS = [250, 1000]
|
|
51397
|
+
const canonicalizeJson = (${canonicalJsonFunction})
|
|
51398
|
+
|
|
51399
|
+
function requireBinding(value, name) {
|
|
51400
|
+
if (typeof value === 'string' && value.length > 0) return value
|
|
51401
|
+
throw new Error('Missing required Worker binding: ' + name)
|
|
51402
|
+
}
|
|
51403
|
+
|
|
51404
|
+
function lower(value) {
|
|
51405
|
+
return typeof value === 'string' ? value.toLowerCase() : ''
|
|
51406
|
+
}
|
|
51407
|
+
|
|
51408
|
+
function uaMatches(ua) {
|
|
51409
|
+
const lc = lower(ua)
|
|
51410
|
+
if (!lc) return false
|
|
51411
|
+
for (const kw of UA_KEYWORDS) {
|
|
51412
|
+
if (lc.indexOf(lower(kw)) !== -1) return true
|
|
51413
|
+
}
|
|
51414
|
+
return false
|
|
51415
|
+
}
|
|
51416
|
+
|
|
51417
|
+
function hostMatchesKnownDomain(host) {
|
|
51418
|
+
let normalized = lower(host)
|
|
51419
|
+
if (normalized.startsWith('www.')) normalized = normalized.slice(4)
|
|
51420
|
+
if (normalized.endsWith('.')) normalized = normalized.slice(0, -1)
|
|
51421
|
+
if (!normalized) return false
|
|
51422
|
+
for (const domain of REFERER_DOMAINS) {
|
|
51423
|
+
if (normalized === domain || normalized.endsWith('.' + domain)) return true
|
|
51424
|
+
}
|
|
51425
|
+
return false
|
|
51426
|
+
}
|
|
51427
|
+
|
|
51428
|
+
function refererMatches(referer) {
|
|
51429
|
+
if (!referer) return false
|
|
51430
|
+
try {
|
|
51431
|
+
return hostMatchesKnownDomain(new URL(referer).hostname)
|
|
51432
|
+
} catch (_) {
|
|
51433
|
+
return false
|
|
51434
|
+
}
|
|
51435
|
+
}
|
|
51436
|
+
|
|
51437
|
+
function utmSourceMatches(value) {
|
|
51438
|
+
let normalized = lower(value).trim()
|
|
51439
|
+
if (normalized.startsWith('www.')) normalized = normalized.slice(4)
|
|
51440
|
+
if (normalized.endsWith('.')) normalized = normalized.slice(0, -1)
|
|
51441
|
+
if (!normalized) return false
|
|
51442
|
+
if (hostMatchesKnownDomain(normalized)) return true
|
|
51443
|
+
return UTM_SOURCE_TOKENS.indexOf(normalized) !== -1
|
|
51444
|
+
}
|
|
51445
|
+
|
|
51446
|
+
function urlUtmMatches(value) {
|
|
51447
|
+
if (!value) return false
|
|
51448
|
+
try {
|
|
51449
|
+
return utmSourceMatches(new URL(value).searchParams.get('utm_source'))
|
|
51450
|
+
} catch (_) {
|
|
51451
|
+
return false
|
|
51452
|
+
}
|
|
51453
|
+
}
|
|
51454
|
+
|
|
51455
|
+
function botSignals(cf) {
|
|
51456
|
+
if (!cf) return false
|
|
51457
|
+
const bm = cf.botManagement
|
|
51458
|
+
if (bm) {
|
|
51459
|
+
if (bm.verifiedBot === true) return true
|
|
51460
|
+
if (typeof bm.score === 'number' && bm.score < BOT_SCORE_MAX_FORWARD) return true
|
|
51461
|
+
}
|
|
51462
|
+
if (typeof cf.botScore === 'number' && cf.botScore < BOT_SCORE_MAX_FORWARD) return true
|
|
51463
|
+
return false
|
|
51464
|
+
}
|
|
51465
|
+
|
|
51466
|
+
function shouldForward(request) {
|
|
51467
|
+
const ua = request.headers.get('user-agent') || ''
|
|
51468
|
+
if (uaMatches(ua)) return true
|
|
51469
|
+
if (urlUtmMatches(request.url)) return true
|
|
51470
|
+
const referer = request.headers.get('referer') || ''
|
|
51471
|
+
if (refererMatches(referer)) return true
|
|
51472
|
+
if (urlUtmMatches(referer)) return true
|
|
51473
|
+
return botSignals(request.cf)
|
|
51474
|
+
}
|
|
51475
|
+
|
|
51476
|
+
function toHex(buffer) {
|
|
51477
|
+
const bytes = new Uint8Array(buffer)
|
|
51478
|
+
let out = ''
|
|
51479
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
51480
|
+
out += bytes[i].toString(16).padStart(2, '0')
|
|
51481
|
+
}
|
|
51482
|
+
return out
|
|
51483
|
+
}
|
|
51484
|
+
|
|
51485
|
+
async function signBody(secret, timestamp, body) {
|
|
51486
|
+
const key = await crypto.subtle.importKey(
|
|
51487
|
+
'raw',
|
|
51488
|
+
new TextEncoder().encode(secret),
|
|
51489
|
+
{ name: 'HMAC', hash: 'SHA-256' },
|
|
51490
|
+
false,
|
|
51491
|
+
['sign'],
|
|
51492
|
+
)
|
|
51493
|
+
const sig = await crypto.subtle.sign(
|
|
51494
|
+
'HMAC',
|
|
51495
|
+
key,
|
|
51496
|
+
new TextEncoder().encode(timestamp + '.' + body),
|
|
51497
|
+
)
|
|
51498
|
+
return toHex(sig)
|
|
51499
|
+
}
|
|
51500
|
+
|
|
51501
|
+
function isRetryableStatus(status) {
|
|
51502
|
+
return status === 408 || status === 425 || status === 429 || status >= 500
|
|
51503
|
+
}
|
|
51504
|
+
|
|
51505
|
+
function sleep(ms) {
|
|
51506
|
+
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
51507
|
+
}
|
|
51508
|
+
|
|
51509
|
+
async function cancelUnusedResponseBody(response) {
|
|
51510
|
+
try {
|
|
51511
|
+
if (response.body) await response.body.cancel()
|
|
51512
|
+
} catch (_) {
|
|
51513
|
+
// The ingest response body is unused. Cancellation failure is non-fatal.
|
|
51514
|
+
}
|
|
51515
|
+
}
|
|
51516
|
+
|
|
51517
|
+
async function withRetry(operation) {
|
|
51518
|
+
for (let attempt = 0; attempt <= RETRY_DELAYS_MS.length; attempt++) {
|
|
51519
|
+
let response
|
|
51520
|
+
try {
|
|
51521
|
+
response = await operation()
|
|
51522
|
+
} catch (error) {
|
|
51523
|
+
if (attempt === RETRY_DELAYS_MS.length) throw error
|
|
51524
|
+
await sleep(RETRY_DELAYS_MS[attempt])
|
|
51525
|
+
continue
|
|
51526
|
+
}
|
|
51527
|
+
|
|
51528
|
+
const status = response.status
|
|
51529
|
+
const ok = response.ok
|
|
51530
|
+
await cancelUnusedResponseBody(response)
|
|
51531
|
+
if (ok) return
|
|
51532
|
+
if (!isRetryableStatus(status) || attempt === RETRY_DELAYS_MS.length) {
|
|
51533
|
+
throw new Error('Canonry ingest returned HTTP ' + status)
|
|
51534
|
+
}
|
|
51535
|
+
await sleep(RETRY_DELAYS_MS[attempt])
|
|
51536
|
+
}
|
|
51537
|
+
}
|
|
51538
|
+
|
|
51539
|
+
function pickCf(cf) {
|
|
51540
|
+
if (!cf) return null
|
|
51541
|
+
const bm = cf.botManagement || {}
|
|
51542
|
+
return {
|
|
51543
|
+
verifiedBot: typeof bm.verifiedBot === 'boolean' ? bm.verifiedBot : null,
|
|
51544
|
+
botScore: typeof bm.score === 'number' ? bm.score : (typeof cf.botScore === 'number' ? cf.botScore : null),
|
|
51545
|
+
country: typeof cf.country === 'string' ? cf.country : null,
|
|
51546
|
+
asn: typeof cf.asn === 'number' ? cf.asn : null,
|
|
51547
|
+
asOrganization: typeof cf.asOrganization === 'string' ? cf.asOrganization : null,
|
|
51548
|
+
}
|
|
51549
|
+
}
|
|
51550
|
+
|
|
51551
|
+
function buildEdgeEvent(request, status, observedAt) {
|
|
51552
|
+
const url = new URL(request.url)
|
|
51553
|
+
return {
|
|
51554
|
+
eventId: request.headers.get('cf-ray') || crypto.randomUUID(),
|
|
51555
|
+
observedAt,
|
|
51556
|
+
method: request.method || null,
|
|
51557
|
+
host: url.hostname || null,
|
|
51558
|
+
path: url.pathname || '/',
|
|
51559
|
+
queryString: url.search ? url.search.slice(1) : null,
|
|
51560
|
+
status: typeof status === 'number' ? status : null,
|
|
51561
|
+
userAgent: request.headers.get('user-agent') || null,
|
|
51562
|
+
remoteIp: request.headers.get('cf-connecting-ip') || null,
|
|
51563
|
+
referer: request.headers.get('referer') || null,
|
|
51564
|
+
cf: pickCf(request.cf),
|
|
51565
|
+
}
|
|
51566
|
+
}
|
|
51567
|
+
|
|
51568
|
+
function buildEdgeEventBatch(env, request, status, observedAt) {
|
|
51569
|
+
return {
|
|
51570
|
+
schemaVersion: 1,
|
|
51571
|
+
workerVersion: requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.workerVersion}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.workerVersion)}),
|
|
51572
|
+
events: [buildEdgeEvent(request, status, observedAt)],
|
|
51573
|
+
}
|
|
51574
|
+
}
|
|
51575
|
+
|
|
51576
|
+
async function deliverViaDirectPush(env, batch) {
|
|
51577
|
+
const sourceId = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.sourceId}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.sourceId)})
|
|
51578
|
+
const ingestUrl = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.ingestUrl}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.ingestUrl)})
|
|
51579
|
+
const bearerToken = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.bearerToken}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.bearerToken)})
|
|
51580
|
+
const hmacSecret = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.hmacSecret}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.hmacSecret)})
|
|
51581
|
+
const body = canonicalizeJson(batch)
|
|
51582
|
+
const timestamp = String(Math.floor(Date.now() / 1000))
|
|
51583
|
+
const signature = await signBody(hmacSecret, timestamp, body)
|
|
51584
|
+
|
|
51585
|
+
await withRetry(() => fetch(ingestUrl, {
|
|
51586
|
+
method: 'POST',
|
|
51587
|
+
headers: {
|
|
51588
|
+
'content-type': 'application/json',
|
|
51589
|
+
'Authorization': 'Bearer ' + bearerToken,
|
|
51590
|
+
'X-Canonry-Timestamp': timestamp,
|
|
51591
|
+
'X-Canonry-Signature': signature,
|
|
51592
|
+
'X-Canonry-Worker-Version': batch.workerVersion,
|
|
51593
|
+
'X-Canonry-Source-Id': sourceId,
|
|
51594
|
+
},
|
|
51595
|
+
body,
|
|
51596
|
+
}))
|
|
51597
|
+
}
|
|
51598
|
+
|
|
51599
|
+
async function deliverEdgeEventBatch(env, batch) {
|
|
51600
|
+
const deliveryMode = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.deliveryMode}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.deliveryMode)})
|
|
51601
|
+
if (deliveryMode === 'direct-push') {
|
|
51602
|
+
return deliverViaDirectPush(env, batch)
|
|
51603
|
+
}
|
|
51604
|
+
throw new Error('Unsupported Canonry delivery mode: ' + deliveryMode)
|
|
51605
|
+
}
|
|
51606
|
+
|
|
51607
|
+
async function deliverSelectedRequest(env, request, status, observedAt) {
|
|
51608
|
+
try {
|
|
51609
|
+
const batch = buildEdgeEventBatch(env, request, status, observedAt)
|
|
51610
|
+
await deliverEdgeEventBatch(env, batch)
|
|
51611
|
+
} catch (err) {
|
|
51612
|
+
// Delivery is statistical and must never mask the customer response.
|
|
51613
|
+
// Emit binding names and source id only; never emit secret values.
|
|
51614
|
+
console.warn('Canonry traffic delivery failed', {
|
|
51615
|
+
sourceId: typeof env.${CLOUDFLARE_WORKER_BINDINGS.sourceId} === 'string'
|
|
51616
|
+
? env.${CLOUDFLARE_WORKER_BINDINGS.sourceId}
|
|
51617
|
+
: null,
|
|
51618
|
+
message: err instanceof Error ? err.message : String(err),
|
|
51619
|
+
})
|
|
51620
|
+
}
|
|
51621
|
+
}
|
|
51622
|
+
|
|
51623
|
+
function shouldForwardSafely(env, request) {
|
|
51624
|
+
try {
|
|
51625
|
+
return shouldForward(request)
|
|
51626
|
+
} catch (err) {
|
|
51627
|
+
console.warn('Canonry traffic filter failed', {
|
|
51628
|
+
sourceId: typeof env.${CLOUDFLARE_WORKER_BINDINGS.sourceId} === 'string'
|
|
51629
|
+
? env.${CLOUDFLARE_WORKER_BINDINGS.sourceId}
|
|
51630
|
+
: null,
|
|
51631
|
+
message: err instanceof Error ? err.message : String(err),
|
|
51632
|
+
})
|
|
51633
|
+
return false
|
|
51634
|
+
}
|
|
51635
|
+
}
|
|
51636
|
+
|
|
51637
|
+
function scheduleSelectedRequest(ctx, env, request, status, observedAt) {
|
|
51638
|
+
try {
|
|
51639
|
+
ctx.waitUntil(deliverSelectedRequest(env, request, status, observedAt))
|
|
51640
|
+
} catch (err) {
|
|
51641
|
+
console.warn('Canonry traffic delivery scheduling failed', {
|
|
51642
|
+
sourceId: typeof env.${CLOUDFLARE_WORKER_BINDINGS.sourceId} === 'string'
|
|
51643
|
+
? env.${CLOUDFLARE_WORKER_BINDINGS.sourceId}
|
|
51644
|
+
: null,
|
|
51645
|
+
message: err instanceof Error ? err.message : String(err),
|
|
51646
|
+
})
|
|
51647
|
+
}
|
|
51648
|
+
}
|
|
51649
|
+
|
|
51650
|
+
export default {
|
|
51651
|
+
async fetch(request, env, ctx) {
|
|
51652
|
+
const observedAt = new Date().toISOString()
|
|
51653
|
+
const shouldLog = shouldForwardSafely(env, request)
|
|
51654
|
+
let response
|
|
51655
|
+
try {
|
|
51656
|
+
response = await fetch(request)
|
|
51657
|
+
} catch (err) {
|
|
51658
|
+
if (shouldLog) {
|
|
51659
|
+
scheduleSelectedRequest(ctx, env, request, null, observedAt)
|
|
51660
|
+
}
|
|
51661
|
+
throw err
|
|
51662
|
+
}
|
|
51663
|
+
if (shouldLog) {
|
|
51664
|
+
scheduleSelectedRequest(ctx, env, request, response.status, observedAt)
|
|
51665
|
+
}
|
|
51666
|
+
return response
|
|
51667
|
+
},
|
|
51668
|
+
}
|
|
51669
|
+
`;
|
|
51670
|
+
}
|
|
51671
|
+
function generateWranglerToml(opts) {
|
|
51672
|
+
const hostname = opts.hostname.trim().toLowerCase().replace(/\.$/, "");
|
|
51673
|
+
const accountConfig = opts.accountId ? `account_id = ${jsString(opts.accountId)}
|
|
51674
|
+
` : "";
|
|
51675
|
+
const zoneHint = opts.zoneId ? `# Target zone id: ${jsString(opts.zoneId)}
|
|
51676
|
+
` : "# Target zone id was not provided. Select the canonical site zone.\n";
|
|
51677
|
+
return `${CLOUDFLARE_WRANGLER_GENERATED_MARKER}
|
|
51678
|
+
name = "canonry-traffic-${opts.sourceId}"
|
|
51679
|
+
${accountConfig}main = "worker.js"
|
|
51680
|
+
compatibility_date = "${WORKER_COMPATIBILITY_DATE}"
|
|
51681
|
+
workers_dev = false
|
|
51682
|
+
|
|
51683
|
+
[vars]
|
|
51684
|
+
${CLOUDFLARE_WORKER_BINDINGS.deliveryMode} = ${jsString(opts.deliveryMode)}
|
|
51685
|
+
${CLOUDFLARE_WORKER_BINDINGS.sourceId} = ${jsString(opts.sourceId)}
|
|
51686
|
+
${CLOUDFLARE_WORKER_BINDINGS.ingestUrl} = ${jsString(opts.ingestUrl)}
|
|
51687
|
+
${CLOUDFLARE_WORKER_BINDINGS.workerVersion} = ${jsString(opts.workerVersion)}
|
|
51688
|
+
|
|
51689
|
+
[secrets]
|
|
51690
|
+
required = ${jsArray(CLOUDFLARE_DIRECT_PUSH_SECRET_BINDINGS)}
|
|
51691
|
+
|
|
51692
|
+
# Install required Worker secrets interactively or through Canonry's deploy
|
|
51693
|
+
# command. Never write their values into this file:
|
|
51694
|
+
${CLOUDFLARE_DIRECT_PUSH_SECRET_BINDINGS.map((name) => `# wrangler secret put ${name}`).join("\n")}
|
|
51695
|
+
|
|
51696
|
+
# Deploy this Worker via:
|
|
51697
|
+
# wrangler deploy
|
|
51698
|
+
# Canonry intentionally does not declare a route in this file.
|
|
51699
|
+
# After deploy, attach this exact route in the Cloudflare dashboard:
|
|
51700
|
+
# ${hostname}/*
|
|
51701
|
+
${zoneHint}# Set the route Request limit failure mode to Fail open before activation.
|
|
51702
|
+
# Wrangler cannot configure this route toggle.
|
|
51703
|
+
`;
|
|
51704
|
+
}
|
|
51705
|
+
|
|
51706
|
+
// ../integration-cloudflare-worker/src/verify.ts
|
|
51707
|
+
import { createHmac, timingSafeEqual } from "crypto";
|
|
51708
|
+
var DEFAULT_MAX_AGE_SECONDS = 300;
|
|
51709
|
+
function verifyRequestSignature(opts) {
|
|
51710
|
+
const { timestamp, signature, secret } = opts;
|
|
51711
|
+
if (timestamp === "" || !/^-?\d+$/.test(timestamp)) {
|
|
51712
|
+
return { ok: false, reason: "timestamp_invalid" };
|
|
51713
|
+
}
|
|
51714
|
+
const ts = Number(timestamp);
|
|
51715
|
+
if (!Number.isFinite(ts)) return { ok: false, reason: "timestamp_invalid" };
|
|
51716
|
+
const now = opts.nowSeconds ?? Math.floor(Date.now() / 1e3);
|
|
51717
|
+
const maxAge = opts.maxAgeSeconds ?? DEFAULT_MAX_AGE_SECONDS;
|
|
51718
|
+
if (Math.abs(now - ts) > maxAge) return { ok: false, reason: "timestamp_expired" };
|
|
51719
|
+
if (signature === "" || !/^[0-9a-f]+$/i.test(signature)) {
|
|
51720
|
+
return { ok: false, reason: "signature_invalid" };
|
|
51721
|
+
}
|
|
51722
|
+
let provided;
|
|
51723
|
+
try {
|
|
51724
|
+
provided = Buffer.from(signature, "hex");
|
|
51725
|
+
} catch {
|
|
51726
|
+
return { ok: false, reason: "signature_invalid" };
|
|
51727
|
+
}
|
|
51728
|
+
let bodies;
|
|
51729
|
+
try {
|
|
51730
|
+
if ("body" in opts) {
|
|
51731
|
+
bodies = [opts.body];
|
|
51732
|
+
} else {
|
|
51733
|
+
const canonical = canonicalizeCloudflareJson(opts.payload);
|
|
51734
|
+
const legacy = JSON.stringify(opts.payload);
|
|
51735
|
+
bodies = [canonical];
|
|
51736
|
+
if (opts.acceptLegacyJson !== false && typeof legacy === "string" && legacy !== canonical) {
|
|
51737
|
+
bodies.push(legacy);
|
|
51738
|
+
}
|
|
51739
|
+
}
|
|
51740
|
+
} catch {
|
|
51741
|
+
return { ok: false, reason: "signature_invalid" };
|
|
51742
|
+
}
|
|
51743
|
+
let comparableLength = false;
|
|
51744
|
+
for (const body of bodies) {
|
|
51745
|
+
const expected = createHmac("sha256", secret).update(`${timestamp}.${body}`).digest();
|
|
51746
|
+
if (provided.length !== expected.length) continue;
|
|
51747
|
+
comparableLength = true;
|
|
51748
|
+
if (timingSafeEqual(provided, expected)) return { ok: true };
|
|
51749
|
+
}
|
|
51750
|
+
if (!comparableLength) return { ok: false, reason: "signature_invalid" };
|
|
51751
|
+
return { ok: false, reason: "signature_mismatch" };
|
|
51752
|
+
}
|
|
51753
|
+
|
|
51754
|
+
// ../api-routes/src/traffic-event-ingest.ts
|
|
51755
|
+
import crypto39 from "crypto";
|
|
51756
|
+
import { and as and38, eq as eq46, gte as gte10, lt as lt7, lte as lte9, sql as sql18 } from "drizzle-orm";
|
|
51757
|
+
var DIRECT_PUSH_RECEIPT_TTL_MS = 10 * 6e4;
|
|
51758
|
+
function observedUtcHourBounds(observedAt) {
|
|
51759
|
+
const instant = new Date(observedAt);
|
|
51760
|
+
if (Number.isNaN(instant.getTime())) return null;
|
|
51761
|
+
instant.setUTCMinutes(0, 0, 0);
|
|
51762
|
+
const start = instant.toISOString();
|
|
51763
|
+
instant.setUTCHours(instant.getUTCHours() + 1);
|
|
51764
|
+
return { start, end: instant.toISOString() };
|
|
51765
|
+
}
|
|
51766
|
+
function writeTrafficEventBatch(opts) {
|
|
51767
|
+
const expiresAt = new Date(Date.parse(opts.receivedAt) + opts.receiptTtlMs).toISOString();
|
|
51768
|
+
let acceptedEvents = 0;
|
|
51769
|
+
let duplicateEvents = 0;
|
|
51770
|
+
let crawlerBucketRows = 0;
|
|
51771
|
+
let aiUserFetchBucketRows = 0;
|
|
51772
|
+
let aiReferralBucketRows = 0;
|
|
51773
|
+
let sampleRows = 0;
|
|
51774
|
+
opts.db.transaction((tx) => {
|
|
51775
|
+
tx.update(trafficSources).set({ updatedAt: sql18`${trafficSources.updatedAt}` }).where(eq46(trafficSources.id, opts.sourceId)).run();
|
|
51776
|
+
const source = tx.select().from(trafficSources).where(eq46(trafficSources.id, opts.sourceId)).get();
|
|
51777
|
+
opts.validateSource(source);
|
|
51778
|
+
tx.delete(trafficEventReceipts).where(and38(
|
|
51779
|
+
eq46(trafficEventReceipts.sourceId, opts.sourceId),
|
|
51780
|
+
lte9(trafficEventReceipts.expiresAt, opts.receivedAt)
|
|
51781
|
+
)).run();
|
|
51782
|
+
const claimedEvents = [];
|
|
51783
|
+
for (const event of opts.events) {
|
|
51784
|
+
const claim = tx.insert(trafficEventReceipts).values({
|
|
51785
|
+
sourceId: opts.sourceId,
|
|
51786
|
+
eventId: event.eventId,
|
|
51787
|
+
receivedAt: opts.receivedAt,
|
|
51788
|
+
expiresAt
|
|
51789
|
+
}).onConflictDoNothing().run();
|
|
51790
|
+
if (claim.changes === 1) claimedEvents.push(event);
|
|
51791
|
+
else duplicateEvents += 1;
|
|
51792
|
+
}
|
|
51793
|
+
acceptedEvents = claimedEvents.length;
|
|
51794
|
+
if (claimedEvents.length > 0) {
|
|
51795
|
+
const report = buildTrafficProbeReport(claimedEvents, { sampleLimit: claimedEvents.length });
|
|
51796
|
+
for (const bucket of report.crawlerEventsHourly) {
|
|
51797
|
+
const status = bucket.status ?? 0;
|
|
51798
|
+
tx.insert(crawlerEventsHourly).values({
|
|
51799
|
+
projectId: opts.projectId,
|
|
51800
|
+
sourceId: opts.sourceId,
|
|
51801
|
+
tsHour: bucket.tsHour,
|
|
51802
|
+
botId: bucket.botId,
|
|
51803
|
+
operator: bucket.operator,
|
|
51804
|
+
verificationStatus: bucket.verificationStatus,
|
|
51805
|
+
pathNormalized: bucket.pathNormalized,
|
|
51806
|
+
status,
|
|
51807
|
+
hits: bucket.hits,
|
|
51808
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
51809
|
+
createdAt: opts.receivedAt,
|
|
51810
|
+
updatedAt: opts.receivedAt
|
|
51811
|
+
}).onConflictDoUpdate({
|
|
51812
|
+
target: [
|
|
51813
|
+
crawlerEventsHourly.projectId,
|
|
51814
|
+
crawlerEventsHourly.sourceId,
|
|
51815
|
+
crawlerEventsHourly.tsHour,
|
|
51816
|
+
crawlerEventsHourly.botId,
|
|
51817
|
+
crawlerEventsHourly.verificationStatus,
|
|
51818
|
+
crawlerEventsHourly.pathNormalized,
|
|
51819
|
+
crawlerEventsHourly.status
|
|
51820
|
+
],
|
|
51821
|
+
set: {
|
|
51822
|
+
hits: sql18`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
51823
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
51824
|
+
updatedAt: opts.receivedAt
|
|
51825
|
+
}
|
|
51826
|
+
}).run();
|
|
51827
|
+
crawlerBucketRows += 1;
|
|
51828
|
+
}
|
|
51829
|
+
for (const bucket of report.aiUserFetchEventsHourly) {
|
|
51830
|
+
const status = bucket.status ?? 0;
|
|
51831
|
+
tx.insert(aiUserFetchEventsHourly).values({
|
|
51832
|
+
projectId: opts.projectId,
|
|
51833
|
+
sourceId: opts.sourceId,
|
|
51834
|
+
tsHour: bucket.tsHour,
|
|
51835
|
+
botId: bucket.botId,
|
|
51836
|
+
operator: bucket.operator,
|
|
51837
|
+
verificationStatus: bucket.verificationStatus,
|
|
51838
|
+
pathNormalized: bucket.pathNormalized,
|
|
51839
|
+
status,
|
|
51840
|
+
hits: bucket.hits,
|
|
51841
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
51842
|
+
createdAt: opts.receivedAt,
|
|
51843
|
+
updatedAt: opts.receivedAt
|
|
51844
|
+
}).onConflictDoUpdate({
|
|
51845
|
+
target: [
|
|
51846
|
+
aiUserFetchEventsHourly.projectId,
|
|
51847
|
+
aiUserFetchEventsHourly.sourceId,
|
|
51848
|
+
aiUserFetchEventsHourly.tsHour,
|
|
51849
|
+
aiUserFetchEventsHourly.botId,
|
|
51850
|
+
aiUserFetchEventsHourly.verificationStatus,
|
|
51851
|
+
aiUserFetchEventsHourly.pathNormalized,
|
|
51852
|
+
aiUserFetchEventsHourly.status
|
|
51853
|
+
],
|
|
51854
|
+
set: {
|
|
51855
|
+
hits: sql18`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
51856
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
51857
|
+
updatedAt: opts.receivedAt
|
|
51858
|
+
}
|
|
51859
|
+
}).run();
|
|
51860
|
+
aiUserFetchBucketRows += 1;
|
|
51861
|
+
}
|
|
51862
|
+
for (const bucket of report.aiReferralEventsHourly) {
|
|
51863
|
+
const status = bucket.status ?? 0;
|
|
51864
|
+
tx.insert(aiReferralEventsHourly).values({
|
|
51865
|
+
projectId: opts.projectId,
|
|
51866
|
+
sourceId: opts.sourceId,
|
|
51867
|
+
tsHour: bucket.tsHour,
|
|
51868
|
+
product: bucket.product,
|
|
51869
|
+
operator: bucket.operator,
|
|
51870
|
+
sourceDomain: bucket.sourceDomain,
|
|
51871
|
+
evidenceType: bucket.evidenceType,
|
|
51872
|
+
landingPathNormalized: bucket.landingPathNormalized,
|
|
51873
|
+
status,
|
|
51874
|
+
sessionsOrHits: bucket.hits,
|
|
51875
|
+
paidSessionsOrHits: bucket.paidHits,
|
|
51876
|
+
organicSessionsOrHits: bucket.organicHits,
|
|
51877
|
+
usersEstimated: null,
|
|
51878
|
+
createdAt: opts.receivedAt,
|
|
51879
|
+
updatedAt: opts.receivedAt
|
|
51880
|
+
}).onConflictDoUpdate({
|
|
51881
|
+
target: [
|
|
51882
|
+
aiReferralEventsHourly.projectId,
|
|
51883
|
+
aiReferralEventsHourly.sourceId,
|
|
51884
|
+
aiReferralEventsHourly.tsHour,
|
|
51885
|
+
aiReferralEventsHourly.product,
|
|
51886
|
+
aiReferralEventsHourly.sourceDomain,
|
|
51887
|
+
aiReferralEventsHourly.evidenceType,
|
|
51888
|
+
aiReferralEventsHourly.landingPathNormalized,
|
|
51889
|
+
aiReferralEventsHourly.status
|
|
51890
|
+
],
|
|
51891
|
+
set: {
|
|
51892
|
+
sessionsOrHits: sql18`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
51893
|
+
paidSessionsOrHits: sql18`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
|
|
51894
|
+
organicSessionsOrHits: sql18`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
|
|
51895
|
+
updatedAt: opts.receivedAt
|
|
51896
|
+
}
|
|
51897
|
+
}).run();
|
|
51898
|
+
aiReferralBucketRows += 1;
|
|
51899
|
+
}
|
|
51900
|
+
const hourlySampleLimit = Math.max(0, Math.floor(opts.sampleLimit));
|
|
51901
|
+
const remainingSamplesByHour = /* @__PURE__ */ new Map();
|
|
51902
|
+
for (const sample of report.samples) {
|
|
51903
|
+
const hour = observedUtcHourBounds(sample.observedAt);
|
|
51904
|
+
if (!hour || hourlySampleLimit === 0) continue;
|
|
51905
|
+
let remaining = remainingSamplesByHour.get(hour.start);
|
|
51906
|
+
if (remaining === void 0) {
|
|
51907
|
+
const existing = tx.select({ total: sql18`COUNT(*)` }).from(rawEventSamples).where(and38(
|
|
51908
|
+
eq46(rawEventSamples.sourceId, opts.sourceId),
|
|
51909
|
+
gte10(rawEventSamples.ts, hour.start),
|
|
51910
|
+
lt7(rawEventSamples.ts, hour.end)
|
|
51911
|
+
)).get();
|
|
51912
|
+
remaining = Math.max(0, hourlySampleLimit - Number(existing?.total ?? 0));
|
|
51913
|
+
}
|
|
51914
|
+
if (remaining === 0) {
|
|
51915
|
+
remainingSamplesByHour.set(hour.start, 0);
|
|
51916
|
+
continue;
|
|
51917
|
+
}
|
|
51918
|
+
const eventType = sample.crawler ? "crawler" : sample.aiUserFetch ? "ai_user_fetch" : sample.aiReferral ? "ai_referral" : "unknown";
|
|
51919
|
+
const refererHost = (() => {
|
|
51920
|
+
if (!sample.referer) return null;
|
|
51921
|
+
try {
|
|
51922
|
+
return new URL(sample.referer).hostname;
|
|
51923
|
+
} catch {
|
|
51924
|
+
return null;
|
|
51925
|
+
}
|
|
51926
|
+
})();
|
|
51927
|
+
tx.insert(rawEventSamples).values({
|
|
51928
|
+
id: crypto39.randomUUID(),
|
|
51929
|
+
projectId: opts.projectId,
|
|
51930
|
+
sourceId: opts.sourceId,
|
|
51931
|
+
ts: sample.observedAt,
|
|
51932
|
+
eventType,
|
|
51933
|
+
ipHash: null,
|
|
51934
|
+
userAgent: sample.userAgent,
|
|
51935
|
+
pathNormalized: sample.pathNormalized,
|
|
51936
|
+
status: sample.status,
|
|
51937
|
+
refererHost,
|
|
51938
|
+
classifierDetailsJson: {
|
|
51939
|
+
crawler: sample.crawler,
|
|
51940
|
+
aiUserFetch: sample.aiUserFetch,
|
|
51941
|
+
aiReferral: sample.aiReferral
|
|
51942
|
+
},
|
|
51943
|
+
createdAt: opts.receivedAt
|
|
51944
|
+
}).run();
|
|
51945
|
+
sampleRows += 1;
|
|
51946
|
+
remainingSamplesByHour.set(hour.start, remaining - 1);
|
|
51947
|
+
}
|
|
51948
|
+
}
|
|
51949
|
+
tx.update(trafficSources).set(opts.sourceUpdate).where(eq46(trafficSources.id, opts.sourceId)).run();
|
|
51950
|
+
});
|
|
51951
|
+
return {
|
|
51952
|
+
acceptedEvents,
|
|
51953
|
+
duplicateEvents,
|
|
51954
|
+
crawlerBucketRows,
|
|
51955
|
+
aiUserFetchBucketRows,
|
|
51956
|
+
aiReferralBucketRows,
|
|
51957
|
+
sampleRows
|
|
51958
|
+
};
|
|
51959
|
+
}
|
|
51960
|
+
|
|
50783
51961
|
// ../api-routes/src/traffic.ts
|
|
50784
51962
|
var DEFAULT_SYNC_WINDOW_MINUTES = 43200;
|
|
50785
51963
|
var DEFAULT_PAGE_SIZE3 = 1e3;
|
|
@@ -50796,6 +51974,126 @@ var DEFAULT_TRAFFIC_SYNC_CRON = "*/30 * * * *";
|
|
|
50796
51974
|
var MAX_BACKFILL_DAYS = 90;
|
|
50797
51975
|
var BACKFILL_MAX_PAGES = 1e3;
|
|
50798
51976
|
var BACKFILL_SAMPLE_LIMIT = 500;
|
|
51977
|
+
var CLOUDFLARE_WORKER_VERSION = "1.0.0";
|
|
51978
|
+
var CLOUDFLARE_INGEST_BODY_LIMIT = 256 * 1024;
|
|
51979
|
+
var DEFAULT_CLOUDFLARE_INGEST_RATE_LIMIT_MAX = 6e3;
|
|
51980
|
+
var SHA256_HEX_PATTERN = /^[0-9a-f]{64}$/i;
|
|
51981
|
+
function sha256Hex3(value) {
|
|
51982
|
+
return crypto40.createHash("sha256").update(value).digest("hex");
|
|
51983
|
+
}
|
|
51984
|
+
function hashCloudflareBearerToken(value) {
|
|
51985
|
+
return sha256Hex3(value);
|
|
51986
|
+
}
|
|
51987
|
+
function timingSafeEqualHex(a, b) {
|
|
51988
|
+
if (!a || !b) return false;
|
|
51989
|
+
if (!SHA256_HEX_PATTERN.test(a) || !SHA256_HEX_PATTERN.test(b)) return false;
|
|
51990
|
+
const left = Buffer.from(a, "hex");
|
|
51991
|
+
const right = Buffer.from(b, "hex");
|
|
51992
|
+
return left.length === right.length && crypto40.timingSafeEqual(left, right);
|
|
51993
|
+
}
|
|
51994
|
+
function parseDirectPushCloudflareSourceConfig(config) {
|
|
51995
|
+
const parsed = cloudflareTrafficSourceConfigSchema.safeParse(config);
|
|
51996
|
+
return parsed.success ? parsed.data : null;
|
|
51997
|
+
}
|
|
51998
|
+
function isDirectPushCloudflareDeliveryMode(value) {
|
|
51999
|
+
return value === CloudflareTrafficDeliveryModes["direct-push"];
|
|
52000
|
+
}
|
|
52001
|
+
function authenticateCloudflareIngest(request, store) {
|
|
52002
|
+
if (!store) return null;
|
|
52003
|
+
const authHeader = request.headers.authorization;
|
|
52004
|
+
const sourceIdHeader = request.headers["x-canonry-source-id"];
|
|
52005
|
+
const timestampHeader = request.headers["x-canonry-timestamp"];
|
|
52006
|
+
const signatureHeader = request.headers["x-canonry-signature"];
|
|
52007
|
+
const bearerToken = typeof authHeader === "string" && authHeader.startsWith("Bearer ") ? authHeader.slice(7) : "";
|
|
52008
|
+
const sourceId = typeof sourceIdHeader === "string" ? sourceIdHeader : "";
|
|
52009
|
+
const timestamp = typeof timestampHeader === "string" ? timestampHeader : "";
|
|
52010
|
+
const signature = typeof signatureHeader === "string" ? signatureHeader : "";
|
|
52011
|
+
if (!bearerToken || !sourceId || !timestamp || !signature) return null;
|
|
52012
|
+
const credential = store.getConnectionBySourceId(sourceId);
|
|
52013
|
+
if (!credential || !isDirectPushCloudflareDeliveryMode(credential.deliveryMode)) return null;
|
|
52014
|
+
const bearerHash = sha256Hex3(bearerToken);
|
|
52015
|
+
if (!timingSafeEqualHex(bearerHash, sha256Hex3(credential.bearerToken))) return null;
|
|
52016
|
+
if (!verifyRequestSignature({
|
|
52017
|
+
timestamp,
|
|
52018
|
+
signature,
|
|
52019
|
+
payload: request.body,
|
|
52020
|
+
secret: credential.hmacSecret
|
|
52021
|
+
}).ok) return null;
|
|
52022
|
+
return { bearerHash, credential };
|
|
52023
|
+
}
|
|
52024
|
+
function cloudflareIngestRateLimitKey(request, store, db) {
|
|
52025
|
+
const authenticated = authenticateCloudflareIngest(request, store);
|
|
52026
|
+
const params = request.params;
|
|
52027
|
+
if (authenticated && typeof params.name === "string" && authenticated.credential.projectName === params.name) {
|
|
52028
|
+
const source = db.select().from(trafficSources).where(eq47(trafficSources.id, authenticated.credential.sourceId)).get();
|
|
52029
|
+
if (source && source.sourceType === TrafficSourceTypes.cloudflare && source.status !== TrafficSourceStatuses.archived && parseDirectPushCloudflareSourceConfig(source.configJson) && timingSafeEqualHex(authenticated.bearerHash, source.ingestTokenHash)) {
|
|
52030
|
+
return `cloudflare-source:${authenticated.credential.sourceId}`;
|
|
52031
|
+
}
|
|
52032
|
+
}
|
|
52033
|
+
return `cloudflare-invalid:${request.ip}`;
|
|
52034
|
+
}
|
|
52035
|
+
function assertCloudflareIngestUrlIsPublicHttps(value) {
|
|
52036
|
+
let url;
|
|
52037
|
+
try {
|
|
52038
|
+
url = new URL(value);
|
|
52039
|
+
} catch {
|
|
52040
|
+
throw validationError("Cloudflare ingest URL must be a valid public HTTPS URL");
|
|
52041
|
+
}
|
|
52042
|
+
const hostname = url.hostname.toLowerCase().replace(/^\[|\]$/g, "");
|
|
52043
|
+
const ipv4 = hostname.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
|
|
52044
|
+
const octets = ipv4?.slice(1).map(Number);
|
|
52045
|
+
const privateIpv4 = octets !== void 0 && (octets.some((part) => part > 255) || octets[0] === 0 || octets[0] === 10 || octets[0] === 127 || octets[0] === 169 && octets[1] === 254 || octets[0] === 172 && octets[1] >= 16 && octets[1] <= 31 || octets[0] === 192 && octets[1] === 168);
|
|
52046
|
+
const privateIpv6 = hostname.includes(":") && (hostname === "::1" || hostname.startsWith("fc") || hostname.startsWith("fd") || hostname.startsWith("fe8") || hostname.startsWith("fe9") || hostname.startsWith("fea") || hostname.startsWith("feb"));
|
|
52047
|
+
const privateHostname = hostname === "localhost" || hostname.endsWith(".localhost") || hostname.endsWith(".local") || privateIpv6 || privateIpv4;
|
|
52048
|
+
if (url.protocol !== "https:" || privateHostname || url.username.length > 0 || url.password.length > 0 || url.search.length > 0 || url.hash.length > 0) {
|
|
52049
|
+
throw validationError(
|
|
52050
|
+
"Cloudflare ingest URL must be a credential-free public HTTPS URL without a query or fragment and reachable from the public Cloudflare edge; configure publicUrl to a public origin"
|
|
52051
|
+
);
|
|
52052
|
+
}
|
|
52053
|
+
}
|
|
52054
|
+
function normalizeDnsHostname(value) {
|
|
52055
|
+
const normalized = value.trim().toLowerCase().replace(/\.+$/, "");
|
|
52056
|
+
if (normalized.length === 0 || normalized.length > 253 || normalized.includes("*") || !normalized.includes(".")) return null;
|
|
52057
|
+
const valid = normalized.split(".").every(
|
|
52058
|
+
(label) => /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/.test(label)
|
|
52059
|
+
);
|
|
52060
|
+
return valid ? normalized : null;
|
|
52061
|
+
}
|
|
52062
|
+
function resolveCloudflareWorkerRouteHost(value) {
|
|
52063
|
+
const trimmed = value.trim();
|
|
52064
|
+
const hasScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed);
|
|
52065
|
+
if (hasScheme && !/^https?:\/\//i.test(trimmed)) {
|
|
52066
|
+
throw validationError("Project canonicalDomain must use an HTTP or HTTPS URL");
|
|
52067
|
+
}
|
|
52068
|
+
let url;
|
|
52069
|
+
try {
|
|
52070
|
+
url = new URL(hasScheme ? trimmed : `https://${trimmed}`);
|
|
52071
|
+
} catch {
|
|
52072
|
+
throw validationError("Project canonicalDomain must resolve to one exact public hostname");
|
|
52073
|
+
}
|
|
52074
|
+
if (url.username.length > 0 || url.password.length > 0 || url.pathname !== "" && url.pathname !== "/" || url.search.length > 0 || url.hash.length > 0 || url.port.length > 0) {
|
|
52075
|
+
throw validationError(
|
|
52076
|
+
"Project canonicalDomain must be a bare hostname or root HTTP(S) URL without credentials, a non-default port, path, query, or fragment"
|
|
52077
|
+
);
|
|
52078
|
+
}
|
|
52079
|
+
const hostname = normalizeDnsHostname(url.hostname);
|
|
52080
|
+
if (!hostname || isIP(hostname) !== 0) {
|
|
52081
|
+
throw validationError("Project canonicalDomain must resolve to one exact public hostname without wildcards");
|
|
52082
|
+
}
|
|
52083
|
+
return hostname;
|
|
52084
|
+
}
|
|
52085
|
+
function normalizeCloudflareEventHost(value) {
|
|
52086
|
+
if (!value || /[/:@?#*]/.test(value)) return null;
|
|
52087
|
+
return normalizeDnsHostname(value);
|
|
52088
|
+
}
|
|
52089
|
+
function assertCloudflareIngestUrlOutsideWorkerRoute(ingestUrl, workerRouteHost) {
|
|
52090
|
+
const ingestHost = normalizeDnsHostname(new URL(ingestUrl).hostname);
|
|
52091
|
+
if (ingestHost && ingestHost === workerRouteHost) {
|
|
52092
|
+
throw validationError(
|
|
52093
|
+
"Cloudflare ingest URL hostname must be outside the generated Worker route hostname to prevent recursive ingestion"
|
|
52094
|
+
);
|
|
52095
|
+
}
|
|
52096
|
+
}
|
|
50799
52097
|
function emptyTrafficSeriesPoint(bucket) {
|
|
50800
52098
|
return { bucket, crawlerHits: 0, aiUserFetchHits: 0, aiReferralHits: 0 };
|
|
50801
52099
|
}
|
|
@@ -50876,8 +52174,8 @@ async function runBackfillTask(options) {
|
|
|
50876
52174
|
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
50877
52175
|
try {
|
|
50878
52176
|
app.db.transaction((tx) => {
|
|
50879
|
-
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(
|
|
50880
|
-
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(
|
|
52177
|
+
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(eq47(runs.id, runId)).run();
|
|
52178
|
+
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(eq47(trafficSources.id, sourceRow2.id)).run();
|
|
50881
52179
|
});
|
|
50882
52180
|
} catch {
|
|
50883
52181
|
}
|
|
@@ -50892,7 +52190,7 @@ async function runBackfillTask(options) {
|
|
|
50892
52190
|
if (allEvents.length === 0) {
|
|
50893
52191
|
const finishedAt2 = (/* @__PURE__ */ new Date()).toISOString();
|
|
50894
52192
|
try {
|
|
50895
|
-
app.db.update(runs).set({ status: RunStatuses.completed, finishedAt: finishedAt2 }).where(
|
|
52193
|
+
app.db.update(runs).set({ status: RunStatuses.completed, finishedAt: finishedAt2 }).where(eq47(runs.id, runId)).run();
|
|
50896
52194
|
} catch {
|
|
50897
52195
|
}
|
|
50898
52196
|
return;
|
|
@@ -50914,31 +52212,31 @@ async function runBackfillTask(options) {
|
|
|
50914
52212
|
try {
|
|
50915
52213
|
app.db.transaction((tx) => {
|
|
50916
52214
|
tx.delete(crawlerEventsHourly).where(
|
|
50917
|
-
|
|
50918
|
-
|
|
50919
|
-
|
|
50920
|
-
|
|
52215
|
+
and39(
|
|
52216
|
+
eq47(crawlerEventsHourly.sourceId, sourceRow2.id),
|
|
52217
|
+
gte11(crawlerEventsHourly.tsHour, windowStartIso),
|
|
52218
|
+
lte10(crawlerEventsHourly.tsHour, windowEndIso)
|
|
50921
52219
|
)
|
|
50922
52220
|
).run();
|
|
50923
52221
|
tx.delete(aiUserFetchEventsHourly).where(
|
|
50924
|
-
|
|
50925
|
-
|
|
50926
|
-
|
|
50927
|
-
|
|
52222
|
+
and39(
|
|
52223
|
+
eq47(aiUserFetchEventsHourly.sourceId, sourceRow2.id),
|
|
52224
|
+
gte11(aiUserFetchEventsHourly.tsHour, windowStartIso),
|
|
52225
|
+
lte10(aiUserFetchEventsHourly.tsHour, windowEndIso)
|
|
50928
52226
|
)
|
|
50929
52227
|
).run();
|
|
50930
52228
|
tx.delete(aiReferralEventsHourly).where(
|
|
50931
|
-
|
|
50932
|
-
|
|
50933
|
-
|
|
50934
|
-
|
|
52229
|
+
and39(
|
|
52230
|
+
eq47(aiReferralEventsHourly.sourceId, sourceRow2.id),
|
|
52231
|
+
gte11(aiReferralEventsHourly.tsHour, windowStartIso),
|
|
52232
|
+
lte10(aiReferralEventsHourly.tsHour, windowEndIso)
|
|
50935
52233
|
)
|
|
50936
52234
|
).run();
|
|
50937
52235
|
tx.delete(rawEventSamples).where(
|
|
50938
|
-
|
|
50939
|
-
|
|
50940
|
-
|
|
50941
|
-
|
|
52236
|
+
and39(
|
|
52237
|
+
eq47(rawEventSamples.sourceId, sourceRow2.id),
|
|
52238
|
+
gte11(rawEventSamples.ts, windowStartIso),
|
|
52239
|
+
lte10(rawEventSamples.ts, windowEndIso)
|
|
50942
52240
|
)
|
|
50943
52241
|
).run();
|
|
50944
52242
|
for (const bucket of report.crawlerEventsHourly) {
|
|
@@ -51003,7 +52301,7 @@ async function runBackfillTask(options) {
|
|
|
51003
52301
|
}
|
|
51004
52302
|
})();
|
|
51005
52303
|
tx.insert(rawEventSamples).values({
|
|
51006
|
-
id:
|
|
52304
|
+
id: crypto40.randomUUID(),
|
|
51007
52305
|
projectId: project.id,
|
|
51008
52306
|
sourceId: sourceRow2.id,
|
|
51009
52307
|
ts: sample.observedAt,
|
|
@@ -51030,8 +52328,8 @@ async function runBackfillTask(options) {
|
|
|
51030
52328
|
lastEventIds: newRingBuffer,
|
|
51031
52329
|
...skipRecovered ? { skippedThroughAt: null } : {},
|
|
51032
52330
|
updatedAt: finishedAt
|
|
51033
|
-
}).where(
|
|
51034
|
-
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
52331
|
+
}).where(eq47(trafficSources.id, sourceRow2.id)).run();
|
|
52332
|
+
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq47(runs.id, runId)).run();
|
|
51035
52333
|
});
|
|
51036
52334
|
} catch (e) {
|
|
51037
52335
|
markFailed(`Backfill rollup write failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
@@ -51076,6 +52374,11 @@ async function trafficRoutes(app, opts) {
|
|
|
51076
52374
|
const pageSize = opts.defaultPageSize ?? DEFAULT_PAGE_SIZE3;
|
|
51077
52375
|
const maxPages = opts.defaultMaxPages ?? DEFAULT_MAX_PAGES4;
|
|
51078
52376
|
const sampleLimit = opts.defaultSampleLimit ?? DEFAULT_SAMPLE_LIMIT2;
|
|
52377
|
+
const checkCloudflareSourceRateLimit = app.createRateLimit({
|
|
52378
|
+
max: opts.cloudflareIngestRateLimitMax ?? DEFAULT_CLOUDFLARE_INGEST_RATE_LIMIT_MAX,
|
|
52379
|
+
timeWindow: "1 minute",
|
|
52380
|
+
keyGenerator: (request) => cloudflareIngestRateLimitKey(request, opts.cloudflareTrafficCredentialStore, app.db)
|
|
52381
|
+
});
|
|
51079
52382
|
app.post("/projects/:name/traffic/connect/cloud-run", async (request) => {
|
|
51080
52383
|
const project = resolveProject(app.db, request.params.name);
|
|
51081
52384
|
const body = request.body ?? {};
|
|
@@ -51113,7 +52416,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51113
52416
|
createdAt: existing?.createdAt ?? now,
|
|
51114
52417
|
updatedAt: now
|
|
51115
52418
|
});
|
|
51116
|
-
const activeSource = app.db.select().from(trafficSources).where(
|
|
52419
|
+
const activeSource = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes["cloud-run"] && row.status !== TrafficSourceStatuses.archived);
|
|
51117
52420
|
const config = {
|
|
51118
52421
|
gcpProjectId,
|
|
51119
52422
|
serviceName: serviceName ?? null,
|
|
@@ -51129,10 +52432,10 @@ async function trafficRoutes(app, opts) {
|
|
|
51129
52432
|
lastError: null,
|
|
51130
52433
|
configJson: config,
|
|
51131
52434
|
updatedAt: now
|
|
51132
|
-
}).where(
|
|
51133
|
-
sourceRow2 = app.db.select().from(trafficSources).where(
|
|
52435
|
+
}).where(eq47(trafficSources.id, activeSource.id)).run();
|
|
52436
|
+
sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, activeSource.id)).get();
|
|
51134
52437
|
} else {
|
|
51135
|
-
const newId =
|
|
52438
|
+
const newId = crypto40.randomUUID();
|
|
51136
52439
|
app.db.insert(trafficSources).values({
|
|
51137
52440
|
id: newId,
|
|
51138
52441
|
projectId: project.id,
|
|
@@ -51147,7 +52450,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51147
52450
|
createdAt: now,
|
|
51148
52451
|
updatedAt: now
|
|
51149
52452
|
}).run();
|
|
51150
|
-
sourceRow2 = app.db.select().from(trafficSources).where(
|
|
52453
|
+
sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, newId)).get();
|
|
51151
52454
|
}
|
|
51152
52455
|
writeAuditLog(app.db, {
|
|
51153
52456
|
projectId: project.id,
|
|
@@ -51199,7 +52502,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51199
52502
|
createdAt: existing?.createdAt ?? now,
|
|
51200
52503
|
updatedAt: now
|
|
51201
52504
|
});
|
|
51202
|
-
const activeSource = app.db.select().from(trafficSources).where(
|
|
52505
|
+
const activeSource = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.wordpress && row.status !== TrafficSourceStatuses.archived);
|
|
51203
52506
|
const config = { baseUrl, username };
|
|
51204
52507
|
const fallbackName = displayName ?? `WordPress \xB7 ${new URL(baseUrl).host}`;
|
|
51205
52508
|
let sourceRow2;
|
|
@@ -51210,10 +52513,10 @@ async function trafficRoutes(app, opts) {
|
|
|
51210
52513
|
lastError: null,
|
|
51211
52514
|
configJson: config,
|
|
51212
52515
|
updatedAt: now
|
|
51213
|
-
}).where(
|
|
51214
|
-
sourceRow2 = app.db.select().from(trafficSources).where(
|
|
52516
|
+
}).where(eq47(trafficSources.id, activeSource.id)).run();
|
|
52517
|
+
sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, activeSource.id)).get();
|
|
51215
52518
|
} else {
|
|
51216
|
-
const newId =
|
|
52519
|
+
const newId = crypto40.randomUUID();
|
|
51217
52520
|
app.db.insert(trafficSources).values({
|
|
51218
52521
|
id: newId,
|
|
51219
52522
|
projectId: project.id,
|
|
@@ -51228,7 +52531,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51228
52531
|
createdAt: now,
|
|
51229
52532
|
updatedAt: now
|
|
51230
52533
|
}).run();
|
|
51231
|
-
sourceRow2 = app.db.select().from(trafficSources).where(
|
|
52534
|
+
sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, newId)).get();
|
|
51232
52535
|
}
|
|
51233
52536
|
writeAuditLog(app.db, {
|
|
51234
52537
|
projectId: project.id,
|
|
@@ -51282,7 +52585,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51282
52585
|
createdAt: existing?.createdAt ?? now,
|
|
51283
52586
|
updatedAt: now
|
|
51284
52587
|
});
|
|
51285
|
-
const activeSource = app.db.select().from(trafficSources).where(
|
|
52588
|
+
const activeSource = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.vercel && row.status !== TrafficSourceStatuses.archived);
|
|
51286
52589
|
const config = { projectId, teamId, environment };
|
|
51287
52590
|
const fallbackName = displayName ?? `Vercel \xB7 ${projectId}`;
|
|
51288
52591
|
const { sourceRow: sourceRow2, scheduleCreated } = app.db.transaction((tx) => {
|
|
@@ -51294,10 +52597,10 @@ async function trafficRoutes(app, opts) {
|
|
|
51294
52597
|
lastError: null,
|
|
51295
52598
|
configJson: config,
|
|
51296
52599
|
updatedAt: now
|
|
51297
|
-
}).where(
|
|
51298
|
-
row = tx.select().from(trafficSources).where(
|
|
52600
|
+
}).where(eq47(trafficSources.id, activeSource.id)).run();
|
|
52601
|
+
row = tx.select().from(trafficSources).where(eq47(trafficSources.id, activeSource.id)).get();
|
|
51299
52602
|
} else {
|
|
51300
|
-
const newId =
|
|
52603
|
+
const newId = crypto40.randomUUID();
|
|
51301
52604
|
tx.insert(trafficSources).values({
|
|
51302
52605
|
id: newId,
|
|
51303
52606
|
projectId: project.id,
|
|
@@ -51320,18 +52623,18 @@ async function trafficRoutes(app, opts) {
|
|
|
51320
52623
|
createdAt: now,
|
|
51321
52624
|
updatedAt: now
|
|
51322
52625
|
}).run();
|
|
51323
|
-
row = tx.select().from(trafficSources).where(
|
|
52626
|
+
row = tx.select().from(trafficSources).where(eq47(trafficSources.id, newId)).get();
|
|
51324
52627
|
}
|
|
51325
52628
|
const existingSchedule = tx.select().from(schedules).where(
|
|
51326
|
-
|
|
51327
|
-
|
|
51328
|
-
|
|
52629
|
+
and39(
|
|
52630
|
+
eq47(schedules.projectId, project.id),
|
|
52631
|
+
eq47(schedules.kind, SchedulableRunKinds["traffic-sync"])
|
|
51329
52632
|
)
|
|
51330
52633
|
).get();
|
|
51331
52634
|
let created = false;
|
|
51332
52635
|
if (!existingSchedule) {
|
|
51333
52636
|
tx.insert(schedules).values({
|
|
51334
|
-
id:
|
|
52637
|
+
id: crypto40.randomUUID(),
|
|
51335
52638
|
projectId: project.id,
|
|
51336
52639
|
kind: SchedulableRunKinds["traffic-sync"],
|
|
51337
52640
|
cronExpr: DEFAULT_TRAFFIC_SYNC_CRON,
|
|
@@ -51372,9 +52675,256 @@ async function trafficRoutes(app, opts) {
|
|
|
51372
52675
|
}
|
|
51373
52676
|
return rowToDto(sourceRow2);
|
|
51374
52677
|
});
|
|
52678
|
+
app.post("/projects/:name/traffic/connect/cloudflare", async (request) => {
|
|
52679
|
+
const project = resolveProject(app.db, request.params.name);
|
|
52680
|
+
if (!opts.cloudflareTrafficCredentialStore) {
|
|
52681
|
+
throw validationError("Cloudflare traffic credential storage is not configured for this deployment");
|
|
52682
|
+
}
|
|
52683
|
+
if (!opts.cloudflareTrafficIngestUrl) {
|
|
52684
|
+
throw validationError("Cloudflare ingest URL is not configured for this deployment");
|
|
52685
|
+
}
|
|
52686
|
+
const credentialStore = opts.cloudflareTrafficCredentialStore;
|
|
52687
|
+
const parsed = trafficConnectCloudflareRequestSchema.safeParse(request.body ?? {});
|
|
52688
|
+
if (!parsed.success) {
|
|
52689
|
+
throw validationError(parsed.error.issues.map((i) => i.message).join("; "));
|
|
52690
|
+
}
|
|
52691
|
+
const { deliveryMode, displayName, zoneId, accountId } = parsed.data;
|
|
52692
|
+
const workerRouteHost = resolveCloudflareWorkerRouteHost(project.canonicalDomain);
|
|
52693
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
52694
|
+
const activeSource = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).all().find(
|
|
52695
|
+
(row) => row.sourceType === TrafficSourceTypes.cloudflare && row.status !== TrafficSourceStatuses.archived
|
|
52696
|
+
);
|
|
52697
|
+
const sourceId = activeSource?.id ?? crypto40.randomUUID();
|
|
52698
|
+
const parsedActiveConfig = activeSource ? parseDirectPushCloudflareSourceConfig(activeSource.configJson) : null;
|
|
52699
|
+
if (activeSource && !parsedActiveConfig) {
|
|
52700
|
+
throw validationError("Existing Cloudflare source is not configured for direct-push delivery");
|
|
52701
|
+
}
|
|
52702
|
+
const previousCredential = credentialStore.getConnection(project.name);
|
|
52703
|
+
if (previousCredential && !isDirectPushCloudflareDeliveryMode(previousCredential.deliveryMode)) {
|
|
52704
|
+
throw validationError(
|
|
52705
|
+
`Cloudflare credential delivery mode "${previousCredential.deliveryMode}" does not match "${deliveryMode}"`
|
|
52706
|
+
);
|
|
52707
|
+
}
|
|
52708
|
+
const reusableCredential = previousCredential?.sourceId === sourceId ? previousCredential : void 0;
|
|
52709
|
+
const previousZoneId = reusableCredential?.zoneId ?? parsedActiveConfig?.zoneId ?? null;
|
|
52710
|
+
const previousAccountId = reusableCredential?.accountId ?? parsedActiveConfig?.accountId ?? null;
|
|
52711
|
+
const effectiveZoneId = zoneId ?? previousZoneId;
|
|
52712
|
+
const effectiveAccountId = accountId ?? previousAccountId;
|
|
52713
|
+
const bearerToken = reusableCredential?.bearerToken ?? `cnry_cfw_${crypto40.randomBytes(32).toString("hex")}`;
|
|
52714
|
+
const hmacSecret = reusableCredential?.hmacSecret ?? crypto40.randomBytes(32).toString("hex");
|
|
52715
|
+
const ingestTokenHash = sha256Hex3(bearerToken);
|
|
52716
|
+
const ingestUrl = opts.cloudflareTrafficIngestUrl.replace("{name}", encodeURIComponent(project.name));
|
|
52717
|
+
assertCloudflareIngestUrlIsPublicHttps(ingestUrl);
|
|
52718
|
+
assertCloudflareIngestUrlOutsideWorkerRoute(ingestUrl, workerRouteHost);
|
|
52719
|
+
const workerVersion = CLOUDFLARE_WORKER_VERSION;
|
|
52720
|
+
const workerScript = generateWorkerScript({
|
|
52721
|
+
deliveryMode,
|
|
52722
|
+
workerVersion,
|
|
52723
|
+
botList: DEFAULT_BOT_LIST
|
|
52724
|
+
});
|
|
52725
|
+
const wranglerToml = generateWranglerToml({
|
|
52726
|
+
deliveryMode,
|
|
52727
|
+
sourceId,
|
|
52728
|
+
hostname: workerRouteHost,
|
|
52729
|
+
ingestUrl,
|
|
52730
|
+
workerVersion,
|
|
52731
|
+
zoneId: effectiveZoneId,
|
|
52732
|
+
accountId: effectiveAccountId
|
|
52733
|
+
});
|
|
52734
|
+
const config = {
|
|
52735
|
+
schemaVersion: 1,
|
|
52736
|
+
deliveryMode,
|
|
52737
|
+
workerVersion,
|
|
52738
|
+
expectedBotListVersion: DEFAULT_BOT_LIST.version,
|
|
52739
|
+
zoneId: effectiveZoneId,
|
|
52740
|
+
accountId: effectiveAccountId
|
|
52741
|
+
};
|
|
52742
|
+
const fallbackName = displayName ?? activeSource?.displayName ?? `Cloudflare \xB7 ${effectiveZoneId ?? sourceId.slice(0, 8)}`;
|
|
52743
|
+
credentialStore.upsertConnection({
|
|
52744
|
+
projectName: project.name,
|
|
52745
|
+
deliveryMode,
|
|
52746
|
+
sourceId,
|
|
52747
|
+
bearerToken,
|
|
52748
|
+
hmacSecret,
|
|
52749
|
+
workerVersion,
|
|
52750
|
+
expectedBotListVersion: DEFAULT_BOT_LIST.version,
|
|
52751
|
+
zoneId: effectiveZoneId,
|
|
52752
|
+
accountId: effectiveAccountId,
|
|
52753
|
+
createdAt: previousCredential?.createdAt ?? activeSource?.createdAt ?? now,
|
|
52754
|
+
updatedAt: now
|
|
52755
|
+
});
|
|
52756
|
+
let sourceRow2;
|
|
52757
|
+
try {
|
|
52758
|
+
if (activeSource) {
|
|
52759
|
+
app.db.update(trafficSources).set({
|
|
52760
|
+
displayName: fallbackName,
|
|
52761
|
+
status: TrafficSourceStatuses.connected,
|
|
52762
|
+
lastError: null,
|
|
52763
|
+
configJson: config,
|
|
52764
|
+
ingestTokenHash,
|
|
52765
|
+
updatedAt: now
|
|
52766
|
+
}).where(eq47(trafficSources.id, activeSource.id)).run();
|
|
52767
|
+
sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, activeSource.id)).get();
|
|
52768
|
+
} else {
|
|
52769
|
+
app.db.insert(trafficSources).values({
|
|
52770
|
+
id: sourceId,
|
|
52771
|
+
projectId: project.id,
|
|
52772
|
+
sourceType: TrafficSourceTypes.cloudflare,
|
|
52773
|
+
displayName: fallbackName,
|
|
52774
|
+
status: TrafficSourceStatuses.connected,
|
|
52775
|
+
// Seed `lastSyncedAt` to NOW so the `traffic.source.recent-data`
|
|
52776
|
+
// doctor check has a non-null baseline. Successful ingest advances
|
|
52777
|
+
// it as the receiver's last-activity timestamp.
|
|
52778
|
+
lastSyncedAt: now,
|
|
52779
|
+
lastCursor: null,
|
|
52780
|
+
lastError: null,
|
|
52781
|
+
archivedAt: null,
|
|
52782
|
+
configJson: config,
|
|
52783
|
+
ingestTokenHash,
|
|
52784
|
+
createdAt: now,
|
|
52785
|
+
updatedAt: now
|
|
52786
|
+
}).run();
|
|
52787
|
+
sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, sourceId)).get();
|
|
52788
|
+
}
|
|
52789
|
+
} catch (err) {
|
|
52790
|
+
try {
|
|
52791
|
+
if (previousCredential) {
|
|
52792
|
+
credentialStore.upsertConnection(previousCredential);
|
|
52793
|
+
} else {
|
|
52794
|
+
credentialStore.deleteConnection(project.name);
|
|
52795
|
+
}
|
|
52796
|
+
} catch {
|
|
52797
|
+
}
|
|
52798
|
+
throw err;
|
|
52799
|
+
}
|
|
52800
|
+
writeAuditLog(app.db, {
|
|
52801
|
+
projectId: project.id,
|
|
52802
|
+
actor: "api",
|
|
52803
|
+
action: "traffic.cloudflare.connected",
|
|
52804
|
+
entityType: "traffic_source",
|
|
52805
|
+
entityId: sourceRow2.id
|
|
52806
|
+
});
|
|
52807
|
+
const routeInstruction = effectiveZoneId ? ` 4. Deploy with \`wrangler deploy\`; wrangler.toml records zone ${effectiveZoneId} but does not claim a route` : " 4. Deploy with `wrangler deploy`; wrangler.toml intentionally does not claim a route";
|
|
52808
|
+
const instructions = [
|
|
52809
|
+
"Deploy this Worker to your Cloudflare zone:",
|
|
52810
|
+
" 1. Save worker.js and wrangler.toml; neither file contains credentials",
|
|
52811
|
+
" 2. Install the required bearer and HMAC Worker secret bindings locally; never paste their values into source or chat",
|
|
52812
|
+
` 3. Inspect existing Cloudflare Worker routes first; confirm the generated \`${workerRouteHost}/*\` catch-all will not replace or collide with another Worker`,
|
|
52813
|
+
routeInstruction,
|
|
52814
|
+
` 5. In the Cloudflare Dashboard route form, select \`${workerRouteHost}/*\` and this Worker`,
|
|
52815
|
+
" 6. Set Request limit failure mode to Fail open, then save the route; Wrangler cannot set this toggle",
|
|
52816
|
+
" 7. Keep the Canonry ingest hostname outside this Worker route to avoid recursion",
|
|
52817
|
+
"",
|
|
52818
|
+
`Source id: ${sourceRow2.id}`,
|
|
52819
|
+
"After deploy, check `canonry doctor --project " + project.name + "` to confirm events are arriving."
|
|
52820
|
+
].join("\n");
|
|
52821
|
+
return {
|
|
52822
|
+
sourceId: sourceRow2.id,
|
|
52823
|
+
deliveryMode,
|
|
52824
|
+
workerScript,
|
|
52825
|
+
wranglerToml,
|
|
52826
|
+
workerVersion,
|
|
52827
|
+
instructions
|
|
52828
|
+
};
|
|
52829
|
+
});
|
|
52830
|
+
app.post("/projects/:name/traffic/cloudflare/ingest", {
|
|
52831
|
+
bodyLimit: CLOUDFLARE_INGEST_BODY_LIMIT,
|
|
52832
|
+
preHandler: async (request, reply) => {
|
|
52833
|
+
const key = cloudflareIngestRateLimitKey(
|
|
52834
|
+
request,
|
|
52835
|
+
opts.cloudflareTrafficCredentialStore,
|
|
52836
|
+
app.db
|
|
52837
|
+
);
|
|
52838
|
+
if (key.startsWith("cloudflare-invalid:")) return;
|
|
52839
|
+
const result = await checkCloudflareSourceRateLimit(request);
|
|
52840
|
+
if (result.isAllowed || !result.isExceeded) return;
|
|
52841
|
+
reply.header("retry-after", result.ttlInSeconds);
|
|
52842
|
+
return reply.status(429).send({
|
|
52843
|
+
statusCode: 429,
|
|
52844
|
+
error: "Too Many Requests",
|
|
52845
|
+
message: `Rate limit exceeded, retry in ${result.ttlInSeconds} second(s)`
|
|
52846
|
+
});
|
|
52847
|
+
},
|
|
52848
|
+
config: {
|
|
52849
|
+
rateLimit: {
|
|
52850
|
+
hook: "onRequest",
|
|
52851
|
+
max: opts.cloudflareIngestIpRateLimitMax ?? DEFAULT_CLOUDFLARE_INGEST_RATE_LIMIT_MAX,
|
|
52852
|
+
timeWindow: "1 minute",
|
|
52853
|
+
groupId: "cloudflare-direct-push-ip",
|
|
52854
|
+
keyGenerator: (request) => `cloudflare-ingest-ip:${request.ip}`
|
|
52855
|
+
}
|
|
52856
|
+
}
|
|
52857
|
+
}, async (request, reply) => {
|
|
52858
|
+
const authenticated = authenticateCloudflareIngest(
|
|
52859
|
+
request,
|
|
52860
|
+
opts.cloudflareTrafficCredentialStore
|
|
52861
|
+
);
|
|
52862
|
+
if (!authenticated || authenticated.credential.projectName !== request.params.name || !isDirectPushCloudflareDeliveryMode(authenticated.credential.deliveryMode)) {
|
|
52863
|
+
throw authRequired();
|
|
52864
|
+
}
|
|
52865
|
+
const { bearerHash, credential } = authenticated;
|
|
52866
|
+
const sourceId = credential.sourceId;
|
|
52867
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, sourceId)).get();
|
|
52868
|
+
if (!sourceRow2 || sourceRow2.sourceType !== TrafficSourceTypes.cloudflare || sourceRow2.status === TrafficSourceStatuses.archived || !parseDirectPushCloudflareSourceConfig(sourceRow2.configJson) || !timingSafeEqualHex(bearerHash, sourceRow2.ingestTokenHash)) {
|
|
52869
|
+
throw authRequired();
|
|
52870
|
+
}
|
|
52871
|
+
const project = resolveProject(app.db, request.params.name);
|
|
52872
|
+
if (sourceRow2.projectId !== project.id) throw authRequired();
|
|
52873
|
+
const parsed = cloudflareWorkerIngestRequestSchema.safeParse(request.body);
|
|
52874
|
+
if (!parsed.success) {
|
|
52875
|
+
throw validationError(parsed.error.issues.map((i) => i.message).join("; "));
|
|
52876
|
+
}
|
|
52877
|
+
const { workerVersion, events } = parsed.data;
|
|
52878
|
+
const canonicalHost = resolveCloudflareWorkerRouteHost(project.canonicalDomain);
|
|
52879
|
+
const wrongHostIndex = events.findIndex(
|
|
52880
|
+
(event) => normalizeCloudflareEventHost(event.host) !== canonicalHost
|
|
52881
|
+
);
|
|
52882
|
+
if (wrongHostIndex !== -1) {
|
|
52883
|
+
throw validationError(
|
|
52884
|
+
`Cloudflare event at index ${wrongHostIndex} does not belong to project domain "${project.canonicalDomain}"`
|
|
52885
|
+
);
|
|
52886
|
+
}
|
|
52887
|
+
const normalized = [];
|
|
52888
|
+
let normalizationDrops = 0;
|
|
52889
|
+
for (const event of events) {
|
|
52890
|
+
const result = normalizeCloudflareWorkerEvent(event);
|
|
52891
|
+
if (result) normalized.push(result);
|
|
52892
|
+
else normalizationDrops += 1;
|
|
52893
|
+
}
|
|
52894
|
+
const receivedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
52895
|
+
const writeResult = writeTrafficEventBatch({
|
|
52896
|
+
db: app.db,
|
|
52897
|
+
projectId: project.id,
|
|
52898
|
+
sourceId,
|
|
52899
|
+
events: normalized,
|
|
52900
|
+
receivedAt,
|
|
52901
|
+
receiptTtlMs: DIRECT_PUSH_RECEIPT_TTL_MS,
|
|
52902
|
+
sampleLimit,
|
|
52903
|
+
validateSource: (latestRow) => {
|
|
52904
|
+
if (!latestRow || latestRow.projectId !== project.id || latestRow.sourceType !== TrafficSourceTypes.cloudflare || latestRow.status === TrafficSourceStatuses.archived || !parseDirectPushCloudflareSourceConfig(latestRow.configJson) || !timingSafeEqualHex(bearerHash, latestRow.ingestTokenHash)) {
|
|
52905
|
+
throw authRequired();
|
|
52906
|
+
}
|
|
52907
|
+
},
|
|
52908
|
+
sourceUpdate: {
|
|
52909
|
+
lastWorkerVersion: workerVersion,
|
|
52910
|
+
lastSyncedAt: receivedAt,
|
|
52911
|
+
lastError: null,
|
|
52912
|
+
updatedAt: receivedAt
|
|
52913
|
+
}
|
|
52914
|
+
});
|
|
52915
|
+
return reply.status(200).send({
|
|
52916
|
+
acceptedEvents: writeResult.acceptedEvents,
|
|
52917
|
+
droppedEvents: normalizationDrops + writeResult.duplicateEvents,
|
|
52918
|
+
workerVersionAck: workerVersion,
|
|
52919
|
+
crawlerBucketRows: writeResult.crawlerBucketRows,
|
|
52920
|
+
aiUserFetchBucketRows: writeResult.aiUserFetchBucketRows,
|
|
52921
|
+
aiReferralBucketRows: writeResult.aiReferralBucketRows,
|
|
52922
|
+
sampleRows: writeResult.sampleRows
|
|
52923
|
+
});
|
|
52924
|
+
});
|
|
51375
52925
|
app.post("/projects/:name/traffic/sources/:id/sync", async (request) => {
|
|
51376
52926
|
const project = resolveProject(app.db, request.params.name);
|
|
51377
|
-
const sourceRow2 = app.db.select().from(trafficSources).where(
|
|
52927
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, request.params.id)).get();
|
|
51378
52928
|
if (!sourceRow2 || sourceRow2.projectId !== project.id) {
|
|
51379
52929
|
throw notFound("Traffic source", request.params.id);
|
|
51380
52930
|
}
|
|
@@ -51386,7 +52936,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51386
52936
|
const windowEnd = /* @__PURE__ */ new Date();
|
|
51387
52937
|
const startedAt = windowEnd.toISOString();
|
|
51388
52938
|
const syncStartedAtMs = windowEnd.getTime();
|
|
51389
|
-
const runId =
|
|
52939
|
+
const runId = crypto40.randomUUID();
|
|
51390
52940
|
app.db.insert(runs).values({
|
|
51391
52941
|
id: runId,
|
|
51392
52942
|
projectId: project.id,
|
|
@@ -51400,8 +52950,8 @@ async function trafficRoutes(app, opts) {
|
|
|
51400
52950
|
const markFailed = (msg, errorCode) => {
|
|
51401
52951
|
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
51402
52952
|
app.db.transaction((tx) => {
|
|
51403
|
-
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(
|
|
51404
|
-
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(
|
|
52953
|
+
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(eq47(runs.id, runId)).run();
|
|
52954
|
+
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(eq47(trafficSources.id, sourceRow2.id)).run();
|
|
51405
52955
|
});
|
|
51406
52956
|
try {
|
|
51407
52957
|
opts.onTrafficSynced?.({
|
|
@@ -51482,7 +53032,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51482
53032
|
}
|
|
51483
53033
|
const credential = credentialStore.getConnection(project.name);
|
|
51484
53034
|
if (!credential) {
|
|
51485
|
-
app.db.delete(runs).where(
|
|
53035
|
+
app.db.delete(runs).where(eq47(runs.id, runId)).run();
|
|
51486
53036
|
throw validationError(
|
|
51487
53037
|
`No WordPress credential found for project "${project.name}". Run "canonry traffic connect wordpress" first.`
|
|
51488
53038
|
);
|
|
@@ -51531,12 +53081,12 @@ async function trafficRoutes(app, opts) {
|
|
|
51531
53081
|
auditAction = "traffic.vercel.synced";
|
|
51532
53082
|
const credentialStore = opts.vercelTrafficCredentialStore;
|
|
51533
53083
|
if (!credentialStore) {
|
|
51534
|
-
app.db.delete(runs).where(
|
|
53084
|
+
app.db.delete(runs).where(eq47(runs.id, runId)).run();
|
|
51535
53085
|
throw validationError("Vercel traffic credential storage is not configured for this deployment");
|
|
51536
53086
|
}
|
|
51537
53087
|
const credential = credentialStore.getConnection(project.name);
|
|
51538
53088
|
if (!credential) {
|
|
51539
|
-
app.db.delete(runs).where(
|
|
53089
|
+
app.db.delete(runs).where(eq47(runs.id, runId)).run();
|
|
51540
53090
|
throw validationError(
|
|
51541
53091
|
`No Vercel credential found for project "${project.name}". Run "canonry traffic connect vercel" first.`
|
|
51542
53092
|
);
|
|
@@ -51554,7 +53104,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51554
53104
|
if (cappedStartMs > clampedStartMs) {
|
|
51555
53105
|
const previousSkip = sourceRow2.skippedThroughAt ? Date.parse(sourceRow2.skippedThroughAt) : Number.NaN;
|
|
51556
53106
|
const skippedThrough = Number.isFinite(previousSkip) ? new Date(Math.max(previousSkip, cappedStartMs)) : new Date(cappedStartMs);
|
|
51557
|
-
app.db.update(trafficSources).set({ skippedThroughAt: skippedThrough.toISOString() }).where(
|
|
53107
|
+
app.db.update(trafficSources).set({ skippedThroughAt: skippedThrough.toISOString() }).where(eq47(trafficSources.id, sourceRow2.id)).run();
|
|
51558
53108
|
request.log.warn(
|
|
51559
53109
|
{
|
|
51560
53110
|
sourceId: sourceRow2.id,
|
|
@@ -51639,7 +53189,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51639
53189
|
let aiReferralHitsCount = 0;
|
|
51640
53190
|
let unknownHitsCount = 0;
|
|
51641
53191
|
app.db.transaction((tx) => {
|
|
51642
|
-
const latestRow = tx.select().from(trafficSources).where(
|
|
53192
|
+
const latestRow = tx.select().from(trafficSources).where(eq47(trafficSources.id, sourceRow2.id)).get();
|
|
51643
53193
|
const previousIds = latestRow.lastEventIds ?? [];
|
|
51644
53194
|
const seenEventIds = new Set(previousIds);
|
|
51645
53195
|
const dedupedEvents = seenEventIds.size === 0 ? allEvents : allEvents.filter((e) => !seenEventIds.has(e.eventId));
|
|
@@ -51687,7 +53237,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51687
53237
|
crawlerEventsHourly.status
|
|
51688
53238
|
],
|
|
51689
53239
|
set: {
|
|
51690
|
-
hits:
|
|
53240
|
+
hits: sql19`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
51691
53241
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
51692
53242
|
updatedAt: finishedAt
|
|
51693
53243
|
}
|
|
@@ -51720,7 +53270,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51720
53270
|
aiUserFetchEventsHourly.status
|
|
51721
53271
|
],
|
|
51722
53272
|
set: {
|
|
51723
|
-
hits:
|
|
53273
|
+
hits: sql19`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
51724
53274
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
51725
53275
|
updatedAt: finishedAt
|
|
51726
53276
|
}
|
|
@@ -51760,9 +53310,9 @@ async function trafficRoutes(app, opts) {
|
|
|
51760
53310
|
// colliding bucket adds its paid and organic arrivals to the ones
|
|
51761
53311
|
// already there rather than overwriting a single label.
|
|
51762
53312
|
set: {
|
|
51763
|
-
sessionsOrHits:
|
|
51764
|
-
paidSessionsOrHits:
|
|
51765
|
-
organicSessionsOrHits:
|
|
53313
|
+
sessionsOrHits: sql19`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
53314
|
+
paidSessionsOrHits: sql19`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
|
|
53315
|
+
organicSessionsOrHits: sql19`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
|
|
51766
53316
|
updatedAt: finishedAt
|
|
51767
53317
|
}
|
|
51768
53318
|
}).run();
|
|
@@ -51779,7 +53329,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51779
53329
|
}
|
|
51780
53330
|
})();
|
|
51781
53331
|
tx.insert(rawEventSamples).values({
|
|
51782
|
-
id:
|
|
53332
|
+
id: crypto40.randomUUID(),
|
|
51783
53333
|
projectId: project.id,
|
|
51784
53334
|
sourceId: sourceRow2.id,
|
|
51785
53335
|
ts: sample.observedAt,
|
|
@@ -51827,8 +53377,8 @@ async function trafficRoutes(app, opts) {
|
|
|
51827
53377
|
if (sourceRow2.sourceType === TrafficSourceTypes.wordpress) {
|
|
51828
53378
|
sourceUpdate.lastCursor = nextCursor ?? null;
|
|
51829
53379
|
}
|
|
51830
|
-
tx.update(trafficSources).set(sourceUpdate).where(
|
|
51831
|
-
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
53380
|
+
tx.update(trafficSources).set(sourceUpdate).where(eq47(trafficSources.id, sourceRow2.id)).run();
|
|
53381
|
+
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq47(runs.id, runId)).run();
|
|
51832
53382
|
});
|
|
51833
53383
|
writeAuditLog(app.db, {
|
|
51834
53384
|
projectId: project.id,
|
|
@@ -51880,7 +53430,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51880
53430
|
});
|
|
51881
53431
|
app.post("/projects/:name/traffic/sources/:id/backfill", async (request) => {
|
|
51882
53432
|
const project = resolveProject(app.db, request.params.name);
|
|
51883
|
-
const sourceRow2 = app.db.select().from(trafficSources).where(
|
|
53433
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, request.params.id)).get();
|
|
51884
53434
|
if (!sourceRow2 || sourceRow2.projectId !== project.id) {
|
|
51885
53435
|
throw notFound("Traffic source", request.params.id);
|
|
51886
53436
|
}
|
|
@@ -52030,7 +53580,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52030
53580
|
};
|
|
52031
53581
|
}
|
|
52032
53582
|
const startedAt = windowEnd.toISOString();
|
|
52033
|
-
const runId =
|
|
53583
|
+
const runId = crypto40.randomUUID();
|
|
52034
53584
|
app.db.insert(runs).values({
|
|
52035
53585
|
id: runId,
|
|
52036
53586
|
projectId: project.id,
|
|
@@ -52066,40 +53616,40 @@ async function trafficRoutes(app, opts) {
|
|
|
52066
53616
|
function buildSourceDetail(projectId, row, since) {
|
|
52067
53617
|
const crawlerPathRows = app.db.select({
|
|
52068
53618
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
52069
|
-
hits:
|
|
53619
|
+
hits: sql19`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
52070
53620
|
}).from(crawlerEventsHourly).where(
|
|
52071
|
-
|
|
52072
|
-
|
|
52073
|
-
|
|
53621
|
+
and39(
|
|
53622
|
+
eq47(crawlerEventsHourly.sourceId, row.id),
|
|
53623
|
+
gte11(crawlerEventsHourly.tsHour, since)
|
|
52074
53624
|
)
|
|
52075
53625
|
).groupBy(crawlerEventsHourly.pathNormalized).all();
|
|
52076
53626
|
const crawlerSegments = segmentCrawlerHits(
|
|
52077
53627
|
crawlerPathRows.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
|
|
52078
53628
|
);
|
|
52079
53629
|
const crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
52080
|
-
const aiUserFetchTotals = app.db.select({ total:
|
|
52081
|
-
|
|
52082
|
-
|
|
52083
|
-
|
|
53630
|
+
const aiUserFetchTotals = app.db.select({ total: sql19`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
53631
|
+
and39(
|
|
53632
|
+
eq47(aiUserFetchEventsHourly.sourceId, row.id),
|
|
53633
|
+
gte11(aiUserFetchEventsHourly.tsHour, since)
|
|
52084
53634
|
)
|
|
52085
53635
|
).get();
|
|
52086
|
-
const aiTotals = app.db.select({ total:
|
|
52087
|
-
|
|
52088
|
-
|
|
52089
|
-
|
|
53636
|
+
const aiTotals = app.db.select({ total: sql19`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
53637
|
+
and39(
|
|
53638
|
+
eq47(aiReferralEventsHourly.sourceId, row.id),
|
|
53639
|
+
gte11(aiReferralEventsHourly.tsHour, since)
|
|
52090
53640
|
)
|
|
52091
53641
|
).get();
|
|
52092
|
-
const sampleTotals = app.db.select({ total:
|
|
52093
|
-
|
|
52094
|
-
|
|
52095
|
-
|
|
53642
|
+
const sampleTotals = app.db.select({ total: sql19`COUNT(*)` }).from(rawEventSamples).where(
|
|
53643
|
+
and39(
|
|
53644
|
+
eq47(rawEventSamples.sourceId, row.id),
|
|
53645
|
+
gte11(rawEventSamples.ts, since)
|
|
52096
53646
|
)
|
|
52097
53647
|
).get();
|
|
52098
53648
|
const latestRun = app.db.select().from(runs).where(
|
|
52099
|
-
|
|
52100
|
-
|
|
52101
|
-
|
|
52102
|
-
|
|
53649
|
+
and39(
|
|
53650
|
+
eq47(runs.projectId, projectId),
|
|
53651
|
+
eq47(runs.kind, RunKinds["traffic-sync"]),
|
|
53652
|
+
eq47(runs.sourceId, row.id)
|
|
52103
53653
|
)
|
|
52104
53654
|
).orderBy(desc22(runs.startedAt)).limit(1).get();
|
|
52105
53655
|
return {
|
|
@@ -52130,7 +53680,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52130
53680
|
"`advanceToNow` must be `true`. There is no implicit reset."
|
|
52131
53681
|
);
|
|
52132
53682
|
}
|
|
52133
|
-
const sourceRow2 = app.db.select().from(trafficSources).where(
|
|
53683
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(and39(eq47(trafficSources.projectId, project.id), eq47(trafficSources.id, request.params.id))).get();
|
|
52134
53684
|
if (!sourceRow2) {
|
|
52135
53685
|
throw notFound("traffic source", request.params.id);
|
|
52136
53686
|
}
|
|
@@ -52147,7 +53697,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52147
53697
|
status: TrafficSourceStatuses.connected,
|
|
52148
53698
|
lastError: null,
|
|
52149
53699
|
updatedAt: now
|
|
52150
|
-
}).where(
|
|
53700
|
+
}).where(eq47(trafficSources.id, sourceRow2.id)).run();
|
|
52151
53701
|
writeAuditLog(tx, auditFromRequest(request, {
|
|
52152
53702
|
projectId: project.id,
|
|
52153
53703
|
actor: "api",
|
|
@@ -52155,20 +53705,20 @@ async function trafficRoutes(app, opts) {
|
|
|
52155
53705
|
entityType: "traffic_source",
|
|
52156
53706
|
entityId: sourceRow2.id
|
|
52157
53707
|
}));
|
|
52158
|
-
updatedRow = tx.select().from(trafficSources).where(
|
|
53708
|
+
updatedRow = tx.select().from(trafficSources).where(eq47(trafficSources.id, sourceRow2.id)).get();
|
|
52159
53709
|
});
|
|
52160
53710
|
return buildSourceDetail(project.id, updatedRow, new Date(Date.now() - 24 * 60 * 6e4).toISOString());
|
|
52161
53711
|
});
|
|
52162
53712
|
app.get("/projects/:name/traffic/sources", async (request) => {
|
|
52163
53713
|
const project = resolveProject(app.db, request.params.name);
|
|
52164
|
-
const rows = app.db.select().from(trafficSources).where(
|
|
53714
|
+
const rows = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).orderBy(desc22(trafficSources.createdAt)).all();
|
|
52165
53715
|
const sources = rows.filter((row) => row.status !== TrafficSourceStatuses.archived).map(rowToDto);
|
|
52166
53716
|
const response = { sources };
|
|
52167
53717
|
return response;
|
|
52168
53718
|
});
|
|
52169
53719
|
app.get("/projects/:name/traffic/status", async (request) => {
|
|
52170
53720
|
const project = resolveProject(app.db, request.params.name);
|
|
52171
|
-
const rows = app.db.select().from(trafficSources).where(
|
|
53721
|
+
const rows = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).orderBy(desc22(trafficSources.createdAt)).all();
|
|
52172
53722
|
const since = new Date(Date.now() - 24 * 60 * 6e4).toISOString();
|
|
52173
53723
|
const sources = rows.filter((row) => row.status !== TrafficSourceStatuses.archived).map((row) => buildSourceDetail(project.id, row, since));
|
|
52174
53724
|
const response = { sources };
|
|
@@ -52178,7 +53728,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52178
53728
|
"/projects/:name/traffic/sources/:id",
|
|
52179
53729
|
async (request) => {
|
|
52180
53730
|
const project = resolveProject(app.db, request.params.name);
|
|
52181
|
-
const row = app.db.select().from(trafficSources).where(
|
|
53731
|
+
const row = app.db.select().from(trafficSources).where(eq47(trafficSources.id, request.params.id)).get();
|
|
52182
53732
|
if (!row || row.projectId !== project.id) {
|
|
52183
53733
|
throw notFound("Traffic source", request.params.id);
|
|
52184
53734
|
}
|
|
@@ -52243,26 +53793,26 @@ async function trafficRoutes(app, opts) {
|
|
|
52243
53793
|
const seriesByBucket = /* @__PURE__ */ new Map();
|
|
52244
53794
|
if (kind === "all" || kind === TrafficEventKinds.crawler) {
|
|
52245
53795
|
const crawlerFilters = [
|
|
52246
|
-
|
|
52247
|
-
|
|
52248
|
-
|
|
53796
|
+
eq47(crawlerEventsHourly.projectId, project.id),
|
|
53797
|
+
gte11(crawlerEventsHourly.tsHour, sinceIso),
|
|
53798
|
+
lte10(crawlerEventsHourly.tsHour, untilIso)
|
|
52249
53799
|
];
|
|
52250
|
-
if (sourceIdParam) crawlerFilters.push(
|
|
52251
|
-
const crawlerWhere =
|
|
53800
|
+
if (sourceIdParam) crawlerFilters.push(eq47(crawlerEventsHourly.sourceId, sourceIdParam));
|
|
53801
|
+
const crawlerWhere = and39(...crawlerFilters);
|
|
52252
53802
|
const pathTotals = app.db.select({
|
|
52253
53803
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
52254
|
-
hits:
|
|
52255
|
-
rows:
|
|
53804
|
+
hits: sql19`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
53805
|
+
rows: sql19`COUNT(*)`
|
|
52256
53806
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerEventsHourly.pathNormalized).all();
|
|
52257
53807
|
crawlerSegments = segmentCrawlerHits(
|
|
52258
53808
|
pathTotals.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
|
|
52259
53809
|
);
|
|
52260
53810
|
crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
52261
53811
|
totalEventRows += pathTotals.reduce((sum, row) => sum + Number(row.rows), 0);
|
|
52262
|
-
const crawlerSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
53812
|
+
const crawlerSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql19`substr(${crawlerEventsHourly.tsHour}, 1, 10)` : crawlerEventsHourly.tsHour;
|
|
52263
53813
|
const crawlerSeries = app.db.select({
|
|
52264
53814
|
bucket: crawlerSeriesBucket,
|
|
52265
|
-
hits:
|
|
53815
|
+
hits: sql19`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
52266
53816
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerSeriesBucket).all();
|
|
52267
53817
|
for (const row of crawlerSeries) {
|
|
52268
53818
|
trafficSeriesPoint(seriesByBucket, row.bucket).crawlerHits = Number(row.hits);
|
|
@@ -52285,22 +53835,22 @@ async function trafficRoutes(app, opts) {
|
|
|
52285
53835
|
}
|
|
52286
53836
|
if (kind === "all" || kind === TrafficEventKinds["ai-user-fetch"]) {
|
|
52287
53837
|
const userFetchFilters = [
|
|
52288
|
-
|
|
52289
|
-
|
|
52290
|
-
|
|
53838
|
+
eq47(aiUserFetchEventsHourly.projectId, project.id),
|
|
53839
|
+
gte11(aiUserFetchEventsHourly.tsHour, sinceIso),
|
|
53840
|
+
lte10(aiUserFetchEventsHourly.tsHour, untilIso)
|
|
52291
53841
|
];
|
|
52292
|
-
if (sourceIdParam) userFetchFilters.push(
|
|
52293
|
-
const userFetchWhere =
|
|
53842
|
+
if (sourceIdParam) userFetchFilters.push(eq47(aiUserFetchEventsHourly.sourceId, sourceIdParam));
|
|
53843
|
+
const userFetchWhere = and39(...userFetchFilters);
|
|
52294
53844
|
const total = app.db.select({
|
|
52295
|
-
total:
|
|
52296
|
-
rows:
|
|
53845
|
+
total: sql19`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`,
|
|
53846
|
+
rows: sql19`COUNT(*)`
|
|
52297
53847
|
}).from(aiUserFetchEventsHourly).where(userFetchWhere).get();
|
|
52298
53848
|
aiUserFetchTotal = Number(total?.total ?? 0);
|
|
52299
53849
|
totalEventRows += Number(total?.rows ?? 0);
|
|
52300
|
-
const userFetchSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
53850
|
+
const userFetchSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql19`substr(${aiUserFetchEventsHourly.tsHour}, 1, 10)` : aiUserFetchEventsHourly.tsHour;
|
|
52301
53851
|
const userFetchSeries = app.db.select({
|
|
52302
53852
|
bucket: userFetchSeriesBucket,
|
|
52303
|
-
hits:
|
|
53853
|
+
hits: sql19`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`
|
|
52304
53854
|
}).from(aiUserFetchEventsHourly).where(userFetchWhere).groupBy(userFetchSeriesBucket).all();
|
|
52305
53855
|
for (const row of userFetchSeries) {
|
|
52306
53856
|
trafficSeriesPoint(seriesByBucket, row.bucket).aiUserFetchHits = Number(row.hits);
|
|
@@ -52322,17 +53872,17 @@ async function trafficRoutes(app, opts) {
|
|
|
52322
53872
|
}
|
|
52323
53873
|
if (kind === "all" || kind === TrafficEventKinds["ai-referral"]) {
|
|
52324
53874
|
const aiFilters = [
|
|
52325
|
-
|
|
52326
|
-
|
|
52327
|
-
|
|
53875
|
+
eq47(aiReferralEventsHourly.projectId, project.id),
|
|
53876
|
+
gte11(aiReferralEventsHourly.tsHour, sinceIso),
|
|
53877
|
+
lte10(aiReferralEventsHourly.tsHour, untilIso)
|
|
52328
53878
|
];
|
|
52329
|
-
if (sourceIdParam) aiFilters.push(
|
|
52330
|
-
const aiWhere =
|
|
53879
|
+
if (sourceIdParam) aiFilters.push(eq47(aiReferralEventsHourly.sourceId, sourceIdParam));
|
|
53880
|
+
const aiWhere = and39(...aiFilters);
|
|
52331
53881
|
const total = app.db.select({
|
|
52332
|
-
total:
|
|
52333
|
-
paid:
|
|
52334
|
-
organic:
|
|
52335
|
-
rows:
|
|
53882
|
+
total: sql19`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
53883
|
+
paid: sql19`COALESCE(SUM(${aiReferralEventsHourly.paidSessionsOrHits}), 0)`,
|
|
53884
|
+
organic: sql19`COALESCE(SUM(${aiReferralEventsHourly.organicSessionsOrHits}), 0)`,
|
|
53885
|
+
rows: sql19`COUNT(*)`
|
|
52336
53886
|
}).from(aiReferralEventsHourly).where(aiWhere).get();
|
|
52337
53887
|
aiReferralCounts = aiReferralClassCounts(
|
|
52338
53888
|
Number(total?.total ?? 0),
|
|
@@ -52340,10 +53890,10 @@ async function trafficRoutes(app, opts) {
|
|
|
52340
53890
|
Number(total?.organic ?? 0)
|
|
52341
53891
|
);
|
|
52342
53892
|
totalEventRows += Number(total?.rows ?? 0);
|
|
52343
|
-
const referralSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
53893
|
+
const referralSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql19`substr(${aiReferralEventsHourly.tsHour}, 1, 10)` : aiReferralEventsHourly.tsHour;
|
|
52344
53894
|
const referralSeries = app.db.select({
|
|
52345
53895
|
bucket: referralSeriesBucket,
|
|
52346
|
-
hits:
|
|
53896
|
+
hits: sql19`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`
|
|
52347
53897
|
}).from(aiReferralEventsHourly).where(aiWhere).groupBy(referralSeriesBucket).all();
|
|
52348
53898
|
for (const row of referralSeries) {
|
|
52349
53899
|
trafficSeriesPoint(seriesByBucket, row.bucket).aiReferralHits = Number(row.hits);
|
|
@@ -52400,7 +53950,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52400
53950
|
}
|
|
52401
53951
|
|
|
52402
53952
|
// ../api-routes/src/doctor/checks/agent.ts
|
|
52403
|
-
import
|
|
53953
|
+
import crypto41 from "crypto";
|
|
52404
53954
|
import fs6 from "fs";
|
|
52405
53955
|
import path7 from "path";
|
|
52406
53956
|
var REQUIRED_SKILLS = ["canonry", "aero"];
|
|
@@ -52631,7 +54181,7 @@ function isInstalled(dir) {
|
|
|
52631
54181
|
}
|
|
52632
54182
|
function hashInstalledFile(filePath) {
|
|
52633
54183
|
try {
|
|
52634
|
-
return
|
|
54184
|
+
return crypto41.createHash("sha256").update(fs6.readFileSync(filePath)).digest("hex");
|
|
52635
54185
|
} catch {
|
|
52636
54186
|
return void 0;
|
|
52637
54187
|
}
|
|
@@ -52646,7 +54196,7 @@ function readInstalledManifest(skillDir) {
|
|
|
52646
54196
|
var AGENT_CHECKS = [skillsInstalledCheck, skillsCurrentCheck];
|
|
52647
54197
|
|
|
52648
54198
|
// ../api-routes/src/doctor/checks/backlinks.ts
|
|
52649
|
-
import { and as
|
|
54199
|
+
import { and as and40, eq as eq48 } from "drizzle-orm";
|
|
52650
54200
|
function skippedNoProject() {
|
|
52651
54201
|
return {
|
|
52652
54202
|
status: CheckStatuses.skipped,
|
|
@@ -52662,8 +54212,8 @@ var BACKLINKS_CHECKS = [
|
|
|
52662
54212
|
title: "Backlinks source connected",
|
|
52663
54213
|
run: (ctx) => {
|
|
52664
54214
|
if (!ctx.project) return skippedNoProject();
|
|
52665
|
-
const projectRow = ctx.db.select({ autoExtract: projects.autoExtractBacklinks }).from(projects).where(
|
|
52666
|
-
const readySync = ctx.db.select({ id: ccReleaseSyncs.id }).from(ccReleaseSyncs).where(
|
|
54215
|
+
const projectRow = ctx.db.select({ autoExtract: projects.autoExtractBacklinks }).from(projects).where(eq48(projects.id, ctx.project.id)).get();
|
|
54216
|
+
const readySync = ctx.db.select({ id: ccReleaseSyncs.id }).from(ccReleaseSyncs).where(eq48(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).limit(1).get();
|
|
52667
54217
|
const ccConnected = projectRow?.autoExtract === true && !!readySync;
|
|
52668
54218
|
const connected = [];
|
|
52669
54219
|
if (ccConnected) connected.push(BacklinkSources.commoncrawl);
|
|
@@ -52676,9 +54226,9 @@ var BACKLINKS_CHECKS = [
|
|
|
52676
54226
|
details: { commoncrawl: ccConnected }
|
|
52677
54227
|
};
|
|
52678
54228
|
}
|
|
52679
|
-
const ccHasData = ccConnected ? !!ctx.db.select({ id: backlinkSummaries.id }).from(backlinkSummaries).where(
|
|
52680
|
-
|
|
52681
|
-
|
|
54229
|
+
const ccHasData = ccConnected ? !!ctx.db.select({ id: backlinkSummaries.id }).from(backlinkSummaries).where(and40(
|
|
54230
|
+
eq48(backlinkSummaries.projectId, ctx.project.id),
|
|
54231
|
+
eq48(backlinkSummaries.source, BacklinkSources.commoncrawl)
|
|
52682
54232
|
)).limit(1).get() : false;
|
|
52683
54233
|
return {
|
|
52684
54234
|
status: CheckStatuses.ok,
|
|
@@ -52838,7 +54388,7 @@ var BING_AUTH_CHECKS = [
|
|
|
52838
54388
|
];
|
|
52839
54389
|
|
|
52840
54390
|
// ../api-routes/src/doctor/checks/content.ts
|
|
52841
|
-
import { eq as
|
|
54391
|
+
import { eq as eq49 } from "drizzle-orm";
|
|
52842
54392
|
var WINNABILITY_COVERAGE_WARN_THRESHOLD = 0.8;
|
|
52843
54393
|
var UNCLASSIFIED_DOMAIN_SAMPLE_LIMIT = 10;
|
|
52844
54394
|
function skippedNoProject2() {
|
|
@@ -52851,7 +54401,7 @@ function skippedNoProject2() {
|
|
|
52851
54401
|
}
|
|
52852
54402
|
function loadProject(ctx) {
|
|
52853
54403
|
if (!ctx.project) return null;
|
|
52854
|
-
return ctx.db.select().from(projects).where(
|
|
54404
|
+
return ctx.db.select().from(projects).where(eq49(projects.id, ctx.project.id)).get() ?? null;
|
|
52855
54405
|
}
|
|
52856
54406
|
function percent(value) {
|
|
52857
54407
|
return Math.round(value * 100);
|
|
@@ -52943,7 +54493,7 @@ var CONTENT_CHECK_BY_ID = Object.fromEntries(
|
|
|
52943
54493
|
);
|
|
52944
54494
|
|
|
52945
54495
|
// ../api-routes/src/doctor/checks/ads.ts
|
|
52946
|
-
import { eq as
|
|
54496
|
+
import { eq as eq50 } from "drizzle-orm";
|
|
52947
54497
|
var RECENT_SYNC_WARN_DAYS = 7;
|
|
52948
54498
|
var RECENT_SYNC_FAIL_DAYS = 30;
|
|
52949
54499
|
var adsConnectionCheck = {
|
|
@@ -52960,7 +54510,7 @@ var adsConnectionCheck = {
|
|
|
52960
54510
|
remediation: null
|
|
52961
54511
|
};
|
|
52962
54512
|
}
|
|
52963
|
-
const row = ctx.db.select().from(adsConnections).where(
|
|
54513
|
+
const row = ctx.db.select().from(adsConnections).where(eq50(adsConnections.projectId, ctx.project.id)).get();
|
|
52964
54514
|
if (!row) {
|
|
52965
54515
|
return {
|
|
52966
54516
|
status: CheckStatuses.skipped,
|
|
@@ -53010,7 +54560,7 @@ var adsRecentSyncCheck = {
|
|
|
53010
54560
|
remediation: null
|
|
53011
54561
|
};
|
|
53012
54562
|
}
|
|
53013
|
-
const row = ctx.db.select().from(adsConnections).where(
|
|
54563
|
+
const row = ctx.db.select().from(adsConnections).where(eq50(adsConnections.projectId, ctx.project.id)).get();
|
|
53014
54564
|
if (!row) {
|
|
53015
54565
|
return {
|
|
53016
54566
|
status: CheckStatuses.skipped,
|
|
@@ -53201,7 +54751,7 @@ var ga4ConnectionCheck = {
|
|
|
53201
54751
|
var GA_AUTH_CHECKS = [ga4ConnectionCheck];
|
|
53202
54752
|
|
|
53203
54753
|
// ../api-routes/src/doctor/checks/gbp-auth.ts
|
|
53204
|
-
import { and as
|
|
54754
|
+
import { and as and41, eq as eq51 } from "drizzle-orm";
|
|
53205
54755
|
var RECENT_SYNC_WARN_DAYS2 = 7;
|
|
53206
54756
|
var RECENT_SYNC_FAIL_DAYS2 = 30;
|
|
53207
54757
|
function skippedNoProject3() {
|
|
@@ -53434,7 +54984,7 @@ var recentSyncCheck = {
|
|
|
53434
54984
|
title: "GBP recent sync",
|
|
53435
54985
|
run: (ctx) => {
|
|
53436
54986
|
if (!ctx.project) return skippedNoProject3();
|
|
53437
|
-
const selected = ctx.db.select({ locationName: gbpLocations.locationName, syncedAt: gbpLocations.syncedAt }).from(gbpLocations).where(
|
|
54987
|
+
const selected = ctx.db.select({ locationName: gbpLocations.locationName, syncedAt: gbpLocations.syncedAt }).from(gbpLocations).where(and41(eq51(gbpLocations.projectId, ctx.project.id), eq51(gbpLocations.selected, true))).all();
|
|
53438
54988
|
if (selected.length === 0) {
|
|
53439
54989
|
return {
|
|
53440
54990
|
status: CheckStatuses.skipped,
|
|
@@ -53494,7 +55044,7 @@ var GBP_AUTH_CHECK_BY_ID = Object.fromEntries(
|
|
|
53494
55044
|
);
|
|
53495
55045
|
|
|
53496
55046
|
// ../api-routes/src/doctor/checks/places.ts
|
|
53497
|
-
import { eq as
|
|
55047
|
+
import { eq as eq52 } from "drizzle-orm";
|
|
53498
55048
|
var apiKeyCheck = {
|
|
53499
55049
|
id: "gbp.places.api-key",
|
|
53500
55050
|
category: CheckCategories.auth,
|
|
@@ -53539,7 +55089,7 @@ var apiKeyCheck = {
|
|
|
53539
55089
|
details: { tier: cfg.tier }
|
|
53540
55090
|
};
|
|
53541
55091
|
}
|
|
53542
|
-
const rows = ctx.db.select({ placeId: gbpLocations.placeId, selected: gbpLocations.selected }).from(gbpLocations).where(
|
|
55092
|
+
const rows = ctx.db.select({ placeId: gbpLocations.placeId, selected: gbpLocations.selected }).from(gbpLocations).where(eq52(gbpLocations.projectId, ctx.project.id)).all();
|
|
53543
55093
|
const selected = rows.filter((r) => r.selected);
|
|
53544
55094
|
const locationsWithPlaceId = selected.filter((r) => Boolean(r.placeId)).length;
|
|
53545
55095
|
const details = {
|
|
@@ -54054,9 +55604,25 @@ var RUNTIME_STATE_CHECKS = [
|
|
|
54054
55604
|
];
|
|
54055
55605
|
|
|
54056
55606
|
// ../api-routes/src/doctor/checks/traffic-source.ts
|
|
54057
|
-
import { and as
|
|
55607
|
+
import { and as and42, eq as eq53, gte as gte12, ne as ne6, sql as sql20 } from "drizzle-orm";
|
|
54058
55608
|
var RECENT_DATA_WARN_DAYS = 7;
|
|
54059
55609
|
var RECENT_DATA_FAIL_DAYS = 30;
|
|
55610
|
+
function isCloudflareDirectPush(source) {
|
|
55611
|
+
if (source.sourceType !== TrafficSourceTypes.cloudflare) return false;
|
|
55612
|
+
const deliveryMode = source.configJson.deliveryMode;
|
|
55613
|
+
return deliveryMode === void 0 || deliveryMode === CloudflareTrafficDeliveryModes["direct-push"];
|
|
55614
|
+
}
|
|
55615
|
+
function recentDataRemediation(sources, lastSyncedAt) {
|
|
55616
|
+
const hasCloudflare = sources.some(isCloudflareDirectPush);
|
|
55617
|
+
const hasPullSource = sources.some((source) => !isCloudflareDirectPush(source));
|
|
55618
|
+
if (hasCloudflare && !hasPullSource) {
|
|
55619
|
+
return "Confirm the Cloudflare Worker is deployed on the zone route and forwarding to canonry; reconnect Cloudflare to regenerate the Worker if needed.";
|
|
55620
|
+
}
|
|
55621
|
+
if (hasCloudflare && hasPullSource) {
|
|
55622
|
+
return "Run `canonry traffic sync <project>` for pull-based sources, and confirm the Cloudflare Worker is deployed on the zone route and forwarding to canonry.";
|
|
55623
|
+
}
|
|
55624
|
+
return lastSyncedAt ? `Last sync: ${lastSyncedAt}. Run \`canonry traffic sync <project>\` to refresh, or check the source connection.` : "Run `canonry traffic sync <project>` to pull recent events.";
|
|
55625
|
+
}
|
|
54060
55626
|
function skippedNoProject5() {
|
|
54061
55627
|
return {
|
|
54062
55628
|
status: CheckStatuses.skipped,
|
|
@@ -54068,8 +55634,8 @@ function skippedNoProject5() {
|
|
|
54068
55634
|
function loadProbes(ctx) {
|
|
54069
55635
|
if (!ctx.project) return [];
|
|
54070
55636
|
const rows = ctx.db.select().from(trafficSources).where(
|
|
54071
|
-
|
|
54072
|
-
|
|
55637
|
+
and42(
|
|
55638
|
+
eq53(trafficSources.projectId, ctx.project.id),
|
|
54073
55639
|
ne6(trafficSources.status, TrafficSourceStatuses.archived)
|
|
54074
55640
|
)
|
|
54075
55641
|
).all();
|
|
@@ -54081,6 +55647,8 @@ function loadProbes(ctx) {
|
|
|
54081
55647
|
displayName: r.displayName,
|
|
54082
55648
|
status: r.status,
|
|
54083
55649
|
lastSyncedAt: r.lastSyncedAt,
|
|
55650
|
+
lastWorkerVersion: r.lastWorkerVersion,
|
|
55651
|
+
ingestTokenHash: r.ingestTokenHash,
|
|
54084
55652
|
skippedThroughAt: r.skippedThroughAt,
|
|
54085
55653
|
lastError: r.lastError,
|
|
54086
55654
|
configJson: r.configJson
|
|
@@ -54149,52 +55717,74 @@ var recentDataCheck = {
|
|
|
54149
55717
|
const warnCutoff = new Date(now.getTime() - RECENT_DATA_WARN_DAYS * 24 * 60 * 6e4).toISOString();
|
|
54150
55718
|
const failCutoff = new Date(now.getTime() - RECENT_DATA_FAIL_DAYS * 24 * 60 * 6e4).toISOString();
|
|
54151
55719
|
const recentCrawlers = Number(
|
|
54152
|
-
ctx.db.select({ total:
|
|
54153
|
-
|
|
54154
|
-
|
|
54155
|
-
|
|
55720
|
+
ctx.db.select({ total: sql20`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
55721
|
+
and42(
|
|
55722
|
+
eq53(crawlerEventsHourly.projectId, ctx.project.id),
|
|
55723
|
+
gte12(crawlerEventsHourly.tsHour, warnCutoff)
|
|
54156
55724
|
)
|
|
54157
55725
|
).get()?.total ?? 0
|
|
54158
55726
|
);
|
|
54159
55727
|
const recentReferrals = Number(
|
|
54160
|
-
ctx.db.select({ total:
|
|
54161
|
-
|
|
54162
|
-
|
|
54163
|
-
|
|
55728
|
+
ctx.db.select({ total: sql20`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
55729
|
+
and42(
|
|
55730
|
+
eq53(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
55731
|
+
gte12(aiReferralEventsHourly.tsHour, warnCutoff)
|
|
54164
55732
|
)
|
|
54165
55733
|
).get()?.total ?? 0
|
|
54166
55734
|
);
|
|
54167
|
-
|
|
55735
|
+
const recentUserFetches = Number(
|
|
55736
|
+
ctx.db.select({ total: sql20`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
55737
|
+
and42(
|
|
55738
|
+
eq53(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
55739
|
+
gte12(aiUserFetchEventsHourly.tsHour, warnCutoff)
|
|
55740
|
+
)
|
|
55741
|
+
).get()?.total ?? 0
|
|
55742
|
+
);
|
|
55743
|
+
if (recentCrawlers > 0 || recentUserFetches > 0 || recentReferrals > 0) {
|
|
54168
55744
|
return {
|
|
54169
55745
|
status: CheckStatuses.ok,
|
|
54170
55746
|
code: "traffic.recent-data.fresh",
|
|
54171
|
-
summary: `${recentCrawlers} crawler hit(s) and ${recentReferrals} AI-referral arrival(s) in the last ${RECENT_DATA_WARN_DAYS} days.`,
|
|
54172
|
-
details: {
|
|
55747
|
+
summary: `${recentCrawlers} crawler hit(s), ${recentUserFetches} AI user-fetch hit(s), and ${recentReferrals} AI-referral arrival(s) in the last ${RECENT_DATA_WARN_DAYS} days.`,
|
|
55748
|
+
details: {
|
|
55749
|
+
crawlerHits: recentCrawlers,
|
|
55750
|
+
aiUserFetchHits: recentUserFetches,
|
|
55751
|
+
referralArrivals: recentReferrals,
|
|
55752
|
+
windowDays: RECENT_DATA_WARN_DAYS
|
|
55753
|
+
}
|
|
54173
55754
|
};
|
|
54174
55755
|
}
|
|
54175
55756
|
const olderCrawlers = Number(
|
|
54176
|
-
ctx.db.select({ total:
|
|
54177
|
-
|
|
54178
|
-
|
|
54179
|
-
|
|
55757
|
+
ctx.db.select({ total: sql20`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
55758
|
+
and42(
|
|
55759
|
+
eq53(crawlerEventsHourly.projectId, ctx.project.id),
|
|
55760
|
+
gte12(crawlerEventsHourly.tsHour, failCutoff)
|
|
54180
55761
|
)
|
|
54181
55762
|
).get()?.total ?? 0
|
|
54182
55763
|
);
|
|
54183
55764
|
const olderReferrals = Number(
|
|
54184
|
-
ctx.db.select({ total:
|
|
54185
|
-
|
|
54186
|
-
|
|
54187
|
-
|
|
55765
|
+
ctx.db.select({ total: sql20`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
55766
|
+
and42(
|
|
55767
|
+
eq53(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
55768
|
+
gte12(aiReferralEventsHourly.tsHour, failCutoff)
|
|
55769
|
+
)
|
|
55770
|
+
).get()?.total ?? 0
|
|
55771
|
+
);
|
|
55772
|
+
const olderUserFetches = Number(
|
|
55773
|
+
ctx.db.select({ total: sql20`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
55774
|
+
and42(
|
|
55775
|
+
eq53(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
55776
|
+
gte12(aiUserFetchEventsHourly.tsHour, failCutoff)
|
|
54188
55777
|
)
|
|
54189
55778
|
).get()?.total ?? 0
|
|
54190
55779
|
);
|
|
54191
55780
|
const lastSyncedAt = sources.map((s) => s.lastSyncedAt).filter(Boolean).sort().at(-1) ?? null;
|
|
54192
|
-
|
|
55781
|
+
const hasOlderData = olderCrawlers > 0 || olderUserFetches > 0 || olderReferrals > 0;
|
|
55782
|
+
if (hasOlderData || lastSyncedAt) {
|
|
54193
55783
|
return {
|
|
54194
55784
|
status: CheckStatuses.warn,
|
|
54195
55785
|
code: "traffic.recent-data.stale",
|
|
54196
|
-
summary: `No crawler
|
|
54197
|
-
remediation:
|
|
55786
|
+
summary: hasOlderData ? `No crawler, AI user-fetch, or AI-referral hits in the last ${RECENT_DATA_WARN_DAYS} days, though older data exists.` : `No crawler, AI user-fetch, or AI-referral hits in the last ${RECENT_DATA_WARN_DAYS} days.`,
|
|
55787
|
+
remediation: recentDataRemediation(sources, lastSyncedAt),
|
|
54198
55788
|
details: { lastSyncedAt, sourceCount: sources.length }
|
|
54199
55789
|
};
|
|
54200
55790
|
}
|
|
@@ -54392,7 +55982,15 @@ var syncLagCheck = {
|
|
|
54392
55982
|
title: "Traffic source sync lag",
|
|
54393
55983
|
run: (ctx) => {
|
|
54394
55984
|
if (!ctx.project) return skippedNoProject5();
|
|
54395
|
-
const
|
|
55985
|
+
const allSources = loadProbes(ctx);
|
|
55986
|
+
const sources = allSources.filter((source) => !isCloudflareDirectPush(source));
|
|
55987
|
+
if (allSources.length > 0 && sources.length === 0) {
|
|
55988
|
+
return {
|
|
55989
|
+
status: CheckStatuses.skipped,
|
|
55990
|
+
code: "traffic.sync-lag.push-only",
|
|
55991
|
+
summary: "Only push traffic sources are connected \u2014 pull sync lag does not apply."
|
|
55992
|
+
};
|
|
55993
|
+
}
|
|
54396
55994
|
if (sources.length === 0) {
|
|
54397
55995
|
return {
|
|
54398
55996
|
status: CheckStatuses.skipped,
|
|
@@ -54482,10 +56080,68 @@ var syncLagCheck = {
|
|
|
54482
56080
|
};
|
|
54483
56081
|
}
|
|
54484
56082
|
};
|
|
56083
|
+
var workerVersionCheck = {
|
|
56084
|
+
id: "traffic.source.worker-version",
|
|
56085
|
+
category: CheckCategories.integrations,
|
|
56086
|
+
scope: CheckScopes.project,
|
|
56087
|
+
title: "Cloudflare Worker version",
|
|
56088
|
+
run: (ctx) => {
|
|
56089
|
+
if (!ctx.project) return skippedNoProject5();
|
|
56090
|
+
const sources = loadProbes(ctx).filter(isCloudflareDirectPush);
|
|
56091
|
+
if (sources.length === 0) {
|
|
56092
|
+
return {
|
|
56093
|
+
status: CheckStatuses.skipped,
|
|
56094
|
+
code: "traffic.worker-version.not-applicable",
|
|
56095
|
+
summary: "No Cloudflare direct-push source is connected."
|
|
56096
|
+
};
|
|
56097
|
+
}
|
|
56098
|
+
const measured = sources.map((source) => {
|
|
56099
|
+
const expected = source.configJson.workerVersion;
|
|
56100
|
+
const expectedVersion = typeof expected === "string" && expected.length > 0 ? expected : null;
|
|
56101
|
+
const observedVersion = source.lastWorkerVersion;
|
|
56102
|
+
const state = observedVersion === null ? "waiting-for-first-event" : expectedVersion !== null && observedVersion === expectedVersion ? "current" : "stale";
|
|
56103
|
+
return {
|
|
56104
|
+
sourceId: source.id,
|
|
56105
|
+
displayName: source.displayName,
|
|
56106
|
+
expectedVersion,
|
|
56107
|
+
observedVersion,
|
|
56108
|
+
state
|
|
56109
|
+
};
|
|
56110
|
+
});
|
|
56111
|
+
const details = { sources: measured };
|
|
56112
|
+
const stale = measured.filter((source) => source.state === "stale");
|
|
56113
|
+
if (stale.length > 0) {
|
|
56114
|
+
return {
|
|
56115
|
+
status: CheckStatuses.warn,
|
|
56116
|
+
code: "traffic.worker-version.stale",
|
|
56117
|
+
summary: `${stale.length} Cloudflare Worker source(s) report a version that differs from the generated version.`,
|
|
56118
|
+
remediation: "Redeploy the generated Worker from the credential-owning host. Use `canonry traffic connect cloudflare <project> --zone-id <id> --deploy --confirm-route --confirm-fail-open`, then attach the route manually.",
|
|
56119
|
+
details
|
|
56120
|
+
};
|
|
56121
|
+
}
|
|
56122
|
+
const waiting = measured.filter((source) => source.state === "waiting-for-first-event");
|
|
56123
|
+
if (waiting.length > 0) {
|
|
56124
|
+
return {
|
|
56125
|
+
status: CheckStatuses.warn,
|
|
56126
|
+
code: "traffic.worker-version.waiting-for-first-event",
|
|
56127
|
+
summary: `${waiting.length} Cloudflare Worker source(s) have not delivered an event yet.`,
|
|
56128
|
+
remediation: "Send a smoke-test request through the Worker. Then run `canonry doctor --project <project>` again.",
|
|
56129
|
+
details
|
|
56130
|
+
};
|
|
56131
|
+
}
|
|
56132
|
+
return {
|
|
56133
|
+
status: CheckStatuses.ok,
|
|
56134
|
+
code: "traffic.worker-version.current",
|
|
56135
|
+
summary: "All Cloudflare direct-push Workers report the generated version.",
|
|
56136
|
+
details
|
|
56137
|
+
};
|
|
56138
|
+
}
|
|
56139
|
+
};
|
|
54485
56140
|
var TRAFFIC_SOURCE_CHECKS = [
|
|
54486
56141
|
sourceConnectedCheck,
|
|
54487
56142
|
recentDataCheck,
|
|
54488
56143
|
syncLagCheck,
|
|
56144
|
+
workerVersionCheck,
|
|
54489
56145
|
credentialsCheck,
|
|
54490
56146
|
scopesCheck3,
|
|
54491
56147
|
cacheBlindSpotCheck
|
|
@@ -54704,8 +56360,8 @@ async function doctorRoutes(app, opts) {
|
|
|
54704
56360
|
}
|
|
54705
56361
|
|
|
54706
56362
|
// ../api-routes/src/discovery/routes.ts
|
|
54707
|
-
import
|
|
54708
|
-
import { and as
|
|
56363
|
+
import crypto42 from "crypto";
|
|
56364
|
+
import { and as and43, desc as desc23, eq as eq54, gte as gte13, inArray as inArray18, isNull as isNull4, or as or10 } from "drizzle-orm";
|
|
54709
56365
|
var MAX_INFLIGHT_DISCOVERY_AGE_MS = 2 * 60 * 60 * 1e3;
|
|
54710
56366
|
async function discoveryRoutes(app, opts) {
|
|
54711
56367
|
app.post("/projects/:name/discover/run", async (request, reply) => {
|
|
@@ -54739,13 +56395,13 @@ async function discoveryRoutes(app, opts) {
|
|
|
54739
56395
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
54740
56396
|
const ageFloorIso = new Date(Date.now() - MAX_INFLIGHT_DISCOVERY_AGE_MS).toISOString();
|
|
54741
56397
|
const decision = app.db.transaction((tx) => {
|
|
54742
|
-
const existing = tx.select({ id: discoverySessions.id, runId: discoverySessions.runId }).from(discoverySessions).where(
|
|
54743
|
-
|
|
54744
|
-
|
|
56398
|
+
const existing = tx.select({ id: discoverySessions.id, runId: discoverySessions.runId }).from(discoverySessions).where(and43(
|
|
56399
|
+
eq54(discoverySessions.projectId, project.id),
|
|
56400
|
+
eq54(discoverySessions.icpDescription, icpDescription),
|
|
54745
56401
|
// Buyer is part of session identity: it changes the seed prompt's
|
|
54746
56402
|
// semantics, so a request with a different (or no) buyer must start
|
|
54747
56403
|
// its own session, never adopt another buyer's probes.
|
|
54748
|
-
parsed.data.buyerDescription == null ? isNull4(discoverySessions.buyerDescription) :
|
|
56404
|
+
parsed.data.buyerDescription == null ? isNull4(discoverySessions.buyerDescription) : eq54(discoverySessions.buyerDescription, parsed.data.buyerDescription),
|
|
54749
56405
|
// Locations are identity too: a different service-area subset seeds
|
|
54750
56406
|
// and probes a different geo, so it must never reuse another geo's
|
|
54751
56407
|
// session. resolveLocations is deterministic (project-config order),
|
|
@@ -54755,23 +56411,23 @@ async function discoveryRoutes(app, opts) {
|
|
|
54755
56411
|
// locations a NULL row's subset is unknowable, so it conservatively
|
|
54756
56412
|
// never reuses — a one-time, bounded (2h window) non-reuse after
|
|
54757
56413
|
// upgrade, never a wrong reuse.
|
|
54758
|
-
locations.length === 0 ? or10(isNull4(discoverySessions.locations),
|
|
56414
|
+
locations.length === 0 ? or10(isNull4(discoverySessions.locations), eq54(discoverySessions.locations, locations)) : eq54(discoverySessions.locations, locations),
|
|
54759
56415
|
// Seed provider set is identity: a different phrasing distribution
|
|
54760
56416
|
// must never reuse another set's session. Null (= Gemini-only
|
|
54761
56417
|
// default, including explicit ['gemini']) matches legacy rows.
|
|
54762
|
-
seedProviders == null ? isNull4(discoverySessions.seedProviders) :
|
|
56418
|
+
seedProviders == null ? isNull4(discoverySessions.seedProviders) : eq54(discoverySessions.seedProviders, seedProviders),
|
|
54763
56419
|
inArray18(discoverySessions.status, [
|
|
54764
56420
|
DiscoverySessionStatuses.queued,
|
|
54765
56421
|
DiscoverySessionStatuses.seeding,
|
|
54766
56422
|
DiscoverySessionStatuses.probing
|
|
54767
56423
|
]),
|
|
54768
|
-
|
|
56424
|
+
gte13(discoverySessions.createdAt, ageFloorIso)
|
|
54769
56425
|
)).orderBy(desc23(discoverySessions.createdAt)).get();
|
|
54770
56426
|
if (existing && existing.runId) {
|
|
54771
56427
|
return { reused: true, sessionId: existing.id, runId: existing.runId };
|
|
54772
56428
|
}
|
|
54773
|
-
const sessionId =
|
|
54774
|
-
const runId =
|
|
56429
|
+
const sessionId = crypto42.randomUUID();
|
|
56430
|
+
const runId = crypto42.randomUUID();
|
|
54775
56431
|
tx.insert(discoverySessions).values({
|
|
54776
56432
|
id: sessionId,
|
|
54777
56433
|
projectId: project.id,
|
|
@@ -54835,7 +56491,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
54835
56491
|
const project = resolveProject(app.db, request.params.name);
|
|
54836
56492
|
const parsedLimit = parseInt(request.query.limit ?? "", 10);
|
|
54837
56493
|
const limit = Number.isNaN(parsedLimit) || parsedLimit <= 0 ? 50 : parsedLimit;
|
|
54838
|
-
const rows = app.db.select().from(discoverySessions).where(
|
|
56494
|
+
const rows = app.db.select().from(discoverySessions).where(eq54(discoverySessions.projectId, project.id)).orderBy(desc23(discoverySessions.createdAt)).limit(limit).all();
|
|
54839
56495
|
return reply.send(rows.map(serializeSession));
|
|
54840
56496
|
}
|
|
54841
56497
|
);
|
|
@@ -54843,11 +56499,11 @@ async function discoveryRoutes(app, opts) {
|
|
|
54843
56499
|
"/projects/:name/discover/sessions/:id",
|
|
54844
56500
|
async (request, reply) => {
|
|
54845
56501
|
const project = resolveProject(app.db, request.params.name);
|
|
54846
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
56502
|
+
const session = app.db.select().from(discoverySessions).where(eq54(discoverySessions.id, request.params.id)).get();
|
|
54847
56503
|
if (!session || session.projectId !== project.id) {
|
|
54848
56504
|
throw notFound("Discovery session", request.params.id);
|
|
54849
56505
|
}
|
|
54850
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
56506
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq54(discoveryProbes.sessionId, session.id)).all();
|
|
54851
56507
|
const detail = {
|
|
54852
56508
|
...serializeSession(session),
|
|
54853
56509
|
probes: probeRows.map(serializeProbe)
|
|
@@ -54859,7 +56515,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
54859
56515
|
"/projects/:name/discover/sessions/:id/harvest",
|
|
54860
56516
|
async (request, reply) => {
|
|
54861
56517
|
const project = resolveProject(app.db, request.params.name);
|
|
54862
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
56518
|
+
const session = app.db.select().from(discoverySessions).where(eq54(discoverySessions.id, request.params.id)).get();
|
|
54863
56519
|
if (!session || session.projectId !== project.id) {
|
|
54864
56520
|
throw notFound("Discovery session", request.params.id);
|
|
54865
56521
|
}
|
|
@@ -54867,7 +56523,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
54867
56523
|
const minProbeHits = Number.isNaN(parsedFloor) || parsedFloor < 1 ? 1 : parsedFloor;
|
|
54868
56524
|
const applyAnchor = request.query.anchor !== "false";
|
|
54869
56525
|
const provider = session.seedProvider ?? "gemini";
|
|
54870
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
56526
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq54(discoveryProbes.sessionId, session.id)).all();
|
|
54871
56527
|
const extract = opts.harvestSearchQueries;
|
|
54872
56528
|
const probesWithQueries = probeRows.map((row) => {
|
|
54873
56529
|
if (!extract || !row.rawResponse) return { searchQueries: [] };
|
|
@@ -54878,7 +56534,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
54878
56534
|
return { searchQueries: [] };
|
|
54879
56535
|
}
|
|
54880
56536
|
});
|
|
54881
|
-
const trackedQueries = app.db.select({ query: queries.query }).from(queries).where(
|
|
56537
|
+
const trackedQueries = app.db.select({ query: queries.query }).from(queries).where(eq54(queries.projectId, project.id)).all().map((r) => r.query);
|
|
54882
56538
|
const anchorTerms = buildHarvestAnchorTerms(
|
|
54883
56539
|
[session.icpDescription ?? "", ...trackedQueries],
|
|
54884
56540
|
effectiveDomains(project)
|
|
@@ -54925,12 +56581,12 @@ async function discoveryRoutes(app, opts) {
|
|
|
54925
56581
|
"/projects/:name/discover/sessions/:id/promote",
|
|
54926
56582
|
async (request, reply) => {
|
|
54927
56583
|
const project = resolveProject(app.db, request.params.name);
|
|
54928
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
56584
|
+
const session = app.db.select().from(discoverySessions).where(eq54(discoverySessions.id, request.params.id)).get();
|
|
54929
56585
|
if (!session || session.projectId !== project.id) {
|
|
54930
56586
|
throw notFound("Discovery session", request.params.id);
|
|
54931
56587
|
}
|
|
54932
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
54933
|
-
const existingCompetitors = app.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
56588
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq54(discoveryProbes.sessionId, session.id)).all();
|
|
56589
|
+
const existingCompetitors = app.db.select({ domain: competitors.domain }).from(competitors).where(eq54(competitors.projectId, project.id)).all().map((r) => r.domain.toLowerCase());
|
|
54934
56590
|
const seenCompetitors = new Set(existingCompetitors);
|
|
54935
56591
|
const cited = /* @__PURE__ */ new Set();
|
|
54936
56592
|
const aspirational = /* @__PURE__ */ new Set();
|
|
@@ -54959,7 +56615,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
54959
56615
|
);
|
|
54960
56616
|
app.post("/projects/:name/discover/sessions/:id/promote", async (request, reply) => {
|
|
54961
56617
|
const project = resolveProject(app.db, request.params.name);
|
|
54962
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
56618
|
+
const session = app.db.select().from(discoverySessions).where(eq54(discoverySessions.id, request.params.id)).get();
|
|
54963
56619
|
if (!session || session.projectId !== project.id) {
|
|
54964
56620
|
throw notFound("Discovery session", request.params.id);
|
|
54965
56621
|
}
|
|
@@ -54982,7 +56638,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
54982
56638
|
const bucketSet = new Set(buckets);
|
|
54983
56639
|
const includeCompetitors = parsed.data.includeCompetitors ?? true;
|
|
54984
56640
|
const competitorTypes = parsed.data.competitorTypes ?? DEFAULT_DISCOVERY_PROMOTE_COMPETITOR_TYPES;
|
|
54985
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
56641
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq54(discoveryProbes.sessionId, session.id)).all();
|
|
54986
56642
|
const candidateQueries = /* @__PURE__ */ new Set();
|
|
54987
56643
|
for (const probe of probeRows) {
|
|
54988
56644
|
if (!probe.bucket) continue;
|
|
@@ -54990,7 +56646,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
54990
56646
|
if (bucket.success && bucketSet.has(bucket.data)) candidateQueries.add(probe.query);
|
|
54991
56647
|
}
|
|
54992
56648
|
const existingQueries = new Set(
|
|
54993
|
-
app.db.select({ query: queries.query }).from(queries).where(
|
|
56649
|
+
app.db.select({ query: queries.query }).from(queries).where(eq54(queries.projectId, project.id)).all().map((r) => r.query.toLowerCase())
|
|
54994
56650
|
);
|
|
54995
56651
|
const promotedQueries = [];
|
|
54996
56652
|
const skippedQueries = [];
|
|
@@ -55006,7 +56662,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55006
56662
|
const skippedCompetitors = [];
|
|
55007
56663
|
if (includeCompetitors) {
|
|
55008
56664
|
const existingCompetitors = new Set(
|
|
55009
|
-
app.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
56665
|
+
app.db.select({ domain: competitors.domain }).from(competitors).where(eq54(competitors.projectId, project.id)).all().map((r) => r.domain.toLowerCase())
|
|
55010
56666
|
);
|
|
55011
56667
|
const competitorMap = parseCompetitorMap(session.competitorMap);
|
|
55012
56668
|
for (const entry of selectEligibleCompetitors(competitorMap, competitorTypes)) {
|
|
@@ -55025,7 +56681,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55025
56681
|
app.db.transaction((tx) => {
|
|
55026
56682
|
for (const query of promotedQueries) {
|
|
55027
56683
|
tx.insert(queries).values({
|
|
55028
|
-
id:
|
|
56684
|
+
id: crypto42.randomUUID(),
|
|
55029
56685
|
projectId: project.id,
|
|
55030
56686
|
query,
|
|
55031
56687
|
provenance,
|
|
@@ -55034,7 +56690,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55034
56690
|
}
|
|
55035
56691
|
for (const domain of promotedCompetitors) {
|
|
55036
56692
|
tx.insert(competitors).values({
|
|
55037
|
-
id:
|
|
56693
|
+
id: crypto42.randomUUID(),
|
|
55038
56694
|
projectId: project.id,
|
|
55039
56695
|
domain,
|
|
55040
56696
|
provenance,
|
|
@@ -55124,8 +56780,8 @@ function selectEligibleCompetitors(competitorMap, competitorTypes) {
|
|
|
55124
56780
|
}
|
|
55125
56781
|
|
|
55126
56782
|
// ../api-routes/src/discovery/orchestrate.ts
|
|
55127
|
-
import
|
|
55128
|
-
import { eq as
|
|
56783
|
+
import crypto43 from "crypto";
|
|
56784
|
+
import { eq as eq55 } from "drizzle-orm";
|
|
55129
56785
|
var DEFAULT_MAX_PROBES = 100;
|
|
55130
56786
|
var ABSOLUTE_MAX_PROBES = 500;
|
|
55131
56787
|
function classifyProbeBucket(input) {
|
|
@@ -55212,7 +56868,7 @@ async function executeDiscovery(opts) {
|
|
|
55212
56868
|
status: DiscoverySessionStatuses.seeding,
|
|
55213
56869
|
dedupThreshold,
|
|
55214
56870
|
startedAt
|
|
55215
|
-
}).where(
|
|
56871
|
+
}).where(eq55(discoverySessions.id, opts.sessionId)).run();
|
|
55216
56872
|
const seedResult = await opts.deps.seed({
|
|
55217
56873
|
project: opts.project,
|
|
55218
56874
|
icpDescription: opts.icpDescription,
|
|
@@ -55273,7 +56929,7 @@ async function executeDiscovery(opts) {
|
|
|
55273
56929
|
dedupBandPairFraction: dedupStats.bandPairFraction,
|
|
55274
56930
|
dedupPairsTotal: dedupStats.pairsTotal,
|
|
55275
56931
|
warning
|
|
55276
|
-
}).where(
|
|
56932
|
+
}).where(eq55(discoverySessions.id, opts.sessionId)).run();
|
|
55277
56933
|
const probeLocation = opts.locations?.[0];
|
|
55278
56934
|
const probeResults = await mapWithConcurrency(
|
|
55279
56935
|
probedCanonicals,
|
|
@@ -55292,7 +56948,7 @@ async function executeDiscovery(opts) {
|
|
|
55292
56948
|
probeRows.push({ citedDomains: probe.citedDomains, bucket });
|
|
55293
56949
|
buckets[bucket]++;
|
|
55294
56950
|
return {
|
|
55295
|
-
id:
|
|
56951
|
+
id: crypto43.randomUUID(),
|
|
55296
56952
|
sessionId: opts.sessionId,
|
|
55297
56953
|
projectId: opts.project.id,
|
|
55298
56954
|
query: probedCanonicals[index2],
|
|
@@ -55327,7 +56983,7 @@ async function executeDiscovery(opts) {
|
|
|
55327
56983
|
wastedCount: buckets["wasted-surface"],
|
|
55328
56984
|
competitorMap,
|
|
55329
56985
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
55330
|
-
}).where(
|
|
56986
|
+
}).where(eq55(discoverySessions.id, opts.sessionId)).run();
|
|
55331
56987
|
upsertDomainClassifications(opts.db, opts.project.id, opts.sessionId, competitorMap);
|
|
55332
56988
|
return {
|
|
55333
56989
|
buckets,
|
|
@@ -55344,7 +57000,7 @@ function upsertDomainClassifications(db, projectId, sessionId, competitorMap) {
|
|
|
55344
57000
|
const domain = hostOf(entry.domain) ?? "";
|
|
55345
57001
|
if (!domain) continue;
|
|
55346
57002
|
db.insert(domainClassifications).values({
|
|
55347
|
-
id:
|
|
57003
|
+
id: crypto43.randomUUID(),
|
|
55348
57004
|
projectId,
|
|
55349
57005
|
domain,
|
|
55350
57006
|
competitorType: entry.competitorType,
|
|
@@ -55367,7 +57023,7 @@ function markSessionFailed(db, sessionId, error) {
|
|
|
55367
57023
|
status: DiscoverySessionStatuses.failed,
|
|
55368
57024
|
error,
|
|
55369
57025
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
55370
|
-
}).where(
|
|
57026
|
+
}).where(eq55(discoverySessions.id, sessionId)).run();
|
|
55371
57027
|
}
|
|
55372
57028
|
function dedupeStrings(input) {
|
|
55373
57029
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -55384,8 +57040,16 @@ function dedupeStrings(input) {
|
|
|
55384
57040
|
}
|
|
55385
57041
|
|
|
55386
57042
|
// ../api-routes/src/technical-aeo.ts
|
|
55387
|
-
import
|
|
55388
|
-
import { and as
|
|
57043
|
+
import crypto44 from "crypto";
|
|
57044
|
+
import { and as and44, asc as asc10, count, desc as desc24, eq as eq56, inArray as inArray19, isNotNull as isNotNull3, lt as lt8, or as or11, sql as sql21 } from "drizzle-orm";
|
|
57045
|
+
import { alias } from "drizzle-orm/sqlite-core";
|
|
57046
|
+
var FETCHED_SITE_CRAWL_STATES = /* @__PURE__ */ new Set([
|
|
57047
|
+
...SiteCrawlFetchedStates,
|
|
57048
|
+
// Snapshots published before the full-crawl contract used this aggregate
|
|
57049
|
+
// state. Keep historical structure counts truthful without admitting it to
|
|
57050
|
+
// the current crawler vocabulary.
|
|
57051
|
+
"fetched"
|
|
57052
|
+
]);
|
|
55389
57053
|
var SURFACEABLE_STATUSES = [RunStatuses.completed, RunStatuses.partial];
|
|
55390
57054
|
function emptyScore(projectName) {
|
|
55391
57055
|
return {
|
|
@@ -55478,7 +57142,32 @@ function mapCrawlPage(row) {
|
|
|
55478
57142
|
inboundOccurrences: row.inboundOccurrences,
|
|
55479
57143
|
outboundOccurrences: row.outboundOccurrences,
|
|
55480
57144
|
linkScoreRaw: row.linkScoreRaw,
|
|
55481
|
-
linkScoreNormalized: row.linkScoreNormalized
|
|
57145
|
+
linkScoreNormalized: row.linkScoreNormalized,
|
|
57146
|
+
healthState: deriveSiteHealthState(row)
|
|
57147
|
+
};
|
|
57148
|
+
}
|
|
57149
|
+
function mapCrawlPageAuditEvidence(row) {
|
|
57150
|
+
const fields = row.auditFields;
|
|
57151
|
+
const rawFactors = Array.isArray(fields.factors) ? fields.factors : [];
|
|
57152
|
+
const rawCriticalDefects = Array.isArray(fields.criticalDefects) ? fields.criticalDefects : [];
|
|
57153
|
+
const parsedFactors = rawFactors.map((factor) => siteCrawlAuditFactorSchema.safeParse(factor));
|
|
57154
|
+
const parsedCriticalDefects = rawCriticalDefects.map((defect) => siteCrawlCriticalDefectSchema.safeParse(defect));
|
|
57155
|
+
const complete = fields.schemaVersion === "1.0" && Array.isArray(fields.factors) && Array.isArray(fields.criticalDefects) && parsedFactors.every((result) => result.success) && parsedCriticalDefects.every((result) => result.success);
|
|
57156
|
+
const factors = parsedFactors.flatMap((result, index2) => {
|
|
57157
|
+
if (result.success) return [result.data];
|
|
57158
|
+
const legacy = siteAuditPageFactorSchema.safeParse(rawFactors[index2]);
|
|
57159
|
+
return legacy.success ? [{
|
|
57160
|
+
...legacy.data,
|
|
57161
|
+
status: factorStatusFromScore(legacy.data.score),
|
|
57162
|
+
applicable: null,
|
|
57163
|
+
findings: [],
|
|
57164
|
+
recommendations: []
|
|
57165
|
+
}] : [];
|
|
57166
|
+
});
|
|
57167
|
+
return {
|
|
57168
|
+
evidenceState: complete ? "complete" : "scores-only",
|
|
57169
|
+
factors,
|
|
57170
|
+
criticalDefects: parsedCriticalDefects.flatMap((result) => result.success ? [result.data] : [])
|
|
55482
57171
|
};
|
|
55483
57172
|
}
|
|
55484
57173
|
function mapCrawlEdge(row) {
|
|
@@ -55497,26 +57186,141 @@ function mapCrawlEdge(row) {
|
|
|
55497
57186
|
anchors: row.anchors
|
|
55498
57187
|
};
|
|
55499
57188
|
}
|
|
57189
|
+
var graphSourceNode = alias(siteCrawlGraphNodes, "site_crawl_graph_source_node");
|
|
57190
|
+
var graphTargetNode = alias(siteCrawlGraphNodes, "site_crawl_graph_target_node");
|
|
57191
|
+
var PAGE_CHANGE_PREDICATE = sql21`(
|
|
57192
|
+
current.url IS NOT previous.url
|
|
57193
|
+
OR current.final_url IS NOT previous.final_url
|
|
57194
|
+
OR current.path IS NOT previous.path
|
|
57195
|
+
OR current.parent_path IS NOT previous.parent_path
|
|
57196
|
+
OR current.discovery_source IS NOT previous.discovery_source
|
|
57197
|
+
OR current.fetch_state IS NOT previous.fetch_state
|
|
57198
|
+
OR current.http_status IS NOT previous.http_status
|
|
57199
|
+
OR current.canonical_url IS NOT previous.canonical_url
|
|
57200
|
+
OR current.indexability_state IS NOT previous.indexability_state
|
|
57201
|
+
OR current.indexability_reasons IS NOT previous.indexability_reasons
|
|
57202
|
+
OR current.audit_state IS NOT previous.audit_state
|
|
57203
|
+
OR current.audit_score IS NOT previous.audit_score
|
|
57204
|
+
OR current.inventory_eligible IS NOT previous.inventory_eligible
|
|
57205
|
+
OR current.depth IS NOT previous.depth
|
|
57206
|
+
OR current.inbound_unique_edges IS NOT previous.inbound_unique_edges
|
|
57207
|
+
OR current.outbound_unique_edges IS NOT previous.outbound_unique_edges
|
|
57208
|
+
OR current.inbound_occurrences IS NOT previous.inbound_occurrences
|
|
57209
|
+
OR current.outbound_occurrences IS NOT previous.outbound_occurrences
|
|
57210
|
+
OR current.link_score_raw IS NOT previous.link_score_raw
|
|
57211
|
+
OR current.link_score_normalized IS NOT previous.link_score_normalized
|
|
57212
|
+
)`;
|
|
57213
|
+
var LINK_CHANGE_PREDICATE = sql21`(
|
|
57214
|
+
current.source_node_key IS NOT previous.source_node_key
|
|
57215
|
+
OR current.source_url IS NOT previous.source_url
|
|
57216
|
+
OR current.target_node_key IS NOT previous.target_node_key
|
|
57217
|
+
OR current.target_url IS NOT previous.target_url
|
|
57218
|
+
OR current.relation IS NOT previous.relation
|
|
57219
|
+
OR current.followable IS NOT previous.followable
|
|
57220
|
+
OR current.occurrences IS NOT previous.occurrences
|
|
57221
|
+
OR current.followable_occurrences IS NOT previous.followable_occurrences
|
|
57222
|
+
OR current.nofollow_occurrences IS NOT previous.nofollow_occurrences
|
|
57223
|
+
OR current.anchors IS NOT previous.anchors
|
|
57224
|
+
)`;
|
|
57225
|
+
function encodeSiteHealthChangesCursor(cursor) {
|
|
57226
|
+
return Buffer.from(JSON.stringify(cursor)).toString("base64url");
|
|
57227
|
+
}
|
|
57228
|
+
function decodeSiteHealthChangesCursor(value, context) {
|
|
57229
|
+
if (value == null || value === "") return null;
|
|
57230
|
+
if (typeof value !== "string" || value.length > 2048) throw validationError("Invalid Site Health changes cursor");
|
|
57231
|
+
try {
|
|
57232
|
+
const parsed = JSON.parse(Buffer.from(value, "base64url").toString("utf8"));
|
|
57233
|
+
const valid = parsed.v === 1 && parsed.fromRunId === context.fromRunId && parsed.toRunId === context.toRunId && parsed.scope === context.scope && parsed.change === context.change && (parsed.entity === "page" || parsed.entity === "link") && typeof parsed.key === "string" && parsed.key.length > 0 && parsed.key.length <= 2048;
|
|
57234
|
+
if (!valid) throw new Error("context mismatch");
|
|
57235
|
+
return parsed;
|
|
57236
|
+
} catch {
|
|
57237
|
+
throw validationError("Site Health changes cursor does not match this snapshot comparison and filter");
|
|
57238
|
+
}
|
|
57239
|
+
}
|
|
57240
|
+
function compareChangeKeys(left, right) {
|
|
57241
|
+
return Buffer.compare(Buffer.from(left.key), Buffer.from(right.key));
|
|
57242
|
+
}
|
|
57243
|
+
function valueChanged(before, after) {
|
|
57244
|
+
if (Array.isArray(before) || Array.isArray(after) || before && typeof before === "object" || after && typeof after === "object") {
|
|
57245
|
+
return JSON.stringify(before) !== JSON.stringify(after);
|
|
57246
|
+
}
|
|
57247
|
+
return before !== after;
|
|
57248
|
+
}
|
|
57249
|
+
var PAGE_CHANGE_FIELDS = [
|
|
57250
|
+
"url",
|
|
57251
|
+
"finalUrl",
|
|
57252
|
+
"path",
|
|
57253
|
+
"parentPath",
|
|
57254
|
+
"discoverySource",
|
|
57255
|
+
"fetchState",
|
|
57256
|
+
"httpStatus",
|
|
57257
|
+
"canonicalUrl",
|
|
57258
|
+
"indexabilityState",
|
|
57259
|
+
"indexabilityReasons",
|
|
57260
|
+
"auditState",
|
|
57261
|
+
"auditScore",
|
|
57262
|
+
"inventoryEligible",
|
|
57263
|
+
"depth",
|
|
57264
|
+
"inboundUniqueEdges",
|
|
57265
|
+
"outboundUniqueEdges",
|
|
57266
|
+
"inboundOccurrences",
|
|
57267
|
+
"outboundOccurrences",
|
|
57268
|
+
"linkScoreRaw",
|
|
57269
|
+
"linkScoreNormalized"
|
|
57270
|
+
];
|
|
57271
|
+
var LINK_CHANGE_FIELDS = [
|
|
57272
|
+
"sourceNodeKey",
|
|
57273
|
+
"sourceUrl",
|
|
57274
|
+
"targetNodeKey",
|
|
57275
|
+
"targetUrl",
|
|
57276
|
+
"relation",
|
|
57277
|
+
"internal",
|
|
57278
|
+
"followable",
|
|
57279
|
+
"occurrences",
|
|
57280
|
+
"followableOccurrences",
|
|
57281
|
+
"nofollowOccurrences",
|
|
57282
|
+
"anchors"
|
|
57283
|
+
];
|
|
57284
|
+
function changedFields(before, after, fields) {
|
|
57285
|
+
if (!before || !after) return [];
|
|
57286
|
+
return fields.filter((field) => valueChanged(before[field], after[field])).map(String);
|
|
57287
|
+
}
|
|
55500
57288
|
async function technicalAeoRoutes(app, opts) {
|
|
55501
57289
|
const resolveCrawl = (projectId, runId) => {
|
|
55502
57290
|
const filters = [
|
|
55503
|
-
|
|
55504
|
-
|
|
55505
|
-
|
|
57291
|
+
eq56(siteCrawlSnapshots.projectId, projectId),
|
|
57292
|
+
eq56(runs.projectId, projectId),
|
|
57293
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
55506
57294
|
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
55507
57295
|
notProbeRun()
|
|
55508
57296
|
];
|
|
55509
57297
|
if (runId) {
|
|
55510
|
-
filters.push(
|
|
57298
|
+
filters.push(eq56(siteCrawlSnapshots.runId, runId));
|
|
55511
57299
|
} else {
|
|
55512
|
-
filters.push(
|
|
55513
|
-
}
|
|
55514
|
-
return app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
55515
|
-
};
|
|
55516
|
-
const
|
|
55517
|
-
|
|
55518
|
-
|
|
55519
|
-
|
|
57300
|
+
filters.push(eq56(siteCrawlSnapshots.complete, true), eq56(runs.status, RunStatuses.completed));
|
|
57301
|
+
}
|
|
57302
|
+
return app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq56(siteCrawlSnapshots.runId, runs.id)).where(and44(...filters)).orderBy(desc24(siteCrawlSnapshots.createdAt), desc24(siteCrawlSnapshots.runId)).limit(1).get();
|
|
57303
|
+
};
|
|
57304
|
+
const detailScopeFor = (projectId, snapshot) => snapshot.detailsAvailable && snapshot.attemptId ? { projectId, runId: snapshot.runId, attemptId: snapshot.attemptId } : null;
|
|
57305
|
+
const pageInScope = (scope, selector) => {
|
|
57306
|
+
if (!selector.nodeKey && !selector.url) return null;
|
|
57307
|
+
return app.db.select().from(siteCrawlPages).where(and44(
|
|
57308
|
+
eq56(siteCrawlPages.projectId, scope.projectId),
|
|
57309
|
+
eq56(siteCrawlPages.runId, scope.runId),
|
|
57310
|
+
eq56(siteCrawlPages.attemptId, scope.attemptId),
|
|
57311
|
+
selector.nodeKey ? eq56(siteCrawlPages.nodeKey, selector.nodeKey) : eq56(siteCrawlPages.url, selector.url)
|
|
57312
|
+
)).orderBy(asc10(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
57313
|
+
};
|
|
57314
|
+
const rootPageInScope = (scope, rootUrl) => pageInScope(scope, { url: rootUrl }) ?? app.db.select().from(siteCrawlPages).where(and44(
|
|
57315
|
+
eq56(siteCrawlPages.projectId, scope.projectId),
|
|
57316
|
+
eq56(siteCrawlPages.runId, scope.runId),
|
|
57317
|
+
eq56(siteCrawlPages.attemptId, scope.attemptId),
|
|
57318
|
+
eq56(siteCrawlPages.depth, 0)
|
|
57319
|
+
)).orderBy(asc10(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
57320
|
+
const hasLegacyAudit = (projectId) => Boolean(app.db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).innerJoin(runs, eq56(siteAuditSnapshots.runId, runs.id)).where(and44(
|
|
57321
|
+
eq56(siteAuditSnapshots.projectId, projectId),
|
|
57322
|
+
eq56(runs.projectId, projectId),
|
|
57323
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
55520
57324
|
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
55521
57325
|
notProbeRun()
|
|
55522
57326
|
)).limit(1).get());
|
|
@@ -55526,6 +57330,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
55526
57330
|
legacyAuditAvailable,
|
|
55527
57331
|
runId: null,
|
|
55528
57332
|
runStatus: null,
|
|
57333
|
+
requestedRootUrl: null,
|
|
55529
57334
|
rootUrl: null,
|
|
55530
57335
|
effectiveOptions: {},
|
|
55531
57336
|
complete: false,
|
|
@@ -55537,20 +57342,20 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
55537
57342
|
app.get("/projects/:name/technical-aeo", async (request) => {
|
|
55538
57343
|
const project = resolveProject(app.db, request.params.name);
|
|
55539
57344
|
const baseFilters = [
|
|
55540
|
-
|
|
55541
|
-
|
|
55542
|
-
|
|
57345
|
+
eq56(siteAuditSnapshots.projectId, project.id),
|
|
57346
|
+
eq56(runs.projectId, project.id),
|
|
57347
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
55543
57348
|
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
55544
57349
|
notProbeRun()
|
|
55545
57350
|
];
|
|
55546
|
-
const targetFilters = request.query.runId ? [...baseFilters,
|
|
55547
|
-
const latest = app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs,
|
|
57351
|
+
const targetFilters = request.query.runId ? [...baseFilters, eq56(siteAuditSnapshots.runId, request.query.runId)] : baseFilters;
|
|
57352
|
+
const latest = app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs, eq56(siteAuditSnapshots.runId, runs.id)).where(and44(...targetFilters)).orderBy(desc24(siteAuditSnapshots.createdAt)).limit(1).get();
|
|
55548
57353
|
if (!latest) {
|
|
55549
57354
|
if (request.query.runId) throw notFound("Site audit run", request.query.runId);
|
|
55550
57355
|
return emptyScore(project.name);
|
|
55551
57356
|
}
|
|
55552
57357
|
const snap = latest.snap;
|
|
55553
|
-
const previous = app.db.select({ snap: siteAuditSnapshots }).from(siteAuditSnapshots).innerJoin(runs,
|
|
57358
|
+
const previous = app.db.select({ snap: siteAuditSnapshots }).from(siteAuditSnapshots).innerJoin(runs, eq56(siteAuditSnapshots.runId, runs.id)).where(and44(...baseFilters, lt8(siteAuditSnapshots.createdAt, snap.createdAt))).orderBy(desc24(siteAuditSnapshots.createdAt)).limit(1).get()?.snap ?? null;
|
|
55554
57359
|
const deltaScore = previous ? snap.aggregateScore - previous.aggregateScore : null;
|
|
55555
57360
|
const trend = deltaScore == null ? null : deltaScore > 0 ? SiteAuditTrendDirections.up : deltaScore < 0 ? SiteAuditTrendDirections.down : SiteAuditTrendDirections.flat;
|
|
55556
57361
|
return {
|
|
@@ -55577,14 +57382,14 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
55577
57382
|
app.get("/projects/:name/technical-aeo/pages", async (request) => {
|
|
55578
57383
|
const project = resolveProject(app.db, request.params.name);
|
|
55579
57384
|
const targetFilters = [
|
|
55580
|
-
|
|
55581
|
-
|
|
55582
|
-
|
|
57385
|
+
eq56(siteAuditSnapshots.projectId, project.id),
|
|
57386
|
+
eq56(runs.projectId, project.id),
|
|
57387
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
55583
57388
|
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
55584
57389
|
notProbeRun()
|
|
55585
57390
|
];
|
|
55586
|
-
if (request.query.runId) targetFilters.push(
|
|
55587
|
-
const latest = app.db.select({ runId: siteAuditSnapshots.runId, auditedAt: siteAuditSnapshots.auditedAt }).from(siteAuditSnapshots).innerJoin(runs,
|
|
57391
|
+
if (request.query.runId) targetFilters.push(eq56(siteAuditSnapshots.runId, request.query.runId));
|
|
57392
|
+
const latest = app.db.select({ runId: siteAuditSnapshots.runId, auditedAt: siteAuditSnapshots.auditedAt }).from(siteAuditSnapshots).innerJoin(runs, eq56(siteAuditSnapshots.runId, runs.id)).where(and44(...targetFilters)).orderBy(desc24(siteAuditSnapshots.createdAt)).limit(1).get();
|
|
55588
57393
|
if (!latest && request.query.runId) {
|
|
55589
57394
|
throw notFound("Site audit run", request.query.runId);
|
|
55590
57395
|
}
|
|
@@ -55592,9 +57397,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
55592
57397
|
return { project: project.name, runId: null, auditedAt: null, total: 0, pages: [] };
|
|
55593
57398
|
}
|
|
55594
57399
|
const statusFilter = request.query.status === "success" || request.query.status === "error" ? request.query.status : null;
|
|
55595
|
-
const conds = [
|
|
55596
|
-
if (statusFilter) conds.push(
|
|
55597
|
-
const where =
|
|
57400
|
+
const conds = [eq56(siteAuditPages.projectId, project.id), eq56(siteAuditPages.runId, latest.runId)];
|
|
57401
|
+
if (statusFilter) conds.push(eq56(siteAuditPages.status, statusFilter));
|
|
57402
|
+
const where = and44(...conds);
|
|
55598
57403
|
const totalRow = app.db.select({ value: count() }).from(siteAuditPages).where(where).get();
|
|
55599
57404
|
const total = totalRow?.value ?? 0;
|
|
55600
57405
|
const limit = parsePositiveInt(request.query.limit, 100, 500);
|
|
@@ -55618,10 +57423,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
55618
57423
|
auditedAt: siteAuditSnapshots.auditedAt,
|
|
55619
57424
|
aggregateScore: siteAuditSnapshots.aggregateScore,
|
|
55620
57425
|
pagesAudited: siteAuditSnapshots.pagesAudited
|
|
55621
|
-
}).from(siteAuditSnapshots).innerJoin(runs,
|
|
55622
|
-
|
|
55623
|
-
|
|
55624
|
-
|
|
57426
|
+
}).from(siteAuditSnapshots).innerJoin(runs, eq56(siteAuditSnapshots.runId, runs.id)).where(and44(
|
|
57427
|
+
eq56(siteAuditSnapshots.projectId, project.id),
|
|
57428
|
+
eq56(runs.projectId, project.id),
|
|
57429
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
55625
57430
|
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
55626
57431
|
notProbeRun()
|
|
55627
57432
|
)).orderBy(desc24(siteAuditSnapshots.createdAt)).limit(limit).all();
|
|
@@ -55643,6 +57448,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
55643
57448
|
legacyAuditAvailable,
|
|
55644
57449
|
runId: snapshot.runId,
|
|
55645
57450
|
runStatus: target.runStatus,
|
|
57451
|
+
requestedRootUrl: snapshot.requestedRootUrl,
|
|
55646
57452
|
rootUrl: snapshot.rootUrl,
|
|
55647
57453
|
crawlSchemaVersion: snapshot.crawlSchemaVersion,
|
|
55648
57454
|
engineVersion: snapshot.engineVersion,
|
|
@@ -55663,6 +57469,827 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
55663
57469
|
deadLinks
|
|
55664
57470
|
};
|
|
55665
57471
|
});
|
|
57472
|
+
app.get("/projects/:name/technical-aeo/graph", async (request) => {
|
|
57473
|
+
const project = resolveProject(app.db, request.params.name);
|
|
57474
|
+
const target = resolveCrawl(project.id, request.query.runId);
|
|
57475
|
+
if (!target) {
|
|
57476
|
+
if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
|
|
57477
|
+
return {
|
|
57478
|
+
project: project.name,
|
|
57479
|
+
hasCrawlData: false,
|
|
57480
|
+
runId: null,
|
|
57481
|
+
layout: { state: "unavailable", version: null, reason: "no-crawl" },
|
|
57482
|
+
totalNodes: 0,
|
|
57483
|
+
totalEdges: 0,
|
|
57484
|
+
nodes: [],
|
|
57485
|
+
edges: [],
|
|
57486
|
+
omittedNodes: 0,
|
|
57487
|
+
omittedEdges: 0,
|
|
57488
|
+
sampled: false
|
|
57489
|
+
};
|
|
57490
|
+
}
|
|
57491
|
+
const snapshot = target.snapshot;
|
|
57492
|
+
if (!snapshot.detailsAvailable || !snapshot.attemptId) {
|
|
57493
|
+
return {
|
|
57494
|
+
project: project.name,
|
|
57495
|
+
hasCrawlData: true,
|
|
57496
|
+
runId: snapshot.runId,
|
|
57497
|
+
layout: { state: "unavailable", version: null, reason: "details-unavailable" },
|
|
57498
|
+
totalNodes: 0,
|
|
57499
|
+
totalEdges: 0,
|
|
57500
|
+
nodes: [],
|
|
57501
|
+
edges: [],
|
|
57502
|
+
omittedNodes: 0,
|
|
57503
|
+
omittedEdges: 0,
|
|
57504
|
+
sampled: false
|
|
57505
|
+
};
|
|
57506
|
+
}
|
|
57507
|
+
const persistedLayout = app.db.select().from(siteCrawlGraphLayouts).where(and44(
|
|
57508
|
+
eq56(siteCrawlGraphLayouts.projectId, project.id),
|
|
57509
|
+
eq56(siteCrawlGraphLayouts.runId, snapshot.runId),
|
|
57510
|
+
eq56(siteCrawlGraphLayouts.attemptId, snapshot.attemptId)
|
|
57511
|
+
)).limit(1).get();
|
|
57512
|
+
if (!persistedLayout) {
|
|
57513
|
+
return {
|
|
57514
|
+
project: project.name,
|
|
57515
|
+
hasCrawlData: true,
|
|
57516
|
+
runId: snapshot.runId,
|
|
57517
|
+
layout: { state: "unavailable", version: null, reason: "legacy-snapshot" },
|
|
57518
|
+
totalNodes: 0,
|
|
57519
|
+
totalEdges: 0,
|
|
57520
|
+
nodes: [],
|
|
57521
|
+
edges: [],
|
|
57522
|
+
omittedNodes: 0,
|
|
57523
|
+
omittedEdges: 0,
|
|
57524
|
+
sampled: false
|
|
57525
|
+
};
|
|
57526
|
+
}
|
|
57527
|
+
if (persistedLayout.state !== "ready" || !persistedLayout.layoutVersion) {
|
|
57528
|
+
const totalNodes2 = persistedLayout.totalNodes;
|
|
57529
|
+
const totalEdges2 = persistedLayout.totalEdges;
|
|
57530
|
+
return {
|
|
57531
|
+
project: project.name,
|
|
57532
|
+
hasCrawlData: true,
|
|
57533
|
+
runId: snapshot.runId,
|
|
57534
|
+
layout: {
|
|
57535
|
+
state: "unavailable",
|
|
57536
|
+
version: null,
|
|
57537
|
+
reason: persistedLayout.failureCode === "empty-crawl" ? "empty-crawl" : "layout-failed"
|
|
57538
|
+
},
|
|
57539
|
+
totalNodes: totalNodes2,
|
|
57540
|
+
totalEdges: totalEdges2,
|
|
57541
|
+
nodes: [],
|
|
57542
|
+
edges: [],
|
|
57543
|
+
omittedNodes: totalNodes2,
|
|
57544
|
+
omittedEdges: totalEdges2,
|
|
57545
|
+
sampled: totalNodes2 > 0 || totalEdges2 > 0
|
|
57546
|
+
};
|
|
57547
|
+
}
|
|
57548
|
+
const maxNodes = parseBoundedLimit(request.query.maxNodes, SITE_CRAWL_GRAPH_DEFAULT_MAX_NODES, SITE_CRAWL_GRAPH_MAX_NODES);
|
|
57549
|
+
const maxEdges = parseBoundedLimit(request.query.maxEdges, SITE_CRAWL_GRAPH_DEFAULT_MAX_EDGES, SITE_CRAWL_GRAPH_MAX_EDGES);
|
|
57550
|
+
const graphScope = [
|
|
57551
|
+
eq56(siteCrawlGraphNodes.projectId, project.id),
|
|
57552
|
+
eq56(siteCrawlGraphNodes.runId, snapshot.runId),
|
|
57553
|
+
eq56(siteCrawlGraphNodes.attemptId, snapshot.attemptId)
|
|
57554
|
+
];
|
|
57555
|
+
const nodeRows = app.db.select({
|
|
57556
|
+
nodeKey: siteCrawlPages.nodeKey,
|
|
57557
|
+
url: siteCrawlPages.url,
|
|
57558
|
+
path: siteCrawlPages.path,
|
|
57559
|
+
depth: siteCrawlPages.depth,
|
|
57560
|
+
indexabilityState: siteCrawlPages.indexabilityState,
|
|
57561
|
+
indexabilityReasons: siteCrawlPages.indexabilityReasons,
|
|
57562
|
+
canonicalNodeKey: siteCrawlPages.canonicalNodeKey,
|
|
57563
|
+
fetchState: siteCrawlPages.fetchState,
|
|
57564
|
+
auditState: siteCrawlPages.auditState,
|
|
57565
|
+
auditScore: siteCrawlPages.auditScore,
|
|
57566
|
+
inventoryEligible: siteCrawlPages.inventoryEligible,
|
|
57567
|
+
inboundUniqueEdges: siteCrawlPages.inboundUniqueEdges,
|
|
57568
|
+
outboundUniqueEdges: siteCrawlPages.outboundUniqueEdges,
|
|
57569
|
+
linkScoreNormalized: siteCrawlPages.linkScoreNormalized,
|
|
57570
|
+
x: siteCrawlGraphNodes.x,
|
|
57571
|
+
y: siteCrawlGraphNodes.y
|
|
57572
|
+
}).from(siteCrawlGraphNodes).innerJoin(siteCrawlPages, and44(
|
|
57573
|
+
eq56(siteCrawlPages.projectId, siteCrawlGraphNodes.projectId),
|
|
57574
|
+
eq56(siteCrawlPages.runId, siteCrawlGraphNodes.runId),
|
|
57575
|
+
eq56(siteCrawlPages.attemptId, siteCrawlGraphNodes.attemptId),
|
|
57576
|
+
eq56(siteCrawlPages.nodeKey, siteCrawlGraphNodes.nodeKey)
|
|
57577
|
+
)).where(and44(...graphScope, lt8(siteCrawlGraphNodes.sampleRank, maxNodes))).orderBy(asc10(siteCrawlGraphNodes.sampleRank)).all();
|
|
57578
|
+
const nodes = nodeRows.map(({ indexabilityReasons, canonicalNodeKey, ...row }) => ({
|
|
57579
|
+
...row,
|
|
57580
|
+
healthState: deriveSiteHealthState({ ...row, indexabilityReasons, canonicalNodeKey })
|
|
57581
|
+
}));
|
|
57582
|
+
const edges = app.db.select({
|
|
57583
|
+
edgeKey: siteCrawlGraphEdges.edgeKey,
|
|
57584
|
+
sourceNodeKey: siteCrawlGraphEdges.sourceNodeKey,
|
|
57585
|
+
targetNodeKey: siteCrawlGraphEdges.targetNodeKey,
|
|
57586
|
+
followable: siteCrawlGraphEdges.followable,
|
|
57587
|
+
occurrences: siteCrawlGraphEdges.occurrences
|
|
57588
|
+
}).from(siteCrawlGraphEdges).innerJoin(graphSourceNode, and44(
|
|
57589
|
+
eq56(graphSourceNode.projectId, siteCrawlGraphEdges.projectId),
|
|
57590
|
+
eq56(graphSourceNode.runId, siteCrawlGraphEdges.runId),
|
|
57591
|
+
eq56(graphSourceNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
57592
|
+
eq56(graphSourceNode.nodeKey, siteCrawlGraphEdges.sourceNodeKey),
|
|
57593
|
+
lt8(graphSourceNode.sampleRank, maxNodes)
|
|
57594
|
+
)).innerJoin(graphTargetNode, and44(
|
|
57595
|
+
eq56(graphTargetNode.projectId, siteCrawlGraphEdges.projectId),
|
|
57596
|
+
eq56(graphTargetNode.runId, siteCrawlGraphEdges.runId),
|
|
57597
|
+
eq56(graphTargetNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
57598
|
+
eq56(graphTargetNode.nodeKey, siteCrawlGraphEdges.targetNodeKey),
|
|
57599
|
+
lt8(graphTargetNode.sampleRank, maxNodes)
|
|
57600
|
+
)).where(and44(
|
|
57601
|
+
eq56(siteCrawlGraphEdges.projectId, project.id),
|
|
57602
|
+
eq56(siteCrawlGraphEdges.runId, snapshot.runId),
|
|
57603
|
+
eq56(siteCrawlGraphEdges.attemptId, snapshot.attemptId),
|
|
57604
|
+
lt8(siteCrawlGraphEdges.sampleRank, maxEdges)
|
|
57605
|
+
)).orderBy(asc10(siteCrawlGraphEdges.sampleRank)).all();
|
|
57606
|
+
const totalNodes = persistedLayout.totalNodes;
|
|
57607
|
+
const totalEdges = persistedLayout.totalEdges;
|
|
57608
|
+
const omittedNodes = Math.max(0, totalNodes - nodes.length);
|
|
57609
|
+
const omittedEdges = Math.max(0, totalEdges - edges.length);
|
|
57610
|
+
return {
|
|
57611
|
+
project: project.name,
|
|
57612
|
+
hasCrawlData: true,
|
|
57613
|
+
runId: snapshot.runId,
|
|
57614
|
+
layout: { state: "ready", version: persistedLayout.layoutVersion, computedAt: persistedLayout.createdAt },
|
|
57615
|
+
totalNodes,
|
|
57616
|
+
totalEdges,
|
|
57617
|
+
nodes,
|
|
57618
|
+
edges,
|
|
57619
|
+
omittedNodes,
|
|
57620
|
+
omittedEdges,
|
|
57621
|
+
sampled: omittedNodes > 0 || omittedEdges > 0
|
|
57622
|
+
};
|
|
57623
|
+
});
|
|
57624
|
+
app.get("/projects/:name/technical-aeo/subgraph", async (request) => {
|
|
57625
|
+
const project = resolveProject(app.db, request.params.name);
|
|
57626
|
+
const hops = Math.min(3, parsePositiveInt(request.query.hops, 1, 3));
|
|
57627
|
+
const maxNodes = parseBoundedLimit(
|
|
57628
|
+
request.query.maxNodes,
|
|
57629
|
+
SITE_HEALTH_SUBGRAPH_DEFAULT_MAX_NODES,
|
|
57630
|
+
SITE_HEALTH_SUBGRAPH_MAX_NODES
|
|
57631
|
+
);
|
|
57632
|
+
const maxEdges = parseBoundedLimit(
|
|
57633
|
+
request.query.maxEdges,
|
|
57634
|
+
SITE_HEALTH_SUBGRAPH_DEFAULT_MAX_EDGES,
|
|
57635
|
+
SITE_HEALTH_SUBGRAPH_MAX_EDGES
|
|
57636
|
+
);
|
|
57637
|
+
const target = resolveCrawl(project.id, request.query.runId);
|
|
57638
|
+
const empty = (state, runId, hasCrawlData, complete, termination) => ({
|
|
57639
|
+
project: project.name,
|
|
57640
|
+
hasCrawlData,
|
|
57641
|
+
runId,
|
|
57642
|
+
complete,
|
|
57643
|
+
termination,
|
|
57644
|
+
state,
|
|
57645
|
+
focusNodeKey: null,
|
|
57646
|
+
focusUrl: null,
|
|
57647
|
+
hops,
|
|
57648
|
+
totalNodes: 0,
|
|
57649
|
+
totalEdges: 0,
|
|
57650
|
+
nodes: [],
|
|
57651
|
+
edges: [],
|
|
57652
|
+
omittedNodes: 0,
|
|
57653
|
+
omittedEdges: 0,
|
|
57654
|
+
countAccuracy: state === "no-crawl" ? "exact" : "lower-bound",
|
|
57655
|
+
truncated: false
|
|
57656
|
+
});
|
|
57657
|
+
if (!target) {
|
|
57658
|
+
if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
|
|
57659
|
+
return empty("no-crawl", null, false, false, null);
|
|
57660
|
+
}
|
|
57661
|
+
const snapshot = target.snapshot;
|
|
57662
|
+
const scope = detailScopeFor(project.id, snapshot);
|
|
57663
|
+
if (!scope) return empty("details-unavailable", snapshot.runId, true, snapshot.complete, snapshot.termination);
|
|
57664
|
+
const explicitFocus = Boolean(request.query.nodeKey || request.query.url);
|
|
57665
|
+
const focus = explicitFocus ? pageInScope(scope, { nodeKey: request.query.nodeKey, url: request.query.url }) : rootPageInScope(scope, snapshot.rootUrl);
|
|
57666
|
+
if (!focus) {
|
|
57667
|
+
if (explicitFocus) throw notFound("Site crawl page", request.query.nodeKey ?? request.query.url);
|
|
57668
|
+
return {
|
|
57669
|
+
...empty("details-unavailable", snapshot.runId, true, snapshot.complete, snapshot.termination),
|
|
57670
|
+
truncated: false
|
|
57671
|
+
};
|
|
57672
|
+
}
|
|
57673
|
+
const distances = /* @__PURE__ */ new Map([[focus.nodeKey, 0]]);
|
|
57674
|
+
const directRelations = /* @__PURE__ */ new Map();
|
|
57675
|
+
const edgeRows = /* @__PURE__ */ new Map();
|
|
57676
|
+
const omittedNodeKeys = /* @__PURE__ */ new Set();
|
|
57677
|
+
const omittedEdgeKeys = /* @__PURE__ */ new Set();
|
|
57678
|
+
const seenEdgeKeys = /* @__PURE__ */ new Set();
|
|
57679
|
+
let frontier = [focus.nodeKey];
|
|
57680
|
+
let queryTruncated = false;
|
|
57681
|
+
for (let distance = 0; distance < hops && frontier.length > 0; distance += 1) {
|
|
57682
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and44(
|
|
57683
|
+
eq56(siteCrawlEdges.projectId, scope.projectId),
|
|
57684
|
+
eq56(siteCrawlEdges.runId, scope.runId),
|
|
57685
|
+
eq56(siteCrawlEdges.attemptId, scope.attemptId),
|
|
57686
|
+
eq56(siteCrawlEdges.internal, true),
|
|
57687
|
+
isNotNull3(siteCrawlEdges.targetNodeKey),
|
|
57688
|
+
or11(
|
|
57689
|
+
inArray19(siteCrawlEdges.sourceNodeKey, frontier),
|
|
57690
|
+
inArray19(siteCrawlEdges.targetNodeKey, frontier)
|
|
57691
|
+
)
|
|
57692
|
+
)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(maxEdges + maxNodes + 1).all();
|
|
57693
|
+
if (candidates.length > maxEdges + maxNodes) queryTruncated = true;
|
|
57694
|
+
const next = /* @__PURE__ */ new Set();
|
|
57695
|
+
for (const edge of candidates.slice(0, maxEdges + maxNodes)) {
|
|
57696
|
+
if (!edge.targetNodeKey || seenEdgeKeys.has(edge.edgeKey)) continue;
|
|
57697
|
+
seenEdgeKeys.add(edge.edgeKey);
|
|
57698
|
+
const sourceInFrontier = frontier.includes(edge.sourceNodeKey);
|
|
57699
|
+
const neighborKey = sourceInFrontier ? edge.targetNodeKey : edge.sourceNodeKey;
|
|
57700
|
+
if (!distances.has(neighborKey)) {
|
|
57701
|
+
if (distances.size < maxNodes) {
|
|
57702
|
+
distances.set(neighborKey, distance + 1);
|
|
57703
|
+
next.add(neighborKey);
|
|
57704
|
+
} else {
|
|
57705
|
+
omittedNodeKeys.add(neighborKey);
|
|
57706
|
+
}
|
|
57707
|
+
}
|
|
57708
|
+
if (distance === 0 && neighborKey !== focus.nodeKey) {
|
|
57709
|
+
const direction = edge.sourceNodeKey === focus.nodeKey ? "outbound" : "inbound";
|
|
57710
|
+
const prior = directRelations.get(neighborKey);
|
|
57711
|
+
directRelations.set(neighborKey, prior && prior !== direction ? "both" : direction);
|
|
57712
|
+
}
|
|
57713
|
+
if (distances.has(edge.sourceNodeKey) && distances.has(edge.targetNodeKey)) {
|
|
57714
|
+
if (edgeRows.size < maxEdges) edgeRows.set(edge.edgeKey, edge);
|
|
57715
|
+
else omittedEdgeKeys.add(edge.edgeKey);
|
|
57716
|
+
} else {
|
|
57717
|
+
omittedEdgeKeys.add(edge.edgeKey);
|
|
57718
|
+
}
|
|
57719
|
+
}
|
|
57720
|
+
frontier = [...next].sort();
|
|
57721
|
+
}
|
|
57722
|
+
const nodeKeys = [...distances.keys()];
|
|
57723
|
+
const pageRows = nodeKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and44(
|
|
57724
|
+
eq56(siteCrawlPages.projectId, scope.projectId),
|
|
57725
|
+
eq56(siteCrawlPages.runId, scope.runId),
|
|
57726
|
+
eq56(siteCrawlPages.attemptId, scope.attemptId),
|
|
57727
|
+
inArray19(siteCrawlPages.nodeKey, nodeKeys)
|
|
57728
|
+
)).all();
|
|
57729
|
+
const persistedNodeKeys = new Set(pageRows.map((row) => row.nodeKey));
|
|
57730
|
+
const outermostNodeKeys = pageRows.filter((row) => distances.get(row.nodeKey) === hops).map((row) => row.nodeKey);
|
|
57731
|
+
const omittedOutermostEdge = outermostNodeKeys.length === 0 ? void 0 : app.db.select({ edgeKey: siteCrawlEdges.edgeKey }).from(siteCrawlEdges).where(and44(
|
|
57732
|
+
eq56(siteCrawlEdges.projectId, scope.projectId),
|
|
57733
|
+
eq56(siteCrawlEdges.runId, scope.runId),
|
|
57734
|
+
eq56(siteCrawlEdges.attemptId, scope.attemptId),
|
|
57735
|
+
eq56(siteCrawlEdges.internal, true),
|
|
57736
|
+
isNotNull3(siteCrawlEdges.targetNodeKey),
|
|
57737
|
+
inArray19(siteCrawlEdges.sourceNodeKey, outermostNodeKeys),
|
|
57738
|
+
inArray19(siteCrawlEdges.targetNodeKey, outermostNodeKeys)
|
|
57739
|
+
)).limit(1).get();
|
|
57740
|
+
if (omittedOutermostEdge && !seenEdgeKeys.has(omittedOutermostEdge.edgeKey)) {
|
|
57741
|
+
omittedEdgeKeys.add(omittedOutermostEdge.edgeKey);
|
|
57742
|
+
}
|
|
57743
|
+
const nodes = pageRows.map((row) => ({
|
|
57744
|
+
...mapCrawlPage(row),
|
|
57745
|
+
distance: distances.get(row.nodeKey) ?? 0,
|
|
57746
|
+
relationToFocus: row.nodeKey === focus.nodeKey ? "focus" : directRelations.get(row.nodeKey) ?? "transitive"
|
|
57747
|
+
})).sort((a, b) => a.distance - b.distance || a.nodeKey.localeCompare(b.nodeKey));
|
|
57748
|
+
const edges = [...edgeRows.values()].filter((edge) => edge.targetNodeKey && persistedNodeKeys.has(edge.sourceNodeKey) && persistedNodeKeys.has(edge.targetNodeKey)).sort((a, b) => a.edgeKey.localeCompare(b.edgeKey)).map(mapCrawlEdge);
|
|
57749
|
+
const omittedNodes = omittedNodeKeys.size + Math.max(0, distances.size - pageRows.length);
|
|
57750
|
+
const omittedEdges = omittedEdgeKeys.size + (queryTruncated ? 1 : 0) + Math.max(0, edgeRows.size - edges.length);
|
|
57751
|
+
const truncated = omittedNodes > 0 || omittedEdges > 0;
|
|
57752
|
+
return {
|
|
57753
|
+
project: project.name,
|
|
57754
|
+
hasCrawlData: true,
|
|
57755
|
+
runId: snapshot.runId,
|
|
57756
|
+
complete: snapshot.complete,
|
|
57757
|
+
termination: snapshot.termination,
|
|
57758
|
+
state: "ready",
|
|
57759
|
+
focusNodeKey: focus.nodeKey,
|
|
57760
|
+
focusUrl: focus.url,
|
|
57761
|
+
hops,
|
|
57762
|
+
totalNodes: nodes.length + omittedNodes,
|
|
57763
|
+
totalEdges: edges.length + omittedEdges,
|
|
57764
|
+
nodes,
|
|
57765
|
+
edges,
|
|
57766
|
+
omittedNodes,
|
|
57767
|
+
omittedEdges,
|
|
57768
|
+
countAccuracy: truncated ? "lower-bound" : "exact",
|
|
57769
|
+
truncated
|
|
57770
|
+
};
|
|
57771
|
+
});
|
|
57772
|
+
app.get("/projects/:name/technical-aeo/path", async (request) => {
|
|
57773
|
+
const project = resolveProject(app.db, request.params.name);
|
|
57774
|
+
const maxDepth = parseBoundedLimit(
|
|
57775
|
+
request.query.maxDepth,
|
|
57776
|
+
SITE_HEALTH_PATH_DEFAULT_MAX_DEPTH,
|
|
57777
|
+
SITE_HEALTH_PATH_MAX_DEPTH
|
|
57778
|
+
);
|
|
57779
|
+
if (!request.query.toNodeKey && !request.query.toUrl) {
|
|
57780
|
+
throw validationError("toNodeKey or toUrl is required");
|
|
57781
|
+
}
|
|
57782
|
+
const target = resolveCrawl(project.id, request.query.runId);
|
|
57783
|
+
if (!target) {
|
|
57784
|
+
if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
|
|
57785
|
+
return {
|
|
57786
|
+
project: project.name,
|
|
57787
|
+
runId: null,
|
|
57788
|
+
state: "no-crawl",
|
|
57789
|
+
from: null,
|
|
57790
|
+
to: null,
|
|
57791
|
+
complete: false,
|
|
57792
|
+
termination: null,
|
|
57793
|
+
maxDepth,
|
|
57794
|
+
visitedNodes: 0,
|
|
57795
|
+
nodes: [],
|
|
57796
|
+
edges: []
|
|
57797
|
+
};
|
|
57798
|
+
}
|
|
57799
|
+
const snapshot = target.snapshot;
|
|
57800
|
+
const scope = detailScopeFor(project.id, snapshot);
|
|
57801
|
+
if (!scope) {
|
|
57802
|
+
return {
|
|
57803
|
+
project: project.name,
|
|
57804
|
+
runId: snapshot.runId,
|
|
57805
|
+
state: "details-unavailable",
|
|
57806
|
+
from: null,
|
|
57807
|
+
to: null,
|
|
57808
|
+
complete: snapshot.complete,
|
|
57809
|
+
termination: snapshot.termination,
|
|
57810
|
+
maxDepth,
|
|
57811
|
+
visitedNodes: 0,
|
|
57812
|
+
nodes: [],
|
|
57813
|
+
edges: []
|
|
57814
|
+
};
|
|
57815
|
+
}
|
|
57816
|
+
const explicitFrom = Boolean(request.query.fromNodeKey || request.query.fromUrl);
|
|
57817
|
+
const fromPage = explicitFrom ? pageInScope(scope, { nodeKey: request.query.fromNodeKey, url: request.query.fromUrl }) : rootPageInScope(scope, snapshot.rootUrl);
|
|
57818
|
+
if (!fromPage) throw notFound("Site crawl page", request.query.fromNodeKey ?? request.query.fromUrl ?? snapshot.rootUrl);
|
|
57819
|
+
const toPage = pageInScope(scope, { nodeKey: request.query.toNodeKey, url: request.query.toUrl });
|
|
57820
|
+
if (!toPage) throw notFound("Site crawl page", request.query.toNodeKey ?? request.query.toUrl);
|
|
57821
|
+
const reference = (row) => ({
|
|
57822
|
+
nodeKey: row.nodeKey,
|
|
57823
|
+
url: row.url,
|
|
57824
|
+
path: row.path
|
|
57825
|
+
});
|
|
57826
|
+
const predecessor = /* @__PURE__ */ new Map();
|
|
57827
|
+
const visited = /* @__PURE__ */ new Set([fromPage.nodeKey]);
|
|
57828
|
+
let frontier = [fromPage.nodeKey];
|
|
57829
|
+
let found = fromPage.nodeKey === toPage.nodeKey;
|
|
57830
|
+
let truncated = false;
|
|
57831
|
+
for (let depth = 0; depth < maxDepth && frontier.length > 0 && !found; depth += 1) {
|
|
57832
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and44(
|
|
57833
|
+
eq56(siteCrawlEdges.projectId, scope.projectId),
|
|
57834
|
+
eq56(siteCrawlEdges.runId, scope.runId),
|
|
57835
|
+
eq56(siteCrawlEdges.attemptId, scope.attemptId),
|
|
57836
|
+
eq56(siteCrawlEdges.internal, true),
|
|
57837
|
+
eq56(siteCrawlEdges.followable, true),
|
|
57838
|
+
eq56(siteCrawlEdges.relation, "anchor"),
|
|
57839
|
+
isNotNull3(siteCrawlEdges.targetNodeKey),
|
|
57840
|
+
inArray19(siteCrawlEdges.sourceNodeKey, frontier)
|
|
57841
|
+
)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(SITE_HEALTH_PATH_MAX_VISITED_NODES + 1).all();
|
|
57842
|
+
if (candidates.length > SITE_HEALTH_PATH_MAX_VISITED_NODES) truncated = true;
|
|
57843
|
+
const candidateTargetKeys = [...new Set(candidates.map((edge) => edge.targetNodeKey).filter((nodeKey) => nodeKey != null))];
|
|
57844
|
+
const persistedTargetKeys = new Set(candidateTargetKeys.length === 0 ? [] : app.db.select({ nodeKey: siteCrawlPages.nodeKey }).from(siteCrawlPages).where(and44(
|
|
57845
|
+
eq56(siteCrawlPages.projectId, scope.projectId),
|
|
57846
|
+
eq56(siteCrawlPages.runId, scope.runId),
|
|
57847
|
+
eq56(siteCrawlPages.attemptId, scope.attemptId),
|
|
57848
|
+
inArray19(siteCrawlPages.nodeKey, candidateTargetKeys)
|
|
57849
|
+
)).all().map((row) => row.nodeKey));
|
|
57850
|
+
const next = /* @__PURE__ */ new Set();
|
|
57851
|
+
for (const edge of candidates.slice(0, SITE_HEALTH_PATH_MAX_VISITED_NODES)) {
|
|
57852
|
+
if (!edge.targetNodeKey || !persistedTargetKeys.has(edge.targetNodeKey) || visited.has(edge.targetNodeKey)) continue;
|
|
57853
|
+
if (visited.size >= SITE_HEALTH_PATH_MAX_VISITED_NODES) {
|
|
57854
|
+
truncated = true;
|
|
57855
|
+
break;
|
|
57856
|
+
}
|
|
57857
|
+
visited.add(edge.targetNodeKey);
|
|
57858
|
+
predecessor.set(edge.targetNodeKey, { nodeKey: edge.sourceNodeKey, edge });
|
|
57859
|
+
next.add(edge.targetNodeKey);
|
|
57860
|
+
if (edge.targetNodeKey === toPage.nodeKey) {
|
|
57861
|
+
found = true;
|
|
57862
|
+
break;
|
|
57863
|
+
}
|
|
57864
|
+
}
|
|
57865
|
+
frontier = [...next].sort();
|
|
57866
|
+
}
|
|
57867
|
+
if (!found) {
|
|
57868
|
+
return {
|
|
57869
|
+
project: project.name,
|
|
57870
|
+
runId: snapshot.runId,
|
|
57871
|
+
complete: snapshot.complete,
|
|
57872
|
+
termination: snapshot.termination,
|
|
57873
|
+
state: truncated || frontier.length > 0 ? "truncated" : "unreachable",
|
|
57874
|
+
from: reference(fromPage),
|
|
57875
|
+
to: reference(toPage),
|
|
57876
|
+
maxDepth,
|
|
57877
|
+
visitedNodes: visited.size,
|
|
57878
|
+
nodes: [],
|
|
57879
|
+
edges: []
|
|
57880
|
+
};
|
|
57881
|
+
}
|
|
57882
|
+
const pathKeys = [toPage.nodeKey];
|
|
57883
|
+
const pathEdges = [];
|
|
57884
|
+
while (pathKeys[0] !== fromPage.nodeKey) {
|
|
57885
|
+
const step = predecessor.get(pathKeys[0]);
|
|
57886
|
+
if (!step) break;
|
|
57887
|
+
pathEdges.unshift(step.edge);
|
|
57888
|
+
pathKeys.unshift(step.nodeKey);
|
|
57889
|
+
}
|
|
57890
|
+
const pathRows = app.db.select().from(siteCrawlPages).where(and44(
|
|
57891
|
+
eq56(siteCrawlPages.projectId, scope.projectId),
|
|
57892
|
+
eq56(siteCrawlPages.runId, scope.runId),
|
|
57893
|
+
eq56(siteCrawlPages.attemptId, scope.attemptId),
|
|
57894
|
+
inArray19(siteCrawlPages.nodeKey, pathKeys)
|
|
57895
|
+
)).all();
|
|
57896
|
+
const pageByKey = new Map(pathRows.map((row) => [row.nodeKey, row]));
|
|
57897
|
+
return {
|
|
57898
|
+
project: project.name,
|
|
57899
|
+
runId: snapshot.runId,
|
|
57900
|
+
complete: snapshot.complete,
|
|
57901
|
+
termination: snapshot.termination,
|
|
57902
|
+
state: "found",
|
|
57903
|
+
from: reference(fromPage),
|
|
57904
|
+
to: reference(toPage),
|
|
57905
|
+
maxDepth,
|
|
57906
|
+
visitedNodes: visited.size,
|
|
57907
|
+
nodes: pathKeys.map((key) => mapCrawlPage(pageByKey.get(key))),
|
|
57908
|
+
edges: pathEdges.map(mapCrawlEdge)
|
|
57909
|
+
};
|
|
57910
|
+
});
|
|
57911
|
+
app.get("/projects/:name/technical-aeo/changes", async (request) => {
|
|
57912
|
+
const project = resolveProject(app.db, request.params.name);
|
|
57913
|
+
const scopeFilter = request.query.scope ?? "all";
|
|
57914
|
+
if (scopeFilter !== "all" && scopeFilter !== "pages" && scopeFilter !== "links") {
|
|
57915
|
+
throw validationError("scope must be all, pages, or links");
|
|
57916
|
+
}
|
|
57917
|
+
const changeFilter = request.query.change ?? "all";
|
|
57918
|
+
if (changeFilter !== "all" && changeFilter !== "added" && changeFilter !== "removed" && changeFilter !== "changed") {
|
|
57919
|
+
throw validationError("change must be all, added, removed, or changed");
|
|
57920
|
+
}
|
|
57921
|
+
const completeSnapshotFilters = [
|
|
57922
|
+
eq56(siteCrawlSnapshots.projectId, project.id),
|
|
57923
|
+
eq56(runs.projectId, project.id),
|
|
57924
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
57925
|
+
eq56(runs.status, RunStatuses.completed),
|
|
57926
|
+
eq56(siteCrawlSnapshots.complete, true),
|
|
57927
|
+
notProbeRun()
|
|
57928
|
+
];
|
|
57929
|
+
const afterTarget = request.query.toRunId ? resolveCrawl(project.id, request.query.toRunId) : app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq56(siteCrawlSnapshots.runId, runs.id)).where(and44(...completeSnapshotFilters)).orderBy(desc24(siteCrawlSnapshots.createdAt), desc24(siteCrawlSnapshots.runId)).limit(1).get();
|
|
57930
|
+
if (request.query.toRunId && !afterTarget) throw notFound("Site crawl run", request.query.toRunId);
|
|
57931
|
+
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq56(siteCrawlSnapshots.runId, runs.id)).where(and44(
|
|
57932
|
+
...completeSnapshotFilters,
|
|
57933
|
+
or11(
|
|
57934
|
+
lt8(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
57935
|
+
and44(
|
|
57936
|
+
eq56(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
57937
|
+
lt8(siteCrawlSnapshots.runId, afterTarget.snapshot.runId)
|
|
57938
|
+
)
|
|
57939
|
+
)
|
|
57940
|
+
)).orderBy(desc24(siteCrawlSnapshots.createdAt), desc24(siteCrawlSnapshots.runId)).limit(1).get() : void 0;
|
|
57941
|
+
if (request.query.fromRunId && !beforeTarget) throw notFound("Site crawl run", request.query.fromRunId);
|
|
57942
|
+
if (!afterTarget) {
|
|
57943
|
+
return { project: project.name, state: "unavailable", reason: "no-crawl", fromRunId: null, toRunId: null };
|
|
57944
|
+
}
|
|
57945
|
+
if (!beforeTarget) {
|
|
57946
|
+
return {
|
|
57947
|
+
project: project.name,
|
|
57948
|
+
state: "unavailable",
|
|
57949
|
+
reason: "insufficient-history",
|
|
57950
|
+
fromRunId: null,
|
|
57951
|
+
toRunId: afterTarget.snapshot.runId
|
|
57952
|
+
};
|
|
57953
|
+
}
|
|
57954
|
+
const beforeSnapshot = beforeTarget.snapshot;
|
|
57955
|
+
const afterSnapshot = afterTarget.snapshot;
|
|
57956
|
+
if (beforeSnapshot.runId === afterSnapshot.runId) {
|
|
57957
|
+
throw validationError("fromRunId and toRunId must identify different crawls");
|
|
57958
|
+
}
|
|
57959
|
+
const crawlOrder = beforeSnapshot.createdAt.localeCompare(afterSnapshot.createdAt);
|
|
57960
|
+
if (crawlOrder > 0 || crawlOrder === 0 && beforeSnapshot.runId >= afterSnapshot.runId) {
|
|
57961
|
+
throw validationError("fromRunId must identify a crawl earlier than toRunId");
|
|
57962
|
+
}
|
|
57963
|
+
if (!beforeSnapshot.complete || !afterSnapshot.complete || beforeTarget.runStatus !== RunStatuses.completed || afterTarget.runStatus !== RunStatuses.completed) {
|
|
57964
|
+
return {
|
|
57965
|
+
project: project.name,
|
|
57966
|
+
state: "unavailable",
|
|
57967
|
+
reason: "partial-not-comparable",
|
|
57968
|
+
fromRunId: beforeSnapshot.runId,
|
|
57969
|
+
toRunId: afterSnapshot.runId
|
|
57970
|
+
};
|
|
57971
|
+
}
|
|
57972
|
+
const beforeScope = detailScopeFor(project.id, beforeSnapshot);
|
|
57973
|
+
const afterScope = detailScopeFor(project.id, afterSnapshot);
|
|
57974
|
+
if (!beforeScope || !afterScope) {
|
|
57975
|
+
return {
|
|
57976
|
+
project: project.name,
|
|
57977
|
+
state: "unavailable",
|
|
57978
|
+
reason: "details-unavailable",
|
|
57979
|
+
fromRunId: beforeSnapshot.runId,
|
|
57980
|
+
toRunId: afterSnapshot.runId
|
|
57981
|
+
};
|
|
57982
|
+
}
|
|
57983
|
+
const versions = {
|
|
57984
|
+
crawlSchema: [beforeSnapshot.crawlSchemaVersion, afterSnapshot.crawlSchemaVersion],
|
|
57985
|
+
normalization: [beforeSnapshot.normalizationVersion, afterSnapshot.normalizationVersion],
|
|
57986
|
+
indexability: [beforeSnapshot.indexabilityVersion, afterSnapshot.indexabilityVersion],
|
|
57987
|
+
linkScore: [beforeSnapshot.linkScoreVersion, afterSnapshot.linkScoreVersion]
|
|
57988
|
+
};
|
|
57989
|
+
const mismatchedVersions = Object.entries(versions).filter(([, pair]) => pair[0] !== pair[1]).map(([name]) => name);
|
|
57990
|
+
if (mismatchedVersions.length > 0) {
|
|
57991
|
+
return {
|
|
57992
|
+
project: project.name,
|
|
57993
|
+
state: "incompatible",
|
|
57994
|
+
reason: "incompatible-versions",
|
|
57995
|
+
fromRunId: beforeSnapshot.runId,
|
|
57996
|
+
toRunId: afterSnapshot.runId,
|
|
57997
|
+
mismatchedVersions
|
|
57998
|
+
};
|
|
57999
|
+
}
|
|
58000
|
+
const cursor = decodeSiteHealthChangesCursor(request.query.cursor, {
|
|
58001
|
+
fromRunId: beforeSnapshot.runId,
|
|
58002
|
+
toRunId: afterSnapshot.runId,
|
|
58003
|
+
scope: scopeFilter,
|
|
58004
|
+
change: changeFilter
|
|
58005
|
+
});
|
|
58006
|
+
const loadSummary = () => {
|
|
58007
|
+
const includeAdded = changeFilter === "all" || changeFilter === "added";
|
|
58008
|
+
const includeRemoved = changeFilter === "all" || changeFilter === "removed";
|
|
58009
|
+
const includeChanged = changeFilter === "all" || changeFilter === "changed";
|
|
58010
|
+
const zero2 = () => ({ added: 0, removed: 0, changed: 0 });
|
|
58011
|
+
const summary2 = { pages: zero2(), links: zero2() };
|
|
58012
|
+
if (scopeFilter !== "links") {
|
|
58013
|
+
if (includeAdded || includeChanged) {
|
|
58014
|
+
const addedExpression = includeAdded ? sql21`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql21`0`;
|
|
58015
|
+
const changedExpression = includeChanged ? sql21`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${PAGE_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql21`0`;
|
|
58016
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql21`
|
|
58017
|
+
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
58018
|
+
FROM site_crawl_pages AS current
|
|
58019
|
+
LEFT JOIN site_crawl_pages AS previous
|
|
58020
|
+
ON previous.project_id = ${beforeScope.projectId}
|
|
58021
|
+
AND previous.run_id = ${beforeScope.runId}
|
|
58022
|
+
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
58023
|
+
AND previous.node_key = current.node_key
|
|
58024
|
+
WHERE current.project_id = ${afterScope.projectId}
|
|
58025
|
+
AND current.run_id = ${afterScope.runId}
|
|
58026
|
+
AND current.attempt_id = ${afterScope.attemptId}
|
|
58027
|
+
`);
|
|
58028
|
+
summary2.pages.added = Number(counts2.added);
|
|
58029
|
+
summary2.pages.changed = Number(counts2.changed);
|
|
58030
|
+
}
|
|
58031
|
+
if (includeRemoved) {
|
|
58032
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql21`
|
|
58033
|
+
SELECT COUNT(*) AS removed
|
|
58034
|
+
FROM site_crawl_pages AS previous
|
|
58035
|
+
LEFT JOIN site_crawl_pages AS current
|
|
58036
|
+
ON current.project_id = ${afterScope.projectId}
|
|
58037
|
+
AND current.run_id = ${afterScope.runId}
|
|
58038
|
+
AND current.attempt_id = ${afterScope.attemptId}
|
|
58039
|
+
AND current.node_key = previous.node_key
|
|
58040
|
+
WHERE previous.project_id = ${beforeScope.projectId}
|
|
58041
|
+
AND previous.run_id = ${beforeScope.runId}
|
|
58042
|
+
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
58043
|
+
AND current.id IS NULL
|
|
58044
|
+
`);
|
|
58045
|
+
summary2.pages.removed = Number(counts2.removed);
|
|
58046
|
+
}
|
|
58047
|
+
}
|
|
58048
|
+
if (scopeFilter !== "pages") {
|
|
58049
|
+
if (includeAdded || includeChanged) {
|
|
58050
|
+
const addedExpression = includeAdded ? sql21`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql21`0`;
|
|
58051
|
+
const changedExpression = includeChanged ? sql21`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${LINK_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql21`0`;
|
|
58052
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql21`
|
|
58053
|
+
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
58054
|
+
FROM site_crawl_edges AS current
|
|
58055
|
+
LEFT JOIN site_crawl_edges AS previous
|
|
58056
|
+
ON previous.project_id = ${beforeScope.projectId}
|
|
58057
|
+
AND previous.run_id = ${beforeScope.runId}
|
|
58058
|
+
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
58059
|
+
AND previous.internal = 1
|
|
58060
|
+
AND previous.edge_key = current.edge_key
|
|
58061
|
+
WHERE current.project_id = ${afterScope.projectId}
|
|
58062
|
+
AND current.run_id = ${afterScope.runId}
|
|
58063
|
+
AND current.attempt_id = ${afterScope.attemptId}
|
|
58064
|
+
AND current.internal = 1
|
|
58065
|
+
`);
|
|
58066
|
+
summary2.links.added = Number(counts2.added);
|
|
58067
|
+
summary2.links.changed = Number(counts2.changed);
|
|
58068
|
+
}
|
|
58069
|
+
if (includeRemoved) {
|
|
58070
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql21`
|
|
58071
|
+
SELECT COUNT(*) AS removed
|
|
58072
|
+
FROM site_crawl_edges AS previous
|
|
58073
|
+
LEFT JOIN site_crawl_edges AS current
|
|
58074
|
+
ON current.project_id = ${afterScope.projectId}
|
|
58075
|
+
AND current.run_id = ${afterScope.runId}
|
|
58076
|
+
AND current.attempt_id = ${afterScope.attemptId}
|
|
58077
|
+
AND current.internal = 1
|
|
58078
|
+
AND current.edge_key = previous.edge_key
|
|
58079
|
+
WHERE previous.project_id = ${beforeScope.projectId}
|
|
58080
|
+
AND previous.run_id = ${beforeScope.runId}
|
|
58081
|
+
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
58082
|
+
AND previous.internal = 1
|
|
58083
|
+
AND current.id IS NULL
|
|
58084
|
+
`);
|
|
58085
|
+
summary2.links.removed = Number(counts2.removed);
|
|
58086
|
+
}
|
|
58087
|
+
}
|
|
58088
|
+
return summary2;
|
|
58089
|
+
};
|
|
58090
|
+
const summary = cursor ? null : loadSummary();
|
|
58091
|
+
const limit = parseBoundedLimit(request.query.limit, SITE_HEALTH_CHANGES_DEFAULT_LIMIT, SITE_HEALTH_CHANGES_MAX_LIMIT);
|
|
58092
|
+
const sumCounts = (counts2) => counts2.added + counts2.removed + counts2.changed;
|
|
58093
|
+
const total = summary ? sumCounts(summary.pages) + sumCounts(summary.links) : null;
|
|
58094
|
+
const collectEntity = (entity, afterKey, wanted2) => {
|
|
58095
|
+
if (wanted2 <= 0) return [];
|
|
58096
|
+
const includeAfter = changeFilter === "all" || changeFilter === "added" || changeFilter === "changed";
|
|
58097
|
+
const includeRemoved = changeFilter === "all" || changeFilter === "removed";
|
|
58098
|
+
const table = entity === "page" ? "site_crawl_pages" : "site_crawl_edges";
|
|
58099
|
+
const keyColumn = entity === "page" ? "node_key" : "edge_key";
|
|
58100
|
+
const internalCurrent = entity === "link" ? sql21`AND current.internal = 1` : sql21``;
|
|
58101
|
+
const internalPreviousJoin = entity === "link" ? sql21`AND previous.internal = 1` : sql21``;
|
|
58102
|
+
const internalPrevious = entity === "link" ? sql21`AND previous.internal = 1` : sql21``;
|
|
58103
|
+
const internalCurrentJoin = entity === "link" ? sql21`AND current.internal = 1` : sql21``;
|
|
58104
|
+
const difference2 = entity === "page" ? PAGE_CHANGE_PREDICATE : LINK_CHANGE_PREDICATE;
|
|
58105
|
+
const afterKind = changeFilter === "added" ? sql21`previous.id IS NULL` : changeFilter === "changed" ? sql21`previous.id IS NOT NULL AND ${difference2}` : sql21`previous.id IS NULL OR (previous.id IS NOT NULL AND ${difference2})`;
|
|
58106
|
+
const afterRows = includeAfter ? app.db.all(sql21`
|
|
58107
|
+
SELECT ${entity} AS entity,
|
|
58108
|
+
CASE WHEN previous.id IS NULL THEN 'added' ELSE 'changed' END AS change,
|
|
58109
|
+
current.${sql21.raw(keyColumn)} AS key
|
|
58110
|
+
FROM ${sql21.raw(table)} AS current
|
|
58111
|
+
LEFT JOIN ${sql21.raw(table)} AS previous
|
|
58112
|
+
ON previous.project_id = ${beforeScope.projectId}
|
|
58113
|
+
AND previous.run_id = ${beforeScope.runId}
|
|
58114
|
+
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
58115
|
+
${internalPreviousJoin}
|
|
58116
|
+
AND previous.${sql21.raw(keyColumn)} = current.${sql21.raw(keyColumn)}
|
|
58117
|
+
WHERE current.project_id = ${afterScope.projectId}
|
|
58118
|
+
AND current.run_id = ${afterScope.runId}
|
|
58119
|
+
AND current.attempt_id = ${afterScope.attemptId}
|
|
58120
|
+
${internalCurrent}
|
|
58121
|
+
AND current.${sql21.raw(keyColumn)} > ${afterKey}
|
|
58122
|
+
AND (${afterKind})
|
|
58123
|
+
ORDER BY current.${sql21.raw(keyColumn)} ASC
|
|
58124
|
+
LIMIT ${wanted2}
|
|
58125
|
+
`) : [];
|
|
58126
|
+
const removedRows = includeRemoved ? app.db.all(sql21`
|
|
58127
|
+
SELECT ${entity} AS entity, 'removed' AS change, previous.${sql21.raw(keyColumn)} AS key
|
|
58128
|
+
FROM ${sql21.raw(table)} AS previous
|
|
58129
|
+
LEFT JOIN ${sql21.raw(table)} AS current
|
|
58130
|
+
ON current.project_id = ${afterScope.projectId}
|
|
58131
|
+
AND current.run_id = ${afterScope.runId}
|
|
58132
|
+
AND current.attempt_id = ${afterScope.attemptId}
|
|
58133
|
+
${internalCurrentJoin}
|
|
58134
|
+
AND current.${sql21.raw(keyColumn)} = previous.${sql21.raw(keyColumn)}
|
|
58135
|
+
WHERE previous.project_id = ${beforeScope.projectId}
|
|
58136
|
+
AND previous.run_id = ${beforeScope.runId}
|
|
58137
|
+
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
58138
|
+
${internalPrevious}
|
|
58139
|
+
AND previous.${sql21.raw(keyColumn)} > ${afterKey}
|
|
58140
|
+
AND current.id IS NULL
|
|
58141
|
+
ORDER BY previous.${sql21.raw(keyColumn)} ASC
|
|
58142
|
+
LIMIT ${wanted2}
|
|
58143
|
+
`) : [];
|
|
58144
|
+
const merged = [];
|
|
58145
|
+
let afterIndex = 0;
|
|
58146
|
+
let removedIndex = 0;
|
|
58147
|
+
while (merged.length < wanted2 && (afterIndex < afterRows.length || removedIndex < removedRows.length)) {
|
|
58148
|
+
const afterRow = afterRows.at(afterIndex);
|
|
58149
|
+
const removedRow = removedRows.at(removedIndex);
|
|
58150
|
+
if (!removedRow || afterRow && compareChangeKeys(afterRow, removedRow) <= 0) {
|
|
58151
|
+
merged.push(afterRow);
|
|
58152
|
+
afterIndex += 1;
|
|
58153
|
+
} else {
|
|
58154
|
+
merged.push(removedRow);
|
|
58155
|
+
removedIndex += 1;
|
|
58156
|
+
}
|
|
58157
|
+
}
|
|
58158
|
+
return merged;
|
|
58159
|
+
};
|
|
58160
|
+
const keyRows = [];
|
|
58161
|
+
const wanted = limit + 1;
|
|
58162
|
+
if (scopeFilter !== "links" && cursor?.entity !== "link") {
|
|
58163
|
+
keyRows.push(...collectEntity("page", cursor?.entity === "page" ? cursor.key : "", wanted));
|
|
58164
|
+
}
|
|
58165
|
+
if (keyRows.length < wanted && scopeFilter !== "pages") {
|
|
58166
|
+
keyRows.push(...collectEntity("link", cursor?.entity === "link" ? cursor.key : "", wanted - keyRows.length));
|
|
58167
|
+
}
|
|
58168
|
+
const visibleKeys = keyRows.slice(0, limit);
|
|
58169
|
+
const pageKeys = visibleKeys.filter((row) => row.entity === "page").map((row) => row.key);
|
|
58170
|
+
const linkKeys = visibleKeys.filter((row) => row.entity === "link").map((row) => row.key);
|
|
58171
|
+
const pageRowsFor = (scope) => pageKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and44(
|
|
58172
|
+
eq56(siteCrawlPages.projectId, scope.projectId),
|
|
58173
|
+
eq56(siteCrawlPages.runId, scope.runId),
|
|
58174
|
+
eq56(siteCrawlPages.attemptId, scope.attemptId),
|
|
58175
|
+
inArray19(siteCrawlPages.nodeKey, pageKeys)
|
|
58176
|
+
)).all();
|
|
58177
|
+
const edgeRowsFor = (scope) => linkKeys.length === 0 ? [] : app.db.select().from(siteCrawlEdges).where(and44(
|
|
58178
|
+
eq56(siteCrawlEdges.projectId, scope.projectId),
|
|
58179
|
+
eq56(siteCrawlEdges.runId, scope.runId),
|
|
58180
|
+
eq56(siteCrawlEdges.attemptId, scope.attemptId),
|
|
58181
|
+
eq56(siteCrawlEdges.internal, true),
|
|
58182
|
+
inArray19(siteCrawlEdges.edgeKey, linkKeys)
|
|
58183
|
+
)).all();
|
|
58184
|
+
const beforePages = new Map(pageRowsFor(beforeScope).map((row) => [row.nodeKey, mapCrawlPage(row)]));
|
|
58185
|
+
const afterPages = new Map(pageRowsFor(afterScope).map((row) => [row.nodeKey, mapCrawlPage(row)]));
|
|
58186
|
+
const beforeLinks = new Map(edgeRowsFor(beforeScope).map((row) => [row.edgeKey, mapCrawlEdge(row)]));
|
|
58187
|
+
const afterLinks = new Map(edgeRowsFor(afterScope).map((row) => [row.edgeKey, mapCrawlEdge(row)]));
|
|
58188
|
+
const changes = visibleKeys.map((row) => {
|
|
58189
|
+
if (row.entity === "page") {
|
|
58190
|
+
const before2 = beforePages.get(row.key) ?? null;
|
|
58191
|
+
const after2 = afterPages.get(row.key) ?? null;
|
|
58192
|
+
const pageChangedFields = changedFields(before2, after2, PAGE_CHANGE_FIELDS);
|
|
58193
|
+
if (before2 && after2 && before2.healthState !== after2.healthState) pageChangedFields.push("healthState");
|
|
58194
|
+
return {
|
|
58195
|
+
entity: "page",
|
|
58196
|
+
change: row.change,
|
|
58197
|
+
key: row.key,
|
|
58198
|
+
changedFields: pageChangedFields,
|
|
58199
|
+
before: before2,
|
|
58200
|
+
after: after2
|
|
58201
|
+
};
|
|
58202
|
+
}
|
|
58203
|
+
const before = beforeLinks.get(row.key) ?? null;
|
|
58204
|
+
const after = afterLinks.get(row.key) ?? null;
|
|
58205
|
+
return {
|
|
58206
|
+
entity: "link",
|
|
58207
|
+
change: row.change,
|
|
58208
|
+
key: row.key,
|
|
58209
|
+
changedFields: changedFields(before, after, LINK_CHANGE_FIELDS),
|
|
58210
|
+
before,
|
|
58211
|
+
after
|
|
58212
|
+
};
|
|
58213
|
+
});
|
|
58214
|
+
return {
|
|
58215
|
+
project: project.name,
|
|
58216
|
+
state: "ready",
|
|
58217
|
+
fromRunId: beforeSnapshot.runId,
|
|
58218
|
+
toRunId: afterSnapshot.runId,
|
|
58219
|
+
versions: {
|
|
58220
|
+
crawlSchema: afterSnapshot.crawlSchemaVersion,
|
|
58221
|
+
normalization: afterSnapshot.normalizationVersion,
|
|
58222
|
+
indexability: afterSnapshot.indexabilityVersion,
|
|
58223
|
+
linkScore: afterSnapshot.linkScoreVersion
|
|
58224
|
+
},
|
|
58225
|
+
filters: { scope: scopeFilter, change: changeFilter },
|
|
58226
|
+
summaryState: cursor ? "omitted-on-continuation" : "exact",
|
|
58227
|
+
summary,
|
|
58228
|
+
total,
|
|
58229
|
+
nextCursor: keyRows.length > limit && visibleKeys.length > 0 ? encodeSiteHealthChangesCursor({
|
|
58230
|
+
v: 1,
|
|
58231
|
+
fromRunId: beforeSnapshot.runId,
|
|
58232
|
+
toRunId: afterSnapshot.runId,
|
|
58233
|
+
scope: scopeFilter,
|
|
58234
|
+
change: changeFilter,
|
|
58235
|
+
entity: visibleKeys.at(-1).entity,
|
|
58236
|
+
key: visibleKeys.at(-1).key
|
|
58237
|
+
}) : null,
|
|
58238
|
+
changes
|
|
58239
|
+
};
|
|
58240
|
+
});
|
|
58241
|
+
app.get("/projects/:name/technical-aeo/crawl/pages/audit", async (request) => {
|
|
58242
|
+
const project = resolveProject(app.db, request.params.name);
|
|
58243
|
+
const scalarQueryString = (value, name) => {
|
|
58244
|
+
if (value === void 0) return void 0;
|
|
58245
|
+
if (typeof value !== "string") throw validationError(`${name} must be provided once`);
|
|
58246
|
+
return value;
|
|
58247
|
+
};
|
|
58248
|
+
const runId = scalarQueryString(request.query.runId, "runId");
|
|
58249
|
+
const nodeKey = scalarQueryString(request.query.nodeKey, "nodeKey")?.trim();
|
|
58250
|
+
const url = scalarQueryString(request.query.url, "url")?.trim();
|
|
58251
|
+
if (Number(Boolean(nodeKey)) + Number(Boolean(url)) !== 1) {
|
|
58252
|
+
throw validationError("Provide exactly one of nodeKey or url");
|
|
58253
|
+
}
|
|
58254
|
+
const target = resolveCrawl(project.id, runId);
|
|
58255
|
+
if (!target) {
|
|
58256
|
+
if (runId) throw notFound("Site crawl run", runId);
|
|
58257
|
+
return { state: "no-crawl", project: project.name, runId: null };
|
|
58258
|
+
}
|
|
58259
|
+
const snapshot = target.snapshot;
|
|
58260
|
+
const provenance = {
|
|
58261
|
+
project: project.name,
|
|
58262
|
+
runId: snapshot.runId,
|
|
58263
|
+
complete: snapshot.complete,
|
|
58264
|
+
termination: snapshot.termination
|
|
58265
|
+
};
|
|
58266
|
+
const scope = detailScopeFor(project.id, snapshot);
|
|
58267
|
+
if (!scope) return { state: "details-unavailable", ...provenance };
|
|
58268
|
+
const page = pageInScope(scope, nodeKey ? { nodeKey } : { url });
|
|
58269
|
+
if (!page) return { state: "not-found", ...provenance };
|
|
58270
|
+
const identity = {
|
|
58271
|
+
nodeKey: page.nodeKey,
|
|
58272
|
+
url: page.url,
|
|
58273
|
+
auditState: page.auditState
|
|
58274
|
+
};
|
|
58275
|
+
if (page.auditScore == null) {
|
|
58276
|
+
return {
|
|
58277
|
+
state: "not-audited",
|
|
58278
|
+
...provenance,
|
|
58279
|
+
...identity,
|
|
58280
|
+
auditScore: null,
|
|
58281
|
+
factors: [],
|
|
58282
|
+
criticalDefects: []
|
|
58283
|
+
};
|
|
58284
|
+
}
|
|
58285
|
+
return {
|
|
58286
|
+
state: "ready",
|
|
58287
|
+
...provenance,
|
|
58288
|
+
...identity,
|
|
58289
|
+
auditScore: page.auditScore,
|
|
58290
|
+
...mapCrawlPageAuditEvidence(page)
|
|
58291
|
+
};
|
|
58292
|
+
});
|
|
55666
58293
|
app.get("/projects/:name/technical-aeo/crawl/pages", async (request) => {
|
|
55667
58294
|
const project = resolveProject(app.db, request.params.name);
|
|
55668
58295
|
const target = resolveCrawl(project.id, request.query.runId);
|
|
@@ -55675,16 +58302,16 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
55675
58302
|
return { project: project.name, hasCrawlData: true, runId: snapshot.runId, total: 0, nextCursor: null, pages: [] };
|
|
55676
58303
|
}
|
|
55677
58304
|
const filters = [
|
|
55678
|
-
|
|
55679
|
-
|
|
55680
|
-
|
|
58305
|
+
eq56(siteCrawlPages.projectId, project.id),
|
|
58306
|
+
eq56(siteCrawlPages.runId, snapshot.runId),
|
|
58307
|
+
eq56(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
55681
58308
|
];
|
|
55682
58309
|
const inventoryEligible = parseBoolean2(request.query.inventoryEligible);
|
|
55683
|
-
if (inventoryEligible != null) filters.push(
|
|
55684
|
-
if (request.query.fetchState) filters.push(
|
|
55685
|
-
if (request.query.indexabilityState) filters.push(
|
|
55686
|
-
if (request.query.auditState) filters.push(
|
|
55687
|
-
const where =
|
|
58310
|
+
if (inventoryEligible != null) filters.push(eq56(siteCrawlPages.inventoryEligible, inventoryEligible));
|
|
58311
|
+
if (request.query.fetchState) filters.push(eq56(siteCrawlPages.fetchState, request.query.fetchState));
|
|
58312
|
+
if (request.query.indexabilityState) filters.push(eq56(siteCrawlPages.indexabilityState, request.query.indexabilityState));
|
|
58313
|
+
if (request.query.auditState) filters.push(eq56(siteCrawlPages.auditState, request.query.auditState));
|
|
58314
|
+
const where = and44(...filters);
|
|
55688
58315
|
const total = app.db.select({ value: count() }).from(siteCrawlPages).where(where).get()?.value ?? 0;
|
|
55689
58316
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
55690
58317
|
const offset = decodeCursor(request.query.cursor);
|
|
@@ -55717,10 +58344,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
55717
58344
|
url: siteCrawlPages.url,
|
|
55718
58345
|
inventoryEligible: siteCrawlPages.inventoryEligible,
|
|
55719
58346
|
fetchState: siteCrawlPages.fetchState
|
|
55720
|
-
}).from(siteCrawlPages).where(
|
|
55721
|
-
|
|
55722
|
-
|
|
55723
|
-
|
|
58347
|
+
}).from(siteCrawlPages).where(and44(
|
|
58348
|
+
eq56(siteCrawlPages.projectId, project.id),
|
|
58349
|
+
eq56(siteCrawlPages.runId, snapshot.runId),
|
|
58350
|
+
eq56(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
55724
58351
|
)).limit(MAX_STRUCTURE_SOURCE_ROWS + 1).all();
|
|
55725
58352
|
if (sourceRows.length > MAX_STRUCTURE_SOURCE_ROWS) {
|
|
55726
58353
|
throw validationError(`Persisted crawl exceeds the ${MAX_STRUCTURE_SOURCE_ROWS}-page structure limit`);
|
|
@@ -55738,7 +58365,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
55738
58365
|
};
|
|
55739
58366
|
child.pageCount++;
|
|
55740
58367
|
if (row.inventoryEligible) child.inventoryEligibleCount++;
|
|
55741
|
-
if (
|
|
58368
|
+
if (FETCHED_SITE_CRAWL_STATES.has(row.fetchState)) child.fetchedCount++;
|
|
55742
58369
|
if (structureHierarchyPath(row.path) === childPath) {
|
|
55743
58370
|
child.hasPage = true;
|
|
55744
58371
|
if (child.url == null || row.url > child.url) child.url = row.url;
|
|
@@ -55779,16 +58406,16 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
55779
58406
|
return { project: project.name, hasCrawlData: true, runId: snapshot.runId, total: 0, nextCursor: null, edges: [] };
|
|
55780
58407
|
}
|
|
55781
58408
|
const filters = [
|
|
55782
|
-
|
|
55783
|
-
|
|
55784
|
-
|
|
55785
|
-
|
|
58409
|
+
eq56(siteCrawlEdges.projectId, project.id),
|
|
58410
|
+
eq56(siteCrawlEdges.runId, snapshot.runId),
|
|
58411
|
+
eq56(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
58412
|
+
eq56(siteCrawlEdges.internal, true)
|
|
55786
58413
|
];
|
|
55787
|
-
if (request.query.sourceUrl) filters.push(
|
|
55788
|
-
if (request.query.targetUrl) filters.push(
|
|
58414
|
+
if (request.query.sourceUrl) filters.push(eq56(siteCrawlEdges.sourceUrl, request.query.sourceUrl));
|
|
58415
|
+
if (request.query.targetUrl) filters.push(eq56(siteCrawlEdges.targetUrl, request.query.targetUrl));
|
|
55789
58416
|
const followable = parseBoolean2(request.query.followable);
|
|
55790
|
-
if (followable != null) filters.push(
|
|
55791
|
-
const where =
|
|
58417
|
+
if (followable != null) filters.push(eq56(siteCrawlEdges.followable, followable));
|
|
58418
|
+
const where = and44(...filters);
|
|
55792
58419
|
const total = app.db.select({ value: count() }).from(siteCrawlEdges).where(where).get()?.value ?? 0;
|
|
55793
58420
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
55794
58421
|
const offset = decodeCursor(request.query.cursor);
|
|
@@ -55838,16 +58465,16 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
55838
58465
|
};
|
|
55839
58466
|
}
|
|
55840
58467
|
const scope = [
|
|
55841
|
-
|
|
55842
|
-
|
|
55843
|
-
|
|
55844
|
-
|
|
58468
|
+
eq56(siteCrawlEdges.projectId, project.id),
|
|
58469
|
+
eq56(siteCrawlEdges.runId, snapshot.runId),
|
|
58470
|
+
eq56(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
58471
|
+
eq56(siteCrawlEdges.internal, true)
|
|
55845
58472
|
];
|
|
55846
|
-
const inboundMatch = request.query.nodeKey && request.query.url ? or11(
|
|
55847
|
-
const outboundMatch = request.query.nodeKey && request.query.url ? or11(
|
|
58473
|
+
const inboundMatch = request.query.nodeKey && request.query.url ? or11(eq56(siteCrawlEdges.targetNodeKey, request.query.nodeKey), eq56(siteCrawlEdges.targetUrl, request.query.url)) : request.query.nodeKey ? eq56(siteCrawlEdges.targetNodeKey, request.query.nodeKey) : eq56(siteCrawlEdges.targetUrl, request.query.url);
|
|
58474
|
+
const outboundMatch = request.query.nodeKey && request.query.url ? or11(eq56(siteCrawlEdges.sourceNodeKey, request.query.nodeKey), eq56(siteCrawlEdges.sourceUrl, request.query.url)) : request.query.nodeKey ? eq56(siteCrawlEdges.sourceNodeKey, request.query.nodeKey) : eq56(siteCrawlEdges.sourceUrl, request.query.url);
|
|
55848
58475
|
const limit = parseBoundedLimit(request.query.limit, 50, 100);
|
|
55849
|
-
const inboundRows = app.db.select().from(siteCrawlEdges).where(
|
|
55850
|
-
const outboundRows = app.db.select().from(siteCrawlEdges).where(
|
|
58476
|
+
const inboundRows = app.db.select().from(siteCrawlEdges).where(and44(...scope, inboundMatch)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
58477
|
+
const outboundRows = app.db.select().from(siteCrawlEdges).where(and44(...scope, outboundMatch)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
55851
58478
|
return {
|
|
55852
58479
|
project: project.name,
|
|
55853
58480
|
hasCrawlData: true,
|
|
@@ -55878,11 +58505,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
55878
58505
|
if (!snapshot.attemptId) {
|
|
55879
58506
|
return { project: project.name, runId: snapshot.runId, state: "partial", checkDeadLinks: true, checked: snapshot.deadLinksChecked, found: snapshot.deadLinksFound, total: 0, nextCursor: null, deadLinks: [] };
|
|
55880
58507
|
}
|
|
55881
|
-
const where =
|
|
55882
|
-
|
|
55883
|
-
|
|
55884
|
-
|
|
55885
|
-
|
|
58508
|
+
const where = and44(
|
|
58509
|
+
eq56(siteCrawlFindings.projectId, project.id),
|
|
58510
|
+
eq56(siteCrawlFindings.runId, snapshot.runId),
|
|
58511
|
+
eq56(siteCrawlFindings.attemptId, snapshot.attemptId),
|
|
58512
|
+
eq56(siteCrawlFindings.findingType, "dead-link")
|
|
55886
58513
|
);
|
|
55887
58514
|
const total = app.db.select({ value: count() }).from(siteCrawlFindings).where(where).get()?.value ?? 0;
|
|
55888
58515
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
@@ -55925,12 +58552,12 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
55925
58552
|
status: runs.status,
|
|
55926
58553
|
identityKey: siteCrawlRunRequests.identityKey,
|
|
55927
58554
|
effectiveOptions: siteCrawlRunRequests.effectiveOptions
|
|
55928
|
-
}).from(runs).leftJoin(siteCrawlRunRequests,
|
|
55929
|
-
|
|
55930
|
-
|
|
55931
|
-
)).where(
|
|
55932
|
-
|
|
55933
|
-
|
|
58555
|
+
}).from(runs).leftJoin(siteCrawlRunRequests, and44(
|
|
58556
|
+
eq56(siteCrawlRunRequests.projectId, runs.projectId),
|
|
58557
|
+
eq56(siteCrawlRunRequests.runId, runs.id)
|
|
58558
|
+
)).where(and44(
|
|
58559
|
+
eq56(runs.projectId, project.id),
|
|
58560
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
55934
58561
|
inArray19(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
55935
58562
|
)).get();
|
|
55936
58563
|
if (existing) {
|
|
@@ -55948,7 +58575,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
55948
58575
|
);
|
|
55949
58576
|
}
|
|
55950
58577
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
55951
|
-
const runId =
|
|
58578
|
+
const runId = crypto44.randomUUID();
|
|
55952
58579
|
tx.insert(runs).values({
|
|
55953
58580
|
id: runId,
|
|
55954
58581
|
projectId: project.id,
|
|
@@ -55981,8 +58608,8 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
55981
58608
|
}
|
|
55982
58609
|
|
|
55983
58610
|
// ../api-routes/src/research.ts
|
|
55984
|
-
import
|
|
55985
|
-
import { and as
|
|
58611
|
+
import crypto45 from "crypto";
|
|
58612
|
+
import { and as and45, desc as desc25, eq as eq57 } from "drizzle-orm";
|
|
55986
58613
|
var sameLocation = (a, b) => a.label === b.label && a.city === b.city && a.region === b.region && a.country === b.country && a.timezone === b.timezone;
|
|
55987
58614
|
async function researchRoutes(app, opts) {
|
|
55988
58615
|
app.post("/projects/:name/research/runs", async (request, reply) => {
|
|
@@ -56008,19 +58635,19 @@ async function researchRoutes(app, opts) {
|
|
|
56008
58635
|
if (!adapter.modelValidationPattern.test(resolvedModel)) throw validationError(`Invalid resolved model "${resolvedModel}" for provider "${providerName}".`, { provider: providerName, model: resolvedModel, hint: adapter.modelValidationHint });
|
|
56009
58636
|
if (new Set(input.queries.map((query) => query.toLocaleLowerCase())).size !== input.queries.length) throw validationError("Research queries must be unique within a batch.");
|
|
56010
58637
|
const normalized = { queries: input.queries, provider: providerName, model: requestedModel, location: location ?? null };
|
|
56011
|
-
const requestHash2 =
|
|
58638
|
+
const requestHash2 = crypto45.createHash("sha256").update(JSON.stringify(normalized)).digest("hex");
|
|
56012
58639
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
56013
58640
|
const decision = app.db.transaction((tx) => {
|
|
56014
58641
|
if (input.idempotencyKey) {
|
|
56015
|
-
const existing = tx.select().from(researchRuns).where(
|
|
58642
|
+
const existing = tx.select().from(researchRuns).where(and45(eq57(researchRuns.projectId, project.id), eq57(researchRuns.idempotencyKey, input.idempotencyKey))).get();
|
|
56016
58643
|
if (existing) {
|
|
56017
58644
|
if (existing.requestHash !== requestHash2) throw alreadyExists("Research idempotency key", input.idempotencyKey);
|
|
56018
58645
|
return { reused: true, id: existing.id, shouldDispatch: existing.status === ResearchRunStatuses.queued };
|
|
56019
58646
|
}
|
|
56020
58647
|
}
|
|
56021
|
-
const id =
|
|
58648
|
+
const id = crypto45.randomUUID();
|
|
56022
58649
|
tx.insert(researchRuns).values({ id, projectId: project.id, status: ResearchRunStatuses.queued, provider: providerName, requestedModel, resolvedModel, location: location ?? null, totalQueries: input.queries.length, idempotencyKey: input.idempotencyKey ?? null, requestHash: input.idempotencyKey ? requestHash2 : null, createdAt: now }).run();
|
|
56023
|
-
for (const [position, query] of input.queries.entries()) tx.insert(researchRunQueries).values({ id:
|
|
58650
|
+
for (const [position, query] of input.queries.entries()) tx.insert(researchRunQueries).values({ id: crypto45.randomUUID(), researchRunId: id, position, queryText: query, status: ResearchQueryStatuses.queued, requestedModel, resolvedModel, groundingSources: [], citedDomains: [], searchQueries: [], createdAt: now }).run();
|
|
56024
58651
|
writeAuditLog(tx, { projectId: project.id, actor: "api", action: "research.created", entityType: "research_run", entityId: id });
|
|
56025
58652
|
return { reused: false, id, shouldDispatch: true };
|
|
56026
58653
|
});
|
|
@@ -56033,7 +58660,7 @@ async function researchRoutes(app, opts) {
|
|
|
56033
58660
|
const project = resolveProject(app.db, request.params.name);
|
|
56034
58661
|
const requested = Number.parseInt(request.query.limit ?? "", 10);
|
|
56035
58662
|
const limit = Number.isInteger(requested) && requested > 0 ? Math.min(requested, 100) : 20;
|
|
56036
|
-
const runs2 = app.db.select().from(researchRuns).where(
|
|
58663
|
+
const runs2 = app.db.select().from(researchRuns).where(eq57(researchRuns.projectId, project.id)).orderBy(desc25(researchRuns.createdAt)).limit(limit).all().map(serializeRun);
|
|
56037
58664
|
return { runs: runs2 };
|
|
56038
58665
|
});
|
|
56039
58666
|
app.get("/projects/:name/research/runs/:runId", async (request) => {
|
|
@@ -56042,9 +58669,9 @@ async function researchRoutes(app, opts) {
|
|
|
56042
58669
|
});
|
|
56043
58670
|
}
|
|
56044
58671
|
function getDetail(app, projectId, id) {
|
|
56045
|
-
const row = app.db.select().from(researchRuns).where(
|
|
58672
|
+
const row = app.db.select().from(researchRuns).where(and45(eq57(researchRuns.id, id), eq57(researchRuns.projectId, projectId))).get();
|
|
56046
58673
|
if (!row) throw notFound("Research run", id);
|
|
56047
|
-
const queries2 = app.db.select().from(researchRunQueries).where(
|
|
58674
|
+
const queries2 = app.db.select().from(researchRunQueries).where(eq57(researchRunQueries.researchRunId, id)).orderBy(researchRunQueries.position).all().map(serializeQuery);
|
|
56048
58675
|
return { ...serializeRun(row), queries: queries2 };
|
|
56049
58676
|
}
|
|
56050
58677
|
function serializeRun(row) {
|
|
@@ -56254,6 +58881,11 @@ async function apiRoutes(app, opts) {
|
|
|
56254
58881
|
vercelTrafficCredentialStore: opts.vercelTrafficCredentialStore,
|
|
56255
58882
|
pullVercelTrafficEvents: opts.pullVercelTrafficEvents,
|
|
56256
58883
|
vercelSyncDeadlineMs: opts.vercelSyncDeadlineMs,
|
|
58884
|
+
cloudflareTrafficCredentialStore: opts.cloudflareTrafficCredentialStore,
|
|
58885
|
+
cloudflareTrafficIngestUrl: opts.cloudflareTrafficIngestUrl,
|
|
58886
|
+
cloudflareIngestRateLimitMax: opts.cloudflareIngestRateLimitMax,
|
|
58887
|
+
cloudflareIngestIpRateLimitMax: opts.cloudflareIngestIpRateLimitMax,
|
|
58888
|
+
defaultSampleLimit: opts.defaultTrafficSampleLimit,
|
|
56257
58889
|
onTrafficSynced: opts.onTrafficSynced,
|
|
56258
58890
|
onScheduleUpdated: opts.onScheduleUpdated,
|
|
56259
58891
|
allowLoopbackWebhooks: opts.allowLoopbackWebhooks
|
|
@@ -56434,11 +59066,62 @@ function buildTrafficSourceValidators(opts) {
|
|
|
56434
59066
|
validateScopes: () => null
|
|
56435
59067
|
};
|
|
56436
59068
|
}
|
|
59069
|
+
if (opts.cloudflareTrafficCredentialStore) {
|
|
59070
|
+
const store = opts.cloudflareTrafficCredentialStore;
|
|
59071
|
+
validators[TrafficSourceTypes.cloudflare] = {
|
|
59072
|
+
validateCredentials: (source) => {
|
|
59073
|
+
const sourceMode = source.configJson.deliveryMode;
|
|
59074
|
+
if (sourceMode !== void 0 && sourceMode !== "direct-push") return null;
|
|
59075
|
+
const record = store.getConnection(source.projectName);
|
|
59076
|
+
if (!record) {
|
|
59077
|
+
return {
|
|
59078
|
+
status: CheckStatuses.fail,
|
|
59079
|
+
code: "traffic.credentials.missing",
|
|
59080
|
+
summary: `No Cloudflare direct-push credential is stored for project "${source.projectName}".`,
|
|
59081
|
+
remediation: "Re-run `canonry traffic connect cloudflare <project> --zone-id <id>` from the credential-owning host."
|
|
59082
|
+
};
|
|
59083
|
+
}
|
|
59084
|
+
const recordMode = record.deliveryMode;
|
|
59085
|
+
if (recordMode !== "direct-push") {
|
|
59086
|
+
return {
|
|
59087
|
+
status: CheckStatuses.fail,
|
|
59088
|
+
code: "traffic.credentials.mode-mismatch",
|
|
59089
|
+
summary: `The stored Cloudflare credential mode does not match direct-push source "${source.displayName}".`,
|
|
59090
|
+
remediation: "Reconnect the Cloudflare source from the credential-owning host."
|
|
59091
|
+
};
|
|
59092
|
+
}
|
|
59093
|
+
if (record.sourceId !== source.id) {
|
|
59094
|
+
return {
|
|
59095
|
+
status: CheckStatuses.fail,
|
|
59096
|
+
code: "traffic.credentials.source-mismatch",
|
|
59097
|
+
summary: `The stored Cloudflare credential belongs to a different source than "${source.displayName}".`,
|
|
59098
|
+
remediation: "Reconnect the Cloudflare source to pair the credential and source row."
|
|
59099
|
+
};
|
|
59100
|
+
}
|
|
59101
|
+
const bearerToken = record.bearerToken;
|
|
59102
|
+
const hmacSecret = record.hmacSecret;
|
|
59103
|
+
if (typeof bearerToken !== "string" || bearerToken.length === 0 || typeof hmacSecret !== "string" || hmacSecret.length === 0 || !source.ingestTokenHash || hashCloudflareBearerToken(bearerToken) !== source.ingestTokenHash) {
|
|
59104
|
+
return {
|
|
59105
|
+
status: CheckStatuses.fail,
|
|
59106
|
+
code: "traffic.credentials.bearer-mismatch",
|
|
59107
|
+
summary: `The stored Cloudflare push credential does not match source "${source.displayName}".`,
|
|
59108
|
+
remediation: "Reconnect the Cloudflare source and redeploy the generated Worker secrets."
|
|
59109
|
+
};
|
|
59110
|
+
}
|
|
59111
|
+
return {
|
|
59112
|
+
status: CheckStatuses.ok,
|
|
59113
|
+
code: "traffic.credentials.resolved",
|
|
59114
|
+
summary: `Cloudflare direct-push credentials match source "${source.displayName}".`
|
|
59115
|
+
};
|
|
59116
|
+
},
|
|
59117
|
+
validateScopes: () => null
|
|
59118
|
+
};
|
|
59119
|
+
}
|
|
56437
59120
|
return Object.keys(validators).length > 0 ? validators : void 0;
|
|
56438
59121
|
}
|
|
56439
59122
|
|
|
56440
59123
|
// src/intelligence-service.ts
|
|
56441
|
-
import
|
|
59124
|
+
import crypto46 from "crypto";
|
|
56442
59125
|
|
|
56443
59126
|
// src/logger.ts
|
|
56444
59127
|
var IS_TTY = process.stdout.isTTY === true;
|
|
@@ -56544,7 +59227,7 @@ function extractRecommendedCompetitors(answerText, ownDomains, citedDomains, com
|
|
|
56544
59227
|
}
|
|
56545
59228
|
const ownKeys = new Set([...ownBrandAliases].map(brandKeyFromText));
|
|
56546
59229
|
const knownCompetitorAliases = new Set(
|
|
56547
|
-
[...citedDomains, ...competitorDomains].flatMap((domain) => collectBrandAliasesFromDomain(domain)).filter((
|
|
59230
|
+
[...citedDomains, ...competitorDomains].flatMap((domain) => collectBrandAliasesFromDomain(domain)).filter((alias2) => !ownKeys.has(brandKeyFromText(alias2)))
|
|
56548
59231
|
);
|
|
56549
59232
|
if (knownCompetitorAliases.size === 0) return [];
|
|
56550
59233
|
const candidatePatterns = [
|
|
@@ -56616,7 +59299,7 @@ function collectBrandAliasesFromDomain(domain) {
|
|
|
56616
59299
|
return [...aliases];
|
|
56617
59300
|
}
|
|
56618
59301
|
function matchesBrandAlias(candidate, aliases) {
|
|
56619
|
-
return [...aliases].some((
|
|
59302
|
+
return [...aliases].some((alias2) => textContainsBrandAlias(candidate, alias2));
|
|
56620
59303
|
}
|
|
56621
59304
|
|
|
56622
59305
|
// src/intelligence-service.ts
|
|
@@ -56646,9 +59329,9 @@ var IntelligenceService = class {
|
|
|
56646
59329
|
*/
|
|
56647
59330
|
analyzeAndPersist(runId, projectId) {
|
|
56648
59331
|
const recentRuns = this.db.select().from(runs).where(
|
|
56649
|
-
|
|
56650
|
-
|
|
56651
|
-
or12(
|
|
59332
|
+
and46(
|
|
59333
|
+
eq58(runs.projectId, projectId),
|
|
59334
|
+
or12(eq58(runs.status, "completed"), eq58(runs.status, "partial")),
|
|
56652
59335
|
// Defensive: RunCoordinator already skips probes before this is
|
|
56653
59336
|
// called, but if a future call site invokes analyzeAndPersist
|
|
56654
59337
|
// directly for a probe, probes still must not pollute the
|
|
@@ -56730,7 +59413,7 @@ var IntelligenceService = class {
|
|
|
56730
59413
|
* Returns the persisted insights so the coordinator can count critical/high.
|
|
56731
59414
|
*/
|
|
56732
59415
|
analyzeAndPersistGbp(runId, projectId) {
|
|
56733
|
-
const runRow = this.db.select({ createdAt: runs.createdAt, startedAt: runs.startedAt, finishedAt: runs.finishedAt }).from(runs).where(
|
|
59416
|
+
const runRow = this.db.select({ createdAt: runs.createdAt, startedAt: runs.startedAt, finishedAt: runs.finishedAt }).from(runs).where(eq58(runs.id, runId)).get();
|
|
56734
59417
|
if (!runRow) {
|
|
56735
59418
|
log.info("gbp-intelligence.skip", { runId, reason: "run not found" });
|
|
56736
59419
|
this.persistGbpInsights(runId, projectId, [], []);
|
|
@@ -56738,11 +59421,11 @@ var IntelligenceService = class {
|
|
|
56738
59421
|
}
|
|
56739
59422
|
const windowStart = runRow.startedAt ?? runRow.createdAt;
|
|
56740
59423
|
const windowEnd = runRow.finishedAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
56741
|
-
const selected = this.db.select().from(gbpLocations).where(
|
|
56742
|
-
|
|
56743
|
-
|
|
56744
|
-
|
|
56745
|
-
|
|
59424
|
+
const selected = this.db.select().from(gbpLocations).where(and46(
|
|
59425
|
+
eq58(gbpLocations.projectId, projectId),
|
|
59426
|
+
eq58(gbpLocations.selected, true),
|
|
59427
|
+
gte14(gbpLocations.syncedAt, windowStart),
|
|
59428
|
+
lte11(gbpLocations.syncedAt, windowEnd)
|
|
56746
59429
|
)).all();
|
|
56747
59430
|
if (selected.length === 0) {
|
|
56748
59431
|
log.info("gbp-intelligence.skip", { runId, reason: "no locations synced during run" });
|
|
@@ -56775,12 +59458,12 @@ var IntelligenceService = class {
|
|
|
56775
59458
|
}
|
|
56776
59459
|
/** Build the per-location signal bundle the GBP analyzer consumes. */
|
|
56777
59460
|
buildGbpLocationSignals(projectId, locationName, displayName, fallbackDate) {
|
|
56778
|
-
const metricRows = this.db.select({ metric: gbpDailyMetrics.metric, date: gbpDailyMetrics.date, value: gbpDailyMetrics.value }).from(gbpDailyMetrics).where(
|
|
56779
|
-
const placeActionRows = this.db.select({ placeActionType: gbpPlaceActions.placeActionType, providerType: gbpPlaceActions.providerType }).from(gbpPlaceActions).where(
|
|
56780
|
-
const lodgingRow = this.db.select({ populatedGroupCount: gbpLodgingSnapshots.populatedGroupCount }).from(gbpLodgingSnapshots).where(
|
|
56781
|
-
const ownerRow = this.db.select({ description: gbpLocations.description }).from(gbpLocations).where(
|
|
59461
|
+
const metricRows = this.db.select({ metric: gbpDailyMetrics.metric, date: gbpDailyMetrics.date, value: gbpDailyMetrics.value }).from(gbpDailyMetrics).where(and46(eq58(gbpDailyMetrics.projectId, projectId), eq58(gbpDailyMetrics.locationName, locationName))).all();
|
|
59462
|
+
const placeActionRows = this.db.select({ placeActionType: gbpPlaceActions.placeActionType, providerType: gbpPlaceActions.providerType }).from(gbpPlaceActions).where(and46(eq58(gbpPlaceActions.projectId, projectId), eq58(gbpPlaceActions.locationName, locationName))).all();
|
|
59463
|
+
const lodgingRow = this.db.select({ populatedGroupCount: gbpLodgingSnapshots.populatedGroupCount }).from(gbpLodgingSnapshots).where(and46(eq58(gbpLodgingSnapshots.projectId, projectId), eq58(gbpLodgingSnapshots.locationName, locationName))).orderBy(desc26(gbpLodgingSnapshots.syncedAt)).limit(1).get();
|
|
59464
|
+
const ownerRow = this.db.select({ description: gbpLocations.description }).from(gbpLocations).where(and46(eq58(gbpLocations.projectId, projectId), eq58(gbpLocations.locationName, locationName))).get();
|
|
56782
59465
|
const descriptionMissing = !(ownerRow?.description ?? "").trim();
|
|
56783
|
-
const placeRow = this.db.select({ attributes: gbpPlaceDetails.attributes }).from(gbpPlaceDetails).where(
|
|
59466
|
+
const placeRow = this.db.select({ attributes: gbpPlaceDetails.attributes }).from(gbpPlaceDetails).where(and46(eq58(gbpPlaceDetails.projectId, projectId), eq58(gbpPlaceDetails.locationName, locationName))).orderBy(desc26(gbpPlaceDetails.syncedAt)).limit(1).get();
|
|
56784
59467
|
const placesAmenities = placeRow ? extractPlaceAmenities(placeRow.attributes) : [];
|
|
56785
59468
|
const summary = buildGbpSummary({
|
|
56786
59469
|
locationName,
|
|
@@ -56816,7 +59499,7 @@ var IntelligenceService = class {
|
|
|
56816
59499
|
/** Build the month-over-month keyword series for a location from the
|
|
56817
59500
|
* accumulating gbp_keyword_monthly table (latest complete month vs prior). */
|
|
56818
59501
|
buildGbpKeywordTrend(projectId, locationName) {
|
|
56819
|
-
const rows = this.db.select({ month: gbpKeywordMonthly.month, keyword: gbpKeywordMonthly.keyword, valueCount: gbpKeywordMonthly.valueCount }).from(gbpKeywordMonthly).where(
|
|
59502
|
+
const rows = this.db.select({ month: gbpKeywordMonthly.month, keyword: gbpKeywordMonthly.keyword, valueCount: gbpKeywordMonthly.valueCount }).from(gbpKeywordMonthly).where(and46(eq58(gbpKeywordMonthly.projectId, projectId), eq58(gbpKeywordMonthly.locationName, locationName))).all();
|
|
56820
59503
|
if (rows.length === 0) return { recentMonth: null, priorMonth: null, points: [] };
|
|
56821
59504
|
const months = [...new Set(rows.map((r) => r.month))].sort().reverse();
|
|
56822
59505
|
const recentMonth = months[0] ?? null;
|
|
@@ -56847,7 +59530,7 @@ var IntelligenceService = class {
|
|
|
56847
59530
|
*/
|
|
56848
59531
|
persistGbpInsights(runId, projectId, gbpInsights, coveredLocationNames) {
|
|
56849
59532
|
const covered = new Set(coveredLocationNames);
|
|
56850
|
-
const existing = this.db.select({ id: insights.id, dismissed: insights.dismissed }).from(insights).where(
|
|
59533
|
+
const existing = this.db.select({ id: insights.id, dismissed: insights.dismissed }).from(insights).where(and46(eq58(insights.projectId, projectId), eq58(insights.provider, GBP_INSIGHT_PROVIDER))).all();
|
|
56851
59534
|
const staleIds = [];
|
|
56852
59535
|
const dismissedSlots = /* @__PURE__ */ new Set();
|
|
56853
59536
|
for (const row of existing) {
|
|
@@ -56858,7 +59541,7 @@ var IntelligenceService = class {
|
|
|
56858
59541
|
}
|
|
56859
59542
|
this.db.transaction((tx) => {
|
|
56860
59543
|
for (const id of staleIds) {
|
|
56861
|
-
tx.delete(insights).where(
|
|
59544
|
+
tx.delete(insights).where(eq58(insights.id, id)).run();
|
|
56862
59545
|
}
|
|
56863
59546
|
for (const insight of gbpInsights) {
|
|
56864
59547
|
const parsed = parseGbpInsightId(insight.id);
|
|
@@ -56936,7 +59619,7 @@ var IntelligenceService = class {
|
|
|
56936
59619
|
* create per run + aggregate). DB is left untouched.
|
|
56937
59620
|
*/
|
|
56938
59621
|
backfill(projectName, opts, onProgress) {
|
|
56939
|
-
const project = this.db.select().from(projects).where(
|
|
59622
|
+
const project = this.db.select().from(projects).where(eq58(projects.name, projectName)).get();
|
|
56940
59623
|
if (!project) {
|
|
56941
59624
|
throw new Error(`Project "${projectName}" not found`);
|
|
56942
59625
|
}
|
|
@@ -56949,9 +59632,9 @@ var IntelligenceService = class {
|
|
|
56949
59632
|
sinceTimestamp = parsed;
|
|
56950
59633
|
}
|
|
56951
59634
|
const allRuns = this.db.select().from(runs).where(
|
|
56952
|
-
|
|
56953
|
-
|
|
56954
|
-
or12(
|
|
59635
|
+
and46(
|
|
59636
|
+
eq58(runs.projectId, project.id),
|
|
59637
|
+
or12(eq58(runs.status, "completed"), eq58(runs.status, "partial")),
|
|
56955
59638
|
// Backfill must not replay probe runs as if they were real sweeps.
|
|
56956
59639
|
ne7(runs.trigger, RunTriggers.probe)
|
|
56957
59640
|
)
|
|
@@ -57030,7 +59713,7 @@ var IntelligenceService = class {
|
|
|
57030
59713
|
return { processed, skipped, totalInsights };
|
|
57031
59714
|
}
|
|
57032
59715
|
loadTrackedCompetitors(projectId) {
|
|
57033
|
-
return this.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
59716
|
+
return this.db.select({ domain: competitors.domain }).from(competitors).where(eq58(competitors.projectId, projectId)).all().map((r) => r.domain);
|
|
57034
59717
|
}
|
|
57035
59718
|
/**
|
|
57036
59719
|
* Wipe transition signals from an analysis result while keeping health.
|
|
@@ -57051,15 +59734,15 @@ var IntelligenceService = class {
|
|
|
57051
59734
|
}
|
|
57052
59735
|
persistResult(result, runId, projectId) {
|
|
57053
59736
|
const previouslyDismissed = /* @__PURE__ */ new Set();
|
|
57054
|
-
const existingInsights = this.db.select({ query: insights.query, provider: insights.provider, type: insights.type, dismissed: insights.dismissed }).from(insights).where(
|
|
59737
|
+
const existingInsights = this.db.select({ query: insights.query, provider: insights.provider, type: insights.type, dismissed: insights.dismissed }).from(insights).where(eq58(insights.runId, runId)).all();
|
|
57055
59738
|
for (const row of existingInsights) {
|
|
57056
59739
|
if (row.dismissed) {
|
|
57057
59740
|
previouslyDismissed.add(`${row.query}:${row.provider}:${row.type}`);
|
|
57058
59741
|
}
|
|
57059
59742
|
}
|
|
57060
59743
|
this.db.transaction((tx) => {
|
|
57061
|
-
tx.delete(insights).where(
|
|
57062
|
-
tx.delete(healthSnapshots).where(
|
|
59744
|
+
tx.delete(insights).where(eq58(insights.runId, runId)).run();
|
|
59745
|
+
tx.delete(healthSnapshots).where(eq58(healthSnapshots.runId, runId)).run();
|
|
57063
59746
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
57064
59747
|
for (const insight of result.insights) {
|
|
57065
59748
|
const wasDismissed = previouslyDismissed.has(`${insight.query}:${insight.provider}:${insight.type}`);
|
|
@@ -57079,7 +59762,7 @@ var IntelligenceService = class {
|
|
|
57079
59762
|
}).run();
|
|
57080
59763
|
}
|
|
57081
59764
|
tx.insert(healthSnapshots).values({
|
|
57082
|
-
id:
|
|
59765
|
+
id: crypto46.randomUUID(),
|
|
57083
59766
|
projectId,
|
|
57084
59767
|
runId,
|
|
57085
59768
|
overallCitedRate: String(result.health.overallCitedRate),
|
|
@@ -57112,24 +59795,24 @@ var IntelligenceService = class {
|
|
|
57112
59795
|
applySeverityTiering(rawInsights, excludeRunId, projectId) {
|
|
57113
59796
|
const regressions = rawInsights.filter((i) => i.type === "regression");
|
|
57114
59797
|
if (regressions.length === 0) return rawInsights;
|
|
57115
|
-
const gscRows = this.db.select({ query: gscSearchData.query, impressions: gscSearchData.impressions }).from(gscSearchData).where(
|
|
59798
|
+
const gscRows = this.db.select({ query: gscSearchData.query, impressions: gscSearchData.impressions }).from(gscSearchData).where(eq58(gscSearchData.projectId, projectId)).all();
|
|
57116
59799
|
const gscConnected = gscRows.length > 0;
|
|
57117
59800
|
const gscImpressionsByQuery = /* @__PURE__ */ new Map();
|
|
57118
59801
|
for (const row of gscRows) {
|
|
57119
59802
|
const key = row.query.toLowerCase();
|
|
57120
59803
|
gscImpressionsByQuery.set(key, (gscImpressionsByQuery.get(key) ?? 0) + row.impressions);
|
|
57121
59804
|
}
|
|
57122
|
-
const projectRow = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
59805
|
+
const projectRow = this.db.select({ locations: projects.locations }).from(projects).where(eq58(projects.id, projectId)).get();
|
|
57123
59806
|
const locationCount = Math.max(
|
|
57124
59807
|
1,
|
|
57125
59808
|
(projectRow?.locations ?? []).length
|
|
57126
59809
|
);
|
|
57127
59810
|
const ROWS_PER_GROUP_BUDGET = Math.max(2, locationCount);
|
|
57128
59811
|
const recentRunRows = this.db.select({ id: runs.id, createdAt: runs.createdAt }).from(runs).where(
|
|
57129
|
-
|
|
57130
|
-
|
|
57131
|
-
|
|
57132
|
-
or12(
|
|
59812
|
+
and46(
|
|
59813
|
+
eq58(runs.projectId, projectId),
|
|
59814
|
+
eq58(runs.kind, RunKinds["answer-visibility"]),
|
|
59815
|
+
or12(eq58(runs.status, "completed"), eq58(runs.status, "partial")),
|
|
57133
59816
|
// Defensive — see top of file.
|
|
57134
59817
|
ne7(runs.trigger, RunTriggers.probe)
|
|
57135
59818
|
)
|
|
@@ -57149,7 +59832,7 @@ var IntelligenceService = class {
|
|
|
57149
59832
|
const haveHistory = recentRunIds.length > 0;
|
|
57150
59833
|
const priorRegressionsByPair = /* @__PURE__ */ new Map();
|
|
57151
59834
|
if (haveHistory) {
|
|
57152
|
-
const priorRows = this.db.select({ query: insights.query, provider: insights.provider, runId: insights.runId }).from(insights).where(
|
|
59835
|
+
const priorRows = this.db.select({ query: insights.query, provider: insights.provider, runId: insights.runId }).from(insights).where(and46(eq58(insights.type, "regression"), inArray20(insights.runId, recentRunIds))).all();
|
|
57153
59836
|
const regressionGroups = /* @__PURE__ */ new Map();
|
|
57154
59837
|
for (const row of priorRows) {
|
|
57155
59838
|
if (!row.runId) continue;
|
|
@@ -57178,7 +59861,7 @@ var IntelligenceService = class {
|
|
|
57178
59861
|
});
|
|
57179
59862
|
}
|
|
57180
59863
|
buildRunData(runId, projectId, completedAt, location = null) {
|
|
57181
|
-
const projectDomainRow = this.db.select({ canonicalDomain: projects.canonicalDomain, ownedDomains: projects.ownedDomains }).from(projects).where(
|
|
59864
|
+
const projectDomainRow = this.db.select({ canonicalDomain: projects.canonicalDomain, ownedDomains: projects.ownedDomains }).from(projects).where(eq58(projects.id, projectId)).get();
|
|
57182
59865
|
const projectDomains = projectDomainRow ? effectiveDomains({
|
|
57183
59866
|
canonicalDomain: projectDomainRow.canonicalDomain,
|
|
57184
59867
|
ownedDomains: projectDomainRow.ownedDomains
|
|
@@ -57195,7 +59878,7 @@ var IntelligenceService = class {
|
|
|
57195
59878
|
citedDomains: querySnapshots.citedDomains,
|
|
57196
59879
|
competitorOverlap: querySnapshots.competitorOverlap,
|
|
57197
59880
|
snapshotLocation: querySnapshots.location
|
|
57198
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
59881
|
+
}).from(querySnapshots).leftJoin(queries, eq58(querySnapshots.queryId, queries.id)).where(eq58(querySnapshots.runId, runId)).all();
|
|
57199
59882
|
const snapshots = [];
|
|
57200
59883
|
let orphanCount = 0;
|
|
57201
59884
|
for (const r of rows) {
|
|
@@ -57260,6 +59943,9 @@ export {
|
|
|
57260
59943
|
siteCrawlSnapshots,
|
|
57261
59944
|
siteCrawlPages,
|
|
57262
59945
|
siteCrawlEdges,
|
|
59946
|
+
siteCrawlGraphLayouts,
|
|
59947
|
+
siteCrawlGraphNodes,
|
|
59948
|
+
siteCrawlGraphEdges,
|
|
57263
59949
|
siteCrawlFindings,
|
|
57264
59950
|
siteCrawlEventReceipts,
|
|
57265
59951
|
bingCoverageSnapshots,
|
|
@@ -57355,6 +60041,12 @@ export {
|
|
|
57355
60041
|
listCachedReleases,
|
|
57356
60042
|
pruneCachedRelease,
|
|
57357
60043
|
resolveVercelSyncDeadlineMs,
|
|
60044
|
+
CLOUDFLARE_WORKER_BINDINGS,
|
|
60045
|
+
CLOUDFLARE_WORKER_GENERATED_MARKER,
|
|
60046
|
+
CLOUDFLARE_WRANGLER_GENERATED_MARKER,
|
|
60047
|
+
DEFAULT_BOT_LIST,
|
|
60048
|
+
generateWorkerScript,
|
|
60049
|
+
generateWranglerToml,
|
|
57358
60050
|
executeDiscovery,
|
|
57359
60051
|
markSessionFailed,
|
|
57360
60052
|
apiRoutes,
|