@canonry/canonry 4.155.0 → 4.156.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agent-workspace/skills/aero/references/orchestration.md +1 -1
- package/assets/agent-workspace/skills/canonry/SKILL.md +1 -1
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +14 -2
- package/assets/assets/{AuditHistoryPanel-BlbMBxOY.js → AuditHistoryPanel-gmJMVlXX.js} +1 -1
- package/assets/assets/{BacklinksPage-DA9EaV9q.js → BacklinksPage-Bv2sTCyN.js} +1 -1
- package/assets/assets/{HistoryPage-BTrIHjMQ.js → HistoryPage-13q7GkiI.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-CN6o5xFg.js → MeasurementPropertyPage-BgBqXT1p.js} +1 -1
- package/assets/assets/ProjectPage-EsGwKhot.js +15 -0
- package/assets/assets/{RunRow-ch_OlCPs.js → RunRow-bPzapIGJ.js} +1 -1
- package/assets/assets/{RunsPage-T89Y_yS8.js → RunsPage-Dqfz18lH.js} +1 -1
- package/assets/assets/{SettingsPage-DnmEq3F9.js → SettingsPage-_wMecn-V.js} +1 -1
- package/assets/assets/{TrafficPage-CW1UhaJt.js → TrafficPage-RYBdsOLB.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-r4Ji3wJm.js → TrafficSourceDetailPage-DpeMMyKU.js} +1 -1
- package/assets/assets/{extract-error-message-a8EErBoZ.js → extract-error-message-DUarAlyD.js} +1 -1
- package/assets/assets/index-BxegEwic.css +1 -0
- package/assets/assets/{index-BKq6k5Of.js → index-DP-fT2uI.js} +29 -29
- package/assets/assets/react-sigma_core.esm.min-DnypJl5w.js +312 -0
- package/assets/assets/vendor-lucide-BtYdosYk.js +1 -0
- package/assets/assets/{vendor-markdown-CpDJch4G.js → vendor-markdown-D274_wsI.js} +1 -1
- package/assets/assets/{vendor-radix-DnzzbllM.js → vendor-radix-BxMXSwbC.js} +1 -1
- package/assets/assets/{vendor-recharts-33ACLxsD.js → vendor-recharts-C4ktw_aE.js} +1 -1
- package/assets/assets/{vendor-tanstack-BjS7iY3g.js → vendor-tanstack-pFxw0eGP.js} +1 -1
- package/assets/index.html +7 -7
- package/dist/{chunk-VOL2BSM5.js → chunk-ABNN2ZVR.js} +1332 -29
- package/dist/{chunk-VIKPFI6F.js → chunk-AMPT2EP7.js} +328 -1
- package/dist/{chunk-OSHRDV36.js → chunk-CPASKGUE.js} +247 -1
- package/dist/{chunk-YSESZNVT.js → chunk-ZWESOFA5.js} +711 -235
- package/dist/cli.js +427 -5
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-G4NV3X36.js → intelligence-service-WHC7B32V.js} +2 -2
- package/dist/mcp.js +4 -4
- package/package.json +11 -9
- package/assets/assets/ProjectPage-CsDbgp_0.js +0 -12
- package/assets/assets/index-DyL54s8x.css +0 -1
- package/assets/assets/vendor-lucide-ByU7vBHt.js +0 -1
|
@@ -72,9 +72,23 @@ import {
|
|
|
72
72
|
RunKinds,
|
|
73
73
|
RunStatuses,
|
|
74
74
|
RunTriggers,
|
|
75
|
+
SITE_CRAWL_GRAPH_DEFAULT_MAX_EDGES,
|
|
76
|
+
SITE_CRAWL_GRAPH_DEFAULT_MAX_NODES,
|
|
77
|
+
SITE_CRAWL_GRAPH_MAX_EDGES,
|
|
78
|
+
SITE_CRAWL_GRAPH_MAX_NODES,
|
|
79
|
+
SITE_HEALTH_CHANGES_DEFAULT_LIMIT,
|
|
80
|
+
SITE_HEALTH_CHANGES_MAX_LIMIT,
|
|
81
|
+
SITE_HEALTH_PATH_DEFAULT_MAX_DEPTH,
|
|
82
|
+
SITE_HEALTH_PATH_MAX_DEPTH,
|
|
83
|
+
SITE_HEALTH_PATH_MAX_VISITED_NODES,
|
|
84
|
+
SITE_HEALTH_SUBGRAPH_DEFAULT_MAX_EDGES,
|
|
85
|
+
SITE_HEALTH_SUBGRAPH_DEFAULT_MAX_NODES,
|
|
86
|
+
SITE_HEALTH_SUBGRAPH_MAX_EDGES,
|
|
87
|
+
SITE_HEALTH_SUBGRAPH_MAX_NODES,
|
|
75
88
|
SKILL_MANIFEST_FILENAME,
|
|
76
89
|
SchedulableRunKinds,
|
|
77
90
|
SiteAuditTrendDirections,
|
|
91
|
+
SiteCrawlFetchedStates,
|
|
78
92
|
TrafficEventConfidences,
|
|
79
93
|
TrafficEventKinds,
|
|
80
94
|
TrafficEvidenceKinds,
|
|
@@ -209,6 +223,7 @@ import {
|
|
|
209
223
|
deltaPercent,
|
|
210
224
|
deltaTone,
|
|
211
225
|
deriveCitedUrlCandidates,
|
|
226
|
+
deriveSiteHealthState,
|
|
212
227
|
deriveWinnabilityClass,
|
|
213
228
|
determineAnswerMentioned,
|
|
214
229
|
discoveryBucketSchema,
|
|
@@ -228,6 +243,7 @@ import {
|
|
|
228
243
|
escapeLikePattern,
|
|
229
244
|
evaluateModelPointerExposure,
|
|
230
245
|
extractAnswerMentions,
|
|
246
|
+
factorStatusFromScore,
|
|
231
247
|
filterBrandedSeedCandidates,
|
|
232
248
|
filterCapturedCitedUrls,
|
|
233
249
|
findDuplicateLocationLabels,
|
|
@@ -469,18 +485,26 @@ import {
|
|
|
469
485
|
segmentCrawlerHits,
|
|
470
486
|
serializeRunError,
|
|
471
487
|
settingsDtoSchema,
|
|
488
|
+
siteAuditPageFactorSchema,
|
|
472
489
|
siteAuditPagesResponseSchema,
|
|
473
490
|
siteAuditRequestIdentity,
|
|
474
491
|
siteAuditRunRequestSchema,
|
|
475
492
|
siteAuditRunResponseSchema,
|
|
476
493
|
siteAuditScoreSchema,
|
|
477
494
|
siteAuditTrendResponseSchema,
|
|
495
|
+
siteCrawlAuditFactorSchema,
|
|
496
|
+
siteCrawlCriticalDefectSchema,
|
|
478
497
|
siteCrawlDeadLinksResponseSchema,
|
|
498
|
+
siteCrawlGraphResponseSchema,
|
|
479
499
|
siteCrawlInternalLinksResponseSchema,
|
|
480
500
|
siteCrawlNeighborsResponseSchema,
|
|
501
|
+
siteCrawlPageAuditSchema,
|
|
481
502
|
siteCrawlPagesResponseSchema,
|
|
482
503
|
siteCrawlStructureResponseSchema,
|
|
483
504
|
siteCrawlSummarySchema,
|
|
505
|
+
siteHealthChangesResponseSchema,
|
|
506
|
+
siteHealthPathResponseSchema,
|
|
507
|
+
siteHealthSubgraphResponseSchema,
|
|
484
508
|
snapshotDiffResponseSchema,
|
|
485
509
|
snapshotListResponseSchema,
|
|
486
510
|
snapshotReportSchema,
|
|
@@ -529,7 +553,7 @@ import {
|
|
|
529
553
|
wordpressSchemaDeployResultDtoSchema,
|
|
530
554
|
wordpressSchemaStatusResultDtoSchema,
|
|
531
555
|
wordpressStatusDtoSchema
|
|
532
|
-
} from "./chunk-
|
|
556
|
+
} from "./chunk-AMPT2EP7.js";
|
|
533
557
|
|
|
534
558
|
// src/intelligence-service.ts
|
|
535
559
|
import { eq as eq57, desc as desc26, asc as asc11, and as and45, ne as ne7, or as or12, inArray as inArray20, gte as gte13, lte as lte10 } from "drizzle-orm";
|
|
@@ -627,6 +651,9 @@ __export(schema_exports, {
|
|
|
627
651
|
siteCrawlEdges: () => siteCrawlEdges,
|
|
628
652
|
siteCrawlEventReceipts: () => siteCrawlEventReceipts,
|
|
629
653
|
siteCrawlFindings: () => siteCrawlFindings,
|
|
654
|
+
siteCrawlGraphEdges: () => siteCrawlGraphEdges,
|
|
655
|
+
siteCrawlGraphLayouts: () => siteCrawlGraphLayouts,
|
|
656
|
+
siteCrawlGraphNodes: () => siteCrawlGraphNodes,
|
|
630
657
|
siteCrawlPages: () => siteCrawlPages,
|
|
631
658
|
siteCrawlRunRequests: () => siteCrawlRunRequests,
|
|
632
659
|
siteCrawlSnapshots: () => siteCrawlSnapshots,
|
|
@@ -1251,6 +1278,8 @@ var siteCrawlSnapshots = sqliteTable("site_crawl_snapshots", {
|
|
|
1251
1278
|
runId: text("run_id").notNull(),
|
|
1252
1279
|
attemptId: text("attempt_id"),
|
|
1253
1280
|
rootUrl: text("root_url").notNull(),
|
|
1281
|
+
/** Original requested crawl root; null for snapshots written before v128. */
|
|
1282
|
+
requestedRootUrl: text("requested_root_url"),
|
|
1254
1283
|
crawlSchemaVersion: text("crawl_schema_version").notNull().default("1.0"),
|
|
1255
1284
|
engineVersion: text("engine_version").notNull().default(""),
|
|
1256
1285
|
normalizationVersion: text("normalization_version").notNull().default(""),
|
|
@@ -1361,12 +1390,116 @@ var siteCrawlEdges = sqliteTable("site_crawl_edges", {
|
|
|
1361
1390
|
index("idx_site_crawl_edges_inbound").on(table.projectId, table.runId, table.attemptId, table.targetNodeKey, table.edgeKey),
|
|
1362
1391
|
index("idx_site_crawl_edges_source_url").on(table.projectId, table.runId, table.attemptId, table.sourceUrl),
|
|
1363
1392
|
index("idx_site_crawl_edges_target_url").on(table.projectId, table.runId, table.attemptId, table.targetUrl),
|
|
1393
|
+
// Publish-time graph sampling filters this exact scope and consumes the
|
|
1394
|
+
// highest-occurrence internal anchors first. The mixed-direction suffix
|
|
1395
|
+
// avoids a million-row temporary ORDER BY at the maximum crawl budget.
|
|
1396
|
+
index("idx_site_crawl_edges_graph_sample").on(
|
|
1397
|
+
table.projectId,
|
|
1398
|
+
table.runId,
|
|
1399
|
+
table.attemptId,
|
|
1400
|
+
table.internal,
|
|
1401
|
+
table.relation,
|
|
1402
|
+
sql`${table.occurrences} desc`,
|
|
1403
|
+
table.edgeKey
|
|
1404
|
+
),
|
|
1364
1405
|
foreignKey({
|
|
1365
1406
|
name: "site_crawl_edges_attempt_fk",
|
|
1366
1407
|
columns: [table.projectId, table.runId, table.attemptId],
|
|
1367
1408
|
foreignColumns: [siteCrawlAttempts.projectId, siteCrawlAttempts.runId, siteCrawlAttempts.id]
|
|
1368
1409
|
}).onDelete("cascade")
|
|
1369
1410
|
]);
|
|
1411
|
+
var siteCrawlGraphLayouts = sqliteTable("site_crawl_graph_layouts", {
|
|
1412
|
+
id: text("id").primaryKey(),
|
|
1413
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1414
|
+
runId: text("run_id").notNull(),
|
|
1415
|
+
attemptId: text("attempt_id").notNull(),
|
|
1416
|
+
state: text("state").notNull(),
|
|
1417
|
+
layoutVersion: text("layout_version"),
|
|
1418
|
+
failureCode: text("failure_code"),
|
|
1419
|
+
totalNodes: integer("total_nodes").notNull().default(0),
|
|
1420
|
+
totalEdges: integer("total_edges").notNull().default(0),
|
|
1421
|
+
nodeCount: integer("node_count").notNull().default(0),
|
|
1422
|
+
edgeCount: integer("edge_count").notNull().default(0),
|
|
1423
|
+
createdAt: text("created_at").notNull(),
|
|
1424
|
+
updatedAt: text("updated_at").notNull()
|
|
1425
|
+
}, (table) => [
|
|
1426
|
+
uniqueIndex("idx_site_crawl_graph_layouts_attempt").on(table.projectId, table.runId, table.attemptId),
|
|
1427
|
+
index("idx_site_crawl_graph_layouts_run").on(table.projectId, table.runId),
|
|
1428
|
+
check("site_crawl_graph_layouts_state_check", sql`${table.state} in ('ready', 'unavailable')`),
|
|
1429
|
+
foreignKey({
|
|
1430
|
+
name: "site_crawl_graph_layouts_attempt_fk",
|
|
1431
|
+
columns: [table.projectId, table.runId, table.attemptId],
|
|
1432
|
+
foreignColumns: [siteCrawlAttempts.projectId, siteCrawlAttempts.runId, siteCrawlAttempts.id]
|
|
1433
|
+
}).onDelete("cascade")
|
|
1434
|
+
]);
|
|
1435
|
+
var siteCrawlGraphNodes = sqliteTable("site_crawl_graph_nodes", {
|
|
1436
|
+
id: text("id").primaryKey(),
|
|
1437
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1438
|
+
runId: text("run_id").notNull(),
|
|
1439
|
+
attemptId: text("attempt_id").notNull(),
|
|
1440
|
+
nodeKey: text("node_key").notNull(),
|
|
1441
|
+
sampleRank: integer("sample_rank").notNull(),
|
|
1442
|
+
x: real("x").notNull(),
|
|
1443
|
+
y: real("y").notNull(),
|
|
1444
|
+
createdAt: text("created_at").notNull()
|
|
1445
|
+
}, (table) => [
|
|
1446
|
+
uniqueIndex("idx_site_crawl_graph_nodes_attempt_node").on(table.projectId, table.runId, table.attemptId, table.nodeKey),
|
|
1447
|
+
uniqueIndex("idx_site_crawl_graph_nodes_attempt_rank").on(table.projectId, table.runId, table.attemptId, table.sampleRank),
|
|
1448
|
+
index("idx_site_crawl_graph_nodes_read").on(table.projectId, table.runId, table.attemptId, table.sampleRank),
|
|
1449
|
+
check("site_crawl_graph_nodes_rank_check", sql`${table.sampleRank} >= 0`),
|
|
1450
|
+
foreignKey({
|
|
1451
|
+
name: "site_crawl_graph_nodes_layout_fk",
|
|
1452
|
+
columns: [table.projectId, table.runId, table.attemptId],
|
|
1453
|
+
foreignColumns: [siteCrawlGraphLayouts.projectId, siteCrawlGraphLayouts.runId, siteCrawlGraphLayouts.attemptId]
|
|
1454
|
+
}).onDelete("cascade"),
|
|
1455
|
+
foreignKey({
|
|
1456
|
+
name: "site_crawl_graph_nodes_page_fk",
|
|
1457
|
+
columns: [table.projectId, table.runId, table.attemptId, table.nodeKey],
|
|
1458
|
+
foreignColumns: [siteCrawlPages.projectId, siteCrawlPages.runId, siteCrawlPages.attemptId, siteCrawlPages.nodeKey]
|
|
1459
|
+
}).onDelete("cascade")
|
|
1460
|
+
]);
|
|
1461
|
+
var siteCrawlGraphEdges = sqliteTable("site_crawl_graph_edges", {
|
|
1462
|
+
id: text("id").primaryKey(),
|
|
1463
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1464
|
+
runId: text("run_id").notNull(),
|
|
1465
|
+
attemptId: text("attempt_id").notNull(),
|
|
1466
|
+
edgeKey: text("edge_key").notNull(),
|
|
1467
|
+
sampleRank: integer("sample_rank").notNull(),
|
|
1468
|
+
sourceNodeKey: text("source_node_key").notNull(),
|
|
1469
|
+
targetNodeKey: text("target_node_key").notNull(),
|
|
1470
|
+
followable: integer("followable", { mode: "boolean" }).notNull(),
|
|
1471
|
+
occurrences: integer("occurrences").notNull(),
|
|
1472
|
+
createdAt: text("created_at").notNull()
|
|
1473
|
+
}, (table) => [
|
|
1474
|
+
uniqueIndex("idx_site_crawl_graph_edges_attempt_edge").on(table.projectId, table.runId, table.attemptId, table.edgeKey),
|
|
1475
|
+
uniqueIndex("idx_site_crawl_graph_edges_attempt_rank").on(table.projectId, table.runId, table.attemptId, table.sampleRank),
|
|
1476
|
+
index("idx_site_crawl_graph_edges_read").on(table.projectId, table.runId, table.attemptId, table.sampleRank),
|
|
1477
|
+
// These FKs otherwise make a layout cascade scan graph edges once per node.
|
|
1478
|
+
index("idx_site_crawl_graph_edges_source_node").on(table.projectId, table.runId, table.attemptId, table.sourceNodeKey),
|
|
1479
|
+
index("idx_site_crawl_graph_edges_target_node").on(table.projectId, table.runId, table.attemptId, table.targetNodeKey),
|
|
1480
|
+
check("site_crawl_graph_edges_rank_check", sql`${table.sampleRank} >= 0`),
|
|
1481
|
+
check("site_crawl_graph_edges_occurrences_check", sql`${table.occurrences} > 0`),
|
|
1482
|
+
foreignKey({
|
|
1483
|
+
name: "site_crawl_graph_edges_layout_fk",
|
|
1484
|
+
columns: [table.projectId, table.runId, table.attemptId],
|
|
1485
|
+
foreignColumns: [siteCrawlGraphLayouts.projectId, siteCrawlGraphLayouts.runId, siteCrawlGraphLayouts.attemptId]
|
|
1486
|
+
}).onDelete("cascade"),
|
|
1487
|
+
foreignKey({
|
|
1488
|
+
name: "site_crawl_graph_edges_source_node_fk",
|
|
1489
|
+
columns: [table.projectId, table.runId, table.attemptId, table.sourceNodeKey],
|
|
1490
|
+
foreignColumns: [siteCrawlGraphNodes.projectId, siteCrawlGraphNodes.runId, siteCrawlGraphNodes.attemptId, siteCrawlGraphNodes.nodeKey]
|
|
1491
|
+
}).onDelete("cascade"),
|
|
1492
|
+
foreignKey({
|
|
1493
|
+
name: "site_crawl_graph_edges_target_node_fk",
|
|
1494
|
+
columns: [table.projectId, table.runId, table.attemptId, table.targetNodeKey],
|
|
1495
|
+
foreignColumns: [siteCrawlGraphNodes.projectId, siteCrawlGraphNodes.runId, siteCrawlGraphNodes.attemptId, siteCrawlGraphNodes.nodeKey]
|
|
1496
|
+
}).onDelete("cascade"),
|
|
1497
|
+
foreignKey({
|
|
1498
|
+
name: "site_crawl_graph_edges_canonical_edge_fk",
|
|
1499
|
+
columns: [table.projectId, table.runId, table.attemptId, table.edgeKey],
|
|
1500
|
+
foreignColumns: [siteCrawlEdges.projectId, siteCrawlEdges.runId, siteCrawlEdges.attemptId, siteCrawlEdges.edgeKey]
|
|
1501
|
+
}).onDelete("cascade")
|
|
1502
|
+
]);
|
|
1370
1503
|
var siteCrawlFindings = sqliteTable("site_crawl_findings", {
|
|
1371
1504
|
id: text("id").primaryKey(),
|
|
1372
1505
|
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
@@ -5526,6 +5659,100 @@ var MIGRATION_VERSIONS = [
|
|
|
5526
5659
|
`CREATE INDEX IF NOT EXISTS idx_site_crawl_receipts_project_run
|
|
5527
5660
|
ON site_crawl_event_receipts(project_id, run_id, attempt_id)`
|
|
5528
5661
|
]
|
|
5662
|
+
},
|
|
5663
|
+
{
|
|
5664
|
+
// Graph coordinates are optional derived snapshot data. Separate tables
|
|
5665
|
+
// preserve the canonical crawl-page shape and make a missing row the
|
|
5666
|
+
// truthful compatibility signal for snapshots published before v127.
|
|
5667
|
+
version: 127,
|
|
5668
|
+
name: "site-crawl-persisted-graph-layout",
|
|
5669
|
+
statements: [
|
|
5670
|
+
`CREATE TABLE IF NOT EXISTS site_crawl_graph_layouts (
|
|
5671
|
+
id TEXT PRIMARY KEY,
|
|
5672
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
5673
|
+
run_id TEXT NOT NULL,
|
|
5674
|
+
attempt_id TEXT NOT NULL,
|
|
5675
|
+
state TEXT NOT NULL CHECK (state IN ('ready', 'unavailable')),
|
|
5676
|
+
layout_version TEXT,
|
|
5677
|
+
failure_code TEXT,
|
|
5678
|
+
total_nodes INTEGER NOT NULL DEFAULT 0,
|
|
5679
|
+
total_edges INTEGER NOT NULL DEFAULT 0,
|
|
5680
|
+
node_count INTEGER NOT NULL DEFAULT 0,
|
|
5681
|
+
edge_count INTEGER NOT NULL DEFAULT 0,
|
|
5682
|
+
created_at TEXT NOT NULL,
|
|
5683
|
+
updated_at TEXT NOT NULL,
|
|
5684
|
+
FOREIGN KEY (project_id, run_id, attempt_id)
|
|
5685
|
+
REFERENCES site_crawl_attempts(project_id, run_id, id) ON DELETE CASCADE
|
|
5686
|
+
)`,
|
|
5687
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_graph_layouts_attempt
|
|
5688
|
+
ON site_crawl_graph_layouts(project_id, run_id, attempt_id)`,
|
|
5689
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_graph_layouts_run
|
|
5690
|
+
ON site_crawl_graph_layouts(project_id, run_id)`,
|
|
5691
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_edges_graph_sample
|
|
5692
|
+
ON site_crawl_edges(project_id, run_id, attempt_id, internal, relation, occurrences DESC, edge_key ASC)`,
|
|
5693
|
+
`CREATE TABLE IF NOT EXISTS site_crawl_graph_nodes (
|
|
5694
|
+
id TEXT PRIMARY KEY,
|
|
5695
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
5696
|
+
run_id TEXT NOT NULL,
|
|
5697
|
+
attempt_id TEXT NOT NULL,
|
|
5698
|
+
node_key TEXT NOT NULL,
|
|
5699
|
+
sample_rank INTEGER NOT NULL CHECK (sample_rank >= 0),
|
|
5700
|
+
x REAL NOT NULL,
|
|
5701
|
+
y REAL NOT NULL,
|
|
5702
|
+
created_at TEXT NOT NULL,
|
|
5703
|
+
FOREIGN KEY (project_id, run_id, attempt_id)
|
|
5704
|
+
REFERENCES site_crawl_graph_layouts(project_id, run_id, attempt_id) ON DELETE CASCADE,
|
|
5705
|
+
FOREIGN KEY (project_id, run_id, attempt_id, node_key)
|
|
5706
|
+
REFERENCES site_crawl_pages(project_id, run_id, attempt_id, node_key) ON DELETE CASCADE
|
|
5707
|
+
)`,
|
|
5708
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_graph_nodes_attempt_node
|
|
5709
|
+
ON site_crawl_graph_nodes(project_id, run_id, attempt_id, node_key)`,
|
|
5710
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_graph_nodes_attempt_rank
|
|
5711
|
+
ON site_crawl_graph_nodes(project_id, run_id, attempt_id, sample_rank)`,
|
|
5712
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_graph_nodes_read
|
|
5713
|
+
ON site_crawl_graph_nodes(project_id, run_id, attempt_id, sample_rank)`,
|
|
5714
|
+
`CREATE TABLE IF NOT EXISTS site_crawl_graph_edges (
|
|
5715
|
+
id TEXT PRIMARY KEY,
|
|
5716
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
5717
|
+
run_id TEXT NOT NULL,
|
|
5718
|
+
attempt_id TEXT NOT NULL,
|
|
5719
|
+
edge_key TEXT NOT NULL,
|
|
5720
|
+
sample_rank INTEGER NOT NULL CHECK (sample_rank >= 0),
|
|
5721
|
+
source_node_key TEXT NOT NULL,
|
|
5722
|
+
target_node_key TEXT NOT NULL,
|
|
5723
|
+
followable INTEGER NOT NULL,
|
|
5724
|
+
occurrences INTEGER NOT NULL CHECK (occurrences > 0),
|
|
5725
|
+
created_at TEXT NOT NULL,
|
|
5726
|
+
FOREIGN KEY (project_id, run_id, attempt_id)
|
|
5727
|
+
REFERENCES site_crawl_graph_layouts(project_id, run_id, attempt_id) ON DELETE CASCADE,
|
|
5728
|
+
FOREIGN KEY (project_id, run_id, attempt_id, source_node_key)
|
|
5729
|
+
REFERENCES site_crawl_graph_nodes(project_id, run_id, attempt_id, node_key) ON DELETE CASCADE,
|
|
5730
|
+
FOREIGN KEY (project_id, run_id, attempt_id, target_node_key)
|
|
5731
|
+
REFERENCES site_crawl_graph_nodes(project_id, run_id, attempt_id, node_key) ON DELETE CASCADE,
|
|
5732
|
+
FOREIGN KEY (project_id, run_id, attempt_id, edge_key)
|
|
5733
|
+
REFERENCES site_crawl_edges(project_id, run_id, attempt_id, edge_key) ON DELETE CASCADE
|
|
5734
|
+
)`,
|
|
5735
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_graph_edges_attempt_edge
|
|
5736
|
+
ON site_crawl_graph_edges(project_id, run_id, attempt_id, edge_key)`,
|
|
5737
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_graph_edges_attempt_rank
|
|
5738
|
+
ON site_crawl_graph_edges(project_id, run_id, attempt_id, sample_rank)`,
|
|
5739
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_graph_edges_read
|
|
5740
|
+
ON site_crawl_graph_edges(project_id, run_id, attempt_id, sample_rank)`
|
|
5741
|
+
]
|
|
5742
|
+
},
|
|
5743
|
+
{
|
|
5744
|
+
// Keep the operator-requested root beside the terminal root without
|
|
5745
|
+
// rewriting immutable historical snapshots. The edge indexes make layout
|
|
5746
|
+
// cleanup/cascades linear at the graph's 20k/50k publish cap.
|
|
5747
|
+
version: 128,
|
|
5748
|
+
name: "site-crawl-requested-root-and-graph-edge-cleanup-indexes",
|
|
5749
|
+
statements: [
|
|
5750
|
+
`ALTER TABLE site_crawl_snapshots ADD COLUMN requested_root_url TEXT`,
|
|
5751
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_graph_edges_source_node
|
|
5752
|
+
ON site_crawl_graph_edges(project_id, run_id, attempt_id, source_node_key)`,
|
|
5753
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_graph_edges_target_node
|
|
5754
|
+
ON site_crawl_graph_edges(project_id, run_id, attempt_id, target_node_key)`
|
|
5755
|
+
]
|
|
5529
5756
|
}
|
|
5530
5757
|
];
|
|
5531
5758
|
function addRunsMeasurementPlanVersionForeignKey(tx) {
|
|
@@ -7465,7 +7692,7 @@ function buildMentionShare(snapshots, options) {
|
|
|
7465
7692
|
competitorCounts.set(competitor.domain, 0);
|
|
7466
7693
|
competitorAliases.set(
|
|
7467
7694
|
competitor.domain,
|
|
7468
|
-
competitor.brandTokens.filter((
|
|
7695
|
+
competitor.brandTokens.filter((alias2) => brandKeyFromText(alias2).length >= 3)
|
|
7469
7696
|
);
|
|
7470
7697
|
}
|
|
7471
7698
|
for (const snap of snapshots) {
|
|
@@ -8398,7 +8625,7 @@ function isOverviewRead(url) {
|
|
|
8398
8625
|
}
|
|
8399
8626
|
function isTechnicalAeoRead(url) {
|
|
8400
8627
|
const rest = projectRouteRest(url);
|
|
8401
|
-
return rest === "technical-aeo" || rest === "technical-aeo/pages" || rest === "technical-aeo/trend" || rest === "technical-aeo/crawl" || rest === "technical-aeo/crawl/pages" || rest === "technical-aeo/structure" || rest === "technical-aeo/internal-links" || rest === "technical-aeo/internal-links/neighbors" || rest === "technical-aeo/dead-links";
|
|
8628
|
+
return rest === "technical-aeo" || rest === "technical-aeo/pages" || rest === "technical-aeo/trend" || rest === "technical-aeo/crawl" || rest === "technical-aeo/graph" || rest === "technical-aeo/subgraph" || rest === "technical-aeo/path" || rest === "technical-aeo/changes" || rest === "technical-aeo/crawl/pages/audit" || rest === "technical-aeo/crawl/pages" || rest === "technical-aeo/structure" || rest === "technical-aeo/internal-links" || rest === "technical-aeo/internal-links/neighbors" || rest === "technical-aeo/dead-links";
|
|
8402
8629
|
}
|
|
8403
8630
|
function isReportRead(url) {
|
|
8404
8631
|
const rest = projectRouteRest(url);
|
|
@@ -9766,9 +9993,9 @@ function aliasMatchesAt(textWords, aliasWords, start) {
|
|
|
9766
9993
|
}
|
|
9767
9994
|
function containsAnyAlias(answerText, aliases) {
|
|
9768
9995
|
const textWords = words(answerText);
|
|
9769
|
-
const candidates = aliases.map(words).filter((
|
|
9996
|
+
const candidates = aliases.map(words).filter((alias2) => alias2.length > 0);
|
|
9770
9997
|
for (let start = 0; start < textWords.length; start++) {
|
|
9771
|
-
if (candidates.some((
|
|
9998
|
+
if (candidates.some((alias2) => aliasMatchesAt(textWords, alias2, start))) return true;
|
|
9772
9999
|
}
|
|
9773
10000
|
return false;
|
|
9774
10001
|
}
|
|
@@ -9776,9 +10003,9 @@ function mentionedTargets(answerText, targets) {
|
|
|
9776
10003
|
const result = /* @__PURE__ */ new Set();
|
|
9777
10004
|
if (answerText === null) return result;
|
|
9778
10005
|
const textWords = words(answerText);
|
|
9779
|
-
const aliases = targets.flatMap((target) => target.aliases.map((
|
|
10006
|
+
const aliases = targets.flatMap((target) => target.aliases.map((alias2) => ({ targetId: target.id, words: words(alias2) }))).filter((alias2) => alias2.words.length > 0);
|
|
9780
10007
|
for (let start = 0; start < textWords.length; ) {
|
|
9781
|
-
const matches = aliases.filter((
|
|
10008
|
+
const matches = aliases.filter((alias2) => aliasMatchesAt(textWords, alias2.words, start));
|
|
9782
10009
|
if (matches.length === 0) {
|
|
9783
10010
|
start++;
|
|
9784
10011
|
continue;
|
|
@@ -9862,7 +10089,7 @@ function prepareReport(input) {
|
|
|
9862
10089
|
}
|
|
9863
10090
|
const answers = [];
|
|
9864
10091
|
const mentionableIds = new Set(
|
|
9865
|
-
input.targets.filter((target) => target.aliases.some((
|
|
10092
|
+
input.targets.filter((target) => target.aliases.some((alias2) => words(alias2).length > 0)).map((target) => target.id)
|
|
9866
10093
|
);
|
|
9867
10094
|
for (const observation of observationsBySlot.values()) {
|
|
9868
10095
|
const edges = [...edgesByExecution.get(observation.slot.executionId) ?? []].sort((left, right) => compareText(left.id, right.id));
|
|
@@ -9985,7 +10212,7 @@ function targetCoverageRate(targetIds, slots, edges, prepared) {
|
|
|
9985
10212
|
}
|
|
9986
10213
|
function mentionRate(target, slots, prepared) {
|
|
9987
10214
|
const numerator = slots.filter((slot) => prepared.observationsBySlot.get(slot.id)?.mentionedTargetIds.has(target.id)).length;
|
|
9988
|
-
if (target.aliases.every((
|
|
10215
|
+
if (target.aliases.every((alias2) => words(alias2).length === 0)) {
|
|
9989
10216
|
return { numerator: null, denominator: null, rate: null, reason: "aliasless" };
|
|
9990
10217
|
}
|
|
9991
10218
|
if (slots.length === 0) return { numerator: null, denominator: null, rate: null, reason: "no-population" };
|
|
@@ -10013,11 +10240,11 @@ function buildSovForSlots(slots, competitors2, projectBrandNames, projectDomain,
|
|
|
10013
10240
|
let reason = null;
|
|
10014
10241
|
if (selected.length === 0) reason = "no-population";
|
|
10015
10242
|
else if (!status.complete || !status.answerComplete) reason = "incomplete";
|
|
10016
|
-
else if (projectBrandNames.every((
|
|
10243
|
+
else if (projectBrandNames.every((alias2) => words(alias2).length === 0)) reason = "no-project-aliases";
|
|
10017
10244
|
else if (sortedCompetitors.length === 0) reason = "no-competitors";
|
|
10018
10245
|
return rows.map((row) => {
|
|
10019
10246
|
if (reason !== null) return { domain: row.domain, own: row.own, presentIn: null, of: null, reason };
|
|
10020
|
-
if (row.aliases.every((
|
|
10247
|
+
if (row.aliases.every((alias2) => words(alias2).length === 0)) {
|
|
10021
10248
|
return { domain: row.domain, own: row.own, presentIn: null, of: null, reason: "aliasless" };
|
|
10022
10249
|
}
|
|
10023
10250
|
const presentIn = selected.filter((slot) => {
|
|
@@ -10167,7 +10394,7 @@ function mentionedForEdge(observation, edge, mentionableIds) {
|
|
|
10167
10394
|
return observation.mentionedTargetIds.has(edge.targetId);
|
|
10168
10395
|
}
|
|
10169
10396
|
function mentionableTargets(input, targetIds) {
|
|
10170
|
-
return input.targets.filter((target) => targetIds.has(target.id) && target.aliases.some((
|
|
10397
|
+
return input.targets.filter((target) => targetIds.has(target.id) && target.aliases.some((alias2) => words(alias2).length > 0));
|
|
10171
10398
|
}
|
|
10172
10399
|
function scopeMentionRate(input, targetIds, slots, answered, prepared) {
|
|
10173
10400
|
const mentionable = mentionableTargets(input, targetIds);
|
|
@@ -10193,7 +10420,7 @@ function indexedScopeCitationRate(slots, edges, indexes) {
|
|
|
10193
10420
|
return { numerator, denominator: basis.length, rate: numerator / basis.length };
|
|
10194
10421
|
}
|
|
10195
10422
|
function targetMentionRate(targets, slots, answered, indexes) {
|
|
10196
|
-
const mentionable = targets?.filter((target) => target.aliases.some((
|
|
10423
|
+
const mentionable = targets?.filter((target) => target.aliases.some((alias2) => words(alias2).length > 0)) ?? [];
|
|
10197
10424
|
if (mentionable.length === 0) return unavailable("aliasless");
|
|
10198
10425
|
if (slots.length === 0) return unavailable("no-population");
|
|
10199
10426
|
if (answered.length === 0) return unavailable("evidence-incomplete");
|
|
@@ -10209,7 +10436,7 @@ function presenceIn(slots, aliases, prepared) {
|
|
|
10209
10436
|
}
|
|
10210
10437
|
function scopeBrandPresence(input, slots, answered, prepared) {
|
|
10211
10438
|
if (slots.length === 0) return unavailable("no-population");
|
|
10212
|
-
if (input.projectBrandNames.every((
|
|
10439
|
+
if (input.projectBrandNames.every((alias2) => words(alias2).length === 0)) return unavailable("no-project-aliases");
|
|
10213
10440
|
if (answered.length === 0) return unavailable("evidence-incomplete");
|
|
10214
10441
|
const numerator = presenceIn(answered, input.projectBrandNames, prepared);
|
|
10215
10442
|
return { numerator, denominator: answered.length, rate: numerator / answered.length };
|
|
@@ -10508,8 +10735,8 @@ function reportInput(revision, plan, manifest, snapshots, legacy) {
|
|
|
10508
10735
|
label: group.label,
|
|
10509
10736
|
targetIds: group.targetKeys,
|
|
10510
10737
|
competitors: (group.competitors ?? []).map((domain) => {
|
|
10511
|
-
const
|
|
10512
|
-
return { domain, aliases: brandKeyFromText(
|
|
10738
|
+
const alias2 = brandLabelFromDomain(domain);
|
|
10739
|
+
return { domain, aliases: brandKeyFromText(alias2).length >= 4 ? [alias2] : [] };
|
|
10513
10740
|
})
|
|
10514
10741
|
})),
|
|
10515
10742
|
expectedSlots: slots,
|
|
@@ -12029,7 +12256,7 @@ function compileRules(input) {
|
|
|
12029
12256
|
const aliases = (input.rules.aliases ?? []).map(
|
|
12030
12257
|
(rule, index2) => compileRouteRule(rule, `rules.aliases[${index2}]`, ownedHosts)
|
|
12031
12258
|
);
|
|
12032
|
-
if (aliases.some((
|
|
12259
|
+
if (aliases.some((alias2) => alias2.host === primary.host)) {
|
|
12033
12260
|
throw new MeasurementDiscoveryConfigurationError("alias rules must use another owned host");
|
|
12034
12261
|
}
|
|
12035
12262
|
const excludedSlugSuffixes = sortedUnique3(
|
|
@@ -12947,14 +13174,14 @@ function compileMeasurementDraft(authoring, context) {
|
|
|
12947
13174
|
matcherClaims.set(identity, target.stableKey);
|
|
12948
13175
|
matchers.push(matcher);
|
|
12949
13176
|
});
|
|
12950
|
-
target.aliases.forEach((
|
|
12951
|
-
const identity = measurementMentionAliasKey(
|
|
13177
|
+
target.aliases.forEach((alias2, aliasIndex) => {
|
|
13178
|
+
const identity = measurementMentionAliasKey(alias2);
|
|
12952
13179
|
if (!identity) return;
|
|
12953
13180
|
const claimedBy = aliasClaims.get(identity);
|
|
12954
13181
|
if (claimedBy !== void 0 && claimedBy !== target.stableKey) {
|
|
12955
13182
|
sink.fail(
|
|
12956
13183
|
"target-alias-ambiguous",
|
|
12957
|
-
`Target alias normalizes to the same mention identity claimed by Target ${claimedBy}: ${
|
|
13184
|
+
`Target alias normalizes to the same mention identity claimed by Target ${claimedBy}: ${alias2}`,
|
|
12958
13185
|
["targets", targetIndex, "aliases", aliasIndex]
|
|
12959
13186
|
);
|
|
12960
13187
|
return;
|
|
@@ -27123,11 +27350,16 @@ var SCHEMA_TABLE = {
|
|
|
27123
27350
|
SiteAuditScoreDto: siteAuditScoreSchema,
|
|
27124
27351
|
SiteAuditTrendResponseDto: siteAuditTrendResponseSchema,
|
|
27125
27352
|
SiteCrawlDeadLinksResponseDto: siteCrawlDeadLinksResponseSchema,
|
|
27353
|
+
SiteCrawlGraphResponseDto: siteCrawlGraphResponseSchema,
|
|
27126
27354
|
SiteCrawlInternalLinksResponseDto: siteCrawlInternalLinksResponseSchema,
|
|
27127
27355
|
SiteCrawlNeighborsResponseDto: siteCrawlNeighborsResponseSchema,
|
|
27356
|
+
SiteCrawlPageAuditDto: siteCrawlPageAuditSchema,
|
|
27128
27357
|
SiteCrawlPagesResponseDto: siteCrawlPagesResponseSchema,
|
|
27129
27358
|
SiteCrawlStructureResponseDto: siteCrawlStructureResponseSchema,
|
|
27130
27359
|
SiteCrawlSummaryDto: siteCrawlSummarySchema,
|
|
27360
|
+
SiteHealthChangesResponseDto: siteHealthChangesResponseSchema,
|
|
27361
|
+
SiteHealthPathResponseDto: siteHealthPathResponseSchema,
|
|
27362
|
+
SiteHealthSubgraphResponseDto: siteHealthSubgraphResponseSchema,
|
|
27131
27363
|
SnapshotDiffResponse: snapshotDiffResponseSchema,
|
|
27132
27364
|
SnapshotListResponse: snapshotListResponseSchema,
|
|
27133
27365
|
SnapshotReportDto: snapshotReportSchema,
|
|
@@ -33062,6 +33294,103 @@ var routeCatalog = [
|
|
|
33062
33294
|
404: errorResponse("Project or crawl-bearing site-audit run not found.")
|
|
33063
33295
|
}
|
|
33064
33296
|
},
|
|
33297
|
+
{
|
|
33298
|
+
method: "get",
|
|
33299
|
+
path: "/api/v1/projects/{name}/technical-aeo/graph",
|
|
33300
|
+
summary: "Get a persisted Site Health graph projection",
|
|
33301
|
+
description: "Returns the deterministic graph projection computed once when the latest complete or selected historical crawl was published. ForceAtlas2 positions and the exact internal-anchor edge sample are persisted, so reads run no layout physics and never rescan the crawl edge table. The response is bounded to 20,000 nodes / 50,000 edges; `layout`, `omittedNodes`, `omittedEdges`, and `sampled` disclose legacy/unavailable layouts and intentional truncation.",
|
|
33302
|
+
tags: ["technical-aeo"],
|
|
33303
|
+
parameters: [
|
|
33304
|
+
nameParameter,
|
|
33305
|
+
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest complete crawl.", schema: stringSchema },
|
|
33306
|
+
{ name: "maxNodes", in: "query", description: "Maximum graph nodes. Defaults to and is capped at 20,000.", schema: { type: "integer", minimum: 1, maximum: 2e4 } },
|
|
33307
|
+
{ name: "maxEdges", in: "query", description: "Maximum graph edges. Defaults to and is capped at 50,000.", schema: { type: "integer", minimum: 1, maximum: 5e4 } }
|
|
33308
|
+
],
|
|
33309
|
+
responses: {
|
|
33310
|
+
200: jsonResponse("Bounded Site Health graph projection returned.", "SiteCrawlGraphResponseDto"),
|
|
33311
|
+
404: errorResponse("Project or crawl-bearing site-audit run not found.")
|
|
33312
|
+
}
|
|
33313
|
+
},
|
|
33314
|
+
{
|
|
33315
|
+
method: "get",
|
|
33316
|
+
path: "/api/v1/projects/{name}/technical-aeo/subgraph",
|
|
33317
|
+
summary: "Get a bounded canonical Site Health neighborhood",
|
|
33318
|
+
description: "Agent-oriented canonical crawl neighborhood around one page. Reads full crawl page/link rows rather than the sampled visualization, excludes layout coordinates, and returns snapshot completeness/termination. When a hard bound is reached, countAccuracy=lower-bound makes totalNodes, totalEdges, omittedNodes, and omittedEdges explicit minimums rather than false exact totals.",
|
|
33319
|
+
tags: ["technical-aeo"],
|
|
33320
|
+
parameters: [
|
|
33321
|
+
nameParameter,
|
|
33322
|
+
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest complete crawl.", schema: stringSchema },
|
|
33323
|
+
{ name: "nodeKey", in: "query", description: "Focus page canonical node key. Omit nodeKey and url to use the crawl root.", schema: stringSchema },
|
|
33324
|
+
{ name: "url", in: "query", description: "Focus page canonical URL. Omit nodeKey and url to use the crawl root.", schema: stringSchema },
|
|
33325
|
+
{ name: "hops", in: "query", description: "Undirected neighborhood radius. Defaults to 1; maximum 3.", schema: { type: "integer", minimum: 0, maximum: 3 } },
|
|
33326
|
+
{ name: "maxNodes", in: "query", description: "Maximum canonical nodes. Defaults to 25; maximum 200.", schema: { type: "integer", minimum: 1, maximum: 200 } },
|
|
33327
|
+
{ name: "maxEdges", in: "query", description: "Maximum canonical edges. Defaults to 50; maximum 500.", schema: { type: "integer", minimum: 1, maximum: 500 } }
|
|
33328
|
+
],
|
|
33329
|
+
responses: {
|
|
33330
|
+
200: jsonResponse("Bounded canonical Site Health neighborhood returned.", "SiteHealthSubgraphResponseDto"),
|
|
33331
|
+
404: errorResponse("Project, crawl-bearing run, or focus page not found.")
|
|
33332
|
+
}
|
|
33333
|
+
},
|
|
33334
|
+
{
|
|
33335
|
+
method: "get",
|
|
33336
|
+
path: "/api/v1/projects/{name}/technical-aeo/path",
|
|
33337
|
+
summary: "Find a directed internal-link path between pages",
|
|
33338
|
+
description: "Returns the shortest directed path over followable internal anchor links. The source defaults to the crawl root; the target must be identified by toNodeKey or toUrl. Exploration is bounded to 5,000 nodes and returns an explicit unreachable or truncated state. Snapshot complete and termination provenance prevents an unreachable result from a selected partial crawl being mistaken for a site-wide claim.",
|
|
33339
|
+
tags: ["technical-aeo"],
|
|
33340
|
+
parameters: [
|
|
33341
|
+
nameParameter,
|
|
33342
|
+
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest complete crawl.", schema: stringSchema },
|
|
33343
|
+
{ name: "fromNodeKey", in: "query", description: "Source canonical node key. Omit both source selectors to start at the crawl root.", schema: stringSchema },
|
|
33344
|
+
{ name: "fromUrl", in: "query", description: "Source canonical URL. Omit both source selectors to start at the crawl root.", schema: stringSchema },
|
|
33345
|
+
{ name: "toNodeKey", in: "query", description: "Target canonical node key. Required when toUrl is omitted.", schema: stringSchema },
|
|
33346
|
+
{ name: "toUrl", in: "query", description: "Target canonical URL. Required when toNodeKey is omitted.", schema: stringSchema },
|
|
33347
|
+
{ name: "maxDepth", in: "query", description: "Maximum directed link depth. Defaults to 12; maximum 24.", schema: { type: "integer", minimum: 1, maximum: 24 } }
|
|
33348
|
+
],
|
|
33349
|
+
responses: {
|
|
33350
|
+
200: jsonResponse("Directed internal-link path result returned.", "SiteHealthPathResponseDto"),
|
|
33351
|
+
400: errorResponse("A target selector is required."),
|
|
33352
|
+
404: errorResponse("Project, crawl-bearing run, source page, or target page not found.")
|
|
33353
|
+
}
|
|
33354
|
+
},
|
|
33355
|
+
{
|
|
33356
|
+
method: "get",
|
|
33357
|
+
path: "/api/v1/projects/{name}/technical-aeo/changes",
|
|
33358
|
+
summary: "Compare canonical Site Health snapshots",
|
|
33359
|
+
description: "Returns exact page and internal-link additions, removals, and semantic field changes between immutable complete crawl snapshots. Omitted toRunId selects the latest complete crawl; omitted fromRunId selects the complete crawl immediately before the target. ForceAtlas2 positions are presentation data and never count as a change. The first page includes exact post-filter summary counts; cursor pages set summary and total to null so they never repeat full snapshot joins or trust caller-carried counts. Results use a snapshot- and filter-bound keyset cursor.",
|
|
33360
|
+
tags: ["technical-aeo"],
|
|
33361
|
+
parameters: [
|
|
33362
|
+
nameParameter,
|
|
33363
|
+
{ name: "fromRunId", in: "query", description: "Baseline complete crawl. Omit for the complete crawl immediately before the target.", schema: stringSchema },
|
|
33364
|
+
{ name: "toRunId", in: "query", description: "Target complete crawl. Omit for the latest complete crawl.", schema: stringSchema },
|
|
33365
|
+
{ name: "scope", in: "query", description: "Restrict records and first-page summary counts to pages, links, or both.", schema: { type: "string", enum: ["all", "pages", "links"], default: "all" } },
|
|
33366
|
+
{ name: "change", in: "query", description: "Restrict records and first-page summary counts to one change kind.", schema: { type: "string", enum: ["all", "added", "removed", "changed"], default: "all" } },
|
|
33367
|
+
{ name: "cursor", in: "query", description: "Opaque keyset cursor bound to both run IDs and the selected filters.", schema: stringSchema },
|
|
33368
|
+
{ name: "limit", in: "query", description: "Maximum change records. Defaults to 25; maximum 100.", schema: { type: "integer", minimum: 1, maximum: 100 } }
|
|
33369
|
+
],
|
|
33370
|
+
responses: {
|
|
33371
|
+
200: jsonResponse("Canonical Site Health snapshot comparison returned.", "SiteHealthChangesResponseDto"),
|
|
33372
|
+
400: errorResponse("Invalid filters, run direction, or cursor context."),
|
|
33373
|
+
404: errorResponse("Project or selected crawl-bearing run not found.")
|
|
33374
|
+
}
|
|
33375
|
+
},
|
|
33376
|
+
{
|
|
33377
|
+
method: "get",
|
|
33378
|
+
path: "/api/v1/projects/{name}/technical-aeo/crawl/pages/audit",
|
|
33379
|
+
summary: "Get exact audit evidence for one persisted crawl page",
|
|
33380
|
+
description: "Returns one page score with the exact weighted-factor findings, recommendations, applicability, and independent critical defects persisted for the selected crawl. Exactly one of nodeKey or url is required. The discriminated state preserves crawl completeness and termination provenance, distinguishes a page that was not audited from one that was not found, and marks legacy score-only rows without fabricating missing evidence. This one-page read keeps verbose evidence out of the bounded interactive graph projection.",
|
|
33381
|
+
tags: ["technical-aeo"],
|
|
33382
|
+
parameters: [
|
|
33383
|
+
nameParameter,
|
|
33384
|
+
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest complete crawl.", schema: stringSchema },
|
|
33385
|
+
{ name: "nodeKey", in: "query", description: "Exact canonical crawl node key. Required when url is omitted.", schema: stringSchema },
|
|
33386
|
+
{ name: "url", in: "query", description: "Exact persisted crawl URL. Required when nodeKey is omitted.", schema: stringSchema }
|
|
33387
|
+
],
|
|
33388
|
+
responses: {
|
|
33389
|
+
200: jsonResponse("Page audit evidence state returned.", "SiteCrawlPageAuditDto"),
|
|
33390
|
+
400: errorResponse("Exactly one page selector is required."),
|
|
33391
|
+
404: errorResponse("Project or explicitly selected crawl-bearing run not found.")
|
|
33392
|
+
}
|
|
33393
|
+
},
|
|
33065
33394
|
{
|
|
33066
33395
|
method: "get",
|
|
33067
33396
|
path: "/api/v1/projects/{name}/technical-aeo/crawl/pages",
|
|
@@ -46482,7 +46811,7 @@ async function queryBacklinks(opts) {
|
|
|
46482
46811
|
const reversed = opts.targets.map(reverseDomain);
|
|
46483
46812
|
const targetList = reversed.map(quote).join(", ");
|
|
46484
46813
|
const limitClause = opts.limitPerTarget ? `QUALIFY row_number() OVER (PARTITION BY t.target_rev_domain ORDER BY v.num_hosts DESC) <= ${Math.floor(opts.limitPerTarget)}` : "";
|
|
46485
|
-
const
|
|
46814
|
+
const sql21 = `
|
|
46486
46815
|
WITH vertices AS (
|
|
46487
46816
|
SELECT * FROM read_csv(
|
|
46488
46817
|
${quote(opts.vertexPath)},
|
|
@@ -46518,7 +46847,7 @@ async function queryBacklinks(opts) {
|
|
|
46518
46847
|
const conn = await instance.connect();
|
|
46519
46848
|
let rows;
|
|
46520
46849
|
try {
|
|
46521
|
-
const reader = await conn.runAndReadAll(
|
|
46850
|
+
const reader = await conn.runAndReadAll(sql21);
|
|
46522
46851
|
rows = reader.getRowObjects();
|
|
46523
46852
|
} finally {
|
|
46524
46853
|
conn.disconnectSync?.();
|
|
@@ -55385,7 +55714,15 @@ function dedupeStrings(input) {
|
|
|
55385
55714
|
|
|
55386
55715
|
// ../api-routes/src/technical-aeo.ts
|
|
55387
55716
|
import crypto43 from "crypto";
|
|
55388
|
-
import { and as and43, asc as asc10, count, desc as desc24, eq as eq55, inArray as inArray19, lt as lt7, or as or11 } from "drizzle-orm";
|
|
55717
|
+
import { and as and43, asc as asc10, count, desc as desc24, eq as eq55, inArray as inArray19, isNotNull as isNotNull3, lt as lt7, or as or11, sql as sql20 } from "drizzle-orm";
|
|
55718
|
+
import { alias } from "drizzle-orm/sqlite-core";
|
|
55719
|
+
var FETCHED_SITE_CRAWL_STATES = /* @__PURE__ */ new Set([
|
|
55720
|
+
...SiteCrawlFetchedStates,
|
|
55721
|
+
// Snapshots published before the full-crawl contract used this aggregate
|
|
55722
|
+
// state. Keep historical structure counts truthful without admitting it to
|
|
55723
|
+
// the current crawler vocabulary.
|
|
55724
|
+
"fetched"
|
|
55725
|
+
]);
|
|
55389
55726
|
var SURFACEABLE_STATUSES = [RunStatuses.completed, RunStatuses.partial];
|
|
55390
55727
|
function emptyScore(projectName) {
|
|
55391
55728
|
return {
|
|
@@ -55478,7 +55815,32 @@ function mapCrawlPage(row) {
|
|
|
55478
55815
|
inboundOccurrences: row.inboundOccurrences,
|
|
55479
55816
|
outboundOccurrences: row.outboundOccurrences,
|
|
55480
55817
|
linkScoreRaw: row.linkScoreRaw,
|
|
55481
|
-
linkScoreNormalized: row.linkScoreNormalized
|
|
55818
|
+
linkScoreNormalized: row.linkScoreNormalized,
|
|
55819
|
+
healthState: deriveSiteHealthState(row)
|
|
55820
|
+
};
|
|
55821
|
+
}
|
|
55822
|
+
function mapCrawlPageAuditEvidence(row) {
|
|
55823
|
+
const fields = row.auditFields;
|
|
55824
|
+
const rawFactors = Array.isArray(fields.factors) ? fields.factors : [];
|
|
55825
|
+
const rawCriticalDefects = Array.isArray(fields.criticalDefects) ? fields.criticalDefects : [];
|
|
55826
|
+
const parsedFactors = rawFactors.map((factor) => siteCrawlAuditFactorSchema.safeParse(factor));
|
|
55827
|
+
const parsedCriticalDefects = rawCriticalDefects.map((defect) => siteCrawlCriticalDefectSchema.safeParse(defect));
|
|
55828
|
+
const complete = fields.schemaVersion === "1.0" && Array.isArray(fields.factors) && Array.isArray(fields.criticalDefects) && parsedFactors.every((result) => result.success) && parsedCriticalDefects.every((result) => result.success);
|
|
55829
|
+
const factors = parsedFactors.flatMap((result, index2) => {
|
|
55830
|
+
if (result.success) return [result.data];
|
|
55831
|
+
const legacy = siteAuditPageFactorSchema.safeParse(rawFactors[index2]);
|
|
55832
|
+
return legacy.success ? [{
|
|
55833
|
+
...legacy.data,
|
|
55834
|
+
status: factorStatusFromScore(legacy.data.score),
|
|
55835
|
+
applicable: null,
|
|
55836
|
+
findings: [],
|
|
55837
|
+
recommendations: []
|
|
55838
|
+
}] : [];
|
|
55839
|
+
});
|
|
55840
|
+
return {
|
|
55841
|
+
evidenceState: complete ? "complete" : "scores-only",
|
|
55842
|
+
factors,
|
|
55843
|
+
criticalDefects: parsedCriticalDefects.flatMap((result) => result.success ? [result.data] : [])
|
|
55482
55844
|
};
|
|
55483
55845
|
}
|
|
55484
55846
|
function mapCrawlEdge(row) {
|
|
@@ -55497,6 +55859,105 @@ function mapCrawlEdge(row) {
|
|
|
55497
55859
|
anchors: row.anchors
|
|
55498
55860
|
};
|
|
55499
55861
|
}
|
|
55862
|
+
var graphSourceNode = alias(siteCrawlGraphNodes, "site_crawl_graph_source_node");
|
|
55863
|
+
var graphTargetNode = alias(siteCrawlGraphNodes, "site_crawl_graph_target_node");
|
|
55864
|
+
var PAGE_CHANGE_PREDICATE = sql20`(
|
|
55865
|
+
current.url IS NOT previous.url
|
|
55866
|
+
OR current.final_url IS NOT previous.final_url
|
|
55867
|
+
OR current.path IS NOT previous.path
|
|
55868
|
+
OR current.parent_path IS NOT previous.parent_path
|
|
55869
|
+
OR current.discovery_source IS NOT previous.discovery_source
|
|
55870
|
+
OR current.fetch_state IS NOT previous.fetch_state
|
|
55871
|
+
OR current.http_status IS NOT previous.http_status
|
|
55872
|
+
OR current.canonical_url IS NOT previous.canonical_url
|
|
55873
|
+
OR current.indexability_state IS NOT previous.indexability_state
|
|
55874
|
+
OR current.indexability_reasons IS NOT previous.indexability_reasons
|
|
55875
|
+
OR current.audit_state IS NOT previous.audit_state
|
|
55876
|
+
OR current.audit_score IS NOT previous.audit_score
|
|
55877
|
+
OR current.inventory_eligible IS NOT previous.inventory_eligible
|
|
55878
|
+
OR current.depth IS NOT previous.depth
|
|
55879
|
+
OR current.inbound_unique_edges IS NOT previous.inbound_unique_edges
|
|
55880
|
+
OR current.outbound_unique_edges IS NOT previous.outbound_unique_edges
|
|
55881
|
+
OR current.inbound_occurrences IS NOT previous.inbound_occurrences
|
|
55882
|
+
OR current.outbound_occurrences IS NOT previous.outbound_occurrences
|
|
55883
|
+
OR current.link_score_raw IS NOT previous.link_score_raw
|
|
55884
|
+
OR current.link_score_normalized IS NOT previous.link_score_normalized
|
|
55885
|
+
)`;
|
|
55886
|
+
var LINK_CHANGE_PREDICATE = sql20`(
|
|
55887
|
+
current.source_node_key IS NOT previous.source_node_key
|
|
55888
|
+
OR current.source_url IS NOT previous.source_url
|
|
55889
|
+
OR current.target_node_key IS NOT previous.target_node_key
|
|
55890
|
+
OR current.target_url IS NOT previous.target_url
|
|
55891
|
+
OR current.relation IS NOT previous.relation
|
|
55892
|
+
OR current.followable IS NOT previous.followable
|
|
55893
|
+
OR current.occurrences IS NOT previous.occurrences
|
|
55894
|
+
OR current.followable_occurrences IS NOT previous.followable_occurrences
|
|
55895
|
+
OR current.nofollow_occurrences IS NOT previous.nofollow_occurrences
|
|
55896
|
+
OR current.anchors IS NOT previous.anchors
|
|
55897
|
+
)`;
|
|
55898
|
+
function encodeSiteHealthChangesCursor(cursor) {
|
|
55899
|
+
return Buffer.from(JSON.stringify(cursor)).toString("base64url");
|
|
55900
|
+
}
|
|
55901
|
+
function decodeSiteHealthChangesCursor(value, context) {
|
|
55902
|
+
if (value == null || value === "") return null;
|
|
55903
|
+
if (typeof value !== "string" || value.length > 2048) throw validationError("Invalid Site Health changes cursor");
|
|
55904
|
+
try {
|
|
55905
|
+
const parsed = JSON.parse(Buffer.from(value, "base64url").toString("utf8"));
|
|
55906
|
+
const valid = parsed.v === 1 && parsed.fromRunId === context.fromRunId && parsed.toRunId === context.toRunId && parsed.scope === context.scope && parsed.change === context.change && (parsed.entity === "page" || parsed.entity === "link") && typeof parsed.key === "string" && parsed.key.length > 0 && parsed.key.length <= 2048;
|
|
55907
|
+
if (!valid) throw new Error("context mismatch");
|
|
55908
|
+
return parsed;
|
|
55909
|
+
} catch {
|
|
55910
|
+
throw validationError("Site Health changes cursor does not match this snapshot comparison and filter");
|
|
55911
|
+
}
|
|
55912
|
+
}
|
|
55913
|
+
function compareChangeKeys(left, right) {
|
|
55914
|
+
return Buffer.compare(Buffer.from(left.key), Buffer.from(right.key));
|
|
55915
|
+
}
|
|
55916
|
+
function valueChanged(before, after) {
|
|
55917
|
+
if (Array.isArray(before) || Array.isArray(after) || before && typeof before === "object" || after && typeof after === "object") {
|
|
55918
|
+
return JSON.stringify(before) !== JSON.stringify(after);
|
|
55919
|
+
}
|
|
55920
|
+
return before !== after;
|
|
55921
|
+
}
|
|
55922
|
+
var PAGE_CHANGE_FIELDS = [
|
|
55923
|
+
"url",
|
|
55924
|
+
"finalUrl",
|
|
55925
|
+
"path",
|
|
55926
|
+
"parentPath",
|
|
55927
|
+
"discoverySource",
|
|
55928
|
+
"fetchState",
|
|
55929
|
+
"httpStatus",
|
|
55930
|
+
"canonicalUrl",
|
|
55931
|
+
"indexabilityState",
|
|
55932
|
+
"indexabilityReasons",
|
|
55933
|
+
"auditState",
|
|
55934
|
+
"auditScore",
|
|
55935
|
+
"inventoryEligible",
|
|
55936
|
+
"depth",
|
|
55937
|
+
"inboundUniqueEdges",
|
|
55938
|
+
"outboundUniqueEdges",
|
|
55939
|
+
"inboundOccurrences",
|
|
55940
|
+
"outboundOccurrences",
|
|
55941
|
+
"linkScoreRaw",
|
|
55942
|
+
"linkScoreNormalized"
|
|
55943
|
+
];
|
|
55944
|
+
var LINK_CHANGE_FIELDS = [
|
|
55945
|
+
"sourceNodeKey",
|
|
55946
|
+
"sourceUrl",
|
|
55947
|
+
"targetNodeKey",
|
|
55948
|
+
"targetUrl",
|
|
55949
|
+
"relation",
|
|
55950
|
+
"internal",
|
|
55951
|
+
"followable",
|
|
55952
|
+
"occurrences",
|
|
55953
|
+
"followableOccurrences",
|
|
55954
|
+
"nofollowOccurrences",
|
|
55955
|
+
"anchors"
|
|
55956
|
+
];
|
|
55957
|
+
function changedFields(before, after, fields) {
|
|
55958
|
+
if (!before || !after) return [];
|
|
55959
|
+
return fields.filter((field) => valueChanged(before[field], after[field])).map(String);
|
|
55960
|
+
}
|
|
55500
55961
|
async function technicalAeoRoutes(app, opts) {
|
|
55501
55962
|
const resolveCrawl = (projectId, runId) => {
|
|
55502
55963
|
const filters = [
|
|
@@ -55511,8 +55972,24 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
55511
55972
|
} else {
|
|
55512
55973
|
filters.push(eq55(siteCrawlSnapshots.complete, true), eq55(runs.status, RunStatuses.completed));
|
|
55513
55974
|
}
|
|
55514
|
-
return app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq55(siteCrawlSnapshots.runId, runs.id)).where(and43(...filters)).orderBy(desc24(siteCrawlSnapshots.createdAt)).limit(1).get();
|
|
55515
|
-
};
|
|
55975
|
+
return app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq55(siteCrawlSnapshots.runId, runs.id)).where(and43(...filters)).orderBy(desc24(siteCrawlSnapshots.createdAt), desc24(siteCrawlSnapshots.runId)).limit(1).get();
|
|
55976
|
+
};
|
|
55977
|
+
const detailScopeFor = (projectId, snapshot) => snapshot.detailsAvailable && snapshot.attemptId ? { projectId, runId: snapshot.runId, attemptId: snapshot.attemptId } : null;
|
|
55978
|
+
const pageInScope = (scope, selector) => {
|
|
55979
|
+
if (!selector.nodeKey && !selector.url) return null;
|
|
55980
|
+
return app.db.select().from(siteCrawlPages).where(and43(
|
|
55981
|
+
eq55(siteCrawlPages.projectId, scope.projectId),
|
|
55982
|
+
eq55(siteCrawlPages.runId, scope.runId),
|
|
55983
|
+
eq55(siteCrawlPages.attemptId, scope.attemptId),
|
|
55984
|
+
selector.nodeKey ? eq55(siteCrawlPages.nodeKey, selector.nodeKey) : eq55(siteCrawlPages.url, selector.url)
|
|
55985
|
+
)).orderBy(asc10(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
55986
|
+
};
|
|
55987
|
+
const rootPageInScope = (scope, rootUrl) => pageInScope(scope, { url: rootUrl }) ?? app.db.select().from(siteCrawlPages).where(and43(
|
|
55988
|
+
eq55(siteCrawlPages.projectId, scope.projectId),
|
|
55989
|
+
eq55(siteCrawlPages.runId, scope.runId),
|
|
55990
|
+
eq55(siteCrawlPages.attemptId, scope.attemptId),
|
|
55991
|
+
eq55(siteCrawlPages.depth, 0)
|
|
55992
|
+
)).orderBy(asc10(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
55516
55993
|
const hasLegacyAudit = (projectId) => Boolean(app.db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).innerJoin(runs, eq55(siteAuditSnapshots.runId, runs.id)).where(and43(
|
|
55517
55994
|
eq55(siteAuditSnapshots.projectId, projectId),
|
|
55518
55995
|
eq55(runs.projectId, projectId),
|
|
@@ -55526,6 +56003,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
55526
56003
|
legacyAuditAvailable,
|
|
55527
56004
|
runId: null,
|
|
55528
56005
|
runStatus: null,
|
|
56006
|
+
requestedRootUrl: null,
|
|
55529
56007
|
rootUrl: null,
|
|
55530
56008
|
effectiveOptions: {},
|
|
55531
56009
|
complete: false,
|
|
@@ -55643,6 +56121,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
55643
56121
|
legacyAuditAvailable,
|
|
55644
56122
|
runId: snapshot.runId,
|
|
55645
56123
|
runStatus: target.runStatus,
|
|
56124
|
+
requestedRootUrl: snapshot.requestedRootUrl,
|
|
55646
56125
|
rootUrl: snapshot.rootUrl,
|
|
55647
56126
|
crawlSchemaVersion: snapshot.crawlSchemaVersion,
|
|
55648
56127
|
engineVersion: snapshot.engineVersion,
|
|
@@ -55663,6 +56142,827 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
55663
56142
|
deadLinks
|
|
55664
56143
|
};
|
|
55665
56144
|
});
|
|
56145
|
+
app.get("/projects/:name/technical-aeo/graph", async (request) => {
|
|
56146
|
+
const project = resolveProject(app.db, request.params.name);
|
|
56147
|
+
const target = resolveCrawl(project.id, request.query.runId);
|
|
56148
|
+
if (!target) {
|
|
56149
|
+
if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
|
|
56150
|
+
return {
|
|
56151
|
+
project: project.name,
|
|
56152
|
+
hasCrawlData: false,
|
|
56153
|
+
runId: null,
|
|
56154
|
+
layout: { state: "unavailable", version: null, reason: "no-crawl" },
|
|
56155
|
+
totalNodes: 0,
|
|
56156
|
+
totalEdges: 0,
|
|
56157
|
+
nodes: [],
|
|
56158
|
+
edges: [],
|
|
56159
|
+
omittedNodes: 0,
|
|
56160
|
+
omittedEdges: 0,
|
|
56161
|
+
sampled: false
|
|
56162
|
+
};
|
|
56163
|
+
}
|
|
56164
|
+
const snapshot = target.snapshot;
|
|
56165
|
+
if (!snapshot.detailsAvailable || !snapshot.attemptId) {
|
|
56166
|
+
return {
|
|
56167
|
+
project: project.name,
|
|
56168
|
+
hasCrawlData: true,
|
|
56169
|
+
runId: snapshot.runId,
|
|
56170
|
+
layout: { state: "unavailable", version: null, reason: "details-unavailable" },
|
|
56171
|
+
totalNodes: 0,
|
|
56172
|
+
totalEdges: 0,
|
|
56173
|
+
nodes: [],
|
|
56174
|
+
edges: [],
|
|
56175
|
+
omittedNodes: 0,
|
|
56176
|
+
omittedEdges: 0,
|
|
56177
|
+
sampled: false
|
|
56178
|
+
};
|
|
56179
|
+
}
|
|
56180
|
+
const persistedLayout = app.db.select().from(siteCrawlGraphLayouts).where(and43(
|
|
56181
|
+
eq55(siteCrawlGraphLayouts.projectId, project.id),
|
|
56182
|
+
eq55(siteCrawlGraphLayouts.runId, snapshot.runId),
|
|
56183
|
+
eq55(siteCrawlGraphLayouts.attemptId, snapshot.attemptId)
|
|
56184
|
+
)).limit(1).get();
|
|
56185
|
+
if (!persistedLayout) {
|
|
56186
|
+
return {
|
|
56187
|
+
project: project.name,
|
|
56188
|
+
hasCrawlData: true,
|
|
56189
|
+
runId: snapshot.runId,
|
|
56190
|
+
layout: { state: "unavailable", version: null, reason: "legacy-snapshot" },
|
|
56191
|
+
totalNodes: 0,
|
|
56192
|
+
totalEdges: 0,
|
|
56193
|
+
nodes: [],
|
|
56194
|
+
edges: [],
|
|
56195
|
+
omittedNodes: 0,
|
|
56196
|
+
omittedEdges: 0,
|
|
56197
|
+
sampled: false
|
|
56198
|
+
};
|
|
56199
|
+
}
|
|
56200
|
+
if (persistedLayout.state !== "ready" || !persistedLayout.layoutVersion) {
|
|
56201
|
+
const totalNodes2 = persistedLayout.totalNodes;
|
|
56202
|
+
const totalEdges2 = persistedLayout.totalEdges;
|
|
56203
|
+
return {
|
|
56204
|
+
project: project.name,
|
|
56205
|
+
hasCrawlData: true,
|
|
56206
|
+
runId: snapshot.runId,
|
|
56207
|
+
layout: {
|
|
56208
|
+
state: "unavailable",
|
|
56209
|
+
version: null,
|
|
56210
|
+
reason: persistedLayout.failureCode === "empty-crawl" ? "empty-crawl" : "layout-failed"
|
|
56211
|
+
},
|
|
56212
|
+
totalNodes: totalNodes2,
|
|
56213
|
+
totalEdges: totalEdges2,
|
|
56214
|
+
nodes: [],
|
|
56215
|
+
edges: [],
|
|
56216
|
+
omittedNodes: totalNodes2,
|
|
56217
|
+
omittedEdges: totalEdges2,
|
|
56218
|
+
sampled: totalNodes2 > 0 || totalEdges2 > 0
|
|
56219
|
+
};
|
|
56220
|
+
}
|
|
56221
|
+
const maxNodes = parseBoundedLimit(request.query.maxNodes, SITE_CRAWL_GRAPH_DEFAULT_MAX_NODES, SITE_CRAWL_GRAPH_MAX_NODES);
|
|
56222
|
+
const maxEdges = parseBoundedLimit(request.query.maxEdges, SITE_CRAWL_GRAPH_DEFAULT_MAX_EDGES, SITE_CRAWL_GRAPH_MAX_EDGES);
|
|
56223
|
+
const graphScope = [
|
|
56224
|
+
eq55(siteCrawlGraphNodes.projectId, project.id),
|
|
56225
|
+
eq55(siteCrawlGraphNodes.runId, snapshot.runId),
|
|
56226
|
+
eq55(siteCrawlGraphNodes.attemptId, snapshot.attemptId)
|
|
56227
|
+
];
|
|
56228
|
+
const nodeRows = app.db.select({
|
|
56229
|
+
nodeKey: siteCrawlPages.nodeKey,
|
|
56230
|
+
url: siteCrawlPages.url,
|
|
56231
|
+
path: siteCrawlPages.path,
|
|
56232
|
+
depth: siteCrawlPages.depth,
|
|
56233
|
+
indexabilityState: siteCrawlPages.indexabilityState,
|
|
56234
|
+
indexabilityReasons: siteCrawlPages.indexabilityReasons,
|
|
56235
|
+
canonicalNodeKey: siteCrawlPages.canonicalNodeKey,
|
|
56236
|
+
fetchState: siteCrawlPages.fetchState,
|
|
56237
|
+
auditState: siteCrawlPages.auditState,
|
|
56238
|
+
auditScore: siteCrawlPages.auditScore,
|
|
56239
|
+
inventoryEligible: siteCrawlPages.inventoryEligible,
|
|
56240
|
+
inboundUniqueEdges: siteCrawlPages.inboundUniqueEdges,
|
|
56241
|
+
outboundUniqueEdges: siteCrawlPages.outboundUniqueEdges,
|
|
56242
|
+
linkScoreNormalized: siteCrawlPages.linkScoreNormalized,
|
|
56243
|
+
x: siteCrawlGraphNodes.x,
|
|
56244
|
+
y: siteCrawlGraphNodes.y
|
|
56245
|
+
}).from(siteCrawlGraphNodes).innerJoin(siteCrawlPages, and43(
|
|
56246
|
+
eq55(siteCrawlPages.projectId, siteCrawlGraphNodes.projectId),
|
|
56247
|
+
eq55(siteCrawlPages.runId, siteCrawlGraphNodes.runId),
|
|
56248
|
+
eq55(siteCrawlPages.attemptId, siteCrawlGraphNodes.attemptId),
|
|
56249
|
+
eq55(siteCrawlPages.nodeKey, siteCrawlGraphNodes.nodeKey)
|
|
56250
|
+
)).where(and43(...graphScope, lt7(siteCrawlGraphNodes.sampleRank, maxNodes))).orderBy(asc10(siteCrawlGraphNodes.sampleRank)).all();
|
|
56251
|
+
const nodes = nodeRows.map(({ indexabilityReasons, canonicalNodeKey, ...row }) => ({
|
|
56252
|
+
...row,
|
|
56253
|
+
healthState: deriveSiteHealthState({ ...row, indexabilityReasons, canonicalNodeKey })
|
|
56254
|
+
}));
|
|
56255
|
+
const edges = app.db.select({
|
|
56256
|
+
edgeKey: siteCrawlGraphEdges.edgeKey,
|
|
56257
|
+
sourceNodeKey: siteCrawlGraphEdges.sourceNodeKey,
|
|
56258
|
+
targetNodeKey: siteCrawlGraphEdges.targetNodeKey,
|
|
56259
|
+
followable: siteCrawlGraphEdges.followable,
|
|
56260
|
+
occurrences: siteCrawlGraphEdges.occurrences
|
|
56261
|
+
}).from(siteCrawlGraphEdges).innerJoin(graphSourceNode, and43(
|
|
56262
|
+
eq55(graphSourceNode.projectId, siteCrawlGraphEdges.projectId),
|
|
56263
|
+
eq55(graphSourceNode.runId, siteCrawlGraphEdges.runId),
|
|
56264
|
+
eq55(graphSourceNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
56265
|
+
eq55(graphSourceNode.nodeKey, siteCrawlGraphEdges.sourceNodeKey),
|
|
56266
|
+
lt7(graphSourceNode.sampleRank, maxNodes)
|
|
56267
|
+
)).innerJoin(graphTargetNode, and43(
|
|
56268
|
+
eq55(graphTargetNode.projectId, siteCrawlGraphEdges.projectId),
|
|
56269
|
+
eq55(graphTargetNode.runId, siteCrawlGraphEdges.runId),
|
|
56270
|
+
eq55(graphTargetNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
56271
|
+
eq55(graphTargetNode.nodeKey, siteCrawlGraphEdges.targetNodeKey),
|
|
56272
|
+
lt7(graphTargetNode.sampleRank, maxNodes)
|
|
56273
|
+
)).where(and43(
|
|
56274
|
+
eq55(siteCrawlGraphEdges.projectId, project.id),
|
|
56275
|
+
eq55(siteCrawlGraphEdges.runId, snapshot.runId),
|
|
56276
|
+
eq55(siteCrawlGraphEdges.attemptId, snapshot.attemptId),
|
|
56277
|
+
lt7(siteCrawlGraphEdges.sampleRank, maxEdges)
|
|
56278
|
+
)).orderBy(asc10(siteCrawlGraphEdges.sampleRank)).all();
|
|
56279
|
+
const totalNodes = persistedLayout.totalNodes;
|
|
56280
|
+
const totalEdges = persistedLayout.totalEdges;
|
|
56281
|
+
const omittedNodes = Math.max(0, totalNodes - nodes.length);
|
|
56282
|
+
const omittedEdges = Math.max(0, totalEdges - edges.length);
|
|
56283
|
+
return {
|
|
56284
|
+
project: project.name,
|
|
56285
|
+
hasCrawlData: true,
|
|
56286
|
+
runId: snapshot.runId,
|
|
56287
|
+
layout: { state: "ready", version: persistedLayout.layoutVersion, computedAt: persistedLayout.createdAt },
|
|
56288
|
+
totalNodes,
|
|
56289
|
+
totalEdges,
|
|
56290
|
+
nodes,
|
|
56291
|
+
edges,
|
|
56292
|
+
omittedNodes,
|
|
56293
|
+
omittedEdges,
|
|
56294
|
+
sampled: omittedNodes > 0 || omittedEdges > 0
|
|
56295
|
+
};
|
|
56296
|
+
});
|
|
56297
|
+
app.get("/projects/:name/technical-aeo/subgraph", async (request) => {
|
|
56298
|
+
const project = resolveProject(app.db, request.params.name);
|
|
56299
|
+
const hops = Math.min(3, parsePositiveInt(request.query.hops, 1, 3));
|
|
56300
|
+
const maxNodes = parseBoundedLimit(
|
|
56301
|
+
request.query.maxNodes,
|
|
56302
|
+
SITE_HEALTH_SUBGRAPH_DEFAULT_MAX_NODES,
|
|
56303
|
+
SITE_HEALTH_SUBGRAPH_MAX_NODES
|
|
56304
|
+
);
|
|
56305
|
+
const maxEdges = parseBoundedLimit(
|
|
56306
|
+
request.query.maxEdges,
|
|
56307
|
+
SITE_HEALTH_SUBGRAPH_DEFAULT_MAX_EDGES,
|
|
56308
|
+
SITE_HEALTH_SUBGRAPH_MAX_EDGES
|
|
56309
|
+
);
|
|
56310
|
+
const target = resolveCrawl(project.id, request.query.runId);
|
|
56311
|
+
const empty = (state, runId, hasCrawlData, complete, termination) => ({
|
|
56312
|
+
project: project.name,
|
|
56313
|
+
hasCrawlData,
|
|
56314
|
+
runId,
|
|
56315
|
+
complete,
|
|
56316
|
+
termination,
|
|
56317
|
+
state,
|
|
56318
|
+
focusNodeKey: null,
|
|
56319
|
+
focusUrl: null,
|
|
56320
|
+
hops,
|
|
56321
|
+
totalNodes: 0,
|
|
56322
|
+
totalEdges: 0,
|
|
56323
|
+
nodes: [],
|
|
56324
|
+
edges: [],
|
|
56325
|
+
omittedNodes: 0,
|
|
56326
|
+
omittedEdges: 0,
|
|
56327
|
+
countAccuracy: state === "no-crawl" ? "exact" : "lower-bound",
|
|
56328
|
+
truncated: false
|
|
56329
|
+
});
|
|
56330
|
+
if (!target) {
|
|
56331
|
+
if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
|
|
56332
|
+
return empty("no-crawl", null, false, false, null);
|
|
56333
|
+
}
|
|
56334
|
+
const snapshot = target.snapshot;
|
|
56335
|
+
const scope = detailScopeFor(project.id, snapshot);
|
|
56336
|
+
if (!scope) return empty("details-unavailable", snapshot.runId, true, snapshot.complete, snapshot.termination);
|
|
56337
|
+
const explicitFocus = Boolean(request.query.nodeKey || request.query.url);
|
|
56338
|
+
const focus = explicitFocus ? pageInScope(scope, { nodeKey: request.query.nodeKey, url: request.query.url }) : rootPageInScope(scope, snapshot.rootUrl);
|
|
56339
|
+
if (!focus) {
|
|
56340
|
+
if (explicitFocus) throw notFound("Site crawl page", request.query.nodeKey ?? request.query.url);
|
|
56341
|
+
return {
|
|
56342
|
+
...empty("details-unavailable", snapshot.runId, true, snapshot.complete, snapshot.termination),
|
|
56343
|
+
truncated: false
|
|
56344
|
+
};
|
|
56345
|
+
}
|
|
56346
|
+
const distances = /* @__PURE__ */ new Map([[focus.nodeKey, 0]]);
|
|
56347
|
+
const directRelations = /* @__PURE__ */ new Map();
|
|
56348
|
+
const edgeRows = /* @__PURE__ */ new Map();
|
|
56349
|
+
const omittedNodeKeys = /* @__PURE__ */ new Set();
|
|
56350
|
+
const omittedEdgeKeys = /* @__PURE__ */ new Set();
|
|
56351
|
+
const seenEdgeKeys = /* @__PURE__ */ new Set();
|
|
56352
|
+
let frontier = [focus.nodeKey];
|
|
56353
|
+
let queryTruncated = false;
|
|
56354
|
+
for (let distance = 0; distance < hops && frontier.length > 0; distance += 1) {
|
|
56355
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and43(
|
|
56356
|
+
eq55(siteCrawlEdges.projectId, scope.projectId),
|
|
56357
|
+
eq55(siteCrawlEdges.runId, scope.runId),
|
|
56358
|
+
eq55(siteCrawlEdges.attemptId, scope.attemptId),
|
|
56359
|
+
eq55(siteCrawlEdges.internal, true),
|
|
56360
|
+
isNotNull3(siteCrawlEdges.targetNodeKey),
|
|
56361
|
+
or11(
|
|
56362
|
+
inArray19(siteCrawlEdges.sourceNodeKey, frontier),
|
|
56363
|
+
inArray19(siteCrawlEdges.targetNodeKey, frontier)
|
|
56364
|
+
)
|
|
56365
|
+
)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(maxEdges + maxNodes + 1).all();
|
|
56366
|
+
if (candidates.length > maxEdges + maxNodes) queryTruncated = true;
|
|
56367
|
+
const next = /* @__PURE__ */ new Set();
|
|
56368
|
+
for (const edge of candidates.slice(0, maxEdges + maxNodes)) {
|
|
56369
|
+
if (!edge.targetNodeKey || seenEdgeKeys.has(edge.edgeKey)) continue;
|
|
56370
|
+
seenEdgeKeys.add(edge.edgeKey);
|
|
56371
|
+
const sourceInFrontier = frontier.includes(edge.sourceNodeKey);
|
|
56372
|
+
const neighborKey = sourceInFrontier ? edge.targetNodeKey : edge.sourceNodeKey;
|
|
56373
|
+
if (!distances.has(neighborKey)) {
|
|
56374
|
+
if (distances.size < maxNodes) {
|
|
56375
|
+
distances.set(neighborKey, distance + 1);
|
|
56376
|
+
next.add(neighborKey);
|
|
56377
|
+
} else {
|
|
56378
|
+
omittedNodeKeys.add(neighborKey);
|
|
56379
|
+
}
|
|
56380
|
+
}
|
|
56381
|
+
if (distance === 0 && neighborKey !== focus.nodeKey) {
|
|
56382
|
+
const direction = edge.sourceNodeKey === focus.nodeKey ? "outbound" : "inbound";
|
|
56383
|
+
const prior = directRelations.get(neighborKey);
|
|
56384
|
+
directRelations.set(neighborKey, prior && prior !== direction ? "both" : direction);
|
|
56385
|
+
}
|
|
56386
|
+
if (distances.has(edge.sourceNodeKey) && distances.has(edge.targetNodeKey)) {
|
|
56387
|
+
if (edgeRows.size < maxEdges) edgeRows.set(edge.edgeKey, edge);
|
|
56388
|
+
else omittedEdgeKeys.add(edge.edgeKey);
|
|
56389
|
+
} else {
|
|
56390
|
+
omittedEdgeKeys.add(edge.edgeKey);
|
|
56391
|
+
}
|
|
56392
|
+
}
|
|
56393
|
+
frontier = [...next].sort();
|
|
56394
|
+
}
|
|
56395
|
+
const nodeKeys = [...distances.keys()];
|
|
56396
|
+
const pageRows = nodeKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and43(
|
|
56397
|
+
eq55(siteCrawlPages.projectId, scope.projectId),
|
|
56398
|
+
eq55(siteCrawlPages.runId, scope.runId),
|
|
56399
|
+
eq55(siteCrawlPages.attemptId, scope.attemptId),
|
|
56400
|
+
inArray19(siteCrawlPages.nodeKey, nodeKeys)
|
|
56401
|
+
)).all();
|
|
56402
|
+
const persistedNodeKeys = new Set(pageRows.map((row) => row.nodeKey));
|
|
56403
|
+
const outermostNodeKeys = pageRows.filter((row) => distances.get(row.nodeKey) === hops).map((row) => row.nodeKey);
|
|
56404
|
+
const omittedOutermostEdge = outermostNodeKeys.length === 0 ? void 0 : app.db.select({ edgeKey: siteCrawlEdges.edgeKey }).from(siteCrawlEdges).where(and43(
|
|
56405
|
+
eq55(siteCrawlEdges.projectId, scope.projectId),
|
|
56406
|
+
eq55(siteCrawlEdges.runId, scope.runId),
|
|
56407
|
+
eq55(siteCrawlEdges.attemptId, scope.attemptId),
|
|
56408
|
+
eq55(siteCrawlEdges.internal, true),
|
|
56409
|
+
isNotNull3(siteCrawlEdges.targetNodeKey),
|
|
56410
|
+
inArray19(siteCrawlEdges.sourceNodeKey, outermostNodeKeys),
|
|
56411
|
+
inArray19(siteCrawlEdges.targetNodeKey, outermostNodeKeys)
|
|
56412
|
+
)).limit(1).get();
|
|
56413
|
+
if (omittedOutermostEdge && !seenEdgeKeys.has(omittedOutermostEdge.edgeKey)) {
|
|
56414
|
+
omittedEdgeKeys.add(omittedOutermostEdge.edgeKey);
|
|
56415
|
+
}
|
|
56416
|
+
const nodes = pageRows.map((row) => ({
|
|
56417
|
+
...mapCrawlPage(row),
|
|
56418
|
+
distance: distances.get(row.nodeKey) ?? 0,
|
|
56419
|
+
relationToFocus: row.nodeKey === focus.nodeKey ? "focus" : directRelations.get(row.nodeKey) ?? "transitive"
|
|
56420
|
+
})).sort((a, b) => a.distance - b.distance || a.nodeKey.localeCompare(b.nodeKey));
|
|
56421
|
+
const edges = [...edgeRows.values()].filter((edge) => edge.targetNodeKey && persistedNodeKeys.has(edge.sourceNodeKey) && persistedNodeKeys.has(edge.targetNodeKey)).sort((a, b) => a.edgeKey.localeCompare(b.edgeKey)).map(mapCrawlEdge);
|
|
56422
|
+
const omittedNodes = omittedNodeKeys.size + Math.max(0, distances.size - pageRows.length);
|
|
56423
|
+
const omittedEdges = omittedEdgeKeys.size + (queryTruncated ? 1 : 0) + Math.max(0, edgeRows.size - edges.length);
|
|
56424
|
+
const truncated = omittedNodes > 0 || omittedEdges > 0;
|
|
56425
|
+
return {
|
|
56426
|
+
project: project.name,
|
|
56427
|
+
hasCrawlData: true,
|
|
56428
|
+
runId: snapshot.runId,
|
|
56429
|
+
complete: snapshot.complete,
|
|
56430
|
+
termination: snapshot.termination,
|
|
56431
|
+
state: "ready",
|
|
56432
|
+
focusNodeKey: focus.nodeKey,
|
|
56433
|
+
focusUrl: focus.url,
|
|
56434
|
+
hops,
|
|
56435
|
+
totalNodes: nodes.length + omittedNodes,
|
|
56436
|
+
totalEdges: edges.length + omittedEdges,
|
|
56437
|
+
nodes,
|
|
56438
|
+
edges,
|
|
56439
|
+
omittedNodes,
|
|
56440
|
+
omittedEdges,
|
|
56441
|
+
countAccuracy: truncated ? "lower-bound" : "exact",
|
|
56442
|
+
truncated
|
|
56443
|
+
};
|
|
56444
|
+
});
|
|
56445
|
+
app.get("/projects/:name/technical-aeo/path", async (request) => {
|
|
56446
|
+
const project = resolveProject(app.db, request.params.name);
|
|
56447
|
+
const maxDepth = parseBoundedLimit(
|
|
56448
|
+
request.query.maxDepth,
|
|
56449
|
+
SITE_HEALTH_PATH_DEFAULT_MAX_DEPTH,
|
|
56450
|
+
SITE_HEALTH_PATH_MAX_DEPTH
|
|
56451
|
+
);
|
|
56452
|
+
if (!request.query.toNodeKey && !request.query.toUrl) {
|
|
56453
|
+
throw validationError("toNodeKey or toUrl is required");
|
|
56454
|
+
}
|
|
56455
|
+
const target = resolveCrawl(project.id, request.query.runId);
|
|
56456
|
+
if (!target) {
|
|
56457
|
+
if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
|
|
56458
|
+
return {
|
|
56459
|
+
project: project.name,
|
|
56460
|
+
runId: null,
|
|
56461
|
+
state: "no-crawl",
|
|
56462
|
+
from: null,
|
|
56463
|
+
to: null,
|
|
56464
|
+
complete: false,
|
|
56465
|
+
termination: null,
|
|
56466
|
+
maxDepth,
|
|
56467
|
+
visitedNodes: 0,
|
|
56468
|
+
nodes: [],
|
|
56469
|
+
edges: []
|
|
56470
|
+
};
|
|
56471
|
+
}
|
|
56472
|
+
const snapshot = target.snapshot;
|
|
56473
|
+
const scope = detailScopeFor(project.id, snapshot);
|
|
56474
|
+
if (!scope) {
|
|
56475
|
+
return {
|
|
56476
|
+
project: project.name,
|
|
56477
|
+
runId: snapshot.runId,
|
|
56478
|
+
state: "details-unavailable",
|
|
56479
|
+
from: null,
|
|
56480
|
+
to: null,
|
|
56481
|
+
complete: snapshot.complete,
|
|
56482
|
+
termination: snapshot.termination,
|
|
56483
|
+
maxDepth,
|
|
56484
|
+
visitedNodes: 0,
|
|
56485
|
+
nodes: [],
|
|
56486
|
+
edges: []
|
|
56487
|
+
};
|
|
56488
|
+
}
|
|
56489
|
+
const explicitFrom = Boolean(request.query.fromNodeKey || request.query.fromUrl);
|
|
56490
|
+
const fromPage = explicitFrom ? pageInScope(scope, { nodeKey: request.query.fromNodeKey, url: request.query.fromUrl }) : rootPageInScope(scope, snapshot.rootUrl);
|
|
56491
|
+
if (!fromPage) throw notFound("Site crawl page", request.query.fromNodeKey ?? request.query.fromUrl ?? snapshot.rootUrl);
|
|
56492
|
+
const toPage = pageInScope(scope, { nodeKey: request.query.toNodeKey, url: request.query.toUrl });
|
|
56493
|
+
if (!toPage) throw notFound("Site crawl page", request.query.toNodeKey ?? request.query.toUrl);
|
|
56494
|
+
const reference = (row) => ({
|
|
56495
|
+
nodeKey: row.nodeKey,
|
|
56496
|
+
url: row.url,
|
|
56497
|
+
path: row.path
|
|
56498
|
+
});
|
|
56499
|
+
const predecessor = /* @__PURE__ */ new Map();
|
|
56500
|
+
const visited = /* @__PURE__ */ new Set([fromPage.nodeKey]);
|
|
56501
|
+
let frontier = [fromPage.nodeKey];
|
|
56502
|
+
let found = fromPage.nodeKey === toPage.nodeKey;
|
|
56503
|
+
let truncated = false;
|
|
56504
|
+
for (let depth = 0; depth < maxDepth && frontier.length > 0 && !found; depth += 1) {
|
|
56505
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and43(
|
|
56506
|
+
eq55(siteCrawlEdges.projectId, scope.projectId),
|
|
56507
|
+
eq55(siteCrawlEdges.runId, scope.runId),
|
|
56508
|
+
eq55(siteCrawlEdges.attemptId, scope.attemptId),
|
|
56509
|
+
eq55(siteCrawlEdges.internal, true),
|
|
56510
|
+
eq55(siteCrawlEdges.followable, true),
|
|
56511
|
+
eq55(siteCrawlEdges.relation, "anchor"),
|
|
56512
|
+
isNotNull3(siteCrawlEdges.targetNodeKey),
|
|
56513
|
+
inArray19(siteCrawlEdges.sourceNodeKey, frontier)
|
|
56514
|
+
)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(SITE_HEALTH_PATH_MAX_VISITED_NODES + 1).all();
|
|
56515
|
+
if (candidates.length > SITE_HEALTH_PATH_MAX_VISITED_NODES) truncated = true;
|
|
56516
|
+
const candidateTargetKeys = [...new Set(candidates.map((edge) => edge.targetNodeKey).filter((nodeKey) => nodeKey != null))];
|
|
56517
|
+
const persistedTargetKeys = new Set(candidateTargetKeys.length === 0 ? [] : app.db.select({ nodeKey: siteCrawlPages.nodeKey }).from(siteCrawlPages).where(and43(
|
|
56518
|
+
eq55(siteCrawlPages.projectId, scope.projectId),
|
|
56519
|
+
eq55(siteCrawlPages.runId, scope.runId),
|
|
56520
|
+
eq55(siteCrawlPages.attemptId, scope.attemptId),
|
|
56521
|
+
inArray19(siteCrawlPages.nodeKey, candidateTargetKeys)
|
|
56522
|
+
)).all().map((row) => row.nodeKey));
|
|
56523
|
+
const next = /* @__PURE__ */ new Set();
|
|
56524
|
+
for (const edge of candidates.slice(0, SITE_HEALTH_PATH_MAX_VISITED_NODES)) {
|
|
56525
|
+
if (!edge.targetNodeKey || !persistedTargetKeys.has(edge.targetNodeKey) || visited.has(edge.targetNodeKey)) continue;
|
|
56526
|
+
if (visited.size >= SITE_HEALTH_PATH_MAX_VISITED_NODES) {
|
|
56527
|
+
truncated = true;
|
|
56528
|
+
break;
|
|
56529
|
+
}
|
|
56530
|
+
visited.add(edge.targetNodeKey);
|
|
56531
|
+
predecessor.set(edge.targetNodeKey, { nodeKey: edge.sourceNodeKey, edge });
|
|
56532
|
+
next.add(edge.targetNodeKey);
|
|
56533
|
+
if (edge.targetNodeKey === toPage.nodeKey) {
|
|
56534
|
+
found = true;
|
|
56535
|
+
break;
|
|
56536
|
+
}
|
|
56537
|
+
}
|
|
56538
|
+
frontier = [...next].sort();
|
|
56539
|
+
}
|
|
56540
|
+
if (!found) {
|
|
56541
|
+
return {
|
|
56542
|
+
project: project.name,
|
|
56543
|
+
runId: snapshot.runId,
|
|
56544
|
+
complete: snapshot.complete,
|
|
56545
|
+
termination: snapshot.termination,
|
|
56546
|
+
state: truncated || frontier.length > 0 ? "truncated" : "unreachable",
|
|
56547
|
+
from: reference(fromPage),
|
|
56548
|
+
to: reference(toPage),
|
|
56549
|
+
maxDepth,
|
|
56550
|
+
visitedNodes: visited.size,
|
|
56551
|
+
nodes: [],
|
|
56552
|
+
edges: []
|
|
56553
|
+
};
|
|
56554
|
+
}
|
|
56555
|
+
const pathKeys = [toPage.nodeKey];
|
|
56556
|
+
const pathEdges = [];
|
|
56557
|
+
while (pathKeys[0] !== fromPage.nodeKey) {
|
|
56558
|
+
const step = predecessor.get(pathKeys[0]);
|
|
56559
|
+
if (!step) break;
|
|
56560
|
+
pathEdges.unshift(step.edge);
|
|
56561
|
+
pathKeys.unshift(step.nodeKey);
|
|
56562
|
+
}
|
|
56563
|
+
const pathRows = app.db.select().from(siteCrawlPages).where(and43(
|
|
56564
|
+
eq55(siteCrawlPages.projectId, scope.projectId),
|
|
56565
|
+
eq55(siteCrawlPages.runId, scope.runId),
|
|
56566
|
+
eq55(siteCrawlPages.attemptId, scope.attemptId),
|
|
56567
|
+
inArray19(siteCrawlPages.nodeKey, pathKeys)
|
|
56568
|
+
)).all();
|
|
56569
|
+
const pageByKey = new Map(pathRows.map((row) => [row.nodeKey, row]));
|
|
56570
|
+
return {
|
|
56571
|
+
project: project.name,
|
|
56572
|
+
runId: snapshot.runId,
|
|
56573
|
+
complete: snapshot.complete,
|
|
56574
|
+
termination: snapshot.termination,
|
|
56575
|
+
state: "found",
|
|
56576
|
+
from: reference(fromPage),
|
|
56577
|
+
to: reference(toPage),
|
|
56578
|
+
maxDepth,
|
|
56579
|
+
visitedNodes: visited.size,
|
|
56580
|
+
nodes: pathKeys.map((key) => mapCrawlPage(pageByKey.get(key))),
|
|
56581
|
+
edges: pathEdges.map(mapCrawlEdge)
|
|
56582
|
+
};
|
|
56583
|
+
});
|
|
56584
|
+
app.get("/projects/:name/technical-aeo/changes", async (request) => {
|
|
56585
|
+
const project = resolveProject(app.db, request.params.name);
|
|
56586
|
+
const scopeFilter = request.query.scope ?? "all";
|
|
56587
|
+
if (scopeFilter !== "all" && scopeFilter !== "pages" && scopeFilter !== "links") {
|
|
56588
|
+
throw validationError("scope must be all, pages, or links");
|
|
56589
|
+
}
|
|
56590
|
+
const changeFilter = request.query.change ?? "all";
|
|
56591
|
+
if (changeFilter !== "all" && changeFilter !== "added" && changeFilter !== "removed" && changeFilter !== "changed") {
|
|
56592
|
+
throw validationError("change must be all, added, removed, or changed");
|
|
56593
|
+
}
|
|
56594
|
+
const completeSnapshotFilters = [
|
|
56595
|
+
eq55(siteCrawlSnapshots.projectId, project.id),
|
|
56596
|
+
eq55(runs.projectId, project.id),
|
|
56597
|
+
eq55(runs.kind, RunKinds["site-audit"]),
|
|
56598
|
+
eq55(runs.status, RunStatuses.completed),
|
|
56599
|
+
eq55(siteCrawlSnapshots.complete, true),
|
|
56600
|
+
notProbeRun()
|
|
56601
|
+
];
|
|
56602
|
+
const afterTarget = request.query.toRunId ? resolveCrawl(project.id, request.query.toRunId) : app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq55(siteCrawlSnapshots.runId, runs.id)).where(and43(...completeSnapshotFilters)).orderBy(desc24(siteCrawlSnapshots.createdAt), desc24(siteCrawlSnapshots.runId)).limit(1).get();
|
|
56603
|
+
if (request.query.toRunId && !afterTarget) throw notFound("Site crawl run", request.query.toRunId);
|
|
56604
|
+
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq55(siteCrawlSnapshots.runId, runs.id)).where(and43(
|
|
56605
|
+
...completeSnapshotFilters,
|
|
56606
|
+
or11(
|
|
56607
|
+
lt7(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
56608
|
+
and43(
|
|
56609
|
+
eq55(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
56610
|
+
lt7(siteCrawlSnapshots.runId, afterTarget.snapshot.runId)
|
|
56611
|
+
)
|
|
56612
|
+
)
|
|
56613
|
+
)).orderBy(desc24(siteCrawlSnapshots.createdAt), desc24(siteCrawlSnapshots.runId)).limit(1).get() : void 0;
|
|
56614
|
+
if (request.query.fromRunId && !beforeTarget) throw notFound("Site crawl run", request.query.fromRunId);
|
|
56615
|
+
if (!afterTarget) {
|
|
56616
|
+
return { project: project.name, state: "unavailable", reason: "no-crawl", fromRunId: null, toRunId: null };
|
|
56617
|
+
}
|
|
56618
|
+
if (!beforeTarget) {
|
|
56619
|
+
return {
|
|
56620
|
+
project: project.name,
|
|
56621
|
+
state: "unavailable",
|
|
56622
|
+
reason: "insufficient-history",
|
|
56623
|
+
fromRunId: null,
|
|
56624
|
+
toRunId: afterTarget.snapshot.runId
|
|
56625
|
+
};
|
|
56626
|
+
}
|
|
56627
|
+
const beforeSnapshot = beforeTarget.snapshot;
|
|
56628
|
+
const afterSnapshot = afterTarget.snapshot;
|
|
56629
|
+
if (beforeSnapshot.runId === afterSnapshot.runId) {
|
|
56630
|
+
throw validationError("fromRunId and toRunId must identify different crawls");
|
|
56631
|
+
}
|
|
56632
|
+
const crawlOrder = beforeSnapshot.createdAt.localeCompare(afterSnapshot.createdAt);
|
|
56633
|
+
if (crawlOrder > 0 || crawlOrder === 0 && beforeSnapshot.runId >= afterSnapshot.runId) {
|
|
56634
|
+
throw validationError("fromRunId must identify a crawl earlier than toRunId");
|
|
56635
|
+
}
|
|
56636
|
+
if (!beforeSnapshot.complete || !afterSnapshot.complete || beforeTarget.runStatus !== RunStatuses.completed || afterTarget.runStatus !== RunStatuses.completed) {
|
|
56637
|
+
return {
|
|
56638
|
+
project: project.name,
|
|
56639
|
+
state: "unavailable",
|
|
56640
|
+
reason: "partial-not-comparable",
|
|
56641
|
+
fromRunId: beforeSnapshot.runId,
|
|
56642
|
+
toRunId: afterSnapshot.runId
|
|
56643
|
+
};
|
|
56644
|
+
}
|
|
56645
|
+
const beforeScope = detailScopeFor(project.id, beforeSnapshot);
|
|
56646
|
+
const afterScope = detailScopeFor(project.id, afterSnapshot);
|
|
56647
|
+
if (!beforeScope || !afterScope) {
|
|
56648
|
+
return {
|
|
56649
|
+
project: project.name,
|
|
56650
|
+
state: "unavailable",
|
|
56651
|
+
reason: "details-unavailable",
|
|
56652
|
+
fromRunId: beforeSnapshot.runId,
|
|
56653
|
+
toRunId: afterSnapshot.runId
|
|
56654
|
+
};
|
|
56655
|
+
}
|
|
56656
|
+
const versions = {
|
|
56657
|
+
crawlSchema: [beforeSnapshot.crawlSchemaVersion, afterSnapshot.crawlSchemaVersion],
|
|
56658
|
+
normalization: [beforeSnapshot.normalizationVersion, afterSnapshot.normalizationVersion],
|
|
56659
|
+
indexability: [beforeSnapshot.indexabilityVersion, afterSnapshot.indexabilityVersion],
|
|
56660
|
+
linkScore: [beforeSnapshot.linkScoreVersion, afterSnapshot.linkScoreVersion]
|
|
56661
|
+
};
|
|
56662
|
+
const mismatchedVersions = Object.entries(versions).filter(([, pair]) => pair[0] !== pair[1]).map(([name]) => name);
|
|
56663
|
+
if (mismatchedVersions.length > 0) {
|
|
56664
|
+
return {
|
|
56665
|
+
project: project.name,
|
|
56666
|
+
state: "incompatible",
|
|
56667
|
+
reason: "incompatible-versions",
|
|
56668
|
+
fromRunId: beforeSnapshot.runId,
|
|
56669
|
+
toRunId: afterSnapshot.runId,
|
|
56670
|
+
mismatchedVersions
|
|
56671
|
+
};
|
|
56672
|
+
}
|
|
56673
|
+
const cursor = decodeSiteHealthChangesCursor(request.query.cursor, {
|
|
56674
|
+
fromRunId: beforeSnapshot.runId,
|
|
56675
|
+
toRunId: afterSnapshot.runId,
|
|
56676
|
+
scope: scopeFilter,
|
|
56677
|
+
change: changeFilter
|
|
56678
|
+
});
|
|
56679
|
+
const loadSummary = () => {
|
|
56680
|
+
const includeAdded = changeFilter === "all" || changeFilter === "added";
|
|
56681
|
+
const includeRemoved = changeFilter === "all" || changeFilter === "removed";
|
|
56682
|
+
const includeChanged = changeFilter === "all" || changeFilter === "changed";
|
|
56683
|
+
const zero2 = () => ({ added: 0, removed: 0, changed: 0 });
|
|
56684
|
+
const summary2 = { pages: zero2(), links: zero2() };
|
|
56685
|
+
if (scopeFilter !== "links") {
|
|
56686
|
+
if (includeAdded || includeChanged) {
|
|
56687
|
+
const addedExpression = includeAdded ? sql20`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql20`0`;
|
|
56688
|
+
const changedExpression = includeChanged ? sql20`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${PAGE_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql20`0`;
|
|
56689
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql20`
|
|
56690
|
+
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
56691
|
+
FROM site_crawl_pages AS current
|
|
56692
|
+
LEFT JOIN site_crawl_pages AS previous
|
|
56693
|
+
ON previous.project_id = ${beforeScope.projectId}
|
|
56694
|
+
AND previous.run_id = ${beforeScope.runId}
|
|
56695
|
+
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
56696
|
+
AND previous.node_key = current.node_key
|
|
56697
|
+
WHERE current.project_id = ${afterScope.projectId}
|
|
56698
|
+
AND current.run_id = ${afterScope.runId}
|
|
56699
|
+
AND current.attempt_id = ${afterScope.attemptId}
|
|
56700
|
+
`);
|
|
56701
|
+
summary2.pages.added = Number(counts2.added);
|
|
56702
|
+
summary2.pages.changed = Number(counts2.changed);
|
|
56703
|
+
}
|
|
56704
|
+
if (includeRemoved) {
|
|
56705
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql20`
|
|
56706
|
+
SELECT COUNT(*) AS removed
|
|
56707
|
+
FROM site_crawl_pages AS previous
|
|
56708
|
+
LEFT JOIN site_crawl_pages AS current
|
|
56709
|
+
ON current.project_id = ${afterScope.projectId}
|
|
56710
|
+
AND current.run_id = ${afterScope.runId}
|
|
56711
|
+
AND current.attempt_id = ${afterScope.attemptId}
|
|
56712
|
+
AND current.node_key = previous.node_key
|
|
56713
|
+
WHERE previous.project_id = ${beforeScope.projectId}
|
|
56714
|
+
AND previous.run_id = ${beforeScope.runId}
|
|
56715
|
+
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
56716
|
+
AND current.id IS NULL
|
|
56717
|
+
`);
|
|
56718
|
+
summary2.pages.removed = Number(counts2.removed);
|
|
56719
|
+
}
|
|
56720
|
+
}
|
|
56721
|
+
if (scopeFilter !== "pages") {
|
|
56722
|
+
if (includeAdded || includeChanged) {
|
|
56723
|
+
const addedExpression = includeAdded ? sql20`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql20`0`;
|
|
56724
|
+
const changedExpression = includeChanged ? sql20`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${LINK_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql20`0`;
|
|
56725
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql20`
|
|
56726
|
+
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
56727
|
+
FROM site_crawl_edges AS current
|
|
56728
|
+
LEFT JOIN site_crawl_edges AS previous
|
|
56729
|
+
ON previous.project_id = ${beforeScope.projectId}
|
|
56730
|
+
AND previous.run_id = ${beforeScope.runId}
|
|
56731
|
+
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
56732
|
+
AND previous.internal = 1
|
|
56733
|
+
AND previous.edge_key = current.edge_key
|
|
56734
|
+
WHERE current.project_id = ${afterScope.projectId}
|
|
56735
|
+
AND current.run_id = ${afterScope.runId}
|
|
56736
|
+
AND current.attempt_id = ${afterScope.attemptId}
|
|
56737
|
+
AND current.internal = 1
|
|
56738
|
+
`);
|
|
56739
|
+
summary2.links.added = Number(counts2.added);
|
|
56740
|
+
summary2.links.changed = Number(counts2.changed);
|
|
56741
|
+
}
|
|
56742
|
+
if (includeRemoved) {
|
|
56743
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql20`
|
|
56744
|
+
SELECT COUNT(*) AS removed
|
|
56745
|
+
FROM site_crawl_edges AS previous
|
|
56746
|
+
LEFT JOIN site_crawl_edges AS current
|
|
56747
|
+
ON current.project_id = ${afterScope.projectId}
|
|
56748
|
+
AND current.run_id = ${afterScope.runId}
|
|
56749
|
+
AND current.attempt_id = ${afterScope.attemptId}
|
|
56750
|
+
AND current.internal = 1
|
|
56751
|
+
AND current.edge_key = previous.edge_key
|
|
56752
|
+
WHERE previous.project_id = ${beforeScope.projectId}
|
|
56753
|
+
AND previous.run_id = ${beforeScope.runId}
|
|
56754
|
+
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
56755
|
+
AND previous.internal = 1
|
|
56756
|
+
AND current.id IS NULL
|
|
56757
|
+
`);
|
|
56758
|
+
summary2.links.removed = Number(counts2.removed);
|
|
56759
|
+
}
|
|
56760
|
+
}
|
|
56761
|
+
return summary2;
|
|
56762
|
+
};
|
|
56763
|
+
const summary = cursor ? null : loadSummary();
|
|
56764
|
+
const limit = parseBoundedLimit(request.query.limit, SITE_HEALTH_CHANGES_DEFAULT_LIMIT, SITE_HEALTH_CHANGES_MAX_LIMIT);
|
|
56765
|
+
const sumCounts = (counts2) => counts2.added + counts2.removed + counts2.changed;
|
|
56766
|
+
const total = summary ? sumCounts(summary.pages) + sumCounts(summary.links) : null;
|
|
56767
|
+
const collectEntity = (entity, afterKey, wanted2) => {
|
|
56768
|
+
if (wanted2 <= 0) return [];
|
|
56769
|
+
const includeAfter = changeFilter === "all" || changeFilter === "added" || changeFilter === "changed";
|
|
56770
|
+
const includeRemoved = changeFilter === "all" || changeFilter === "removed";
|
|
56771
|
+
const table = entity === "page" ? "site_crawl_pages" : "site_crawl_edges";
|
|
56772
|
+
const keyColumn = entity === "page" ? "node_key" : "edge_key";
|
|
56773
|
+
const internalCurrent = entity === "link" ? sql20`AND current.internal = 1` : sql20``;
|
|
56774
|
+
const internalPreviousJoin = entity === "link" ? sql20`AND previous.internal = 1` : sql20``;
|
|
56775
|
+
const internalPrevious = entity === "link" ? sql20`AND previous.internal = 1` : sql20``;
|
|
56776
|
+
const internalCurrentJoin = entity === "link" ? sql20`AND current.internal = 1` : sql20``;
|
|
56777
|
+
const difference2 = entity === "page" ? PAGE_CHANGE_PREDICATE : LINK_CHANGE_PREDICATE;
|
|
56778
|
+
const afterKind = changeFilter === "added" ? sql20`previous.id IS NULL` : changeFilter === "changed" ? sql20`previous.id IS NOT NULL AND ${difference2}` : sql20`previous.id IS NULL OR (previous.id IS NOT NULL AND ${difference2})`;
|
|
56779
|
+
const afterRows = includeAfter ? app.db.all(sql20`
|
|
56780
|
+
SELECT ${entity} AS entity,
|
|
56781
|
+
CASE WHEN previous.id IS NULL THEN 'added' ELSE 'changed' END AS change,
|
|
56782
|
+
current.${sql20.raw(keyColumn)} AS key
|
|
56783
|
+
FROM ${sql20.raw(table)} AS current
|
|
56784
|
+
LEFT JOIN ${sql20.raw(table)} AS previous
|
|
56785
|
+
ON previous.project_id = ${beforeScope.projectId}
|
|
56786
|
+
AND previous.run_id = ${beforeScope.runId}
|
|
56787
|
+
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
56788
|
+
${internalPreviousJoin}
|
|
56789
|
+
AND previous.${sql20.raw(keyColumn)} = current.${sql20.raw(keyColumn)}
|
|
56790
|
+
WHERE current.project_id = ${afterScope.projectId}
|
|
56791
|
+
AND current.run_id = ${afterScope.runId}
|
|
56792
|
+
AND current.attempt_id = ${afterScope.attemptId}
|
|
56793
|
+
${internalCurrent}
|
|
56794
|
+
AND current.${sql20.raw(keyColumn)} > ${afterKey}
|
|
56795
|
+
AND (${afterKind})
|
|
56796
|
+
ORDER BY current.${sql20.raw(keyColumn)} ASC
|
|
56797
|
+
LIMIT ${wanted2}
|
|
56798
|
+
`) : [];
|
|
56799
|
+
const removedRows = includeRemoved ? app.db.all(sql20`
|
|
56800
|
+
SELECT ${entity} AS entity, 'removed' AS change, previous.${sql20.raw(keyColumn)} AS key
|
|
56801
|
+
FROM ${sql20.raw(table)} AS previous
|
|
56802
|
+
LEFT JOIN ${sql20.raw(table)} AS current
|
|
56803
|
+
ON current.project_id = ${afterScope.projectId}
|
|
56804
|
+
AND current.run_id = ${afterScope.runId}
|
|
56805
|
+
AND current.attempt_id = ${afterScope.attemptId}
|
|
56806
|
+
${internalCurrentJoin}
|
|
56807
|
+
AND current.${sql20.raw(keyColumn)} = previous.${sql20.raw(keyColumn)}
|
|
56808
|
+
WHERE previous.project_id = ${beforeScope.projectId}
|
|
56809
|
+
AND previous.run_id = ${beforeScope.runId}
|
|
56810
|
+
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
56811
|
+
${internalPrevious}
|
|
56812
|
+
AND previous.${sql20.raw(keyColumn)} > ${afterKey}
|
|
56813
|
+
AND current.id IS NULL
|
|
56814
|
+
ORDER BY previous.${sql20.raw(keyColumn)} ASC
|
|
56815
|
+
LIMIT ${wanted2}
|
|
56816
|
+
`) : [];
|
|
56817
|
+
const merged = [];
|
|
56818
|
+
let afterIndex = 0;
|
|
56819
|
+
let removedIndex = 0;
|
|
56820
|
+
while (merged.length < wanted2 && (afterIndex < afterRows.length || removedIndex < removedRows.length)) {
|
|
56821
|
+
const afterRow = afterRows.at(afterIndex);
|
|
56822
|
+
const removedRow = removedRows.at(removedIndex);
|
|
56823
|
+
if (!removedRow || afterRow && compareChangeKeys(afterRow, removedRow) <= 0) {
|
|
56824
|
+
merged.push(afterRow);
|
|
56825
|
+
afterIndex += 1;
|
|
56826
|
+
} else {
|
|
56827
|
+
merged.push(removedRow);
|
|
56828
|
+
removedIndex += 1;
|
|
56829
|
+
}
|
|
56830
|
+
}
|
|
56831
|
+
return merged;
|
|
56832
|
+
};
|
|
56833
|
+
const keyRows = [];
|
|
56834
|
+
const wanted = limit + 1;
|
|
56835
|
+
if (scopeFilter !== "links" && cursor?.entity !== "link") {
|
|
56836
|
+
keyRows.push(...collectEntity("page", cursor?.entity === "page" ? cursor.key : "", wanted));
|
|
56837
|
+
}
|
|
56838
|
+
if (keyRows.length < wanted && scopeFilter !== "pages") {
|
|
56839
|
+
keyRows.push(...collectEntity("link", cursor?.entity === "link" ? cursor.key : "", wanted - keyRows.length));
|
|
56840
|
+
}
|
|
56841
|
+
const visibleKeys = keyRows.slice(0, limit);
|
|
56842
|
+
const pageKeys = visibleKeys.filter((row) => row.entity === "page").map((row) => row.key);
|
|
56843
|
+
const linkKeys = visibleKeys.filter((row) => row.entity === "link").map((row) => row.key);
|
|
56844
|
+
const pageRowsFor = (scope) => pageKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and43(
|
|
56845
|
+
eq55(siteCrawlPages.projectId, scope.projectId),
|
|
56846
|
+
eq55(siteCrawlPages.runId, scope.runId),
|
|
56847
|
+
eq55(siteCrawlPages.attemptId, scope.attemptId),
|
|
56848
|
+
inArray19(siteCrawlPages.nodeKey, pageKeys)
|
|
56849
|
+
)).all();
|
|
56850
|
+
const edgeRowsFor = (scope) => linkKeys.length === 0 ? [] : app.db.select().from(siteCrawlEdges).where(and43(
|
|
56851
|
+
eq55(siteCrawlEdges.projectId, scope.projectId),
|
|
56852
|
+
eq55(siteCrawlEdges.runId, scope.runId),
|
|
56853
|
+
eq55(siteCrawlEdges.attemptId, scope.attemptId),
|
|
56854
|
+
eq55(siteCrawlEdges.internal, true),
|
|
56855
|
+
inArray19(siteCrawlEdges.edgeKey, linkKeys)
|
|
56856
|
+
)).all();
|
|
56857
|
+
const beforePages = new Map(pageRowsFor(beforeScope).map((row) => [row.nodeKey, mapCrawlPage(row)]));
|
|
56858
|
+
const afterPages = new Map(pageRowsFor(afterScope).map((row) => [row.nodeKey, mapCrawlPage(row)]));
|
|
56859
|
+
const beforeLinks = new Map(edgeRowsFor(beforeScope).map((row) => [row.edgeKey, mapCrawlEdge(row)]));
|
|
56860
|
+
const afterLinks = new Map(edgeRowsFor(afterScope).map((row) => [row.edgeKey, mapCrawlEdge(row)]));
|
|
56861
|
+
const changes = visibleKeys.map((row) => {
|
|
56862
|
+
if (row.entity === "page") {
|
|
56863
|
+
const before2 = beforePages.get(row.key) ?? null;
|
|
56864
|
+
const after2 = afterPages.get(row.key) ?? null;
|
|
56865
|
+
const pageChangedFields = changedFields(before2, after2, PAGE_CHANGE_FIELDS);
|
|
56866
|
+
if (before2 && after2 && before2.healthState !== after2.healthState) pageChangedFields.push("healthState");
|
|
56867
|
+
return {
|
|
56868
|
+
entity: "page",
|
|
56869
|
+
change: row.change,
|
|
56870
|
+
key: row.key,
|
|
56871
|
+
changedFields: pageChangedFields,
|
|
56872
|
+
before: before2,
|
|
56873
|
+
after: after2
|
|
56874
|
+
};
|
|
56875
|
+
}
|
|
56876
|
+
const before = beforeLinks.get(row.key) ?? null;
|
|
56877
|
+
const after = afterLinks.get(row.key) ?? null;
|
|
56878
|
+
return {
|
|
56879
|
+
entity: "link",
|
|
56880
|
+
change: row.change,
|
|
56881
|
+
key: row.key,
|
|
56882
|
+
changedFields: changedFields(before, after, LINK_CHANGE_FIELDS),
|
|
56883
|
+
before,
|
|
56884
|
+
after
|
|
56885
|
+
};
|
|
56886
|
+
});
|
|
56887
|
+
return {
|
|
56888
|
+
project: project.name,
|
|
56889
|
+
state: "ready",
|
|
56890
|
+
fromRunId: beforeSnapshot.runId,
|
|
56891
|
+
toRunId: afterSnapshot.runId,
|
|
56892
|
+
versions: {
|
|
56893
|
+
crawlSchema: afterSnapshot.crawlSchemaVersion,
|
|
56894
|
+
normalization: afterSnapshot.normalizationVersion,
|
|
56895
|
+
indexability: afterSnapshot.indexabilityVersion,
|
|
56896
|
+
linkScore: afterSnapshot.linkScoreVersion
|
|
56897
|
+
},
|
|
56898
|
+
filters: { scope: scopeFilter, change: changeFilter },
|
|
56899
|
+
summaryState: cursor ? "omitted-on-continuation" : "exact",
|
|
56900
|
+
summary,
|
|
56901
|
+
total,
|
|
56902
|
+
nextCursor: keyRows.length > limit && visibleKeys.length > 0 ? encodeSiteHealthChangesCursor({
|
|
56903
|
+
v: 1,
|
|
56904
|
+
fromRunId: beforeSnapshot.runId,
|
|
56905
|
+
toRunId: afterSnapshot.runId,
|
|
56906
|
+
scope: scopeFilter,
|
|
56907
|
+
change: changeFilter,
|
|
56908
|
+
entity: visibleKeys.at(-1).entity,
|
|
56909
|
+
key: visibleKeys.at(-1).key
|
|
56910
|
+
}) : null,
|
|
56911
|
+
changes
|
|
56912
|
+
};
|
|
56913
|
+
});
|
|
56914
|
+
app.get("/projects/:name/technical-aeo/crawl/pages/audit", async (request) => {
|
|
56915
|
+
const project = resolveProject(app.db, request.params.name);
|
|
56916
|
+
const scalarQueryString = (value, name) => {
|
|
56917
|
+
if (value === void 0) return void 0;
|
|
56918
|
+
if (typeof value !== "string") throw validationError(`${name} must be provided once`);
|
|
56919
|
+
return value;
|
|
56920
|
+
};
|
|
56921
|
+
const runId = scalarQueryString(request.query.runId, "runId");
|
|
56922
|
+
const nodeKey = scalarQueryString(request.query.nodeKey, "nodeKey")?.trim();
|
|
56923
|
+
const url = scalarQueryString(request.query.url, "url")?.trim();
|
|
56924
|
+
if (Number(Boolean(nodeKey)) + Number(Boolean(url)) !== 1) {
|
|
56925
|
+
throw validationError("Provide exactly one of nodeKey or url");
|
|
56926
|
+
}
|
|
56927
|
+
const target = resolveCrawl(project.id, runId);
|
|
56928
|
+
if (!target) {
|
|
56929
|
+
if (runId) throw notFound("Site crawl run", runId);
|
|
56930
|
+
return { state: "no-crawl", project: project.name, runId: null };
|
|
56931
|
+
}
|
|
56932
|
+
const snapshot = target.snapshot;
|
|
56933
|
+
const provenance = {
|
|
56934
|
+
project: project.name,
|
|
56935
|
+
runId: snapshot.runId,
|
|
56936
|
+
complete: snapshot.complete,
|
|
56937
|
+
termination: snapshot.termination
|
|
56938
|
+
};
|
|
56939
|
+
const scope = detailScopeFor(project.id, snapshot);
|
|
56940
|
+
if (!scope) return { state: "details-unavailable", ...provenance };
|
|
56941
|
+
const page = pageInScope(scope, nodeKey ? { nodeKey } : { url });
|
|
56942
|
+
if (!page) return { state: "not-found", ...provenance };
|
|
56943
|
+
const identity = {
|
|
56944
|
+
nodeKey: page.nodeKey,
|
|
56945
|
+
url: page.url,
|
|
56946
|
+
auditState: page.auditState
|
|
56947
|
+
};
|
|
56948
|
+
if (page.auditScore == null) {
|
|
56949
|
+
return {
|
|
56950
|
+
state: "not-audited",
|
|
56951
|
+
...provenance,
|
|
56952
|
+
...identity,
|
|
56953
|
+
auditScore: null,
|
|
56954
|
+
factors: [],
|
|
56955
|
+
criticalDefects: []
|
|
56956
|
+
};
|
|
56957
|
+
}
|
|
56958
|
+
return {
|
|
56959
|
+
state: "ready",
|
|
56960
|
+
...provenance,
|
|
56961
|
+
...identity,
|
|
56962
|
+
auditScore: page.auditScore,
|
|
56963
|
+
...mapCrawlPageAuditEvidence(page)
|
|
56964
|
+
};
|
|
56965
|
+
});
|
|
55666
56966
|
app.get("/projects/:name/technical-aeo/crawl/pages", async (request) => {
|
|
55667
56967
|
const project = resolveProject(app.db, request.params.name);
|
|
55668
56968
|
const target = resolveCrawl(project.id, request.query.runId);
|
|
@@ -55738,7 +57038,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
55738
57038
|
};
|
|
55739
57039
|
child.pageCount++;
|
|
55740
57040
|
if (row.inventoryEligible) child.inventoryEligibleCount++;
|
|
55741
|
-
if (
|
|
57041
|
+
if (FETCHED_SITE_CRAWL_STATES.has(row.fetchState)) child.fetchedCount++;
|
|
55742
57042
|
if (structureHierarchyPath(row.path) === childPath) {
|
|
55743
57043
|
child.hasPage = true;
|
|
55744
57044
|
if (child.url == null || row.url > child.url) child.url = row.url;
|
|
@@ -56544,7 +57844,7 @@ function extractRecommendedCompetitors(answerText, ownDomains, citedDomains, com
|
|
|
56544
57844
|
}
|
|
56545
57845
|
const ownKeys = new Set([...ownBrandAliases].map(brandKeyFromText));
|
|
56546
57846
|
const knownCompetitorAliases = new Set(
|
|
56547
|
-
[...citedDomains, ...competitorDomains].flatMap((domain) => collectBrandAliasesFromDomain(domain)).filter((
|
|
57847
|
+
[...citedDomains, ...competitorDomains].flatMap((domain) => collectBrandAliasesFromDomain(domain)).filter((alias2) => !ownKeys.has(brandKeyFromText(alias2)))
|
|
56548
57848
|
);
|
|
56549
57849
|
if (knownCompetitorAliases.size === 0) return [];
|
|
56550
57850
|
const candidatePatterns = [
|
|
@@ -56616,7 +57916,7 @@ function collectBrandAliasesFromDomain(domain) {
|
|
|
56616
57916
|
return [...aliases];
|
|
56617
57917
|
}
|
|
56618
57918
|
function matchesBrandAlias(candidate, aliases) {
|
|
56619
|
-
return [...aliases].some((
|
|
57919
|
+
return [...aliases].some((alias2) => textContainsBrandAlias(candidate, alias2));
|
|
56620
57920
|
}
|
|
56621
57921
|
|
|
56622
57922
|
// src/intelligence-service.ts
|
|
@@ -57260,6 +58560,9 @@ export {
|
|
|
57260
58560
|
siteCrawlSnapshots,
|
|
57261
58561
|
siteCrawlPages,
|
|
57262
58562
|
siteCrawlEdges,
|
|
58563
|
+
siteCrawlGraphLayouts,
|
|
58564
|
+
siteCrawlGraphNodes,
|
|
58565
|
+
siteCrawlGraphEdges,
|
|
57263
58566
|
siteCrawlFindings,
|
|
57264
58567
|
siteCrawlEventReceipts,
|
|
57265
58568
|
bingCoverageSnapshots,
|