@canonry/canonry 4.157.0 → 4.159.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/THIRD_PARTY_NOTICES.md +102 -0
- package/assets/THIRD_PARTY_NOTICES.md +102 -0
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +1 -0
- package/assets/assets/{AuditHistoryPanel-D12lHs6n.js → AuditHistoryPanel-BiloGcQA.js} +1 -1
- package/assets/assets/{BacklinksPage-D2ScFYru.js → BacklinksPage-CPM1O2bC.js} +1 -1
- package/assets/assets/{HistoryPage-jZ_I4rwX.js → HistoryPage-1YOjtdJy.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-BIdQjXJT.js → MeasurementPropertyPage-CVDTY_gC.js} +1 -1
- package/assets/assets/ProjectPage-DnGXMx1n.js +12 -0
- package/assets/assets/{RunRow-Q2X0NR74.js → RunRow-fIaadH4h.js} +1 -1
- package/assets/assets/{RunsPage-Cw7YVsbJ.js → RunsPage-DfifdM5z.js} +1 -1
- package/assets/assets/{SettingsPage-bFl2EXOD.js → SettingsPage-efA2EEmS.js} +1 -1
- package/assets/assets/SiteHealthSection-BROtQGfu.js +4 -0
- package/assets/assets/{TrafficPage-DJjhk7Jw.js → TrafficPage-Wk43Bm_q.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-CGKwIu9A.js → TrafficSourceDetailPage-BhBvoLWu.js} +1 -1
- package/assets/assets/extract-error-message-YGzZFvWs.js +1 -0
- package/assets/assets/geist-cyrillic-ext-wght-normal-DjL33-gN.woff2 +0 -0
- package/assets/assets/geist-cyrillic-wght-normal-BEAKL7Jp.woff2 +0 -0
- package/assets/assets/geist-latin-ext-wght-normal-DC-KSUi6.woff2 +0 -0
- package/assets/assets/geist-latin-wght-normal-BgDaEnEv.woff2 +0 -0
- package/assets/assets/geist-mono-cyrillic-ext-wght-normal-X_5orZeX.woff2 +0 -0
- package/assets/assets/geist-mono-cyrillic-wght-normal-DiZS0aHC.woff2 +0 -0
- package/assets/assets/geist-mono-latin-ext-wght-normal-Bwz-egvJ.woff2 +0 -0
- package/assets/assets/geist-mono-latin-wght-normal-XN7g48iV.woff2 +0 -0
- package/assets/assets/geist-mono-symbols2-wght-normal-CO5SzqOn.woff2 +0 -0
- package/assets/assets/geist-mono-vietnamese-wght-normal-DadHysG0.woff2 +0 -0
- package/assets/assets/geist-vietnamese-wght-normal-6IgcOCM7.woff2 +0 -0
- package/assets/assets/index-CACRaOmZ.css +1 -0
- package/assets/assets/index-DmsIZQAm.js +83 -0
- package/assets/assets/{react-sigma_core.esm.min-EzswMBkr.js → react-sigma_core.esm.min-DM9XptbM.js} +2 -2
- package/assets/assets/{vendor-lucide-BtYdosYk.js → vendor-lucide-nYKv-BB7.js} +1 -1
- package/assets/assets/{vendor-markdown-D274_wsI.js → vendor-markdown-Bn7fRwrx.js} +1 -1
- package/assets/assets/{vendor-radix-BxMXSwbC.js → vendor-radix-4UOaVWza.js} +1 -1
- package/assets/assets/{vendor-recharts-C4ktw_aE.js → vendor-recharts-D8rt5lKn.js} +1 -1
- package/assets/assets/{vendor-tanstack-pFxw0eGP.js → vendor-tanstack-Dock0eH1.js} +1 -1
- package/assets/index.html +7 -7
- package/dist/{chunk-MEBL4EJO.js → chunk-5AFWHKKI.js} +875 -38
- package/dist/{chunk-AEIKAWOW.js → chunk-BZPBVOAM.js} +753 -212
- package/dist/{chunk-KMTRO56O.js → chunk-ZEG43TY2.js} +45 -15
- package/dist/{chunk-TRRVBTU5.js → chunk-ZL3VY435.js} +264 -10
- package/dist/cli.js +42 -4
- package/dist/index.d.ts +9 -0
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-YUUPSJRC.js → intelligence-service-DHY5CYWX.js} +2 -2
- package/dist/mcp.js +2 -2
- package/package.json +13 -11
- package/assets/assets/ProjectPage-CLvLOOtj.js +0 -15
- package/assets/assets/extract-error-message-BHm98tXH.js +0 -1
- package/assets/assets/index-BxegEwic.css +0 -1
- package/assets/assets/index-VO6nKq5b.js +0 -83
|
@@ -67,6 +67,7 @@ import {
|
|
|
67
67
|
MeasurementEvidenceShapes,
|
|
68
68
|
MeasurementPlanValidationError,
|
|
69
69
|
MeasurementRunScopeError,
|
|
70
|
+
PROJECTS_WRITE_SCOPE,
|
|
70
71
|
READ_ONLY_SCOPE,
|
|
71
72
|
ResearchQueryStatuses,
|
|
72
73
|
ResearchRunStatuses,
|
|
@@ -82,6 +83,8 @@ import {
|
|
|
82
83
|
SITE_HEALTH_PATH_DEFAULT_MAX_DEPTH,
|
|
83
84
|
SITE_HEALTH_PATH_MAX_DEPTH,
|
|
84
85
|
SITE_HEALTH_PATH_MAX_VISITED_NODES,
|
|
86
|
+
SITE_HEALTH_SCANS_DEFAULT_LIMIT,
|
|
87
|
+
SITE_HEALTH_SCANS_MAX_LIMIT,
|
|
85
88
|
SITE_HEALTH_SUBGRAPH_DEFAULT_MAX_EDGES,
|
|
86
89
|
SITE_HEALTH_SUBGRAPH_DEFAULT_MAX_NODES,
|
|
87
90
|
SITE_HEALTH_SUBGRAPH_MAX_EDGES,
|
|
@@ -90,6 +93,8 @@ import {
|
|
|
90
93
|
SchedulableRunKinds,
|
|
91
94
|
SiteAuditTrendDirections,
|
|
92
95
|
SiteCrawlFetchedStates,
|
|
96
|
+
SiteHealthLinkKinds,
|
|
97
|
+
SiteHealthTemplateDetections,
|
|
93
98
|
TrafficEventConfidences,
|
|
94
99
|
TrafficEventKinds,
|
|
95
100
|
TrafficEvidenceKinds,
|
|
@@ -219,6 +224,7 @@ import {
|
|
|
219
224
|
contentTargetsResponseDtoSchema,
|
|
220
225
|
cosineSimilarity,
|
|
221
226
|
createApiKeyRequestSchema,
|
|
227
|
+
createTemplateLinkPairIndex,
|
|
222
228
|
createUserRequestSchema,
|
|
223
229
|
createdApiKeyDtoSchema,
|
|
224
230
|
dedupeReportActions,
|
|
@@ -311,6 +317,7 @@ import {
|
|
|
311
317
|
isBrowserProvider,
|
|
312
318
|
isReadOnlyKey,
|
|
313
319
|
isRetryableHttpError,
|
|
320
|
+
isTemplateLinkRatio,
|
|
314
321
|
isTrendBaseline,
|
|
315
322
|
isVertexGroundingRedirect,
|
|
316
323
|
isoDateDaysBeforeInTimeZone,
|
|
@@ -422,6 +429,7 @@ import {
|
|
|
422
429
|
normalizeModelId,
|
|
423
430
|
normalizeProjectAliases,
|
|
424
431
|
normalizeProjectDomain,
|
|
432
|
+
normalizeProjectName,
|
|
425
433
|
normalizeQueryText,
|
|
426
434
|
normalizeSiteAuditRunRequest,
|
|
427
435
|
normalizeUrlPath,
|
|
@@ -429,6 +437,7 @@ import {
|
|
|
429
437
|
notFound,
|
|
430
438
|
notImplemented,
|
|
431
439
|
notificationDtoSchema,
|
|
440
|
+
observeTemplateLinkEdges,
|
|
432
441
|
onboardingTelemetryEventSchema,
|
|
433
442
|
operationInProgress,
|
|
434
443
|
orderLocationsDefaultFirst,
|
|
@@ -444,6 +453,7 @@ import {
|
|
|
444
453
|
parseWindow,
|
|
445
454
|
pickClusterRepresentative,
|
|
446
455
|
projectConfigSchema,
|
|
456
|
+
projectCreateRequestSchema,
|
|
447
457
|
projectDtoSchema,
|
|
448
458
|
projectOverviewDtoSchema,
|
|
449
459
|
projectReportDtoSchema,
|
|
@@ -489,9 +499,11 @@ import {
|
|
|
489
499
|
segmentCrawlerHits,
|
|
490
500
|
serializeRunError,
|
|
491
501
|
settingsDtoSchema,
|
|
502
|
+
siteAuditLivePageHealthSchema,
|
|
492
503
|
siteAuditPageFactorSchema,
|
|
493
504
|
siteAuditPagesResponseSchema,
|
|
494
505
|
siteAuditRequestIdentity,
|
|
506
|
+
siteAuditRunProgressSchema,
|
|
495
507
|
siteAuditRunRequestSchema,
|
|
496
508
|
siteAuditRunResponseSchema,
|
|
497
509
|
siteAuditScoreSchema,
|
|
@@ -507,8 +519,12 @@ import {
|
|
|
507
519
|
siteCrawlStructureResponseSchema,
|
|
508
520
|
siteCrawlSummarySchema,
|
|
509
521
|
siteHealthChangesResponseSchema,
|
|
522
|
+
siteHealthLinkKindSchema,
|
|
510
523
|
siteHealthPathResponseSchema,
|
|
524
|
+
siteHealthScansResponseSchema,
|
|
525
|
+
siteHealthStateSchema,
|
|
511
526
|
siteHealthSubgraphResponseSchema,
|
|
527
|
+
siteHealthTemplateDetectionSchema,
|
|
512
528
|
snapshotDiffResponseSchema,
|
|
513
529
|
snapshotListResponseSchema,
|
|
514
530
|
snapshotReportSchema,
|
|
@@ -520,6 +536,8 @@ import {
|
|
|
520
536
|
surfaceClassFromCompetitorType,
|
|
521
537
|
surfaceClassLabel,
|
|
522
538
|
telemetryEventAcceptedDtoSchema,
|
|
539
|
+
templateLinkDetection,
|
|
540
|
+
templateLinkRatio,
|
|
523
541
|
textContainsAnyBrandAlias,
|
|
524
542
|
textContainsBrandAlias,
|
|
525
543
|
textContainsDomain,
|
|
@@ -559,7 +577,7 @@ import {
|
|
|
559
577
|
wordpressSchemaDeployResultDtoSchema,
|
|
560
578
|
wordpressSchemaStatusResultDtoSchema,
|
|
561
579
|
wordpressStatusDtoSchema
|
|
562
|
-
} from "./chunk-
|
|
580
|
+
} from "./chunk-ZL3VY435.js";
|
|
563
581
|
|
|
564
582
|
// src/intelligence-service.ts
|
|
565
583
|
import { eq as eq58, desc as desc26, asc as asc11, and as and46, ne as ne7, or as or12, inArray as inArray20, gte as gte14, lte as lte11 } from "drizzle-orm";
|
|
@@ -1309,6 +1327,13 @@ var siteCrawlSnapshots = sqliteTable("site_crawl_snapshots", {
|
|
|
1309
1327
|
deadLinkState: text("dead_link_state").notNull().default("disabled"),
|
|
1310
1328
|
deadLinksChecked: integer("dead_links_checked").notNull().default(0),
|
|
1311
1329
|
deadLinksFound: integer("dead_links_found").notNull().default(0),
|
|
1330
|
+
/**
|
|
1331
|
+
* Whether nav/header/footer link detection ran for this scan, and if not,
|
|
1332
|
+
* why. NULL means the scan predates detection, which reads report as
|
|
1333
|
+
* `unavailable-legacy-scan` rather than letting an empty template-link list
|
|
1334
|
+
* pass for "this site has no nav".
|
|
1335
|
+
*/
|
|
1336
|
+
templateDetection: text("template_detection"),
|
|
1312
1337
|
createdAt: text("created_at").notNull(),
|
|
1313
1338
|
updatedAt: text("updated_at").notNull()
|
|
1314
1339
|
}, (table) => [
|
|
@@ -1348,6 +1373,13 @@ var siteCrawlPages = sqliteTable("site_crawl_pages", {
|
|
|
1348
1373
|
canonicalNodeKey: text("canonical_node_key"),
|
|
1349
1374
|
indexabilityState: text("indexability_state").notNull().default("unknown"),
|
|
1350
1375
|
indexabilityReasons: text("indexability_reasons", { mode: "json" }).$type().notNull().default([]),
|
|
1376
|
+
/**
|
|
1377
|
+
* Derived Site Health state, computed once at publish time by the contract's
|
|
1378
|
+
* `deriveSiteHealthState`. Stored so reads can filter on an index instead of
|
|
1379
|
+
* materializing every page row; NULL means a snapshot published before this
|
|
1380
|
+
* column existed, which reads must report rather than treat as a state.
|
|
1381
|
+
*/
|
|
1382
|
+
healthState: text("health_state"),
|
|
1351
1383
|
auditState: text("audit_state").notNull().default("pending"),
|
|
1352
1384
|
auditScore: real("audit_score"),
|
|
1353
1385
|
auditFields: text("audit_fields", { mode: "json" }).$type().notNull().default({}),
|
|
@@ -1366,6 +1398,11 @@ var siteCrawlPages = sqliteTable("site_crawl_pages", {
|
|
|
1366
1398
|
index("idx_site_crawl_pages_read").on(table.projectId, table.runId, table.attemptId, table.inventoryEligible, table.auditScore, table.url),
|
|
1367
1399
|
index("idx_site_crawl_pages_parent").on(table.projectId, table.runId, table.attemptId, table.parentPath, table.path),
|
|
1368
1400
|
index("idx_site_crawl_pages_url").on(table.projectId, table.runId, table.attemptId, table.url),
|
|
1401
|
+
index("idx_site_crawl_pages_health").on(table.projectId, table.runId, table.attemptId, table.healthState, table.path, table.nodeKey),
|
|
1402
|
+
// The in-progress Page Health preview reads only completed audits below its
|
|
1403
|
+
// threshold, ordered worst-first. Keep its stable tie-breaker in the index
|
|
1404
|
+
// so a growing crawl never needs a transient sort before its small limit.
|
|
1405
|
+
index("idx_site_crawl_pages_live_preview").on(table.projectId, table.runId, table.attemptId, table.auditState, table.auditScore, table.nodeKey),
|
|
1369
1406
|
foreignKey({
|
|
1370
1407
|
name: "site_crawl_pages_attempt_fk",
|
|
1371
1408
|
columns: [table.projectId, table.runId, table.attemptId],
|
|
@@ -1389,6 +1426,29 @@ var siteCrawlEdges = sqliteTable("site_crawl_edges", {
|
|
|
1389
1426
|
followableOccurrences: integer("followable_occurrences").notNull().default(1),
|
|
1390
1427
|
nofollowOccurrences: integer("nofollow_occurrences").notNull().default(0),
|
|
1391
1428
|
anchors: text("anchors", { mode: "json" }).$type().notNull().default([]),
|
|
1429
|
+
/**
|
|
1430
|
+
* True when EVERY (target page, anchor) pair on this link appears on at
|
|
1431
|
+
* least `TEMPLATE_LINK_RATIO_THRESHOLD` of the attempt's fetched pages: a
|
|
1432
|
+
* nav, header, or footer link rather than an editorial one. Computed once at
|
|
1433
|
+
* publish time by the contract's `classifyTemplateLinks`, so the map, the
|
|
1434
|
+
* API filters, and the agents all read the same decision.
|
|
1435
|
+
*
|
|
1436
|
+
* One row aggregates every anchor between the same two pages, so a link that
|
|
1437
|
+
* is BOTH a footer link and an in-prose link is one row with two anchors. It
|
|
1438
|
+
* is editorial, because the in-prose link exists.
|
|
1439
|
+
*
|
|
1440
|
+
* NULL means this row was never classified, which reads report as
|
|
1441
|
+
* `unavailable-legacy-scan`. It never means "not a template link": the
|
|
1442
|
+
* too-few-pages guard writes an explicit `false` and records its reason on
|
|
1443
|
+
* the snapshot instead.
|
|
1444
|
+
*/
|
|
1445
|
+
isTemplate: integer("is_template", { mode: "boolean" }),
|
|
1446
|
+
/**
|
|
1447
|
+
* Share of fetched pages carrying this link's LEAST ubiquitous anchor, which
|
|
1448
|
+
* is the one that decides `is_template`. NULL when the link has no
|
|
1449
|
+
* measurable pair (an unresolved target or no anchor).
|
|
1450
|
+
*/
|
|
1451
|
+
templateRatio: real("template_ratio"),
|
|
1392
1452
|
createdAt: text("created_at").notNull(),
|
|
1393
1453
|
updatedAt: text("updated_at").notNull()
|
|
1394
1454
|
}, (table) => [
|
|
@@ -1397,6 +1457,17 @@ var siteCrawlEdges = sqliteTable("site_crawl_edges", {
|
|
|
1397
1457
|
index("idx_site_crawl_edges_inbound").on(table.projectId, table.runId, table.attemptId, table.targetNodeKey, table.edgeKey),
|
|
1398
1458
|
index("idx_site_crawl_edges_source_url").on(table.projectId, table.runId, table.attemptId, table.sourceUrl),
|
|
1399
1459
|
index("idx_site_crawl_edges_target_url").on(table.projectId, table.runId, table.attemptId, table.targetUrl),
|
|
1460
|
+
// Backs the `linkKind` filter on the internal-link reads, whose ORDER BY is
|
|
1461
|
+
// the trailing edge key, so a filtered page terminates at LIMIT instead of
|
|
1462
|
+
// sorting every match in a temp b-tree.
|
|
1463
|
+
index("idx_site_crawl_edges_template").on(
|
|
1464
|
+
table.projectId,
|
|
1465
|
+
table.runId,
|
|
1466
|
+
table.attemptId,
|
|
1467
|
+
table.internal,
|
|
1468
|
+
table.isTemplate,
|
|
1469
|
+
table.edgeKey
|
|
1470
|
+
),
|
|
1400
1471
|
// Publish-time graph sampling filters this exact scope and consumes the
|
|
1401
1472
|
// highest-occurrence internal anchors first. The mixed-direction suffix
|
|
1402
1473
|
// avoids a million-row temporary ORDER BY at the maximum crawl budget.
|
|
@@ -1425,8 +1496,17 @@ var siteCrawlGraphLayouts = sqliteTable("site_crawl_graph_layouts", {
|
|
|
1425
1496
|
failureCode: text("failure_code"),
|
|
1426
1497
|
totalNodes: integer("total_nodes").notNull().default(0),
|
|
1427
1498
|
totalEdges: integer("total_edges").notNull().default(0),
|
|
1499
|
+
/** Share of `totalEdges` classified as nav, header, or footer links. */
|
|
1500
|
+
totalTemplateEdges: integer("total_template_edges").notNull().default(0),
|
|
1428
1501
|
nodeCount: integer("node_count").notNull().default(0),
|
|
1429
1502
|
edgeCount: integer("edge_count").notNull().default(0),
|
|
1503
|
+
/**
|
|
1504
|
+
* True when template links were kept out of the ForceAtlas2 physics, so
|
|
1505
|
+
* these positions describe content structure. Layouts published before
|
|
1506
|
+
* template detection keep `false`: the migration classifies their links, but
|
|
1507
|
+
* it deliberately does not rewrite immutable coordinates.
|
|
1508
|
+
*/
|
|
1509
|
+
templateLinksExcluded: integer("template_links_excluded", { mode: "boolean" }).notNull().default(false),
|
|
1430
1510
|
createdAt: text("created_at").notNull(),
|
|
1431
1511
|
updatedAt: text("updated_at").notNull()
|
|
1432
1512
|
}, (table) => [
|
|
@@ -1476,6 +1556,13 @@ var siteCrawlGraphEdges = sqliteTable("site_crawl_graph_edges", {
|
|
|
1476
1556
|
targetNodeKey: text("target_node_key").notNull(),
|
|
1477
1557
|
followable: integer("followable", { mode: "boolean" }).notNull(),
|
|
1478
1558
|
occurrences: integer("occurrences").notNull(),
|
|
1559
|
+
/**
|
|
1560
|
+
* Denormalized from `site_crawl_edges` the same way `followable` and
|
|
1561
|
+
* `occurrences` are, so a bounded map read never joins the full link table.
|
|
1562
|
+
* Template links stay IN the sample: the map hides them by default, and a
|
|
1563
|
+
* viewer switching them on must not trigger a refetch or a re-layout.
|
|
1564
|
+
*/
|
|
1565
|
+
isTemplate: integer("is_template", { mode: "boolean" }).notNull().default(false),
|
|
1479
1566
|
createdAt: text("created_at").notNull()
|
|
1480
1567
|
}, (table) => [
|
|
1481
1568
|
uniqueIndex("idx_site_crawl_graph_edges_attempt_edge").on(table.projectId, table.runId, table.attemptId, table.edgeKey),
|
|
@@ -2863,6 +2950,138 @@ function relinkOrphanedSnapshotQueryIds(tx) {
|
|
|
2863
2950
|
}
|
|
2864
2951
|
}
|
|
2865
2952
|
}
|
|
2953
|
+
function dropSiteCrawlSelfLinks(tx) {
|
|
2954
|
+
tx.run(sql2.raw("DELETE FROM site_crawl_edges WHERE source_url = target_url"));
|
|
2955
|
+
tx.run(sql2.raw("DELETE FROM site_crawl_graph_edges WHERE source_node_key = target_node_key"));
|
|
2956
|
+
}
|
|
2957
|
+
function backfillSiteCrawlPageHealthState(tx) {
|
|
2958
|
+
const BATCH = 500;
|
|
2959
|
+
for (; ; ) {
|
|
2960
|
+
const rows = tx.all(sql2.raw(`
|
|
2961
|
+
SELECT id, node_key, fetch_state, indexability_state, indexability_reasons, canonical_node_key
|
|
2962
|
+
FROM site_crawl_pages
|
|
2963
|
+
WHERE health_state IS NULL
|
|
2964
|
+
LIMIT ${BATCH}
|
|
2965
|
+
`));
|
|
2966
|
+
if (rows.length === 0) return;
|
|
2967
|
+
for (const row of rows) {
|
|
2968
|
+
const healthState = deriveSiteHealthState({
|
|
2969
|
+
fetchState: row.fetch_state,
|
|
2970
|
+
indexabilityState: row.indexability_state,
|
|
2971
|
+
indexabilityReasons: parseJsonColumn(row.indexability_reasons, []),
|
|
2972
|
+
canonicalNodeKey: row.canonical_node_key,
|
|
2973
|
+
nodeKey: row.node_key
|
|
2974
|
+
});
|
|
2975
|
+
tx.run(sql2`UPDATE site_crawl_pages SET health_state = ${healthState} WHERE id = ${row.id}`);
|
|
2976
|
+
}
|
|
2977
|
+
if (rows.length < BATCH) return;
|
|
2978
|
+
}
|
|
2979
|
+
}
|
|
2980
|
+
function* streamSiteCrawlTemplateLinkEdges(tx, attemptId, batchSize = 2e3) {
|
|
2981
|
+
let after = "";
|
|
2982
|
+
for (; ; ) {
|
|
2983
|
+
const rows = tx.all(sql2`
|
|
2984
|
+
SELECT edge_key, source_node_key, target_node_key, anchors
|
|
2985
|
+
FROM site_crawl_edges
|
|
2986
|
+
WHERE attempt_id = ${attemptId} AND edge_key > ${after}
|
|
2987
|
+
ORDER BY edge_key
|
|
2988
|
+
LIMIT ${batchSize}
|
|
2989
|
+
`);
|
|
2990
|
+
if (rows.length === 0) return;
|
|
2991
|
+
yield rows.map((row) => ({
|
|
2992
|
+
edgeKey: row.edge_key,
|
|
2993
|
+
sourceNodeKey: row.source_node_key,
|
|
2994
|
+
targetNodeKey: row.target_node_key,
|
|
2995
|
+
anchors: parseJsonColumn(row.anchors, [])
|
|
2996
|
+
}));
|
|
2997
|
+
after = rows[rows.length - 1].edge_key;
|
|
2998
|
+
if (rows.length < batchSize) return;
|
|
2999
|
+
}
|
|
3000
|
+
}
|
|
3001
|
+
function* chunked(values, size = 500) {
|
|
3002
|
+
for (let offset = 0; offset < values.length; offset += size) {
|
|
3003
|
+
yield values.slice(offset, offset + size);
|
|
3004
|
+
}
|
|
3005
|
+
}
|
|
3006
|
+
function backfillSiteCrawlTemplateLinks(tx) {
|
|
3007
|
+
const attempts = tx.all(sql2.raw(`
|
|
3008
|
+
SELECT
|
|
3009
|
+
a.id AS attempt_id,
|
|
3010
|
+
a.project_id AS project_id,
|
|
3011
|
+
a.run_id AS run_id,
|
|
3012
|
+
COALESCE(NULLIF(a.pages_fetched, 0), s.pages_fetched, 0) AS pages_fetched
|
|
3013
|
+
FROM site_crawl_attempts AS a
|
|
3014
|
+
LEFT JOIN site_crawl_snapshots AS s
|
|
3015
|
+
ON s.project_id = a.project_id AND s.run_id = a.run_id AND s.attempt_id = a.id
|
|
3016
|
+
`));
|
|
3017
|
+
for (const attempt of attempts) {
|
|
3018
|
+
const detection = templateLinkDetection(attempt.pages_fetched);
|
|
3019
|
+
const scope = { attemptId: attempt.attempt_id, projectId: attempt.project_id, runId: attempt.run_id };
|
|
3020
|
+
tx.run(sql2`
|
|
3021
|
+
UPDATE site_crawl_edges SET is_template = 0, template_ratio = NULL
|
|
3022
|
+
WHERE project_id = ${scope.projectId} AND run_id = ${scope.runId} AND attempt_id = ${scope.attemptId}
|
|
3023
|
+
`);
|
|
3024
|
+
tx.run(sql2`
|
|
3025
|
+
UPDATE site_crawl_graph_edges SET is_template = 0
|
|
3026
|
+
WHERE project_id = ${scope.projectId} AND run_id = ${scope.runId} AND attempt_id = ${scope.attemptId}
|
|
3027
|
+
`);
|
|
3028
|
+
if (detection === SiteHealthTemplateDetections.applied) {
|
|
3029
|
+
const index2 = createTemplateLinkPairIndex();
|
|
3030
|
+
for (const batch of streamSiteCrawlTemplateLinkEdges(tx, scope.attemptId)) {
|
|
3031
|
+
observeTemplateLinkEdges(index2, batch);
|
|
3032
|
+
}
|
|
3033
|
+
const keysByRatio = /* @__PURE__ */ new Map();
|
|
3034
|
+
const templateEdgeKeys = [];
|
|
3035
|
+
for (const batch of streamSiteCrawlTemplateLinkEdges(tx, scope.attemptId)) {
|
|
3036
|
+
for (const edge of batch) {
|
|
3037
|
+
const ratio = templateLinkRatio(index2, attempt.pages_fetched, edge);
|
|
3038
|
+
if (ratio == null) continue;
|
|
3039
|
+
const group = keysByRatio.get(ratio);
|
|
3040
|
+
if (group) group.push(edge.edgeKey);
|
|
3041
|
+
else keysByRatio.set(ratio, [edge.edgeKey]);
|
|
3042
|
+
if (isTemplateLinkRatio(ratio)) templateEdgeKeys.push(edge.edgeKey);
|
|
3043
|
+
}
|
|
3044
|
+
}
|
|
3045
|
+
for (const [ratio, edgeKeys] of keysByRatio) {
|
|
3046
|
+
const isTemplate = isTemplateLinkRatio(ratio) ? 1 : 0;
|
|
3047
|
+
for (const chunk of chunked(edgeKeys)) {
|
|
3048
|
+
tx.run(sql2`
|
|
3049
|
+
UPDATE site_crawl_edges SET is_template = ${isTemplate}, template_ratio = ${ratio}
|
|
3050
|
+
WHERE project_id = ${scope.projectId} AND run_id = ${scope.runId} AND attempt_id = ${scope.attemptId}
|
|
3051
|
+
AND edge_key IN (SELECT value FROM json_each(${JSON.stringify(chunk)}))
|
|
3052
|
+
`);
|
|
3053
|
+
}
|
|
3054
|
+
}
|
|
3055
|
+
for (const chunk of chunked(templateEdgeKeys)) {
|
|
3056
|
+
tx.run(sql2`
|
|
3057
|
+
UPDATE site_crawl_graph_edges SET is_template = 1
|
|
3058
|
+
WHERE project_id = ${scope.projectId} AND run_id = ${scope.runId} AND attempt_id = ${scope.attemptId}
|
|
3059
|
+
AND edge_key IN (SELECT value FROM json_each(${JSON.stringify(chunk)}))
|
|
3060
|
+
`);
|
|
3061
|
+
}
|
|
3062
|
+
}
|
|
3063
|
+
tx.run(sql2`
|
|
3064
|
+
UPDATE site_crawl_graph_layouts
|
|
3065
|
+
SET total_template_edges = (
|
|
3066
|
+
SELECT COUNT(*)
|
|
3067
|
+
FROM site_crawl_edges AS e
|
|
3068
|
+
INNER JOIN site_crawl_pages AS source_page
|
|
3069
|
+
ON source_page.project_id = e.project_id AND source_page.run_id = e.run_id
|
|
3070
|
+
AND source_page.attempt_id = e.attempt_id AND source_page.node_key = e.source_node_key
|
|
3071
|
+
INNER JOIN site_crawl_pages AS target_page
|
|
3072
|
+
ON target_page.project_id = e.project_id AND target_page.run_id = e.run_id
|
|
3073
|
+
AND target_page.attempt_id = e.attempt_id AND target_page.node_key = e.target_node_key
|
|
3074
|
+
WHERE e.project_id = ${scope.projectId} AND e.run_id = ${scope.runId} AND e.attempt_id = ${scope.attemptId}
|
|
3075
|
+
AND e.internal = 1 AND e.relation = 'anchor' AND e.is_template = 1
|
|
3076
|
+
)
|
|
3077
|
+
WHERE project_id = ${scope.projectId} AND run_id = ${scope.runId} AND attempt_id = ${scope.attemptId}
|
|
3078
|
+
`);
|
|
3079
|
+
tx.run(sql2`
|
|
3080
|
+
UPDATE site_crawl_snapshots SET template_detection = ${detection}
|
|
3081
|
+
WHERE project_id = ${scope.projectId} AND run_id = ${scope.runId} AND attempt_id = ${scope.attemptId}
|
|
3082
|
+
`);
|
|
3083
|
+
}
|
|
3084
|
+
}
|
|
2866
3085
|
function backfillQuerySnapshotServedModel(tx) {
|
|
2867
3086
|
tx.run(sql2.raw(`
|
|
2868
3087
|
UPDATE query_snapshots
|
|
@@ -5805,6 +6024,127 @@ var MIGRATION_VERSIONS = [
|
|
|
5805
6024
|
`CREATE INDEX IF NOT EXISTS idx_traffic_event_receipts_expires
|
|
5806
6025
|
ON traffic_event_receipts(source_id, expires_at)`
|
|
5807
6026
|
]
|
|
6027
|
+
},
|
|
6028
|
+
{
|
|
6029
|
+
version: 130,
|
|
6030
|
+
name: "site-crawl-pages-persisted-health-state",
|
|
6031
|
+
// Site Health state is derived from fetch state, indexability, the
|
|
6032
|
+
// crawler's reasons, and canonical identity together. Filtering it by
|
|
6033
|
+
// recomputing in JS meant reading every page row on every request. It is
|
|
6034
|
+
// now written once at publish time by the SAME contract function the map
|
|
6035
|
+
// and the agents use, so reads are an ordinary indexed WHERE.
|
|
6036
|
+
//
|
|
6037
|
+
// Backfilled in TypeScript inside this transaction (see
|
|
6038
|
+
// `backfillSiteCrawlPageHealthState`) so existing scans are filterable
|
|
6039
|
+
// immediately. The derivation is never rewritten in SQL: the backfill
|
|
6040
|
+
// calls the contract's own function, so there is no second implementation
|
|
6041
|
+
// to drift. `unavailable-legacy-scan` therefore survives only as the
|
|
6042
|
+
// honest answer for a row this could not reach.
|
|
6043
|
+
//
|
|
6044
|
+
// Idempotent: `ALTER TABLE ADD COLUMN` errors with "duplicate column
|
|
6045
|
+
// name" on retry, which the runner already swallows, and the backfill
|
|
6046
|
+
// only touches NULL rows.
|
|
6047
|
+
statements: [
|
|
6048
|
+
`ALTER TABLE site_crawl_pages ADD COLUMN health_state TEXT`,
|
|
6049
|
+
// Trailing columns match the ORDER BY the page list actually issues
|
|
6050
|
+
// (`sort=path`), so a filtered read terminates at LIMIT instead of
|
|
6051
|
+
// sorting every match in a temp b-tree on every cursor page.
|
|
6052
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_pages_health
|
|
6053
|
+
ON site_crawl_pages(project_id, run_id, attempt_id, health_state, path, node_key)`
|
|
6054
|
+
],
|
|
6055
|
+
run: backfillSiteCrawlPageHealthState
|
|
6056
|
+
},
|
|
6057
|
+
{
|
|
6058
|
+
version: 131,
|
|
6059
|
+
name: "site-crawl-template-link-classification",
|
|
6060
|
+
// A real site's internal links are bimodal: a (target page, anchor) pair
|
|
6061
|
+
// sits on nearly every page or on nearly none. The ubiquitous half is nav,
|
|
6062
|
+
// header, and footer chrome, and drawing it buries the content structure
|
|
6063
|
+
// the map exists to show. Classification is computed once at publish time
|
|
6064
|
+
// by the contract's own function, so the map, the API filters, and the
|
|
6065
|
+
// agents cannot disagree about which links are chrome.
|
|
6066
|
+
//
|
|
6067
|
+
// `is_template` and `template_detection` are deliberately nullable:
|
|
6068
|
+
// NULL means "never classified" and reads report it as
|
|
6069
|
+
// `unavailable-legacy-scan`. The too-few-pages guard writes an explicit
|
|
6070
|
+
// `false` plus its reason on the snapshot, so an empty template-link list
|
|
6071
|
+
// can never pass for "this site has no nav".
|
|
6072
|
+
//
|
|
6073
|
+
// Idempotent: `ALTER TABLE ADD COLUMN` errors with "duplicate column
|
|
6074
|
+
// name", which the runner already swallows, and the backfill reclassifies
|
|
6075
|
+
// each attempt from its own stored rows.
|
|
6076
|
+
statements: [
|
|
6077
|
+
`ALTER TABLE site_crawl_edges ADD COLUMN is_template INTEGER`,
|
|
6078
|
+
`ALTER TABLE site_crawl_edges ADD COLUMN template_ratio REAL`,
|
|
6079
|
+
`ALTER TABLE site_crawl_graph_edges ADD COLUMN is_template INTEGER NOT NULL DEFAULT 0`,
|
|
6080
|
+
`ALTER TABLE site_crawl_graph_layouts ADD COLUMN total_template_edges INTEGER NOT NULL DEFAULT 0`,
|
|
6081
|
+
`ALTER TABLE site_crawl_graph_layouts ADD COLUMN template_links_excluded INTEGER NOT NULL DEFAULT 0`,
|
|
6082
|
+
`ALTER TABLE site_crawl_snapshots ADD COLUMN template_detection TEXT`,
|
|
6083
|
+
// Trailing edge key matches the ORDER BY the internal-link reads issue,
|
|
6084
|
+
// so a filtered page terminates at LIMIT.
|
|
6085
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_edges_template
|
|
6086
|
+
ON site_crawl_edges(project_id, run_id, attempt_id, internal, is_template, edge_key)`
|
|
6087
|
+
],
|
|
6088
|
+
run: backfillSiteCrawlTemplateLinks
|
|
6089
|
+
},
|
|
6090
|
+
{
|
|
6091
|
+
version: 132,
|
|
6092
|
+
name: "site-crawl-drop-self-links",
|
|
6093
|
+
// A page linking to itself is not a link to or from another page, and the
|
|
6094
|
+
// crawl engine already excludes self-loops from a page's inbound and
|
|
6095
|
+
// outbound metrics. The edge tables kept them, so the stored edges
|
|
6096
|
+
// disagreed with the page rows produced by the same crawl: a self-loop
|
|
6097
|
+
// appeared in BOTH neighbour lists, so every self-linking page read one
|
|
6098
|
+
// higher in each direction than its own tiles.
|
|
6099
|
+
//
|
|
6100
|
+
// The writer now drops them, which fixes new scans. This clears the ones
|
|
6101
|
+
// already stored so an existing scan is correct without a re-crawl.
|
|
6102
|
+
// Comparing the stored normalized URLs is exactly the rule the writer
|
|
6103
|
+
// applies (`isSelfLink`), expressed as the equality SQL can do without
|
|
6104
|
+
// reimplementing any derivation.
|
|
6105
|
+
//
|
|
6106
|
+
// Idempotent: a re-run deletes nothing once the rows are gone.
|
|
6107
|
+
statements: [],
|
|
6108
|
+
run: dropSiteCrawlSelfLinks
|
|
6109
|
+
},
|
|
6110
|
+
{
|
|
6111
|
+
version: 133,
|
|
6112
|
+
name: "site-crawl-reclassify-template-links-per-anchor",
|
|
6113
|
+
// v131 classified a link as chrome when its MOST ubiquitous anchor was
|
|
6114
|
+
// ubiquitous. One stored link row aggregates every anchor the crawl saw
|
|
6115
|
+
// between the same two pages, so on any site with a comprehensive footer
|
|
6116
|
+
// (which is most sites) an in-prose link to a footer-linked target shared
|
|
6117
|
+
// a row with that page's footer link and inherited its ratio. The
|
|
6118
|
+
// editorial link was marked chrome, hidden from the map, and dropped from
|
|
6119
|
+
// every content count. Observed on canonry.ai: /aeo-methodology had 24
|
|
6120
|
+
// outbound links, all marked chrome at ratio 0.96, with anchors like
|
|
6121
|
+
// "Explore the Canonry platform" that are plainly editorial.
|
|
6122
|
+
//
|
|
6123
|
+
// The rule is now the LEAST ubiquitous anchor: chrome only when every
|
|
6124
|
+
// anchor on the link is chrome. This re-runs the same backfill hook, which
|
|
6125
|
+
// reclassifies each attempt from its own stored rows through the corrected
|
|
6126
|
+
// contract function, so an existing scan is right without a re-crawl.
|
|
6127
|
+
//
|
|
6128
|
+
// Statements stay empty because v131 already added every column; only the
|
|
6129
|
+
// values were wrong. Re-running is safe: the hook resets each attempt to
|
|
6130
|
+
// "classified, not chrome" before it writes, so it is idempotent and its
|
|
6131
|
+
// result depends on nothing but the stored links.
|
|
6132
|
+
statements: [],
|
|
6133
|
+
run: backfillSiteCrawlTemplateLinks
|
|
6134
|
+
},
|
|
6135
|
+
{
|
|
6136
|
+
version: 134,
|
|
6137
|
+
name: "site-crawl-live-page-health-preview-index",
|
|
6138
|
+
// The live Page Health preview is deliberately small and stable while a
|
|
6139
|
+
// crawl writes more pages: completed audits below the display threshold,
|
|
6140
|
+
// ordered worst-first by score and then node key. Cover the exact WHERE +
|
|
6141
|
+
// ORDER BY shape so the read stops at LIMIT without sorting every current
|
|
6142
|
+
// crawl row in a temporary b-tree. Its standard composite-index SQL is
|
|
6143
|
+
// also PostgreSQL-compatible; the query shape has no SQLite-only syntax.
|
|
6144
|
+
statements: [
|
|
6145
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_pages_live_preview
|
|
6146
|
+
ON site_crawl_pages(project_id, run_id, attempt_id, audit_state, audit_score, node_key)`
|
|
6147
|
+
]
|
|
5808
6148
|
}
|
|
5809
6149
|
];
|
|
5810
6150
|
function addRunsMeasurementPlanVersionForeignKey(tx) {
|
|
@@ -8676,9 +9016,15 @@ function isOverviewRead(url) {
|
|
|
8676
9016
|
"citations/visibility"
|
|
8677
9017
|
])).has(rest);
|
|
8678
9018
|
}
|
|
8679
|
-
function isTechnicalAeoRead(url) {
|
|
9019
|
+
function isTechnicalAeoRead(request, url) {
|
|
8680
9020
|
const rest = projectRouteRest(url);
|
|
8681
|
-
|
|
9021
|
+
if (rest === "runs") {
|
|
9022
|
+
return queryValue(request, "kind") === RunKinds["site-audit"];
|
|
9023
|
+
}
|
|
9024
|
+
if (rest && /^technical-aeo\/runs\/[^/]+\/(?:progress|page-health-preview)$/.test(rest)) {
|
|
9025
|
+
return true;
|
|
9026
|
+
}
|
|
9027
|
+
return rest === "technical-aeo" || rest === "technical-aeo/pages" || rest === "technical-aeo/trend" || rest === "technical-aeo/crawl" || rest === "technical-aeo/graph" || rest === "technical-aeo/subgraph" || rest === "technical-aeo/path" || rest === "technical-aeo/changes" || rest === "technical-aeo/crawl/pages/audit" || rest === "technical-aeo/crawl/pages" || rest === "technical-aeo/runs" || rest === "technical-aeo/structure" || rest === "technical-aeo/internal-links" || rest === "technical-aeo/internal-links/neighbors" || rest === "technical-aeo/dead-links";
|
|
8682
9028
|
}
|
|
8683
9029
|
function isReportRead(url) {
|
|
8684
9030
|
const rest = projectRouteRest(url);
|
|
@@ -8694,7 +9040,7 @@ function enforceEmbedProjectTabs(request, configuredTabs) {
|
|
|
8694
9040
|
const url = request.url.split("?")[0];
|
|
8695
9041
|
if (isProjectShellRead(request, url)) return;
|
|
8696
9042
|
if (tabs.includes("overview") && isOverviewRead(url)) return;
|
|
8697
|
-
if (tabs.includes("technical-aeo") && isTechnicalAeoRead(url)) return;
|
|
9043
|
+
if (tabs.includes("technical-aeo") && isTechnicalAeoRead(request, url)) return;
|
|
8698
9044
|
if (tabs.includes("report") && isReportRead(url)) return;
|
|
8699
9045
|
throw forbidden("This endpoint is not available for the configured embed tabs.");
|
|
8700
9046
|
}
|
|
@@ -9043,6 +9389,104 @@ function pruneProviderModelsForProviders(models, providers) {
|
|
|
9043
9389
|
|
|
9044
9390
|
// ../api-routes/src/projects.ts
|
|
9045
9391
|
async function projectRoutes(app, opts) {
|
|
9392
|
+
app.post("/projects", async (request, reply) => {
|
|
9393
|
+
requireAdminSession(request);
|
|
9394
|
+
if (request.principal?.projectId) {
|
|
9395
|
+
throw forbidden("This API key is limited to one project, and cannot create projects for this install.");
|
|
9396
|
+
}
|
|
9397
|
+
requireScope(request, PROJECTS_WRITE_SCOPE);
|
|
9398
|
+
const parsedBody = projectCreateRequestSchema.safeParse(request.body);
|
|
9399
|
+
if (!parsedBody.success) {
|
|
9400
|
+
throw validationError("Invalid project payload", {
|
|
9401
|
+
issues: parsedBody.error.issues.map((issue) => ({
|
|
9402
|
+
path: issue.path.join("."),
|
|
9403
|
+
message: issue.message
|
|
9404
|
+
}))
|
|
9405
|
+
});
|
|
9406
|
+
}
|
|
9407
|
+
const body = parsedBody.data;
|
|
9408
|
+
const name = normalizeProjectName(body.name);
|
|
9409
|
+
if (!name) {
|
|
9410
|
+
throw validationError("Project name must contain at least one letter or number after normalization.");
|
|
9411
|
+
}
|
|
9412
|
+
const canonicalDomain = hostOf(body.canonicalDomain);
|
|
9413
|
+
if (!canonicalDomain) {
|
|
9414
|
+
throw validationError("canonicalDomain must be a valid hostname or http(s) URL.");
|
|
9415
|
+
}
|
|
9416
|
+
const validNames = opts.providerAdapters?.map((adapter) => adapter.name) ?? [];
|
|
9417
|
+
if (validNames.length && body.providers?.length) {
|
|
9418
|
+
const invalid = body.providers.filter((p) => !validNames.includes(p));
|
|
9419
|
+
if (invalid.length) {
|
|
9420
|
+
throw validationError(`Invalid provider(s): ${invalid.join(", ")}. Must be one of: ${validNames.join(", ")}`, {
|
|
9421
|
+
invalidProviders: invalid,
|
|
9422
|
+
validProviders: validNames
|
|
9423
|
+
});
|
|
9424
|
+
}
|
|
9425
|
+
}
|
|
9426
|
+
const nextProviders = body.providers ?? [];
|
|
9427
|
+
const providerModels = pruneProviderModelsForProviders(
|
|
9428
|
+
validateProviderModels(body.providerModels ?? {}, opts.providerAdapters),
|
|
9429
|
+
nextProviders
|
|
9430
|
+
);
|
|
9431
|
+
assertProviderModelScope(request, {}, providerModels, nextProviders);
|
|
9432
|
+
const nextLocations = body.locations ?? [];
|
|
9433
|
+
const duplicateLabels = findDuplicateLocationLabels(nextLocations);
|
|
9434
|
+
if (duplicateLabels.length > 0) {
|
|
9435
|
+
throw validationError(`Duplicate location labels are not allowed: ${duplicateLabels.join(", ")}`, {
|
|
9436
|
+
duplicateLabels
|
|
9437
|
+
});
|
|
9438
|
+
}
|
|
9439
|
+
const nextDefaultLocation = body.defaultLocation ?? null;
|
|
9440
|
+
if (!hasLocationLabel(nextLocations, nextDefaultLocation)) {
|
|
9441
|
+
throw validationError(`defaultLocation "${nextDefaultLocation}" must match a configured location label`, {
|
|
9442
|
+
defaultLocation: nextDefaultLocation
|
|
9443
|
+
});
|
|
9444
|
+
}
|
|
9445
|
+
const normalizedCollision = app.db.select({
|
|
9446
|
+
name: projects.name
|
|
9447
|
+
}).from(projects).all().find((project) => normalizeProjectName(project.name) === name);
|
|
9448
|
+
if (normalizedCollision) throw alreadyExists("Project", name);
|
|
9449
|
+
const id = crypto6.randomUUID();
|
|
9450
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9451
|
+
const nextAliases = normalizeProjectAliases(body.displayName, body.aliases ?? []);
|
|
9452
|
+
const inserted = app.db.transaction((tx) => {
|
|
9453
|
+
const result = tx.insert(projects).values({
|
|
9454
|
+
id,
|
|
9455
|
+
name,
|
|
9456
|
+
displayName: body.displayName,
|
|
9457
|
+
canonicalDomain,
|
|
9458
|
+
ownedDomains: body.ownedDomains ?? [],
|
|
9459
|
+
aliases: nextAliases,
|
|
9460
|
+
country: body.country,
|
|
9461
|
+
language: body.language,
|
|
9462
|
+
tags: body.tags ?? [],
|
|
9463
|
+
labels: body.labels ?? {},
|
|
9464
|
+
providers: nextProviders,
|
|
9465
|
+
providerModels,
|
|
9466
|
+
measurement: body.measurement ?? DEFAULT_MEASUREMENT_CONFIG,
|
|
9467
|
+
locations: nextLocations,
|
|
9468
|
+
defaultLocation: nextDefaultLocation,
|
|
9469
|
+
autoExtractBacklinks: body.autoExtractBacklinks ?? false,
|
|
9470
|
+
configSource: body.configSource ?? "api",
|
|
9471
|
+
configRevision: 1,
|
|
9472
|
+
createdAt: now,
|
|
9473
|
+
updatedAt: now
|
|
9474
|
+
}).onConflictDoNothing().run();
|
|
9475
|
+
if (result.changes !== 1) return false;
|
|
9476
|
+
writeAuditLog(tx, {
|
|
9477
|
+
projectId: id,
|
|
9478
|
+
actor: "api",
|
|
9479
|
+
action: "project.created",
|
|
9480
|
+
entityType: "project",
|
|
9481
|
+
entityId: id
|
|
9482
|
+
});
|
|
9483
|
+
return true;
|
|
9484
|
+
});
|
|
9485
|
+
if (!inserted) throw alreadyExists("Project", name);
|
|
9486
|
+
opts.onProjectUpserted?.(id, name);
|
|
9487
|
+
const created = app.db.select().from(projects).where(eq4(projects.id, id)).get();
|
|
9488
|
+
return reply.status(201).send(formatProject(created));
|
|
9489
|
+
});
|
|
9046
9490
|
app.put("/projects/:name", async (request, reply) => {
|
|
9047
9491
|
const { name } = request.params;
|
|
9048
9492
|
const parsedBody = projectUpsertRequestSchema.safeParse(request.body);
|
|
@@ -27385,6 +27829,7 @@ var SCHEMA_TABLE = {
|
|
|
27385
27829
|
NotificationDto: notificationDtoSchema,
|
|
27386
27830
|
OnboardingTelemetryEvent: onboardingTelemetryEventSchema,
|
|
27387
27831
|
OrganicEvidenceDto: organicEvidenceDtoSchema,
|
|
27832
|
+
ProjectCreateRequest: projectCreateRequestSchema,
|
|
27388
27833
|
ProjectDto: projectDtoSchema,
|
|
27389
27834
|
ProjectUpsertRequest: projectUpsertRequestSchema,
|
|
27390
27835
|
ProjectConfig: projectConfigSchema,
|
|
@@ -27398,7 +27843,9 @@ var SCHEMA_TABLE = {
|
|
|
27398
27843
|
SchedulableRunKind: schedulableRunKindSchema,
|
|
27399
27844
|
ScheduleDto: scheduleDtoSchema,
|
|
27400
27845
|
SettingsDto: settingsDtoSchema,
|
|
27846
|
+
SiteAuditLivePageHealthDto: siteAuditLivePageHealthSchema,
|
|
27401
27847
|
SiteAuditPagesResponseDto: siteAuditPagesResponseSchema,
|
|
27848
|
+
SiteAuditRunProgressDto: siteAuditRunProgressSchema,
|
|
27402
27849
|
SiteAuditRunResponseDto: siteAuditRunResponseSchema,
|
|
27403
27850
|
SiteAuditScoreDto: siteAuditScoreSchema,
|
|
27404
27851
|
SiteAuditTrendResponseDto: siteAuditTrendResponseSchema,
|
|
@@ -27412,6 +27859,7 @@ var SCHEMA_TABLE = {
|
|
|
27412
27859
|
SiteCrawlSummaryDto: siteCrawlSummarySchema,
|
|
27413
27860
|
SiteHealthChangesResponseDto: siteHealthChangesResponseSchema,
|
|
27414
27861
|
SiteHealthPathResponseDto: siteHealthPathResponseSchema,
|
|
27862
|
+
SiteHealthScansResponseDto: siteHealthScansResponseSchema,
|
|
27415
27863
|
SiteHealthSubgraphResponseDto: siteHealthSubgraphResponseSchema,
|
|
27416
27864
|
SnapshotDiffResponse: snapshotDiffResponseSchema,
|
|
27417
27865
|
SnapshotListResponse: snapshotListResponseSchema,
|
|
@@ -27885,6 +28333,12 @@ var crawlLimitParameter = {
|
|
|
27885
28333
|
description: "Page size. Defaults to 100 (50 for structure/neighbors); maximum 200 (100 for structure/neighbors).",
|
|
27886
28334
|
schema: { type: "integer", minimum: 1, maximum: 200 }
|
|
27887
28335
|
};
|
|
28336
|
+
var linkKindParameter = {
|
|
28337
|
+
name: "linkKind",
|
|
28338
|
+
in: "query",
|
|
28339
|
+
description: 'Restrict links to `content` (excludes nav, header, and footer links) or `template` (only those). Defaults to `all`. A scan whose links were never classified matches neither, which is why every link response also carries `templateDetection`: an empty `content` list means "could not tell" when detection is not `applied`.',
|
|
28340
|
+
schema: { type: "string", enum: ["all", "content", "template"], default: "all" }
|
|
28341
|
+
};
|
|
27888
28342
|
var measurementOverviewSortParameter = {
|
|
27889
28343
|
name: "sort",
|
|
27890
28344
|
in: "query",
|
|
@@ -28026,6 +28480,27 @@ var routeCatalog = [
|
|
|
28026
28480
|
404: errorResponse("Project or connection not found.")
|
|
28027
28481
|
}
|
|
28028
28482
|
},
|
|
28483
|
+
{
|
|
28484
|
+
method: "post",
|
|
28485
|
+
path: "/api/v1/projects",
|
|
28486
|
+
summary: "Create a project",
|
|
28487
|
+
description: "Creates a new project with create-only semantics for the domain-first launchpad. The server normalizes the project name and canonical domain; a normalized-name collision returns 409 and never updates the existing project.",
|
|
28488
|
+
tags: ["projects"],
|
|
28489
|
+
requestBody: {
|
|
28490
|
+
required: true,
|
|
28491
|
+
content: {
|
|
28492
|
+
"application/json": {
|
|
28493
|
+
schema: { $ref: "#/components/schemas/ProjectCreateRequest" }
|
|
28494
|
+
}
|
|
28495
|
+
}
|
|
28496
|
+
},
|
|
28497
|
+
responses: {
|
|
28498
|
+
201: jsonResponse("Project created.", "ProjectDto"),
|
|
28499
|
+
400: errorResponse("Invalid project payload, normalized name, or canonical domain."),
|
|
28500
|
+
403: errorResponse("The caller lacks broad projects.write authority."),
|
|
28501
|
+
409: errorResponse("A project with the normalized name already exists.")
|
|
28502
|
+
}
|
|
28503
|
+
},
|
|
28029
28504
|
{
|
|
28030
28505
|
method: "put",
|
|
28031
28506
|
path: "/api/v1/projects/{name}",
|
|
@@ -33396,24 +33871,25 @@ var routeCatalog = [
|
|
|
33396
33871
|
],
|
|
33397
33872
|
responses: {
|
|
33398
33873
|
200: jsonResponse("Persisted crawl summary returned.", "SiteCrawlSummaryDto"),
|
|
33399
|
-
404: errorResponse("Project or
|
|
33874
|
+
404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
|
|
33400
33875
|
}
|
|
33401
33876
|
},
|
|
33402
33877
|
{
|
|
33403
33878
|
method: "get",
|
|
33404
33879
|
path: "/api/v1/projects/{name}/technical-aeo/graph",
|
|
33405
33880
|
summary: "Get a persisted Site Health graph projection",
|
|
33406
|
-
description: "Returns the deterministic graph projection computed once when the latest complete or selected historical crawl was published. ForceAtlas2 positions and the exact internal-anchor edge sample are persisted, so reads run no layout physics and never rescan the crawl edge table. The response is bounded to 20,000 nodes / 50,000 edges; `layout`, `omittedNodes`, `omittedEdges`, and `sampled` disclose legacy/unavailable layouts and intentional truncation.",
|
|
33881
|
+
description: "Returns the deterministic graph projection computed once when the latest complete or selected historical crawl was published. ForceAtlas2 positions and the exact internal-anchor edge sample are persisted, so reads run no layout physics and never rescan the crawl edge table. Nav, header, and footer links are excluded from the layout physics but retained in the sample and tagged `isTemplate`, so a viewer can draw them without a refetch and without any node moving. The response is bounded to 20,000 nodes / 50,000 edges; `layout`, `omittedNodes`, `omittedEdges`, and `sampled` disclose legacy/unavailable layouts and intentional truncation, and `templateDetection` says whether template links could be told apart at all.",
|
|
33407
33882
|
tags: ["technical-aeo"],
|
|
33408
33883
|
parameters: [
|
|
33409
33884
|
nameParameter,
|
|
33410
33885
|
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest complete crawl.", schema: stringSchema },
|
|
33411
33886
|
{ name: "maxNodes", in: "query", description: "Maximum graph nodes. Defaults to and is capped at 20,000.", schema: { type: "integer", minimum: 1, maximum: 2e4 } },
|
|
33412
|
-
{ name: "maxEdges", in: "query", description: "Maximum graph edges. Defaults to and is capped at 50,000.", schema: { type: "integer", minimum: 1, maximum: 5e4 } }
|
|
33887
|
+
{ name: "maxEdges", in: "query", description: "Maximum graph edges. Defaults to and is capped at 50,000.", schema: { type: "integer", minimum: 1, maximum: 5e4 } },
|
|
33888
|
+
linkKindParameter
|
|
33413
33889
|
],
|
|
33414
33890
|
responses: {
|
|
33415
33891
|
200: jsonResponse("Bounded Site Health graph projection returned.", "SiteCrawlGraphResponseDto"),
|
|
33416
|
-
404: errorResponse("Project or
|
|
33892
|
+
404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
|
|
33417
33893
|
}
|
|
33418
33894
|
},
|
|
33419
33895
|
{
|
|
@@ -33508,6 +33984,8 @@ var routeCatalog = [
|
|
|
33508
33984
|
{ name: "inventoryEligible", in: "query", description: "Filter Canonry technical inventory eligibility (`true` or `false`).", schema: booleanSchema },
|
|
33509
33985
|
{ name: "fetchState", in: "query", description: "Filter by persisted fetch state.", schema: stringSchema },
|
|
33510
33986
|
{ name: "indexabilityState", in: "query", description: "Filter by crawler-derived indexability state; this is not Google index coverage.", schema: stringSchema },
|
|
33987
|
+
{ name: "nodeKey", in: "query", description: "Return only the page with this node key. Combined with a filter it answers whether that exact page is in the filtered set, without paging through the list.", schema: stringSchema },
|
|
33988
|
+
{ name: "healthState", in: "query", description: "Filter by the derived Site Health state shared with the dashboard and agents. Unlike `indexabilityState` this folds fetch state, canonical identity, and the crawler reasons into one decision. `hidden` means the SITE told answer engines not to index the page (noindex, canonical-away, robots.txt); a non-HTML file is `resource` and a redirect is `redirect`, because neither is suppressed. Values: `eligible`, `hidden`, `resource`, `redirect`, `failed`, `unchecked`.", schema: { type: "string", enum: ["eligible", "hidden", "resource", "redirect", "failed", "unchecked"] } },
|
|
33511
33989
|
{ name: "auditState", in: "query", description: "Filter by audit state.", schema: stringSchema },
|
|
33512
33990
|
{ name: "sort", in: "query", description: "Sort order. Defaults to `url`.", schema: { type: "string", enum: ["url", "path", "score-asc", "score-desc"] } },
|
|
33513
33991
|
crawlCursorParameter,
|
|
@@ -33515,7 +33993,7 @@ var routeCatalog = [
|
|
|
33515
33993
|
],
|
|
33516
33994
|
responses: {
|
|
33517
33995
|
200: jsonResponse("Crawl pages returned.", "SiteCrawlPagesResponseDto"),
|
|
33518
|
-
404: errorResponse("Project or
|
|
33996
|
+
404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
|
|
33519
33997
|
}
|
|
33520
33998
|
},
|
|
33521
33999
|
{
|
|
@@ -33533,14 +34011,14 @@ var routeCatalog = [
|
|
|
33533
34011
|
],
|
|
33534
34012
|
responses: {
|
|
33535
34013
|
200: jsonResponse("Immediate crawl children returned.", "SiteCrawlStructureResponseDto"),
|
|
33536
|
-
404: errorResponse("Project or
|
|
34014
|
+
404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
|
|
33537
34015
|
}
|
|
33538
34016
|
},
|
|
33539
34017
|
{
|
|
33540
34018
|
method: "get",
|
|
33541
34019
|
path: "/api/v1/projects/{name}/technical-aeo/internal-links",
|
|
33542
34020
|
summary: "List persisted internal crawl links",
|
|
33543
|
-
description: "Cursor-paged internal edges for the latest or selected crawl. Optional source/target/followability filters remain project-, run-, and attempt-scoped.",
|
|
34021
|
+
description: "Cursor-paged internal edges for the latest or selected crawl. Optional source/target/followability/link-kind filters remain project-, run-, and attempt-scoped. `total` counts exactly what the requested filters match, and `templateDetection` says whether nav and footer links could be told apart for this scan.",
|
|
33544
34022
|
tags: ["technical-aeo"],
|
|
33545
34023
|
parameters: [
|
|
33546
34024
|
nameParameter,
|
|
@@ -33548,12 +34026,13 @@ var routeCatalog = [
|
|
|
33548
34026
|
{ name: "sourceUrl", in: "query", description: "Restrict to a source URL.", schema: stringSchema },
|
|
33549
34027
|
{ name: "targetUrl", in: "query", description: "Restrict to a target URL.", schema: stringSchema },
|
|
33550
34028
|
{ name: "followable", in: "query", description: "Restrict to followable or nofollow link observations.", schema: booleanSchema },
|
|
34029
|
+
linkKindParameter,
|
|
33551
34030
|
crawlCursorParameter,
|
|
33552
34031
|
crawlLimitParameter
|
|
33553
34032
|
],
|
|
33554
34033
|
responses: {
|
|
33555
34034
|
200: jsonResponse("Internal crawl links returned.", "SiteCrawlInternalLinksResponseDto"),
|
|
33556
|
-
404: errorResponse("Project or
|
|
34035
|
+
404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
|
|
33557
34036
|
}
|
|
33558
34037
|
},
|
|
33559
34038
|
{
|
|
@@ -33567,12 +34046,13 @@ var routeCatalog = [
|
|
|
33567
34046
|
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest crawl.", schema: stringSchema },
|
|
33568
34047
|
{ name: "nodeKey", in: "query", description: "Canonical crawl node key.", schema: stringSchema },
|
|
33569
34048
|
{ name: "url", in: "query", description: "Canonical crawl URL.", schema: stringSchema },
|
|
34049
|
+
linkKindParameter,
|
|
33570
34050
|
{ name: "limit", in: "query", description: "Maximum inbound and outbound edges independently. Defaults to 50; maximum 100.", schema: { type: "integer", minimum: 1, maximum: 100 } }
|
|
33571
34051
|
],
|
|
33572
34052
|
responses: {
|
|
33573
34053
|
200: jsonResponse("Bounded internal-link neighborhood returned.", "SiteCrawlNeighborsResponseDto"),
|
|
33574
34054
|
400: errorResponse("A nodeKey or URL is required."),
|
|
33575
|
-
404: errorResponse("Project or
|
|
34055
|
+
404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
|
|
33576
34056
|
}
|
|
33577
34057
|
},
|
|
33578
34058
|
{
|
|
@@ -33589,7 +34069,52 @@ var routeCatalog = [
|
|
|
33589
34069
|
],
|
|
33590
34070
|
responses: {
|
|
33591
34071
|
200: jsonResponse("Dead-link status and findings returned.", "SiteCrawlDeadLinksResponseDto"),
|
|
33592
|
-
404: errorResponse("Project or
|
|
34072
|
+
404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
|
|
34073
|
+
}
|
|
34074
|
+
},
|
|
34075
|
+
{
|
|
34076
|
+
method: "get",
|
|
34077
|
+
path: "/api/v1/projects/{name}/technical-aeo/runs",
|
|
34078
|
+
summary: "List Site Health scan history",
|
|
34079
|
+
description: "Returns every non-probe `site-audit` run for the project, newest first, including runs still queued or running and runs that failed. `hasCrawlData` says whether that scan published a page and internal-link crawl: a legacy score-only scan is listed with `hasCrawlData=false` rather than hidden, and the crawl-scoped reads answer it with their no-crawl state instead of a 404.",
|
|
34080
|
+
tags: ["technical-aeo"],
|
|
34081
|
+
parameters: [
|
|
34082
|
+
nameParameter,
|
|
34083
|
+
{ name: "limit", in: "query", description: "Max scans returned (most recent first). Defaults to 20; max 100.", schema: { type: "integer", minimum: 1, maximum: 100 } }
|
|
34084
|
+
],
|
|
34085
|
+
responses: {
|
|
34086
|
+
200: jsonResponse("Site Health scan history returned.", "SiteHealthScansResponseDto"),
|
|
34087
|
+
404: errorResponse("Project not found.")
|
|
34088
|
+
}
|
|
34089
|
+
},
|
|
34090
|
+
{
|
|
34091
|
+
method: "get",
|
|
34092
|
+
path: "/api/v1/projects/{name}/technical-aeo/runs/{runId}/progress",
|
|
34093
|
+
summary: "Get exact stored Site Health run progress",
|
|
34094
|
+
description: "Returns durable run, crawl-attempt, and terminal graph-layout state for one exact non-probe site-audit run. It performs no network work and intentionally returns raw counters instead of a completion percentage.",
|
|
34095
|
+
tags: ["technical-aeo"],
|
|
34096
|
+
parameters: [
|
|
34097
|
+
nameParameter,
|
|
34098
|
+
{ name: "runId", in: "path", required: true, description: "Exact site-audit run ID.", schema: stringSchema }
|
|
34099
|
+
],
|
|
34100
|
+
responses: {
|
|
34101
|
+
200: jsonResponse("Exact stored Site Health progress returned.", "SiteAuditRunProgressDto"),
|
|
34102
|
+
404: errorResponse("Project or non-probe site-audit run not found.")
|
|
34103
|
+
}
|
|
34104
|
+
},
|
|
34105
|
+
{
|
|
34106
|
+
method: "get",
|
|
34107
|
+
path: "/api/v1/projects/{name}/technical-aeo/runs/{runId}/page-health-preview",
|
|
34108
|
+
summary: "Get a bounded, provisional Page Health preview",
|
|
34109
|
+
description: "Returns a small durable, provisional preview for one exact non-probe site-audit run while it is running: the count of audited pages plus at most 12 actionable low-score examples without finding prose. Queued scans return waiting. Terminal scans return no examples so callers use the immutable Page Health results instead. It performs no network work.",
|
|
34110
|
+
tags: ["technical-aeo"],
|
|
34111
|
+
parameters: [
|
|
34112
|
+
nameParameter,
|
|
34113
|
+
{ name: "runId", in: "path", required: true, description: "Exact site-audit run ID.", schema: stringSchema }
|
|
34114
|
+
],
|
|
34115
|
+
responses: {
|
|
34116
|
+
200: jsonResponse("Bounded provisional Page Health preview returned.", "SiteAuditLivePageHealthDto"),
|
|
34117
|
+
404: errorResponse("Project or non-probe site-audit run not found.")
|
|
33593
34118
|
}
|
|
33594
34119
|
},
|
|
33595
34120
|
{
|
|
@@ -33621,6 +34146,7 @@ var routeCatalog = [
|
|
|
33621
34146
|
200: jsonResponse("Site-audit run queued (or the in-flight run returned).", "SiteAuditRunResponseDto"),
|
|
33622
34147
|
400: errorResponse("Invalid site-audit request."),
|
|
33623
34148
|
409: errorResponse("A crawl with different effective options is already queued or running."),
|
|
34149
|
+
422: errorResponse("The site-audit executor is unavailable on this deployment."),
|
|
33624
34150
|
404: errorResponse("Project not found.")
|
|
33625
34151
|
}
|
|
33626
34152
|
}
|
|
@@ -57041,7 +57567,7 @@ function dedupeStrings(input) {
|
|
|
57041
57567
|
|
|
57042
57568
|
// ../api-routes/src/technical-aeo.ts
|
|
57043
57569
|
import crypto44 from "crypto";
|
|
57044
|
-
import { and as and44, asc as asc10, count, desc as desc24, eq as eq56, inArray as inArray19, isNotNull as isNotNull3, lt as lt8, or as or11, sql as sql21 } from "drizzle-orm";
|
|
57570
|
+
import { and as and44, asc as asc10, count, desc as desc24, eq as eq56, inArray as inArray19, isNotNull as isNotNull3, isNull as isNull5, lt as lt8, or as or11, sql as sql21 } from "drizzle-orm";
|
|
57045
57571
|
import { alias } from "drizzle-orm/sqlite-core";
|
|
57046
57572
|
var FETCHED_SITE_CRAWL_STATES = /* @__PURE__ */ new Set([
|
|
57047
57573
|
...SiteCrawlFetchedStates,
|
|
@@ -57051,6 +57577,22 @@ var FETCHED_SITE_CRAWL_STATES = /* @__PURE__ */ new Set([
|
|
|
57051
57577
|
"fetched"
|
|
57052
57578
|
]);
|
|
57053
57579
|
var SURFACEABLE_STATUSES = [RunStatuses.completed, RunStatuses.partial];
|
|
57580
|
+
function siteAuditRunPhase(status, attempt, layoutState) {
|
|
57581
|
+
switch (status) {
|
|
57582
|
+
case RunStatuses.queued:
|
|
57583
|
+
return "queued";
|
|
57584
|
+
case RunStatuses.running:
|
|
57585
|
+
return (attempt?.pagesFetched ?? 0) > 0 ? "checking" : "discovering";
|
|
57586
|
+
case RunStatuses.completed:
|
|
57587
|
+
return attempt && layoutState === "pending" ? "arranging-map" : "completed";
|
|
57588
|
+
case RunStatuses.partial:
|
|
57589
|
+
return attempt && layoutState === "pending" ? "arranging-map" : "partial";
|
|
57590
|
+
case RunStatuses.failed:
|
|
57591
|
+
return "failed";
|
|
57592
|
+
case RunStatuses.cancelled:
|
|
57593
|
+
return "cancelled";
|
|
57594
|
+
}
|
|
57595
|
+
}
|
|
57054
57596
|
function emptyScore(projectName) {
|
|
57055
57597
|
return {
|
|
57056
57598
|
project: projectName,
|
|
@@ -57082,6 +57624,14 @@ function parseBoundedLimit(value, fallback, max) {
|
|
|
57082
57624
|
const parsed = parsePositiveInt(value, fallback, max);
|
|
57083
57625
|
return Math.max(1, parsed);
|
|
57084
57626
|
}
|
|
57627
|
+
function parseSiteHealthState(value) {
|
|
57628
|
+
if (value === void 0 || value === "") return null;
|
|
57629
|
+
const allowed = siteHealthStateSchema.safeParse(value);
|
|
57630
|
+
if (!allowed.success) {
|
|
57631
|
+
throw validationError(`"healthState" must be one of: ${siteHealthStateSchema.options.join(", ")}`);
|
|
57632
|
+
}
|
|
57633
|
+
return allowed.data;
|
|
57634
|
+
}
|
|
57085
57635
|
function parseBoolean2(value) {
|
|
57086
57636
|
if (value === true || value === "true" || value === "1") return true;
|
|
57087
57637
|
if (value === false || value === "false" || value === "0") return false;
|
|
@@ -57170,6 +57720,21 @@ function mapCrawlPageAuditEvidence(row) {
|
|
|
57170
57720
|
criticalDefects: parsedCriticalDefects.flatMap((result) => result.success ? [result.data] : [])
|
|
57171
57721
|
};
|
|
57172
57722
|
}
|
|
57723
|
+
var LIVE_PAGE_HEALTH_CANDIDATE_LIMIT = 48;
|
|
57724
|
+
var LIVE_PAGE_HEALTH_EXAMPLE_LIMIT = 12;
|
|
57725
|
+
function countLivePageHealthChecks(auditFields) {
|
|
57726
|
+
const rawFactors = Array.isArray(auditFields.factors) ? auditFields.factors : [];
|
|
57727
|
+
const rawCriticalDefects = Array.isArray(auditFields.criticalDefects) ? auditFields.criticalDefects : [];
|
|
57728
|
+
let checks = 0;
|
|
57729
|
+
for (const rawFactor of rawFactors) {
|
|
57730
|
+
const factor = siteCrawlAuditFactorSchema.safeParse(rawFactor);
|
|
57731
|
+
if (factor.success && factor.data.applicable !== false && factor.data.status !== "pass") checks++;
|
|
57732
|
+
}
|
|
57733
|
+
for (const rawCriticalDefect of rawCriticalDefects) {
|
|
57734
|
+
if (siteCrawlCriticalDefectSchema.safeParse(rawCriticalDefect).success) checks++;
|
|
57735
|
+
}
|
|
57736
|
+
return checks;
|
|
57737
|
+
}
|
|
57173
57738
|
function mapCrawlEdge(row) {
|
|
57174
57739
|
return {
|
|
57175
57740
|
edgeKey: row.edgeKey,
|
|
@@ -57183,9 +57748,28 @@ function mapCrawlEdge(row) {
|
|
|
57183
57748
|
occurrences: row.occurrences,
|
|
57184
57749
|
followableOccurrences: row.followableOccurrences,
|
|
57185
57750
|
nofollowOccurrences: row.nofollowOccurrences,
|
|
57186
|
-
anchors: row.anchors
|
|
57751
|
+
anchors: row.anchors,
|
|
57752
|
+
isTemplate: row.isTemplate,
|
|
57753
|
+
templateRatio: row.templateRatio
|
|
57187
57754
|
};
|
|
57188
57755
|
}
|
|
57756
|
+
function templateDetectionOf(value) {
|
|
57757
|
+
const parsed = siteHealthTemplateDetectionSchema.safeParse(value);
|
|
57758
|
+
return parsed.success ? parsed.data : SiteHealthTemplateDetections["unavailable-legacy-scan"];
|
|
57759
|
+
}
|
|
57760
|
+
function parseLinkKind(value) {
|
|
57761
|
+
if (value === void 0 || value === "") return SiteHealthLinkKinds.all;
|
|
57762
|
+
const allowed = siteHealthLinkKindSchema.safeParse(value);
|
|
57763
|
+
if (!allowed.success) {
|
|
57764
|
+
throw validationError(`"linkKind" must be one of: ${siteHealthLinkKindSchema.options.join(", ")}`);
|
|
57765
|
+
}
|
|
57766
|
+
return allowed.data;
|
|
57767
|
+
}
|
|
57768
|
+
function linkKindFilter(linkKind) {
|
|
57769
|
+
if (linkKind === SiteHealthLinkKinds.content) return eq56(siteCrawlEdges.isTemplate, false);
|
|
57770
|
+
if (linkKind === SiteHealthLinkKinds.template) return eq56(siteCrawlEdges.isTemplate, true);
|
|
57771
|
+
return void 0;
|
|
57772
|
+
}
|
|
57189
57773
|
var graphSourceNode = alias(siteCrawlGraphNodes, "site_crawl_graph_source_node");
|
|
57190
57774
|
var graphTargetNode = alias(siteCrawlGraphNodes, "site_crawl_graph_target_node");
|
|
57191
57775
|
var PAGE_CHANGE_PREDICATE = sql21`(
|
|
@@ -57317,6 +57901,16 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57317
57901
|
eq56(siteCrawlPages.attemptId, scope.attemptId),
|
|
57318
57902
|
eq56(siteCrawlPages.depth, 0)
|
|
57319
57903
|
)).orderBy(asc10(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
57904
|
+
const isSurfaceableAuditRun = (projectId, runId) => Boolean(app.db.select({ id: runs.id }).from(runs).where(and44(
|
|
57905
|
+
eq56(runs.id, runId),
|
|
57906
|
+
eq56(runs.projectId, projectId),
|
|
57907
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
57908
|
+
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
57909
|
+
notProbeRun()
|
|
57910
|
+
)).limit(1).get());
|
|
57911
|
+
const assertKnownAuditRun = (projectId, runId) => {
|
|
57912
|
+
if (runId && !isSurfaceableAuditRun(projectId, runId)) throw notFound("Site crawl run", runId);
|
|
57913
|
+
};
|
|
57320
57914
|
const hasLegacyAudit = (projectId) => Boolean(app.db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).innerJoin(runs, eq56(siteAuditSnapshots.runId, runs.id)).where(and44(
|
|
57321
57915
|
eq56(siteAuditSnapshots.projectId, projectId),
|
|
57322
57916
|
eq56(runs.projectId, projectId),
|
|
@@ -57437,7 +58031,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57437
58031
|
const target = resolveCrawl(project.id, request.query.runId);
|
|
57438
58032
|
const legacyAuditAvailable = hasLegacyAudit(project.id);
|
|
57439
58033
|
if (!target) {
|
|
57440
|
-
|
|
58034
|
+
assertKnownAuditRun(project.id, request.query.runId);
|
|
57441
58035
|
return emptyCrawlSummary(project.name, legacyAuditAvailable);
|
|
57442
58036
|
}
|
|
57443
58037
|
const snapshot = target.snapshot;
|
|
@@ -57471,16 +58065,23 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57471
58065
|
});
|
|
57472
58066
|
app.get("/projects/:name/technical-aeo/graph", async (request) => {
|
|
57473
58067
|
const project = resolveProject(app.db, request.params.name);
|
|
58068
|
+
const linkKind = parseLinkKind(request.query.linkKind);
|
|
57474
58069
|
const target = resolveCrawl(project.id, request.query.runId);
|
|
57475
58070
|
if (!target) {
|
|
57476
|
-
|
|
58071
|
+
assertKnownAuditRun(project.id, request.query.runId);
|
|
57477
58072
|
return {
|
|
57478
58073
|
project: project.name,
|
|
57479
58074
|
hasCrawlData: false,
|
|
57480
58075
|
runId: null,
|
|
58076
|
+
rootNodeKey: null,
|
|
57481
58077
|
layout: { state: "unavailable", version: null, reason: "no-crawl" },
|
|
58078
|
+
// Nothing was classified because nothing was crawled.
|
|
58079
|
+
templateDetection: SiteHealthTemplateDetections["unavailable-legacy-scan"],
|
|
58080
|
+
linkKind,
|
|
57482
58081
|
totalNodes: 0,
|
|
57483
58082
|
totalEdges: 0,
|
|
58083
|
+
totalTemplateEdges: 0,
|
|
58084
|
+
totalContentEdges: 0,
|
|
57484
58085
|
nodes: [],
|
|
57485
58086
|
edges: [],
|
|
57486
58087
|
omittedNodes: 0,
|
|
@@ -57489,14 +58090,20 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57489
58090
|
};
|
|
57490
58091
|
}
|
|
57491
58092
|
const snapshot = target.snapshot;
|
|
58093
|
+
const templateDetection = templateDetectionOf(snapshot.templateDetection);
|
|
57492
58094
|
if (!snapshot.detailsAvailable || !snapshot.attemptId) {
|
|
57493
58095
|
return {
|
|
57494
58096
|
project: project.name,
|
|
57495
58097
|
hasCrawlData: true,
|
|
57496
58098
|
runId: snapshot.runId,
|
|
58099
|
+
rootNodeKey: null,
|
|
57497
58100
|
layout: { state: "unavailable", version: null, reason: "details-unavailable" },
|
|
58101
|
+
templateDetection,
|
|
58102
|
+
linkKind,
|
|
57498
58103
|
totalNodes: 0,
|
|
57499
58104
|
totalEdges: 0,
|
|
58105
|
+
totalTemplateEdges: 0,
|
|
58106
|
+
totalContentEdges: 0,
|
|
57500
58107
|
nodes: [],
|
|
57501
58108
|
edges: [],
|
|
57502
58109
|
omittedNodes: 0,
|
|
@@ -57504,6 +58111,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57504
58111
|
sampled: false
|
|
57505
58112
|
};
|
|
57506
58113
|
}
|
|
58114
|
+
const rootNodeKey = rootPageInScope(
|
|
58115
|
+
{ projectId: project.id, runId: snapshot.runId, attemptId: snapshot.attemptId },
|
|
58116
|
+
snapshot.rootUrl
|
|
58117
|
+
)?.nodeKey ?? null;
|
|
57507
58118
|
const persistedLayout = app.db.select().from(siteCrawlGraphLayouts).where(and44(
|
|
57508
58119
|
eq56(siteCrawlGraphLayouts.projectId, project.id),
|
|
57509
58120
|
eq56(siteCrawlGraphLayouts.runId, snapshot.runId),
|
|
@@ -57514,9 +58125,14 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57514
58125
|
project: project.name,
|
|
57515
58126
|
hasCrawlData: true,
|
|
57516
58127
|
runId: snapshot.runId,
|
|
58128
|
+
rootNodeKey,
|
|
57517
58129
|
layout: { state: "unavailable", version: null, reason: "legacy-snapshot" },
|
|
58130
|
+
templateDetection,
|
|
58131
|
+
linkKind,
|
|
57518
58132
|
totalNodes: 0,
|
|
57519
58133
|
totalEdges: 0,
|
|
58134
|
+
totalTemplateEdges: 0,
|
|
58135
|
+
totalContentEdges: 0,
|
|
57520
58136
|
nodes: [],
|
|
57521
58137
|
edges: [],
|
|
57522
58138
|
omittedNodes: 0,
|
|
@@ -57531,13 +58147,18 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57531
58147
|
project: project.name,
|
|
57532
58148
|
hasCrawlData: true,
|
|
57533
58149
|
runId: snapshot.runId,
|
|
58150
|
+
rootNodeKey,
|
|
57534
58151
|
layout: {
|
|
57535
58152
|
state: "unavailable",
|
|
57536
58153
|
version: null,
|
|
57537
58154
|
reason: persistedLayout.failureCode === "empty-crawl" ? "empty-crawl" : "layout-failed"
|
|
57538
58155
|
},
|
|
58156
|
+
templateDetection,
|
|
58157
|
+
linkKind,
|
|
57539
58158
|
totalNodes: totalNodes2,
|
|
57540
58159
|
totalEdges: totalEdges2,
|
|
58160
|
+
totalTemplateEdges: persistedLayout.totalTemplateEdges,
|
|
58161
|
+
totalContentEdges: Math.max(0, totalEdges2 - persistedLayout.totalTemplateEdges),
|
|
57541
58162
|
nodes: [],
|
|
57542
58163
|
edges: [],
|
|
57543
58164
|
omittedNodes: totalNodes2,
|
|
@@ -57579,12 +58200,14 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57579
58200
|
...row,
|
|
57580
58201
|
healthState: deriveSiteHealthState({ ...row, indexabilityReasons, canonicalNodeKey })
|
|
57581
58202
|
}));
|
|
58203
|
+
const graphLinkKindFilter = linkKind === SiteHealthLinkKinds.all ? void 0 : eq56(siteCrawlGraphEdges.isTemplate, linkKind === SiteHealthLinkKinds.template);
|
|
57582
58204
|
const edges = app.db.select({
|
|
57583
58205
|
edgeKey: siteCrawlGraphEdges.edgeKey,
|
|
57584
58206
|
sourceNodeKey: siteCrawlGraphEdges.sourceNodeKey,
|
|
57585
58207
|
targetNodeKey: siteCrawlGraphEdges.targetNodeKey,
|
|
57586
58208
|
followable: siteCrawlGraphEdges.followable,
|
|
57587
|
-
occurrences: siteCrawlGraphEdges.occurrences
|
|
58209
|
+
occurrences: siteCrawlGraphEdges.occurrences,
|
|
58210
|
+
isTemplate: siteCrawlGraphEdges.isTemplate
|
|
57588
58211
|
}).from(siteCrawlGraphEdges).innerJoin(graphSourceNode, and44(
|
|
57589
58212
|
eq56(graphSourceNode.projectId, siteCrawlGraphEdges.projectId),
|
|
57590
58213
|
eq56(graphSourceNode.runId, siteCrawlGraphEdges.runId),
|
|
@@ -57601,19 +58224,34 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57601
58224
|
eq56(siteCrawlGraphEdges.projectId, project.id),
|
|
57602
58225
|
eq56(siteCrawlGraphEdges.runId, snapshot.runId),
|
|
57603
58226
|
eq56(siteCrawlGraphEdges.attemptId, snapshot.attemptId),
|
|
57604
|
-
lt8(siteCrawlGraphEdges.sampleRank, maxEdges)
|
|
58227
|
+
lt8(siteCrawlGraphEdges.sampleRank, maxEdges),
|
|
58228
|
+
graphLinkKindFilter
|
|
57605
58229
|
)).orderBy(asc10(siteCrawlGraphEdges.sampleRank)).all();
|
|
57606
58230
|
const totalNodes = persistedLayout.totalNodes;
|
|
57607
58231
|
const totalEdges = persistedLayout.totalEdges;
|
|
58232
|
+
const totalTemplateEdges = persistedLayout.totalTemplateEdges;
|
|
57608
58233
|
const omittedNodes = Math.max(0, totalNodes - nodes.length);
|
|
57609
58234
|
const omittedEdges = Math.max(0, totalEdges - edges.length);
|
|
57610
58235
|
return {
|
|
57611
58236
|
project: project.name,
|
|
57612
58237
|
hasCrawlData: true,
|
|
57613
58238
|
runId: snapshot.runId,
|
|
57614
|
-
|
|
58239
|
+
rootNodeKey,
|
|
58240
|
+
layout: {
|
|
58241
|
+
state: "ready",
|
|
58242
|
+
version: persistedLayout.layoutVersion,
|
|
58243
|
+
computedAt: persistedLayout.createdAt,
|
|
58244
|
+
templateLinksExcluded: persistedLayout.templateLinksExcluded
|
|
58245
|
+
},
|
|
58246
|
+
templateDetection,
|
|
58247
|
+
linkKind,
|
|
57615
58248
|
totalNodes,
|
|
58249
|
+
// Deliberately still every graph-compatible link, whatever `linkKind`
|
|
58250
|
+
// asked for. The two counts below split it; none of the three moves
|
|
58251
|
+
// because a caller narrowed the payload.
|
|
57616
58252
|
totalEdges,
|
|
58253
|
+
totalTemplateEdges,
|
|
58254
|
+
totalContentEdges: Math.max(0, totalEdges - totalTemplateEdges),
|
|
57617
58255
|
nodes,
|
|
57618
58256
|
edges,
|
|
57619
58257
|
omittedNodes,
|
|
@@ -57655,7 +58293,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57655
58293
|
truncated: false
|
|
57656
58294
|
});
|
|
57657
58295
|
if (!target) {
|
|
57658
|
-
|
|
58296
|
+
assertKnownAuditRun(project.id, request.query.runId);
|
|
57659
58297
|
return empty("no-crawl", null, false, false, null);
|
|
57660
58298
|
}
|
|
57661
58299
|
const snapshot = target.snapshot;
|
|
@@ -57781,7 +58419,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57781
58419
|
}
|
|
57782
58420
|
const target = resolveCrawl(project.id, request.query.runId);
|
|
57783
58421
|
if (!target) {
|
|
57784
|
-
|
|
58422
|
+
assertKnownAuditRun(project.id, request.query.runId);
|
|
57785
58423
|
return {
|
|
57786
58424
|
project: project.name,
|
|
57787
58425
|
runId: null,
|
|
@@ -58253,7 +58891,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58253
58891
|
}
|
|
58254
58892
|
const target = resolveCrawl(project.id, runId);
|
|
58255
58893
|
if (!target) {
|
|
58256
|
-
|
|
58894
|
+
assertKnownAuditRun(project.id, runId);
|
|
58257
58895
|
return { state: "no-crawl", project: project.name, runId: null };
|
|
58258
58896
|
}
|
|
58259
58897
|
const snapshot = target.snapshot;
|
|
@@ -58294,28 +58932,52 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58294
58932
|
const project = resolveProject(app.db, request.params.name);
|
|
58295
58933
|
const target = resolveCrawl(project.id, request.query.runId);
|
|
58296
58934
|
if (!target) {
|
|
58297
|
-
|
|
58298
|
-
return { project: project.name, hasCrawlData: false, runId: null, total: 0, nextCursor: null, pages: [] };
|
|
58935
|
+
assertKnownAuditRun(project.id, request.query.runId);
|
|
58936
|
+
return { project: project.name, hasCrawlData: false, runId: null, total: 0, nextCursor: null, healthStateFilter: null, pages: [] };
|
|
58299
58937
|
}
|
|
58300
58938
|
const snapshot = target.snapshot;
|
|
58301
58939
|
if (!snapshot.detailsAvailable || !snapshot.attemptId) {
|
|
58302
|
-
return { project: project.name, hasCrawlData: true, runId: snapshot.runId, total: 0, nextCursor: null, pages: [] };
|
|
58940
|
+
return { project: project.name, hasCrawlData: true, runId: snapshot.runId, total: 0, nextCursor: null, healthStateFilter: null, pages: [] };
|
|
58303
58941
|
}
|
|
58304
58942
|
const filters = [
|
|
58305
58943
|
eq56(siteCrawlPages.projectId, project.id),
|
|
58306
58944
|
eq56(siteCrawlPages.runId, snapshot.runId),
|
|
58307
58945
|
eq56(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
58308
58946
|
];
|
|
58947
|
+
if (request.query.nodeKey) filters.push(eq56(siteCrawlPages.nodeKey, request.query.nodeKey));
|
|
58309
58948
|
const inventoryEligible = parseBoolean2(request.query.inventoryEligible);
|
|
58310
58949
|
if (inventoryEligible != null) filters.push(eq56(siteCrawlPages.inventoryEligible, inventoryEligible));
|
|
58311
58950
|
if (request.query.fetchState) filters.push(eq56(siteCrawlPages.fetchState, request.query.fetchState));
|
|
58312
58951
|
if (request.query.indexabilityState) filters.push(eq56(siteCrawlPages.indexabilityState, request.query.indexabilityState));
|
|
58313
58952
|
if (request.query.auditState) filters.push(eq56(siteCrawlPages.auditState, request.query.auditState));
|
|
58314
|
-
const
|
|
58315
|
-
const total = app.db.select({ value: count() }).from(siteCrawlPages).where(where).get()?.value ?? 0;
|
|
58953
|
+
const healthState = parseSiteHealthState(request.query.healthState);
|
|
58316
58954
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
58317
58955
|
const offset = decodeCursor(request.query.cursor);
|
|
58318
58956
|
const orderBy = request.query.sort === "score-desc" ? [desc24(siteCrawlPages.auditScore), asc10(siteCrawlPages.nodeKey)] : request.query.sort === "score-asc" ? [asc10(siteCrawlPages.auditScore), asc10(siteCrawlPages.nodeKey)] : request.query.sort === "path" ? [asc10(siteCrawlPages.path), asc10(siteCrawlPages.nodeKey)] : [asc10(siteCrawlPages.url), asc10(siteCrawlPages.nodeKey)];
|
|
58957
|
+
let healthStateFilter = null;
|
|
58958
|
+
if (healthState) {
|
|
58959
|
+
const legacyRow = app.db.select({ id: siteCrawlPages.id }).from(siteCrawlPages).where(and44(
|
|
58960
|
+
eq56(siteCrawlPages.projectId, project.id),
|
|
58961
|
+
eq56(siteCrawlPages.runId, snapshot.runId),
|
|
58962
|
+
eq56(siteCrawlPages.attemptId, snapshot.attemptId),
|
|
58963
|
+
isNull5(siteCrawlPages.healthState)
|
|
58964
|
+
)).limit(1).get();
|
|
58965
|
+
healthStateFilter = legacyRow ? "unavailable-legacy-scan" : "applied";
|
|
58966
|
+
if (healthStateFilter === "applied") filters.push(eq56(siteCrawlPages.healthState, healthState));
|
|
58967
|
+
}
|
|
58968
|
+
if (healthStateFilter === "unavailable-legacy-scan") {
|
|
58969
|
+
return {
|
|
58970
|
+
project: project.name,
|
|
58971
|
+
hasCrawlData: true,
|
|
58972
|
+
runId: snapshot.runId,
|
|
58973
|
+
total: 0,
|
|
58974
|
+
nextCursor: null,
|
|
58975
|
+
healthStateFilter,
|
|
58976
|
+
pages: []
|
|
58977
|
+
};
|
|
58978
|
+
}
|
|
58979
|
+
const where = and44(...filters);
|
|
58980
|
+
const total = app.db.select({ value: count() }).from(siteCrawlPages).where(where).get()?.value ?? 0;
|
|
58319
58981
|
const rows = app.db.select().from(siteCrawlPages).where(where).orderBy(...orderBy).limit(limit).offset(offset).all();
|
|
58320
58982
|
const nextOffset = offset + rows.length;
|
|
58321
58983
|
return {
|
|
@@ -58324,6 +58986,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58324
58986
|
runId: snapshot.runId,
|
|
58325
58987
|
total,
|
|
58326
58988
|
nextCursor: nextOffset < total ? encodeCursor2(nextOffset) : null,
|
|
58989
|
+
healthStateFilter,
|
|
58327
58990
|
pages: rows.map(mapCrawlPage)
|
|
58328
58991
|
};
|
|
58329
58992
|
});
|
|
@@ -58332,7 +58995,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58332
58995
|
const parentPath = normalizeParentPath(request.query.parentPath);
|
|
58333
58996
|
const target = resolveCrawl(project.id, request.query.runId);
|
|
58334
58997
|
if (!target) {
|
|
58335
|
-
|
|
58998
|
+
assertKnownAuditRun(project.id, request.query.runId);
|
|
58336
58999
|
return { project: project.name, hasCrawlData: false, runId: null, parentPath, nextCursor: null, children: [] };
|
|
58337
59000
|
}
|
|
58338
59001
|
const snapshot = target.snapshot;
|
|
@@ -58396,20 +59059,41 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58396
59059
|
});
|
|
58397
59060
|
app.get("/projects/:name/technical-aeo/internal-links", async (request) => {
|
|
58398
59061
|
const project = resolveProject(app.db, request.params.name);
|
|
59062
|
+
const linkKind = parseLinkKind(request.query.linkKind);
|
|
58399
59063
|
const target = resolveCrawl(project.id, request.query.runId);
|
|
58400
59064
|
if (!target) {
|
|
58401
|
-
|
|
58402
|
-
return {
|
|
59065
|
+
assertKnownAuditRun(project.id, request.query.runId);
|
|
59066
|
+
return {
|
|
59067
|
+
project: project.name,
|
|
59068
|
+
hasCrawlData: false,
|
|
59069
|
+
runId: null,
|
|
59070
|
+
total: 0,
|
|
59071
|
+
nextCursor: null,
|
|
59072
|
+
templateDetection: SiteHealthTemplateDetections["unavailable-legacy-scan"],
|
|
59073
|
+
linkKind,
|
|
59074
|
+
edges: []
|
|
59075
|
+
};
|
|
58403
59076
|
}
|
|
58404
59077
|
const snapshot = target.snapshot;
|
|
59078
|
+
const templateDetection = templateDetectionOf(snapshot.templateDetection);
|
|
58405
59079
|
if (!snapshot.detailsAvailable || !snapshot.attemptId) {
|
|
58406
|
-
return {
|
|
59080
|
+
return {
|
|
59081
|
+
project: project.name,
|
|
59082
|
+
hasCrawlData: true,
|
|
59083
|
+
runId: snapshot.runId,
|
|
59084
|
+
total: 0,
|
|
59085
|
+
nextCursor: null,
|
|
59086
|
+
templateDetection,
|
|
59087
|
+
linkKind,
|
|
59088
|
+
edges: []
|
|
59089
|
+
};
|
|
58407
59090
|
}
|
|
58408
59091
|
const filters = [
|
|
58409
59092
|
eq56(siteCrawlEdges.projectId, project.id),
|
|
58410
59093
|
eq56(siteCrawlEdges.runId, snapshot.runId),
|
|
58411
59094
|
eq56(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
58412
|
-
eq56(siteCrawlEdges.internal, true)
|
|
59095
|
+
eq56(siteCrawlEdges.internal, true),
|
|
59096
|
+
linkKindFilter(linkKind)
|
|
58413
59097
|
];
|
|
58414
59098
|
if (request.query.sourceUrl) filters.push(eq56(siteCrawlEdges.sourceUrl, request.query.sourceUrl));
|
|
58415
59099
|
if (request.query.targetUrl) filters.push(eq56(siteCrawlEdges.targetUrl, request.query.targetUrl));
|
|
@@ -58427,23 +59111,28 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58427
59111
|
runId: snapshot.runId,
|
|
58428
59112
|
total,
|
|
58429
59113
|
nextCursor: nextOffset < total ? encodeCursor2(nextOffset) : null,
|
|
59114
|
+
templateDetection,
|
|
59115
|
+
linkKind,
|
|
58430
59116
|
edges: rows.map(mapCrawlEdge)
|
|
58431
59117
|
};
|
|
58432
59118
|
});
|
|
58433
59119
|
app.get("/projects/:name/technical-aeo/internal-links/neighbors", async (request) => {
|
|
58434
59120
|
const project = resolveProject(app.db, request.params.name);
|
|
59121
|
+
const linkKind = parseLinkKind(request.query.linkKind);
|
|
58435
59122
|
if (!request.query.nodeKey && !request.query.url) {
|
|
58436
59123
|
throw validationError("Provide nodeKey or url");
|
|
58437
59124
|
}
|
|
58438
59125
|
const target = resolveCrawl(project.id, request.query.runId);
|
|
58439
59126
|
if (!target) {
|
|
58440
|
-
|
|
59127
|
+
assertKnownAuditRun(project.id, request.query.runId);
|
|
58441
59128
|
return {
|
|
58442
59129
|
project: project.name,
|
|
58443
59130
|
hasCrawlData: false,
|
|
58444
59131
|
runId: null,
|
|
58445
59132
|
nodeKey: request.query.nodeKey ?? null,
|
|
58446
59133
|
url: request.query.url ?? null,
|
|
59134
|
+
templateDetection: SiteHealthTemplateDetections["unavailable-legacy-scan"],
|
|
59135
|
+
linkKind,
|
|
58447
59136
|
inbound: [],
|
|
58448
59137
|
outbound: [],
|
|
58449
59138
|
inboundTruncated: false,
|
|
@@ -58451,6 +59140,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58451
59140
|
};
|
|
58452
59141
|
}
|
|
58453
59142
|
const snapshot = target.snapshot;
|
|
59143
|
+
const templateDetection = templateDetectionOf(snapshot.templateDetection);
|
|
58454
59144
|
if (!snapshot.detailsAvailable || !snapshot.attemptId) {
|
|
58455
59145
|
return {
|
|
58456
59146
|
project: project.name,
|
|
@@ -58458,6 +59148,8 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58458
59148
|
runId: snapshot.runId,
|
|
58459
59149
|
nodeKey: request.query.nodeKey ?? null,
|
|
58460
59150
|
url: request.query.url ?? null,
|
|
59151
|
+
templateDetection,
|
|
59152
|
+
linkKind,
|
|
58461
59153
|
inbound: [],
|
|
58462
59154
|
outbound: [],
|
|
58463
59155
|
inboundTruncated: false,
|
|
@@ -58468,7 +59160,8 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58468
59160
|
eq56(siteCrawlEdges.projectId, project.id),
|
|
58469
59161
|
eq56(siteCrawlEdges.runId, snapshot.runId),
|
|
58470
59162
|
eq56(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
58471
|
-
eq56(siteCrawlEdges.internal, true)
|
|
59163
|
+
eq56(siteCrawlEdges.internal, true),
|
|
59164
|
+
linkKindFilter(linkKind)
|
|
58472
59165
|
];
|
|
58473
59166
|
const inboundMatch = request.query.nodeKey && request.query.url ? or11(eq56(siteCrawlEdges.targetNodeKey, request.query.nodeKey), eq56(siteCrawlEdges.targetUrl, request.query.url)) : request.query.nodeKey ? eq56(siteCrawlEdges.targetNodeKey, request.query.nodeKey) : eq56(siteCrawlEdges.targetUrl, request.query.url);
|
|
58474
59167
|
const outboundMatch = request.query.nodeKey && request.query.url ? or11(eq56(siteCrawlEdges.sourceNodeKey, request.query.nodeKey), eq56(siteCrawlEdges.sourceUrl, request.query.url)) : request.query.nodeKey ? eq56(siteCrawlEdges.sourceNodeKey, request.query.nodeKey) : eq56(siteCrawlEdges.sourceUrl, request.query.url);
|
|
@@ -58481,6 +59174,8 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58481
59174
|
runId: snapshot.runId,
|
|
58482
59175
|
nodeKey: request.query.nodeKey ?? null,
|
|
58483
59176
|
url: request.query.url ?? null,
|
|
59177
|
+
templateDetection,
|
|
59178
|
+
linkKind,
|
|
58484
59179
|
inbound: inboundRows.slice(0, limit).map(mapCrawlEdge),
|
|
58485
59180
|
outbound: outboundRows.slice(0, limit).map(mapCrawlEdge),
|
|
58486
59181
|
inboundTruncated: inboundRows.length > limit,
|
|
@@ -58492,7 +59187,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58492
59187
|
const target = resolveCrawl(project.id, request.query.runId);
|
|
58493
59188
|
const legacyAuditAvailable = hasLegacyAudit(project.id);
|
|
58494
59189
|
if (!target) {
|
|
58495
|
-
|
|
59190
|
+
assertKnownAuditRun(project.id, request.query.runId);
|
|
58496
59191
|
return { project: project.name, runId: null, state: "unavailable", legacyAuditAvailable };
|
|
58497
59192
|
}
|
|
58498
59193
|
const snapshot = target.snapshot;
|
|
@@ -58538,12 +59233,153 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58538
59233
|
deadLinks
|
|
58539
59234
|
};
|
|
58540
59235
|
});
|
|
59236
|
+
app.get("/projects/:name/technical-aeo/runs", async (request) => {
|
|
59237
|
+
const project = resolveProject(app.db, request.params.name);
|
|
59238
|
+
const limit = parseBoundedLimit(request.query.limit, SITE_HEALTH_SCANS_DEFAULT_LIMIT, SITE_HEALTH_SCANS_MAX_LIMIT);
|
|
59239
|
+
const rows = app.db.select({
|
|
59240
|
+
runId: runs.id,
|
|
59241
|
+
status: runs.status,
|
|
59242
|
+
createdAt: runs.createdAt,
|
|
59243
|
+
startedAt: runs.startedAt,
|
|
59244
|
+
finishedAt: runs.finishedAt
|
|
59245
|
+
}).from(runs).where(and44(
|
|
59246
|
+
eq56(runs.projectId, project.id),
|
|
59247
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
59248
|
+
notProbeRun()
|
|
59249
|
+
)).orderBy(desc24(runs.createdAt), desc24(runs.id)).limit(limit).all();
|
|
59250
|
+
const runIds = rows.map((row) => row.runId);
|
|
59251
|
+
const crawlRunIds = new Set(runIds.length === 0 ? [] : app.db.select({ runId: siteCrawlSnapshots.runId }).from(siteCrawlSnapshots).innerJoin(runs, eq56(siteCrawlSnapshots.runId, runs.id)).where(and44(
|
|
59252
|
+
eq56(siteCrawlSnapshots.projectId, project.id),
|
|
59253
|
+
eq56(runs.projectId, project.id),
|
|
59254
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
59255
|
+
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
59256
|
+
notProbeRun(),
|
|
59257
|
+
inArray19(siteCrawlSnapshots.runId, runIds)
|
|
59258
|
+
)).all().map((row) => row.runId));
|
|
59259
|
+
return {
|
|
59260
|
+
project: project.name,
|
|
59261
|
+
scans: rows.map((row) => ({
|
|
59262
|
+
...row,
|
|
59263
|
+
status: row.status,
|
|
59264
|
+
hasCrawlData: crawlRunIds.has(row.runId)
|
|
59265
|
+
}))
|
|
59266
|
+
};
|
|
59267
|
+
});
|
|
59268
|
+
app.get("/projects/:name/technical-aeo/runs/:runId/progress", async (request) => {
|
|
59269
|
+
const project = resolveProject(app.db, request.params.name);
|
|
59270
|
+
const run = app.db.select().from(runs).where(and44(
|
|
59271
|
+
eq56(runs.id, request.params.runId),
|
|
59272
|
+
eq56(runs.projectId, project.id),
|
|
59273
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
59274
|
+
notProbeRun()
|
|
59275
|
+
)).get();
|
|
59276
|
+
if (!run) throw notFound("Site audit run", request.params.runId);
|
|
59277
|
+
const attempt = app.db.select().from(siteCrawlAttempts).where(and44(
|
|
59278
|
+
eq56(siteCrawlAttempts.projectId, project.id),
|
|
59279
|
+
eq56(siteCrawlAttempts.runId, run.id)
|
|
59280
|
+
)).orderBy(desc24(siteCrawlAttempts.attemptNumber), desc24(siteCrawlAttempts.updatedAt)).limit(1).get();
|
|
59281
|
+
const persistedLayout = attempt ? app.db.select().from(siteCrawlGraphLayouts).where(and44(
|
|
59282
|
+
eq56(siteCrawlGraphLayouts.projectId, project.id),
|
|
59283
|
+
eq56(siteCrawlGraphLayouts.runId, run.id),
|
|
59284
|
+
eq56(siteCrawlGraphLayouts.attemptId, attempt.id)
|
|
59285
|
+
)).get() : void 0;
|
|
59286
|
+
const layoutState = persistedLayout?.state === "ready" || persistedLayout?.state === "unavailable" ? persistedLayout.state : run.status === RunStatuses.completed || run.status === RunStatuses.partial ? "unavailable" : "pending";
|
|
59287
|
+
return {
|
|
59288
|
+
project: project.name,
|
|
59289
|
+
runId: run.id,
|
|
59290
|
+
status: run.status,
|
|
59291
|
+
phase: siteAuditRunPhase(run.status, attempt, layoutState),
|
|
59292
|
+
attempt: attempt ? {
|
|
59293
|
+
id: attempt.id,
|
|
59294
|
+
state: attempt.state,
|
|
59295
|
+
pagesDiscovered: attempt.pagesDiscovered,
|
|
59296
|
+
pagesFetched: attempt.pagesFetched,
|
|
59297
|
+
pagesEligible: attempt.pagesEligible,
|
|
59298
|
+
pagesErrored: attempt.pagesErrored,
|
|
59299
|
+
edgesDiscovered: attempt.edgesDiscovered,
|
|
59300
|
+
lastUpdatedAt: attempt.updatedAt,
|
|
59301
|
+
startedAt: attempt.startedAt,
|
|
59302
|
+
finishedAt: attempt.finishedAt,
|
|
59303
|
+
error: attempt.error
|
|
59304
|
+
} : null,
|
|
59305
|
+
layout: {
|
|
59306
|
+
state: layoutState,
|
|
59307
|
+
layoutVersion: persistedLayout?.layoutVersion ?? null,
|
|
59308
|
+
failureCode: persistedLayout?.failureCode ?? null,
|
|
59309
|
+
updatedAt: persistedLayout?.updatedAt ?? null
|
|
59310
|
+
},
|
|
59311
|
+
error: run.error ?? attempt?.error ?? null
|
|
59312
|
+
};
|
|
59313
|
+
});
|
|
59314
|
+
app.get("/projects/:name/technical-aeo/runs/:runId/page-health-preview", async (request) => app.db.transaction((tx) => {
|
|
59315
|
+
const project = tx.select().from(projects).where(eq56(projects.name, request.params.name)).get();
|
|
59316
|
+
if (!project) throw notFound("Project", request.params.name);
|
|
59317
|
+
const run = tx.select().from(runs).where(and44(
|
|
59318
|
+
eq56(runs.id, request.params.runId),
|
|
59319
|
+
eq56(runs.projectId, project.id),
|
|
59320
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
59321
|
+
notProbeRun()
|
|
59322
|
+
)).get();
|
|
59323
|
+
if (!run) throw notFound("Site audit run", request.params.runId);
|
|
59324
|
+
const attempt = tx.select().from(siteCrawlAttempts).where(and44(
|
|
59325
|
+
eq56(siteCrawlAttempts.projectId, project.id),
|
|
59326
|
+
eq56(siteCrawlAttempts.runId, run.id)
|
|
59327
|
+
)).orderBy(desc24(siteCrawlAttempts.attemptNumber), desc24(siteCrawlAttempts.updatedAt)).limit(1).get();
|
|
59328
|
+
const state = run.status === RunStatuses.queued ? "waiting" : run.status === RunStatuses.running ? "collecting" : "terminal";
|
|
59329
|
+
const base = {
|
|
59330
|
+
project: project.name,
|
|
59331
|
+
runId: run.id,
|
|
59332
|
+
status: run.status,
|
|
59333
|
+
state,
|
|
59334
|
+
attemptId: attempt?.id ?? null,
|
|
59335
|
+
updatedAt: attempt?.updatedAt ?? null
|
|
59336
|
+
};
|
|
59337
|
+
if (!attempt) {
|
|
59338
|
+
return { ...base, pagesAudited: 0, examples: [] };
|
|
59339
|
+
}
|
|
59340
|
+
const auditedWhere = and44(
|
|
59341
|
+
eq56(siteCrawlPages.projectId, project.id),
|
|
59342
|
+
eq56(siteCrawlPages.runId, run.id),
|
|
59343
|
+
eq56(siteCrawlPages.attemptId, attempt.id),
|
|
59344
|
+
eq56(siteCrawlPages.auditState, "success")
|
|
59345
|
+
);
|
|
59346
|
+
const pagesAudited = tx.select({ value: count() }).from(siteCrawlPages).where(auditedWhere).get()?.value ?? 0;
|
|
59347
|
+
if (state !== "collecting") {
|
|
59348
|
+
return { ...base, pagesAudited, examples: [] };
|
|
59349
|
+
}
|
|
59350
|
+
const candidates = tx.select({
|
|
59351
|
+
nodeKey: siteCrawlPages.nodeKey,
|
|
59352
|
+
url: siteCrawlPages.url,
|
|
59353
|
+
auditScore: siteCrawlPages.auditScore,
|
|
59354
|
+
auditFields: siteCrawlPages.auditFields
|
|
59355
|
+
}).from(siteCrawlPages).where(and44(
|
|
59356
|
+
auditedWhere,
|
|
59357
|
+
isNotNull3(siteCrawlPages.auditScore),
|
|
59358
|
+
lt8(siteCrawlPages.auditScore, 70)
|
|
59359
|
+
)).orderBy(asc10(siteCrawlPages.auditScore), asc10(siteCrawlPages.nodeKey)).limit(LIVE_PAGE_HEALTH_CANDIDATE_LIMIT).all();
|
|
59360
|
+
const examples = candidates.flatMap((candidate) => {
|
|
59361
|
+
if (candidate.auditScore == null) return [];
|
|
59362
|
+
const checksNeedingAttention = countLivePageHealthChecks(candidate.auditFields);
|
|
59363
|
+
return checksNeedingAttention > 0 ? [{
|
|
59364
|
+
nodeKey: candidate.nodeKey,
|
|
59365
|
+
url: candidate.url,
|
|
59366
|
+
auditScore: candidate.auditScore,
|
|
59367
|
+
checksNeedingAttention
|
|
59368
|
+
}] : [];
|
|
59369
|
+
}).slice(0, LIVE_PAGE_HEALTH_EXAMPLE_LIMIT);
|
|
59370
|
+
return { ...base, pagesAudited, examples };
|
|
59371
|
+
}));
|
|
58541
59372
|
app.post("/projects/:name/technical-aeo/runs", async (request) => {
|
|
58542
59373
|
const project = resolveProject(app.db, request.params.name);
|
|
58543
59374
|
const parsed = siteAuditRunRequestSchema.safeParse(request.body ?? {});
|
|
58544
59375
|
if (!parsed.success) {
|
|
58545
59376
|
throw validationError(parsed.error.issues[0]?.message ?? "Invalid site-audit request");
|
|
58546
59377
|
}
|
|
59378
|
+
if (!opts.onSiteAuditRequested) {
|
|
59379
|
+
throw missingDependency("Site Health execution is not available on this deployment.", {
|
|
59380
|
+
reason: "no-site-audit-handler"
|
|
59381
|
+
});
|
|
59382
|
+
}
|
|
58547
59383
|
const effectiveRequest = normalizeSiteAuditRunRequest(parsed.data);
|
|
58548
59384
|
const identityKey = siteAuditRequestIdentity(effectiveRequest);
|
|
58549
59385
|
const result = app.db.transaction((tx) => {
|
|
@@ -58594,7 +59430,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58594
59430
|
return { runId, status: RunStatuses.queued, created: true };
|
|
58595
59431
|
});
|
|
58596
59432
|
if (result.created) {
|
|
58597
|
-
opts.onSiteAuditRequested
|
|
59433
|
+
opts.onSiteAuditRequested(result.runId, project.id, {
|
|
58598
59434
|
sitemapUrl: effectiveRequest.sitemapUrl ?? void 0,
|
|
58599
59435
|
limit: parsed.data.limit,
|
|
58600
59436
|
maxPages: effectiveRequest.maxPages,
|
|
@@ -59999,6 +60835,7 @@ export {
|
|
|
59999
60835
|
latestQueryBasketRevision,
|
|
60000
60836
|
ensureCurrentQueryBasketRevision,
|
|
60001
60837
|
queueRunIfProjectIdle,
|
|
60838
|
+
resolveMeasurementSitemapTarget,
|
|
60002
60839
|
measurementRunCompleteness,
|
|
60003
60840
|
nextRunFromCron,
|
|
60004
60841
|
resolveWebhookTarget,
|