@canonry/canonry 4.157.0 → 4.159.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/README.md +2 -2
  2. package/THIRD_PARTY_NOTICES.md +102 -0
  3. package/assets/THIRD_PARTY_NOTICES.md +102 -0
  4. package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +1 -0
  5. package/assets/assets/{AuditHistoryPanel-D12lHs6n.js → AuditHistoryPanel-BiloGcQA.js} +1 -1
  6. package/assets/assets/{BacklinksPage-D2ScFYru.js → BacklinksPage-CPM1O2bC.js} +1 -1
  7. package/assets/assets/{HistoryPage-jZ_I4rwX.js → HistoryPage-1YOjtdJy.js} +1 -1
  8. package/assets/assets/{MeasurementPropertyPage-BIdQjXJT.js → MeasurementPropertyPage-CVDTY_gC.js} +1 -1
  9. package/assets/assets/ProjectPage-DnGXMx1n.js +12 -0
  10. package/assets/assets/{RunRow-Q2X0NR74.js → RunRow-fIaadH4h.js} +1 -1
  11. package/assets/assets/{RunsPage-Cw7YVsbJ.js → RunsPage-DfifdM5z.js} +1 -1
  12. package/assets/assets/{SettingsPage-bFl2EXOD.js → SettingsPage-efA2EEmS.js} +1 -1
  13. package/assets/assets/SiteHealthSection-BROtQGfu.js +4 -0
  14. package/assets/assets/{TrafficPage-DJjhk7Jw.js → TrafficPage-Wk43Bm_q.js} +1 -1
  15. package/assets/assets/{TrafficSourceDetailPage-CGKwIu9A.js → TrafficSourceDetailPage-BhBvoLWu.js} +1 -1
  16. package/assets/assets/extract-error-message-YGzZFvWs.js +1 -0
  17. package/assets/assets/geist-cyrillic-ext-wght-normal-DjL33-gN.woff2 +0 -0
  18. package/assets/assets/geist-cyrillic-wght-normal-BEAKL7Jp.woff2 +0 -0
  19. package/assets/assets/geist-latin-ext-wght-normal-DC-KSUi6.woff2 +0 -0
  20. package/assets/assets/geist-latin-wght-normal-BgDaEnEv.woff2 +0 -0
  21. package/assets/assets/geist-mono-cyrillic-ext-wght-normal-X_5orZeX.woff2 +0 -0
  22. package/assets/assets/geist-mono-cyrillic-wght-normal-DiZS0aHC.woff2 +0 -0
  23. package/assets/assets/geist-mono-latin-ext-wght-normal-Bwz-egvJ.woff2 +0 -0
  24. package/assets/assets/geist-mono-latin-wght-normal-XN7g48iV.woff2 +0 -0
  25. package/assets/assets/geist-mono-symbols2-wght-normal-CO5SzqOn.woff2 +0 -0
  26. package/assets/assets/geist-mono-vietnamese-wght-normal-DadHysG0.woff2 +0 -0
  27. package/assets/assets/geist-vietnamese-wght-normal-6IgcOCM7.woff2 +0 -0
  28. package/assets/assets/index-CACRaOmZ.css +1 -0
  29. package/assets/assets/index-DmsIZQAm.js +83 -0
  30. package/assets/assets/{react-sigma_core.esm.min-EzswMBkr.js → react-sigma_core.esm.min-DM9XptbM.js} +2 -2
  31. package/assets/assets/{vendor-lucide-BtYdosYk.js → vendor-lucide-nYKv-BB7.js} +1 -1
  32. package/assets/assets/{vendor-markdown-D274_wsI.js → vendor-markdown-Bn7fRwrx.js} +1 -1
  33. package/assets/assets/{vendor-radix-BxMXSwbC.js → vendor-radix-4UOaVWza.js} +1 -1
  34. package/assets/assets/{vendor-recharts-C4ktw_aE.js → vendor-recharts-D8rt5lKn.js} +1 -1
  35. package/assets/assets/{vendor-tanstack-pFxw0eGP.js → vendor-tanstack-Dock0eH1.js} +1 -1
  36. package/assets/index.html +7 -7
  37. package/dist/{chunk-MEBL4EJO.js → chunk-5AFWHKKI.js} +875 -38
  38. package/dist/{chunk-AEIKAWOW.js → chunk-BZPBVOAM.js} +753 -212
  39. package/dist/{chunk-KMTRO56O.js → chunk-ZEG43TY2.js} +45 -15
  40. package/dist/{chunk-TRRVBTU5.js → chunk-ZL3VY435.js} +264 -10
  41. package/dist/cli.js +42 -4
  42. package/dist/index.d.ts +9 -0
  43. package/dist/index.js +4 -4
  44. package/dist/{intelligence-service-YUUPSJRC.js → intelligence-service-DHY5CYWX.js} +2 -2
  45. package/dist/mcp.js +2 -2
  46. package/package.json +13 -11
  47. package/assets/assets/ProjectPage-CLvLOOtj.js +0 -15
  48. package/assets/assets/extract-error-message-BHm98tXH.js +0 -1
  49. package/assets/assets/index-BxegEwic.css +0 -1
  50. package/assets/assets/index-VO6nKq5b.js +0 -83
@@ -67,6 +67,7 @@ import {
67
67
  MeasurementEvidenceShapes,
68
68
  MeasurementPlanValidationError,
69
69
  MeasurementRunScopeError,
70
+ PROJECTS_WRITE_SCOPE,
70
71
  READ_ONLY_SCOPE,
71
72
  ResearchQueryStatuses,
72
73
  ResearchRunStatuses,
@@ -82,6 +83,8 @@ import {
82
83
  SITE_HEALTH_PATH_DEFAULT_MAX_DEPTH,
83
84
  SITE_HEALTH_PATH_MAX_DEPTH,
84
85
  SITE_HEALTH_PATH_MAX_VISITED_NODES,
86
+ SITE_HEALTH_SCANS_DEFAULT_LIMIT,
87
+ SITE_HEALTH_SCANS_MAX_LIMIT,
85
88
  SITE_HEALTH_SUBGRAPH_DEFAULT_MAX_EDGES,
86
89
  SITE_HEALTH_SUBGRAPH_DEFAULT_MAX_NODES,
87
90
  SITE_HEALTH_SUBGRAPH_MAX_EDGES,
@@ -90,6 +93,8 @@ import {
90
93
  SchedulableRunKinds,
91
94
  SiteAuditTrendDirections,
92
95
  SiteCrawlFetchedStates,
96
+ SiteHealthLinkKinds,
97
+ SiteHealthTemplateDetections,
93
98
  TrafficEventConfidences,
94
99
  TrafficEventKinds,
95
100
  TrafficEvidenceKinds,
@@ -219,6 +224,7 @@ import {
219
224
  contentTargetsResponseDtoSchema,
220
225
  cosineSimilarity,
221
226
  createApiKeyRequestSchema,
227
+ createTemplateLinkPairIndex,
222
228
  createUserRequestSchema,
223
229
  createdApiKeyDtoSchema,
224
230
  dedupeReportActions,
@@ -311,6 +317,7 @@ import {
311
317
  isBrowserProvider,
312
318
  isReadOnlyKey,
313
319
  isRetryableHttpError,
320
+ isTemplateLinkRatio,
314
321
  isTrendBaseline,
315
322
  isVertexGroundingRedirect,
316
323
  isoDateDaysBeforeInTimeZone,
@@ -422,6 +429,7 @@ import {
422
429
  normalizeModelId,
423
430
  normalizeProjectAliases,
424
431
  normalizeProjectDomain,
432
+ normalizeProjectName,
425
433
  normalizeQueryText,
426
434
  normalizeSiteAuditRunRequest,
427
435
  normalizeUrlPath,
@@ -429,6 +437,7 @@ import {
429
437
  notFound,
430
438
  notImplemented,
431
439
  notificationDtoSchema,
440
+ observeTemplateLinkEdges,
432
441
  onboardingTelemetryEventSchema,
433
442
  operationInProgress,
434
443
  orderLocationsDefaultFirst,
@@ -444,6 +453,7 @@ import {
444
453
  parseWindow,
445
454
  pickClusterRepresentative,
446
455
  projectConfigSchema,
456
+ projectCreateRequestSchema,
447
457
  projectDtoSchema,
448
458
  projectOverviewDtoSchema,
449
459
  projectReportDtoSchema,
@@ -489,9 +499,11 @@ import {
489
499
  segmentCrawlerHits,
490
500
  serializeRunError,
491
501
  settingsDtoSchema,
502
+ siteAuditLivePageHealthSchema,
492
503
  siteAuditPageFactorSchema,
493
504
  siteAuditPagesResponseSchema,
494
505
  siteAuditRequestIdentity,
506
+ siteAuditRunProgressSchema,
495
507
  siteAuditRunRequestSchema,
496
508
  siteAuditRunResponseSchema,
497
509
  siteAuditScoreSchema,
@@ -507,8 +519,12 @@ import {
507
519
  siteCrawlStructureResponseSchema,
508
520
  siteCrawlSummarySchema,
509
521
  siteHealthChangesResponseSchema,
522
+ siteHealthLinkKindSchema,
510
523
  siteHealthPathResponseSchema,
524
+ siteHealthScansResponseSchema,
525
+ siteHealthStateSchema,
511
526
  siteHealthSubgraphResponseSchema,
527
+ siteHealthTemplateDetectionSchema,
512
528
  snapshotDiffResponseSchema,
513
529
  snapshotListResponseSchema,
514
530
  snapshotReportSchema,
@@ -520,6 +536,8 @@ import {
520
536
  surfaceClassFromCompetitorType,
521
537
  surfaceClassLabel,
522
538
  telemetryEventAcceptedDtoSchema,
539
+ templateLinkDetection,
540
+ templateLinkRatio,
523
541
  textContainsAnyBrandAlias,
524
542
  textContainsBrandAlias,
525
543
  textContainsDomain,
@@ -559,7 +577,7 @@ import {
559
577
  wordpressSchemaDeployResultDtoSchema,
560
578
  wordpressSchemaStatusResultDtoSchema,
561
579
  wordpressStatusDtoSchema
562
- } from "./chunk-TRRVBTU5.js";
580
+ } from "./chunk-ZL3VY435.js";
563
581
 
564
582
  // src/intelligence-service.ts
565
583
  import { eq as eq58, desc as desc26, asc as asc11, and as and46, ne as ne7, or as or12, inArray as inArray20, gte as gte14, lte as lte11 } from "drizzle-orm";
@@ -1309,6 +1327,13 @@ var siteCrawlSnapshots = sqliteTable("site_crawl_snapshots", {
1309
1327
  deadLinkState: text("dead_link_state").notNull().default("disabled"),
1310
1328
  deadLinksChecked: integer("dead_links_checked").notNull().default(0),
1311
1329
  deadLinksFound: integer("dead_links_found").notNull().default(0),
1330
+ /**
1331
+ * Whether nav/header/footer link detection ran for this scan, and if not,
1332
+ * why. NULL means the scan predates detection, which reads report as
1333
+ * `unavailable-legacy-scan` rather than letting an empty template-link list
1334
+ * pass for "this site has no nav".
1335
+ */
1336
+ templateDetection: text("template_detection"),
1312
1337
  createdAt: text("created_at").notNull(),
1313
1338
  updatedAt: text("updated_at").notNull()
1314
1339
  }, (table) => [
@@ -1348,6 +1373,13 @@ var siteCrawlPages = sqliteTable("site_crawl_pages", {
1348
1373
  canonicalNodeKey: text("canonical_node_key"),
1349
1374
  indexabilityState: text("indexability_state").notNull().default("unknown"),
1350
1375
  indexabilityReasons: text("indexability_reasons", { mode: "json" }).$type().notNull().default([]),
1376
+ /**
1377
+ * Derived Site Health state, computed once at publish time by the contract's
1378
+ * `deriveSiteHealthState`. Stored so reads can filter on an index instead of
1379
+ * materializing every page row; NULL means a snapshot published before this
1380
+ * column existed, which reads must report rather than treat as a state.
1381
+ */
1382
+ healthState: text("health_state"),
1351
1383
  auditState: text("audit_state").notNull().default("pending"),
1352
1384
  auditScore: real("audit_score"),
1353
1385
  auditFields: text("audit_fields", { mode: "json" }).$type().notNull().default({}),
@@ -1366,6 +1398,11 @@ var siteCrawlPages = sqliteTable("site_crawl_pages", {
1366
1398
  index("idx_site_crawl_pages_read").on(table.projectId, table.runId, table.attemptId, table.inventoryEligible, table.auditScore, table.url),
1367
1399
  index("idx_site_crawl_pages_parent").on(table.projectId, table.runId, table.attemptId, table.parentPath, table.path),
1368
1400
  index("idx_site_crawl_pages_url").on(table.projectId, table.runId, table.attemptId, table.url),
1401
+ index("idx_site_crawl_pages_health").on(table.projectId, table.runId, table.attemptId, table.healthState, table.path, table.nodeKey),
1402
+ // The in-progress Page Health preview reads only completed audits below its
1403
+ // threshold, ordered worst-first. Keep its stable tie-breaker in the index
1404
+ // so a growing crawl never needs a transient sort before its small limit.
1405
+ index("idx_site_crawl_pages_live_preview").on(table.projectId, table.runId, table.attemptId, table.auditState, table.auditScore, table.nodeKey),
1369
1406
  foreignKey({
1370
1407
  name: "site_crawl_pages_attempt_fk",
1371
1408
  columns: [table.projectId, table.runId, table.attemptId],
@@ -1389,6 +1426,29 @@ var siteCrawlEdges = sqliteTable("site_crawl_edges", {
1389
1426
  followableOccurrences: integer("followable_occurrences").notNull().default(1),
1390
1427
  nofollowOccurrences: integer("nofollow_occurrences").notNull().default(0),
1391
1428
  anchors: text("anchors", { mode: "json" }).$type().notNull().default([]),
1429
+ /**
1430
+ * True when EVERY (target page, anchor) pair on this link appears on at
1431
+ * least `TEMPLATE_LINK_RATIO_THRESHOLD` of the attempt's fetched pages: a
1432
+ * nav, header, or footer link rather than an editorial one. Computed once at
1433
+ * publish time by the contract's `classifyTemplateLinks`, so the map, the
1434
+ * API filters, and the agents all read the same decision.
1435
+ *
1436
+ * One row aggregates every anchor between the same two pages, so a link that
1437
+ * is BOTH a footer link and an in-prose link is one row with two anchors. It
1438
+ * is editorial, because the in-prose link exists.
1439
+ *
1440
+ * NULL means this row was never classified, which reads report as
1441
+ * `unavailable-legacy-scan`. It never means "not a template link": the
1442
+ * too-few-pages guard writes an explicit `false` and records its reason on
1443
+ * the snapshot instead.
1444
+ */
1445
+ isTemplate: integer("is_template", { mode: "boolean" }),
1446
+ /**
1447
+ * Share of fetched pages carrying this link's LEAST ubiquitous anchor, which
1448
+ * is the one that decides `is_template`. NULL when the link has no
1449
+ * measurable pair (an unresolved target or no anchor).
1450
+ */
1451
+ templateRatio: real("template_ratio"),
1392
1452
  createdAt: text("created_at").notNull(),
1393
1453
  updatedAt: text("updated_at").notNull()
1394
1454
  }, (table) => [
@@ -1397,6 +1457,17 @@ var siteCrawlEdges = sqliteTable("site_crawl_edges", {
1397
1457
  index("idx_site_crawl_edges_inbound").on(table.projectId, table.runId, table.attemptId, table.targetNodeKey, table.edgeKey),
1398
1458
  index("idx_site_crawl_edges_source_url").on(table.projectId, table.runId, table.attemptId, table.sourceUrl),
1399
1459
  index("idx_site_crawl_edges_target_url").on(table.projectId, table.runId, table.attemptId, table.targetUrl),
1460
+ // Backs the `linkKind` filter on the internal-link reads, whose ORDER BY is
1461
+ // the trailing edge key, so a filtered page terminates at LIMIT instead of
1462
+ // sorting every match in a temp b-tree.
1463
+ index("idx_site_crawl_edges_template").on(
1464
+ table.projectId,
1465
+ table.runId,
1466
+ table.attemptId,
1467
+ table.internal,
1468
+ table.isTemplate,
1469
+ table.edgeKey
1470
+ ),
1400
1471
  // Publish-time graph sampling filters this exact scope and consumes the
1401
1472
  // highest-occurrence internal anchors first. The mixed-direction suffix
1402
1473
  // avoids a million-row temporary ORDER BY at the maximum crawl budget.
@@ -1425,8 +1496,17 @@ var siteCrawlGraphLayouts = sqliteTable("site_crawl_graph_layouts", {
1425
1496
  failureCode: text("failure_code"),
1426
1497
  totalNodes: integer("total_nodes").notNull().default(0),
1427
1498
  totalEdges: integer("total_edges").notNull().default(0),
1499
+ /** Share of `totalEdges` classified as nav, header, or footer links. */
1500
+ totalTemplateEdges: integer("total_template_edges").notNull().default(0),
1428
1501
  nodeCount: integer("node_count").notNull().default(0),
1429
1502
  edgeCount: integer("edge_count").notNull().default(0),
1503
+ /**
1504
+ * True when template links were kept out of the ForceAtlas2 physics, so
1505
+ * these positions describe content structure. Layouts published before
1506
+ * template detection keep `false`: the migration classifies their links, but
1507
+ * it deliberately does not rewrite immutable coordinates.
1508
+ */
1509
+ templateLinksExcluded: integer("template_links_excluded", { mode: "boolean" }).notNull().default(false),
1430
1510
  createdAt: text("created_at").notNull(),
1431
1511
  updatedAt: text("updated_at").notNull()
1432
1512
  }, (table) => [
@@ -1476,6 +1556,13 @@ var siteCrawlGraphEdges = sqliteTable("site_crawl_graph_edges", {
1476
1556
  targetNodeKey: text("target_node_key").notNull(),
1477
1557
  followable: integer("followable", { mode: "boolean" }).notNull(),
1478
1558
  occurrences: integer("occurrences").notNull(),
1559
+ /**
1560
+ * Denormalized from `site_crawl_edges` the same way `followable` and
1561
+ * `occurrences` are, so a bounded map read never joins the full link table.
1562
+ * Template links stay IN the sample: the map hides them by default, and a
1563
+ * viewer switching them on must not trigger a refetch or a re-layout.
1564
+ */
1565
+ isTemplate: integer("is_template", { mode: "boolean" }).notNull().default(false),
1479
1566
  createdAt: text("created_at").notNull()
1480
1567
  }, (table) => [
1481
1568
  uniqueIndex("idx_site_crawl_graph_edges_attempt_edge").on(table.projectId, table.runId, table.attemptId, table.edgeKey),
@@ -2863,6 +2950,138 @@ function relinkOrphanedSnapshotQueryIds(tx) {
2863
2950
  }
2864
2951
  }
2865
2952
  }
2953
+ function dropSiteCrawlSelfLinks(tx) {
2954
+ tx.run(sql2.raw("DELETE FROM site_crawl_edges WHERE source_url = target_url"));
2955
+ tx.run(sql2.raw("DELETE FROM site_crawl_graph_edges WHERE source_node_key = target_node_key"));
2956
+ }
2957
+ function backfillSiteCrawlPageHealthState(tx) {
2958
+ const BATCH = 500;
2959
+ for (; ; ) {
2960
+ const rows = tx.all(sql2.raw(`
2961
+ SELECT id, node_key, fetch_state, indexability_state, indexability_reasons, canonical_node_key
2962
+ FROM site_crawl_pages
2963
+ WHERE health_state IS NULL
2964
+ LIMIT ${BATCH}
2965
+ `));
2966
+ if (rows.length === 0) return;
2967
+ for (const row of rows) {
2968
+ const healthState = deriveSiteHealthState({
2969
+ fetchState: row.fetch_state,
2970
+ indexabilityState: row.indexability_state,
2971
+ indexabilityReasons: parseJsonColumn(row.indexability_reasons, []),
2972
+ canonicalNodeKey: row.canonical_node_key,
2973
+ nodeKey: row.node_key
2974
+ });
2975
+ tx.run(sql2`UPDATE site_crawl_pages SET health_state = ${healthState} WHERE id = ${row.id}`);
2976
+ }
2977
+ if (rows.length < BATCH) return;
2978
+ }
2979
+ }
2980
+ function* streamSiteCrawlTemplateLinkEdges(tx, attemptId, batchSize = 2e3) {
2981
+ let after = "";
2982
+ for (; ; ) {
2983
+ const rows = tx.all(sql2`
2984
+ SELECT edge_key, source_node_key, target_node_key, anchors
2985
+ FROM site_crawl_edges
2986
+ WHERE attempt_id = ${attemptId} AND edge_key > ${after}
2987
+ ORDER BY edge_key
2988
+ LIMIT ${batchSize}
2989
+ `);
2990
+ if (rows.length === 0) return;
2991
+ yield rows.map((row) => ({
2992
+ edgeKey: row.edge_key,
2993
+ sourceNodeKey: row.source_node_key,
2994
+ targetNodeKey: row.target_node_key,
2995
+ anchors: parseJsonColumn(row.anchors, [])
2996
+ }));
2997
+ after = rows[rows.length - 1].edge_key;
2998
+ if (rows.length < batchSize) return;
2999
+ }
3000
+ }
3001
+ function* chunked(values, size = 500) {
3002
+ for (let offset = 0; offset < values.length; offset += size) {
3003
+ yield values.slice(offset, offset + size);
3004
+ }
3005
+ }
3006
+ function backfillSiteCrawlTemplateLinks(tx) {
3007
+ const attempts = tx.all(sql2.raw(`
3008
+ SELECT
3009
+ a.id AS attempt_id,
3010
+ a.project_id AS project_id,
3011
+ a.run_id AS run_id,
3012
+ COALESCE(NULLIF(a.pages_fetched, 0), s.pages_fetched, 0) AS pages_fetched
3013
+ FROM site_crawl_attempts AS a
3014
+ LEFT JOIN site_crawl_snapshots AS s
3015
+ ON s.project_id = a.project_id AND s.run_id = a.run_id AND s.attempt_id = a.id
3016
+ `));
3017
+ for (const attempt of attempts) {
3018
+ const detection = templateLinkDetection(attempt.pages_fetched);
3019
+ const scope = { attemptId: attempt.attempt_id, projectId: attempt.project_id, runId: attempt.run_id };
3020
+ tx.run(sql2`
3021
+ UPDATE site_crawl_edges SET is_template = 0, template_ratio = NULL
3022
+ WHERE project_id = ${scope.projectId} AND run_id = ${scope.runId} AND attempt_id = ${scope.attemptId}
3023
+ `);
3024
+ tx.run(sql2`
3025
+ UPDATE site_crawl_graph_edges SET is_template = 0
3026
+ WHERE project_id = ${scope.projectId} AND run_id = ${scope.runId} AND attempt_id = ${scope.attemptId}
3027
+ `);
3028
+ if (detection === SiteHealthTemplateDetections.applied) {
3029
+ const index2 = createTemplateLinkPairIndex();
3030
+ for (const batch of streamSiteCrawlTemplateLinkEdges(tx, scope.attemptId)) {
3031
+ observeTemplateLinkEdges(index2, batch);
3032
+ }
3033
+ const keysByRatio = /* @__PURE__ */ new Map();
3034
+ const templateEdgeKeys = [];
3035
+ for (const batch of streamSiteCrawlTemplateLinkEdges(tx, scope.attemptId)) {
3036
+ for (const edge of batch) {
3037
+ const ratio = templateLinkRatio(index2, attempt.pages_fetched, edge);
3038
+ if (ratio == null) continue;
3039
+ const group = keysByRatio.get(ratio);
3040
+ if (group) group.push(edge.edgeKey);
3041
+ else keysByRatio.set(ratio, [edge.edgeKey]);
3042
+ if (isTemplateLinkRatio(ratio)) templateEdgeKeys.push(edge.edgeKey);
3043
+ }
3044
+ }
3045
+ for (const [ratio, edgeKeys] of keysByRatio) {
3046
+ const isTemplate = isTemplateLinkRatio(ratio) ? 1 : 0;
3047
+ for (const chunk of chunked(edgeKeys)) {
3048
+ tx.run(sql2`
3049
+ UPDATE site_crawl_edges SET is_template = ${isTemplate}, template_ratio = ${ratio}
3050
+ WHERE project_id = ${scope.projectId} AND run_id = ${scope.runId} AND attempt_id = ${scope.attemptId}
3051
+ AND edge_key IN (SELECT value FROM json_each(${JSON.stringify(chunk)}))
3052
+ `);
3053
+ }
3054
+ }
3055
+ for (const chunk of chunked(templateEdgeKeys)) {
3056
+ tx.run(sql2`
3057
+ UPDATE site_crawl_graph_edges SET is_template = 1
3058
+ WHERE project_id = ${scope.projectId} AND run_id = ${scope.runId} AND attempt_id = ${scope.attemptId}
3059
+ AND edge_key IN (SELECT value FROM json_each(${JSON.stringify(chunk)}))
3060
+ `);
3061
+ }
3062
+ }
3063
+ tx.run(sql2`
3064
+ UPDATE site_crawl_graph_layouts
3065
+ SET total_template_edges = (
3066
+ SELECT COUNT(*)
3067
+ FROM site_crawl_edges AS e
3068
+ INNER JOIN site_crawl_pages AS source_page
3069
+ ON source_page.project_id = e.project_id AND source_page.run_id = e.run_id
3070
+ AND source_page.attempt_id = e.attempt_id AND source_page.node_key = e.source_node_key
3071
+ INNER JOIN site_crawl_pages AS target_page
3072
+ ON target_page.project_id = e.project_id AND target_page.run_id = e.run_id
3073
+ AND target_page.attempt_id = e.attempt_id AND target_page.node_key = e.target_node_key
3074
+ WHERE e.project_id = ${scope.projectId} AND e.run_id = ${scope.runId} AND e.attempt_id = ${scope.attemptId}
3075
+ AND e.internal = 1 AND e.relation = 'anchor' AND e.is_template = 1
3076
+ )
3077
+ WHERE project_id = ${scope.projectId} AND run_id = ${scope.runId} AND attempt_id = ${scope.attemptId}
3078
+ `);
3079
+ tx.run(sql2`
3080
+ UPDATE site_crawl_snapshots SET template_detection = ${detection}
3081
+ WHERE project_id = ${scope.projectId} AND run_id = ${scope.runId} AND attempt_id = ${scope.attemptId}
3082
+ `);
3083
+ }
3084
+ }
2866
3085
  function backfillQuerySnapshotServedModel(tx) {
2867
3086
  tx.run(sql2.raw(`
2868
3087
  UPDATE query_snapshots
@@ -5805,6 +6024,127 @@ var MIGRATION_VERSIONS = [
5805
6024
  `CREATE INDEX IF NOT EXISTS idx_traffic_event_receipts_expires
5806
6025
  ON traffic_event_receipts(source_id, expires_at)`
5807
6026
  ]
6027
+ },
6028
+ {
6029
+ version: 130,
6030
+ name: "site-crawl-pages-persisted-health-state",
6031
+ // Site Health state is derived from fetch state, indexability, the
6032
+ // crawler's reasons, and canonical identity together. Filtering it by
6033
+ // recomputing in JS meant reading every page row on every request. It is
6034
+ // now written once at publish time by the SAME contract function the map
6035
+ // and the agents use, so reads are an ordinary indexed WHERE.
6036
+ //
6037
+ // Backfilled in TypeScript inside this transaction (see
6038
+ // `backfillSiteCrawlPageHealthState`) so existing scans are filterable
6039
+ // immediately. The derivation is never rewritten in SQL: the backfill
6040
+ // calls the contract's own function, so there is no second implementation
6041
+ // to drift. `unavailable-legacy-scan` therefore survives only as the
6042
+ // honest answer for a row this could not reach.
6043
+ //
6044
+ // Idempotent: `ALTER TABLE ADD COLUMN` errors with "duplicate column
6045
+ // name" on retry, which the runner already swallows, and the backfill
6046
+ // only touches NULL rows.
6047
+ statements: [
6048
+ `ALTER TABLE site_crawl_pages ADD COLUMN health_state TEXT`,
6049
+ // Trailing columns match the ORDER BY the page list actually issues
6050
+ // (`sort=path`), so a filtered read terminates at LIMIT instead of
6051
+ // sorting every match in a temp b-tree on every cursor page.
6052
+ `CREATE INDEX IF NOT EXISTS idx_site_crawl_pages_health
6053
+ ON site_crawl_pages(project_id, run_id, attempt_id, health_state, path, node_key)`
6054
+ ],
6055
+ run: backfillSiteCrawlPageHealthState
6056
+ },
6057
+ {
6058
+ version: 131,
6059
+ name: "site-crawl-template-link-classification",
6060
+ // A real site's internal links are bimodal: a (target page, anchor) pair
6061
+ // sits on nearly every page or on nearly none. The ubiquitous half is nav,
6062
+ // header, and footer chrome, and drawing it buries the content structure
6063
+ // the map exists to show. Classification is computed once at publish time
6064
+ // by the contract's own function, so the map, the API filters, and the
6065
+ // agents cannot disagree about which links are chrome.
6066
+ //
6067
+ // `is_template` and `template_detection` are deliberately nullable:
6068
+ // NULL means "never classified" and reads report it as
6069
+ // `unavailable-legacy-scan`. The too-few-pages guard writes an explicit
6070
+ // `false` plus its reason on the snapshot, so an empty template-link list
6071
+ // can never pass for "this site has no nav".
6072
+ //
6073
+ // Idempotent: `ALTER TABLE ADD COLUMN` errors with "duplicate column
6074
+ // name", which the runner already swallows, and the backfill reclassifies
6075
+ // each attempt from its own stored rows.
6076
+ statements: [
6077
+ `ALTER TABLE site_crawl_edges ADD COLUMN is_template INTEGER`,
6078
+ `ALTER TABLE site_crawl_edges ADD COLUMN template_ratio REAL`,
6079
+ `ALTER TABLE site_crawl_graph_edges ADD COLUMN is_template INTEGER NOT NULL DEFAULT 0`,
6080
+ `ALTER TABLE site_crawl_graph_layouts ADD COLUMN total_template_edges INTEGER NOT NULL DEFAULT 0`,
6081
+ `ALTER TABLE site_crawl_graph_layouts ADD COLUMN template_links_excluded INTEGER NOT NULL DEFAULT 0`,
6082
+ `ALTER TABLE site_crawl_snapshots ADD COLUMN template_detection TEXT`,
6083
+ // Trailing edge key matches the ORDER BY the internal-link reads issue,
6084
+ // so a filtered page terminates at LIMIT.
6085
+ `CREATE INDEX IF NOT EXISTS idx_site_crawl_edges_template
6086
+ ON site_crawl_edges(project_id, run_id, attempt_id, internal, is_template, edge_key)`
6087
+ ],
6088
+ run: backfillSiteCrawlTemplateLinks
6089
+ },
6090
+ {
6091
+ version: 132,
6092
+ name: "site-crawl-drop-self-links",
6093
+ // A page linking to itself is not a link to or from another page, and the
6094
+ // crawl engine already excludes self-loops from a page's inbound and
6095
+ // outbound metrics. The edge tables kept them, so the stored edges
6096
+ // disagreed with the page rows produced by the same crawl: a self-loop
6097
+ // appeared in BOTH neighbour lists, so every self-linking page read one
6098
+ // higher in each direction than its own tiles.
6099
+ //
6100
+ // The writer now drops them, which fixes new scans. This clears the ones
6101
+ // already stored so an existing scan is correct without a re-crawl.
6102
+ // Comparing the stored normalized URLs is exactly the rule the writer
6103
+ // applies (`isSelfLink`), expressed as the equality SQL can do without
6104
+ // reimplementing any derivation.
6105
+ //
6106
+ // Idempotent: a re-run deletes nothing once the rows are gone.
6107
+ statements: [],
6108
+ run: dropSiteCrawlSelfLinks
6109
+ },
6110
+ {
6111
+ version: 133,
6112
+ name: "site-crawl-reclassify-template-links-per-anchor",
6113
+ // v131 classified a link as chrome when its MOST ubiquitous anchor was
6114
+ // ubiquitous. One stored link row aggregates every anchor the crawl saw
6115
+ // between the same two pages, so on any site with a comprehensive footer
6116
+ // (which is most sites) an in-prose link to a footer-linked target shared
6117
+ // a row with that page's footer link and inherited its ratio. The
6118
+ // editorial link was marked chrome, hidden from the map, and dropped from
6119
+ // every content count. Observed on canonry.ai: /aeo-methodology had 24
6120
+ // outbound links, all marked chrome at ratio 0.96, with anchors like
6121
+ // "Explore the Canonry platform" that are plainly editorial.
6122
+ //
6123
+ // The rule is now the LEAST ubiquitous anchor: chrome only when every
6124
+ // anchor on the link is chrome. This re-runs the same backfill hook, which
6125
+ // reclassifies each attempt from its own stored rows through the corrected
6126
+ // contract function, so an existing scan is right without a re-crawl.
6127
+ //
6128
+ // Statements stay empty because v131 already added every column; only the
6129
+ // values were wrong. Re-running is safe: the hook resets each attempt to
6130
+ // "classified, not chrome" before it writes, so it is idempotent and its
6131
+ // result depends on nothing but the stored links.
6132
+ statements: [],
6133
+ run: backfillSiteCrawlTemplateLinks
6134
+ },
6135
+ {
6136
+ version: 134,
6137
+ name: "site-crawl-live-page-health-preview-index",
6138
+ // The live Page Health preview is deliberately small and stable while a
6139
+ // crawl writes more pages: completed audits below the display threshold,
6140
+ // ordered worst-first by score and then node key. Cover the exact WHERE +
6141
+ // ORDER BY shape so the read stops at LIMIT without sorting every current
6142
+ // crawl row in a temporary b-tree. Its standard composite-index SQL is
6143
+ // also PostgreSQL-compatible; the query shape has no SQLite-only syntax.
6144
+ statements: [
6145
+ `CREATE INDEX IF NOT EXISTS idx_site_crawl_pages_live_preview
6146
+ ON site_crawl_pages(project_id, run_id, attempt_id, audit_state, audit_score, node_key)`
6147
+ ]
5808
6148
  }
5809
6149
  ];
5810
6150
  function addRunsMeasurementPlanVersionForeignKey(tx) {
@@ -8676,9 +9016,15 @@ function isOverviewRead(url) {
8676
9016
  "citations/visibility"
8677
9017
  ])).has(rest);
8678
9018
  }
8679
- function isTechnicalAeoRead(url) {
9019
+ function isTechnicalAeoRead(request, url) {
8680
9020
  const rest = projectRouteRest(url);
8681
- return rest === "technical-aeo" || rest === "technical-aeo/pages" || rest === "technical-aeo/trend" || rest === "technical-aeo/crawl" || rest === "technical-aeo/graph" || rest === "technical-aeo/subgraph" || rest === "technical-aeo/path" || rest === "technical-aeo/changes" || rest === "technical-aeo/crawl/pages/audit" || rest === "technical-aeo/crawl/pages" || rest === "technical-aeo/structure" || rest === "technical-aeo/internal-links" || rest === "technical-aeo/internal-links/neighbors" || rest === "technical-aeo/dead-links";
9021
+ if (rest === "runs") {
9022
+ return queryValue(request, "kind") === RunKinds["site-audit"];
9023
+ }
9024
+ if (rest && /^technical-aeo\/runs\/[^/]+\/(?:progress|page-health-preview)$/.test(rest)) {
9025
+ return true;
9026
+ }
9027
+ return rest === "technical-aeo" || rest === "technical-aeo/pages" || rest === "technical-aeo/trend" || rest === "technical-aeo/crawl" || rest === "technical-aeo/graph" || rest === "technical-aeo/subgraph" || rest === "technical-aeo/path" || rest === "technical-aeo/changes" || rest === "technical-aeo/crawl/pages/audit" || rest === "technical-aeo/crawl/pages" || rest === "technical-aeo/runs" || rest === "technical-aeo/structure" || rest === "technical-aeo/internal-links" || rest === "technical-aeo/internal-links/neighbors" || rest === "technical-aeo/dead-links";
8682
9028
  }
8683
9029
  function isReportRead(url) {
8684
9030
  const rest = projectRouteRest(url);
@@ -8694,7 +9040,7 @@ function enforceEmbedProjectTabs(request, configuredTabs) {
8694
9040
  const url = request.url.split("?")[0];
8695
9041
  if (isProjectShellRead(request, url)) return;
8696
9042
  if (tabs.includes("overview") && isOverviewRead(url)) return;
8697
- if (tabs.includes("technical-aeo") && isTechnicalAeoRead(url)) return;
9043
+ if (tabs.includes("technical-aeo") && isTechnicalAeoRead(request, url)) return;
8698
9044
  if (tabs.includes("report") && isReportRead(url)) return;
8699
9045
  throw forbidden("This endpoint is not available for the configured embed tabs.");
8700
9046
  }
@@ -9043,6 +9389,104 @@ function pruneProviderModelsForProviders(models, providers) {
9043
9389
 
9044
9390
  // ../api-routes/src/projects.ts
9045
9391
  async function projectRoutes(app, opts) {
9392
+ app.post("/projects", async (request, reply) => {
9393
+ requireAdminSession(request);
9394
+ if (request.principal?.projectId) {
9395
+ throw forbidden("This API key is limited to one project, and cannot create projects for this install.");
9396
+ }
9397
+ requireScope(request, PROJECTS_WRITE_SCOPE);
9398
+ const parsedBody = projectCreateRequestSchema.safeParse(request.body);
9399
+ if (!parsedBody.success) {
9400
+ throw validationError("Invalid project payload", {
9401
+ issues: parsedBody.error.issues.map((issue) => ({
9402
+ path: issue.path.join("."),
9403
+ message: issue.message
9404
+ }))
9405
+ });
9406
+ }
9407
+ const body = parsedBody.data;
9408
+ const name = normalizeProjectName(body.name);
9409
+ if (!name) {
9410
+ throw validationError("Project name must contain at least one letter or number after normalization.");
9411
+ }
9412
+ const canonicalDomain = hostOf(body.canonicalDomain);
9413
+ if (!canonicalDomain) {
9414
+ throw validationError("canonicalDomain must be a valid hostname or http(s) URL.");
9415
+ }
9416
+ const validNames = opts.providerAdapters?.map((adapter) => adapter.name) ?? [];
9417
+ if (validNames.length && body.providers?.length) {
9418
+ const invalid = body.providers.filter((p) => !validNames.includes(p));
9419
+ if (invalid.length) {
9420
+ throw validationError(`Invalid provider(s): ${invalid.join(", ")}. Must be one of: ${validNames.join(", ")}`, {
9421
+ invalidProviders: invalid,
9422
+ validProviders: validNames
9423
+ });
9424
+ }
9425
+ }
9426
+ const nextProviders = body.providers ?? [];
9427
+ const providerModels = pruneProviderModelsForProviders(
9428
+ validateProviderModels(body.providerModels ?? {}, opts.providerAdapters),
9429
+ nextProviders
9430
+ );
9431
+ assertProviderModelScope(request, {}, providerModels, nextProviders);
9432
+ const nextLocations = body.locations ?? [];
9433
+ const duplicateLabels = findDuplicateLocationLabels(nextLocations);
9434
+ if (duplicateLabels.length > 0) {
9435
+ throw validationError(`Duplicate location labels are not allowed: ${duplicateLabels.join(", ")}`, {
9436
+ duplicateLabels
9437
+ });
9438
+ }
9439
+ const nextDefaultLocation = body.defaultLocation ?? null;
9440
+ if (!hasLocationLabel(nextLocations, nextDefaultLocation)) {
9441
+ throw validationError(`defaultLocation "${nextDefaultLocation}" must match a configured location label`, {
9442
+ defaultLocation: nextDefaultLocation
9443
+ });
9444
+ }
9445
+ const normalizedCollision = app.db.select({
9446
+ name: projects.name
9447
+ }).from(projects).all().find((project) => normalizeProjectName(project.name) === name);
9448
+ if (normalizedCollision) throw alreadyExists("Project", name);
9449
+ const id = crypto6.randomUUID();
9450
+ const now = (/* @__PURE__ */ new Date()).toISOString();
9451
+ const nextAliases = normalizeProjectAliases(body.displayName, body.aliases ?? []);
9452
+ const inserted = app.db.transaction((tx) => {
9453
+ const result = tx.insert(projects).values({
9454
+ id,
9455
+ name,
9456
+ displayName: body.displayName,
9457
+ canonicalDomain,
9458
+ ownedDomains: body.ownedDomains ?? [],
9459
+ aliases: nextAliases,
9460
+ country: body.country,
9461
+ language: body.language,
9462
+ tags: body.tags ?? [],
9463
+ labels: body.labels ?? {},
9464
+ providers: nextProviders,
9465
+ providerModels,
9466
+ measurement: body.measurement ?? DEFAULT_MEASUREMENT_CONFIG,
9467
+ locations: nextLocations,
9468
+ defaultLocation: nextDefaultLocation,
9469
+ autoExtractBacklinks: body.autoExtractBacklinks ?? false,
9470
+ configSource: body.configSource ?? "api",
9471
+ configRevision: 1,
9472
+ createdAt: now,
9473
+ updatedAt: now
9474
+ }).onConflictDoNothing().run();
9475
+ if (result.changes !== 1) return false;
9476
+ writeAuditLog(tx, {
9477
+ projectId: id,
9478
+ actor: "api",
9479
+ action: "project.created",
9480
+ entityType: "project",
9481
+ entityId: id
9482
+ });
9483
+ return true;
9484
+ });
9485
+ if (!inserted) throw alreadyExists("Project", name);
9486
+ opts.onProjectUpserted?.(id, name);
9487
+ const created = app.db.select().from(projects).where(eq4(projects.id, id)).get();
9488
+ return reply.status(201).send(formatProject(created));
9489
+ });
9046
9490
  app.put("/projects/:name", async (request, reply) => {
9047
9491
  const { name } = request.params;
9048
9492
  const parsedBody = projectUpsertRequestSchema.safeParse(request.body);
@@ -27385,6 +27829,7 @@ var SCHEMA_TABLE = {
27385
27829
  NotificationDto: notificationDtoSchema,
27386
27830
  OnboardingTelemetryEvent: onboardingTelemetryEventSchema,
27387
27831
  OrganicEvidenceDto: organicEvidenceDtoSchema,
27832
+ ProjectCreateRequest: projectCreateRequestSchema,
27388
27833
  ProjectDto: projectDtoSchema,
27389
27834
  ProjectUpsertRequest: projectUpsertRequestSchema,
27390
27835
  ProjectConfig: projectConfigSchema,
@@ -27398,7 +27843,9 @@ var SCHEMA_TABLE = {
27398
27843
  SchedulableRunKind: schedulableRunKindSchema,
27399
27844
  ScheduleDto: scheduleDtoSchema,
27400
27845
  SettingsDto: settingsDtoSchema,
27846
+ SiteAuditLivePageHealthDto: siteAuditLivePageHealthSchema,
27401
27847
  SiteAuditPagesResponseDto: siteAuditPagesResponseSchema,
27848
+ SiteAuditRunProgressDto: siteAuditRunProgressSchema,
27402
27849
  SiteAuditRunResponseDto: siteAuditRunResponseSchema,
27403
27850
  SiteAuditScoreDto: siteAuditScoreSchema,
27404
27851
  SiteAuditTrendResponseDto: siteAuditTrendResponseSchema,
@@ -27412,6 +27859,7 @@ var SCHEMA_TABLE = {
27412
27859
  SiteCrawlSummaryDto: siteCrawlSummarySchema,
27413
27860
  SiteHealthChangesResponseDto: siteHealthChangesResponseSchema,
27414
27861
  SiteHealthPathResponseDto: siteHealthPathResponseSchema,
27862
+ SiteHealthScansResponseDto: siteHealthScansResponseSchema,
27415
27863
  SiteHealthSubgraphResponseDto: siteHealthSubgraphResponseSchema,
27416
27864
  SnapshotDiffResponse: snapshotDiffResponseSchema,
27417
27865
  SnapshotListResponse: snapshotListResponseSchema,
@@ -27885,6 +28333,12 @@ var crawlLimitParameter = {
27885
28333
  description: "Page size. Defaults to 100 (50 for structure/neighbors); maximum 200 (100 for structure/neighbors).",
27886
28334
  schema: { type: "integer", minimum: 1, maximum: 200 }
27887
28335
  };
28336
+ var linkKindParameter = {
28337
+ name: "linkKind",
28338
+ in: "query",
28339
+ description: 'Restrict links to `content` (excludes nav, header, and footer links) or `template` (only those). Defaults to `all`. A scan whose links were never classified matches neither, which is why every link response also carries `templateDetection`: an empty `content` list means "could not tell" when detection is not `applied`.',
28340
+ schema: { type: "string", enum: ["all", "content", "template"], default: "all" }
28341
+ };
27888
28342
  var measurementOverviewSortParameter = {
27889
28343
  name: "sort",
27890
28344
  in: "query",
@@ -28026,6 +28480,27 @@ var routeCatalog = [
28026
28480
  404: errorResponse("Project or connection not found.")
28027
28481
  }
28028
28482
  },
28483
+ {
28484
+ method: "post",
28485
+ path: "/api/v1/projects",
28486
+ summary: "Create a project",
28487
+ description: "Creates a new project with create-only semantics for the domain-first launchpad. The server normalizes the project name and canonical domain; a normalized-name collision returns 409 and never updates the existing project.",
28488
+ tags: ["projects"],
28489
+ requestBody: {
28490
+ required: true,
28491
+ content: {
28492
+ "application/json": {
28493
+ schema: { $ref: "#/components/schemas/ProjectCreateRequest" }
28494
+ }
28495
+ }
28496
+ },
28497
+ responses: {
28498
+ 201: jsonResponse("Project created.", "ProjectDto"),
28499
+ 400: errorResponse("Invalid project payload, normalized name, or canonical domain."),
28500
+ 403: errorResponse("The caller lacks broad projects.write authority."),
28501
+ 409: errorResponse("A project with the normalized name already exists.")
28502
+ }
28503
+ },
28029
28504
  {
28030
28505
  method: "put",
28031
28506
  path: "/api/v1/projects/{name}",
@@ -33396,24 +33871,25 @@ var routeCatalog = [
33396
33871
  ],
33397
33872
  responses: {
33398
33873
  200: jsonResponse("Persisted crawl summary returned.", "SiteCrawlSummaryDto"),
33399
- 404: errorResponse("Project or crawl-bearing site-audit run not found.")
33874
+ 404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
33400
33875
  }
33401
33876
  },
33402
33877
  {
33403
33878
  method: "get",
33404
33879
  path: "/api/v1/projects/{name}/technical-aeo/graph",
33405
33880
  summary: "Get a persisted Site Health graph projection",
33406
- description: "Returns the deterministic graph projection computed once when the latest complete or selected historical crawl was published. ForceAtlas2 positions and the exact internal-anchor edge sample are persisted, so reads run no layout physics and never rescan the crawl edge table. The response is bounded to 20,000 nodes / 50,000 edges; `layout`, `omittedNodes`, `omittedEdges`, and `sampled` disclose legacy/unavailable layouts and intentional truncation.",
33881
+ description: "Returns the deterministic graph projection computed once when the latest complete or selected historical crawl was published. ForceAtlas2 positions and the exact internal-anchor edge sample are persisted, so reads run no layout physics and never rescan the crawl edge table. Nav, header, and footer links are excluded from the layout physics but retained in the sample and tagged `isTemplate`, so a viewer can draw them without a refetch and without any node moving. The response is bounded to 20,000 nodes / 50,000 edges; `layout`, `omittedNodes`, `omittedEdges`, and `sampled` disclose legacy/unavailable layouts and intentional truncation, and `templateDetection` says whether template links could be told apart at all.",
33407
33882
  tags: ["technical-aeo"],
33408
33883
  parameters: [
33409
33884
  nameParameter,
33410
33885
  { name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest complete crawl.", schema: stringSchema },
33411
33886
  { name: "maxNodes", in: "query", description: "Maximum graph nodes. Defaults to and is capped at 20,000.", schema: { type: "integer", minimum: 1, maximum: 2e4 } },
33412
- { name: "maxEdges", in: "query", description: "Maximum graph edges. Defaults to and is capped at 50,000.", schema: { type: "integer", minimum: 1, maximum: 5e4 } }
33887
+ { name: "maxEdges", in: "query", description: "Maximum graph edges. Defaults to and is capped at 50,000.", schema: { type: "integer", minimum: 1, maximum: 5e4 } },
33888
+ linkKindParameter
33413
33889
  ],
33414
33890
  responses: {
33415
33891
  200: jsonResponse("Bounded Site Health graph projection returned.", "SiteCrawlGraphResponseDto"),
33416
- 404: errorResponse("Project or crawl-bearing site-audit run not found.")
33892
+ 404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
33417
33893
  }
33418
33894
  },
33419
33895
  {
@@ -33508,6 +33984,8 @@ var routeCatalog = [
33508
33984
  { name: "inventoryEligible", in: "query", description: "Filter Canonry technical inventory eligibility (`true` or `false`).", schema: booleanSchema },
33509
33985
  { name: "fetchState", in: "query", description: "Filter by persisted fetch state.", schema: stringSchema },
33510
33986
  { name: "indexabilityState", in: "query", description: "Filter by crawler-derived indexability state; this is not Google index coverage.", schema: stringSchema },
33987
+ { name: "nodeKey", in: "query", description: "Return only the page with this node key. Combined with a filter it answers whether that exact page is in the filtered set, without paging through the list.", schema: stringSchema },
33988
+ { name: "healthState", in: "query", description: "Filter by the derived Site Health state shared with the dashboard and agents. Unlike `indexabilityState` this folds fetch state, canonical identity, and the crawler reasons into one decision. `hidden` means the SITE told answer engines not to index the page (noindex, canonical-away, robots.txt); a non-HTML file is `resource` and a redirect is `redirect`, because neither is suppressed. Values: `eligible`, `hidden`, `resource`, `redirect`, `failed`, `unchecked`.", schema: { type: "string", enum: ["eligible", "hidden", "resource", "redirect", "failed", "unchecked"] } },
33511
33989
  { name: "auditState", in: "query", description: "Filter by audit state.", schema: stringSchema },
33512
33990
  { name: "sort", in: "query", description: "Sort order. Defaults to `url`.", schema: { type: "string", enum: ["url", "path", "score-asc", "score-desc"] } },
33513
33991
  crawlCursorParameter,
@@ -33515,7 +33993,7 @@ var routeCatalog = [
33515
33993
  ],
33516
33994
  responses: {
33517
33995
  200: jsonResponse("Crawl pages returned.", "SiteCrawlPagesResponseDto"),
33518
- 404: errorResponse("Project or crawl-bearing site-audit run not found.")
33996
+ 404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
33519
33997
  }
33520
33998
  },
33521
33999
  {
@@ -33533,14 +34011,14 @@ var routeCatalog = [
33533
34011
  ],
33534
34012
  responses: {
33535
34013
  200: jsonResponse("Immediate crawl children returned.", "SiteCrawlStructureResponseDto"),
33536
- 404: errorResponse("Project or crawl-bearing site-audit run not found.")
34014
+ 404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
33537
34015
  }
33538
34016
  },
33539
34017
  {
33540
34018
  method: "get",
33541
34019
  path: "/api/v1/projects/{name}/technical-aeo/internal-links",
33542
34020
  summary: "List persisted internal crawl links",
33543
- description: "Cursor-paged internal edges for the latest or selected crawl. Optional source/target/followability filters remain project-, run-, and attempt-scoped.",
34021
+ description: "Cursor-paged internal edges for the latest or selected crawl. Optional source/target/followability/link-kind filters remain project-, run-, and attempt-scoped. `total` counts exactly what the requested filters match, and `templateDetection` says whether nav and footer links could be told apart for this scan.",
33544
34022
  tags: ["technical-aeo"],
33545
34023
  parameters: [
33546
34024
  nameParameter,
@@ -33548,12 +34026,13 @@ var routeCatalog = [
33548
34026
  { name: "sourceUrl", in: "query", description: "Restrict to a source URL.", schema: stringSchema },
33549
34027
  { name: "targetUrl", in: "query", description: "Restrict to a target URL.", schema: stringSchema },
33550
34028
  { name: "followable", in: "query", description: "Restrict to followable or nofollow link observations.", schema: booleanSchema },
34029
+ linkKindParameter,
33551
34030
  crawlCursorParameter,
33552
34031
  crawlLimitParameter
33553
34032
  ],
33554
34033
  responses: {
33555
34034
  200: jsonResponse("Internal crawl links returned.", "SiteCrawlInternalLinksResponseDto"),
33556
- 404: errorResponse("Project or crawl-bearing site-audit run not found.")
34035
+ 404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
33557
34036
  }
33558
34037
  },
33559
34038
  {
@@ -33567,12 +34046,13 @@ var routeCatalog = [
33567
34046
  { name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest crawl.", schema: stringSchema },
33568
34047
  { name: "nodeKey", in: "query", description: "Canonical crawl node key.", schema: stringSchema },
33569
34048
  { name: "url", in: "query", description: "Canonical crawl URL.", schema: stringSchema },
34049
+ linkKindParameter,
33570
34050
  { name: "limit", in: "query", description: "Maximum inbound and outbound edges independently. Defaults to 50; maximum 100.", schema: { type: "integer", minimum: 1, maximum: 100 } }
33571
34051
  ],
33572
34052
  responses: {
33573
34053
  200: jsonResponse("Bounded internal-link neighborhood returned.", "SiteCrawlNeighborsResponseDto"),
33574
34054
  400: errorResponse("A nodeKey or URL is required."),
33575
- 404: errorResponse("Project or crawl-bearing site-audit run not found.")
34055
+ 404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
33576
34056
  }
33577
34057
  },
33578
34058
  {
@@ -33589,7 +34069,52 @@ var routeCatalog = [
33589
34069
  ],
33590
34070
  responses: {
33591
34071
  200: jsonResponse("Dead-link status and findings returned.", "SiteCrawlDeadLinksResponseDto"),
33592
- 404: errorResponse("Project or crawl-bearing site-audit run not found.")
34072
+ 404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
34073
+ }
34074
+ },
34075
+ {
34076
+ method: "get",
34077
+ path: "/api/v1/projects/{name}/technical-aeo/runs",
34078
+ summary: "List Site Health scan history",
34079
+ description: "Returns every non-probe `site-audit` run for the project, newest first, including runs still queued or running and runs that failed. `hasCrawlData` says whether that scan published a page and internal-link crawl: a legacy score-only scan is listed with `hasCrawlData=false` rather than hidden, and the crawl-scoped reads answer it with their no-crawl state instead of a 404.",
34080
+ tags: ["technical-aeo"],
34081
+ parameters: [
34082
+ nameParameter,
34083
+ { name: "limit", in: "query", description: "Max scans returned (most recent first). Defaults to 20; max 100.", schema: { type: "integer", minimum: 1, maximum: 100 } }
34084
+ ],
34085
+ responses: {
34086
+ 200: jsonResponse("Site Health scan history returned.", "SiteHealthScansResponseDto"),
34087
+ 404: errorResponse("Project not found.")
34088
+ }
34089
+ },
34090
+ {
34091
+ method: "get",
34092
+ path: "/api/v1/projects/{name}/technical-aeo/runs/{runId}/progress",
34093
+ summary: "Get exact stored Site Health run progress",
34094
+ description: "Returns durable run, crawl-attempt, and terminal graph-layout state for one exact non-probe site-audit run. It performs no network work and intentionally returns raw counters instead of a completion percentage.",
34095
+ tags: ["technical-aeo"],
34096
+ parameters: [
34097
+ nameParameter,
34098
+ { name: "runId", in: "path", required: true, description: "Exact site-audit run ID.", schema: stringSchema }
34099
+ ],
34100
+ responses: {
34101
+ 200: jsonResponse("Exact stored Site Health progress returned.", "SiteAuditRunProgressDto"),
34102
+ 404: errorResponse("Project or non-probe site-audit run not found.")
34103
+ }
34104
+ },
34105
+ {
34106
+ method: "get",
34107
+ path: "/api/v1/projects/{name}/technical-aeo/runs/{runId}/page-health-preview",
34108
+ summary: "Get a bounded, provisional Page Health preview",
34109
+ description: "Returns a small durable, provisional preview for one exact non-probe site-audit run while it is running: the count of audited pages plus at most 12 actionable low-score examples without finding prose. Queued scans return waiting. Terminal scans return no examples so callers use the immutable Page Health results instead. It performs no network work.",
34110
+ tags: ["technical-aeo"],
34111
+ parameters: [
34112
+ nameParameter,
34113
+ { name: "runId", in: "path", required: true, description: "Exact site-audit run ID.", schema: stringSchema }
34114
+ ],
34115
+ responses: {
34116
+ 200: jsonResponse("Bounded provisional Page Health preview returned.", "SiteAuditLivePageHealthDto"),
34117
+ 404: errorResponse("Project or non-probe site-audit run not found.")
33593
34118
  }
33594
34119
  },
33595
34120
  {
@@ -33621,6 +34146,7 @@ var routeCatalog = [
33621
34146
  200: jsonResponse("Site-audit run queued (or the in-flight run returned).", "SiteAuditRunResponseDto"),
33622
34147
  400: errorResponse("Invalid site-audit request."),
33623
34148
  409: errorResponse("A crawl with different effective options is already queued or running."),
34149
+ 422: errorResponse("The site-audit executor is unavailable on this deployment."),
33624
34150
  404: errorResponse("Project not found.")
33625
34151
  }
33626
34152
  }
@@ -57041,7 +57567,7 @@ function dedupeStrings(input) {
57041
57567
 
57042
57568
  // ../api-routes/src/technical-aeo.ts
57043
57569
  import crypto44 from "crypto";
57044
- import { and as and44, asc as asc10, count, desc as desc24, eq as eq56, inArray as inArray19, isNotNull as isNotNull3, lt as lt8, or as or11, sql as sql21 } from "drizzle-orm";
57570
+ import { and as and44, asc as asc10, count, desc as desc24, eq as eq56, inArray as inArray19, isNotNull as isNotNull3, isNull as isNull5, lt as lt8, or as or11, sql as sql21 } from "drizzle-orm";
57045
57571
  import { alias } from "drizzle-orm/sqlite-core";
57046
57572
  var FETCHED_SITE_CRAWL_STATES = /* @__PURE__ */ new Set([
57047
57573
  ...SiteCrawlFetchedStates,
@@ -57051,6 +57577,22 @@ var FETCHED_SITE_CRAWL_STATES = /* @__PURE__ */ new Set([
57051
57577
  "fetched"
57052
57578
  ]);
57053
57579
  var SURFACEABLE_STATUSES = [RunStatuses.completed, RunStatuses.partial];
57580
+ function siteAuditRunPhase(status, attempt, layoutState) {
57581
+ switch (status) {
57582
+ case RunStatuses.queued:
57583
+ return "queued";
57584
+ case RunStatuses.running:
57585
+ return (attempt?.pagesFetched ?? 0) > 0 ? "checking" : "discovering";
57586
+ case RunStatuses.completed:
57587
+ return attempt && layoutState === "pending" ? "arranging-map" : "completed";
57588
+ case RunStatuses.partial:
57589
+ return attempt && layoutState === "pending" ? "arranging-map" : "partial";
57590
+ case RunStatuses.failed:
57591
+ return "failed";
57592
+ case RunStatuses.cancelled:
57593
+ return "cancelled";
57594
+ }
57595
+ }
57054
57596
  function emptyScore(projectName) {
57055
57597
  return {
57056
57598
  project: projectName,
@@ -57082,6 +57624,14 @@ function parseBoundedLimit(value, fallback, max) {
57082
57624
  const parsed = parsePositiveInt(value, fallback, max);
57083
57625
  return Math.max(1, parsed);
57084
57626
  }
57627
+ function parseSiteHealthState(value) {
57628
+ if (value === void 0 || value === "") return null;
57629
+ const allowed = siteHealthStateSchema.safeParse(value);
57630
+ if (!allowed.success) {
57631
+ throw validationError(`"healthState" must be one of: ${siteHealthStateSchema.options.join(", ")}`);
57632
+ }
57633
+ return allowed.data;
57634
+ }
57085
57635
  function parseBoolean2(value) {
57086
57636
  if (value === true || value === "true" || value === "1") return true;
57087
57637
  if (value === false || value === "false" || value === "0") return false;
@@ -57170,6 +57720,21 @@ function mapCrawlPageAuditEvidence(row) {
57170
57720
  criticalDefects: parsedCriticalDefects.flatMap((result) => result.success ? [result.data] : [])
57171
57721
  };
57172
57722
  }
57723
+ var LIVE_PAGE_HEALTH_CANDIDATE_LIMIT = 48;
57724
+ var LIVE_PAGE_HEALTH_EXAMPLE_LIMIT = 12;
57725
+ function countLivePageHealthChecks(auditFields) {
57726
+ const rawFactors = Array.isArray(auditFields.factors) ? auditFields.factors : [];
57727
+ const rawCriticalDefects = Array.isArray(auditFields.criticalDefects) ? auditFields.criticalDefects : [];
57728
+ let checks = 0;
57729
+ for (const rawFactor of rawFactors) {
57730
+ const factor = siteCrawlAuditFactorSchema.safeParse(rawFactor);
57731
+ if (factor.success && factor.data.applicable !== false && factor.data.status !== "pass") checks++;
57732
+ }
57733
+ for (const rawCriticalDefect of rawCriticalDefects) {
57734
+ if (siteCrawlCriticalDefectSchema.safeParse(rawCriticalDefect).success) checks++;
57735
+ }
57736
+ return checks;
57737
+ }
57173
57738
  function mapCrawlEdge(row) {
57174
57739
  return {
57175
57740
  edgeKey: row.edgeKey,
@@ -57183,9 +57748,28 @@ function mapCrawlEdge(row) {
57183
57748
  occurrences: row.occurrences,
57184
57749
  followableOccurrences: row.followableOccurrences,
57185
57750
  nofollowOccurrences: row.nofollowOccurrences,
57186
- anchors: row.anchors
57751
+ anchors: row.anchors,
57752
+ isTemplate: row.isTemplate,
57753
+ templateRatio: row.templateRatio
57187
57754
  };
57188
57755
  }
57756
+ function templateDetectionOf(value) {
57757
+ const parsed = siteHealthTemplateDetectionSchema.safeParse(value);
57758
+ return parsed.success ? parsed.data : SiteHealthTemplateDetections["unavailable-legacy-scan"];
57759
+ }
57760
+ function parseLinkKind(value) {
57761
+ if (value === void 0 || value === "") return SiteHealthLinkKinds.all;
57762
+ const allowed = siteHealthLinkKindSchema.safeParse(value);
57763
+ if (!allowed.success) {
57764
+ throw validationError(`"linkKind" must be one of: ${siteHealthLinkKindSchema.options.join(", ")}`);
57765
+ }
57766
+ return allowed.data;
57767
+ }
57768
+ function linkKindFilter(linkKind) {
57769
+ if (linkKind === SiteHealthLinkKinds.content) return eq56(siteCrawlEdges.isTemplate, false);
57770
+ if (linkKind === SiteHealthLinkKinds.template) return eq56(siteCrawlEdges.isTemplate, true);
57771
+ return void 0;
57772
+ }
57189
57773
  var graphSourceNode = alias(siteCrawlGraphNodes, "site_crawl_graph_source_node");
57190
57774
  var graphTargetNode = alias(siteCrawlGraphNodes, "site_crawl_graph_target_node");
57191
57775
  var PAGE_CHANGE_PREDICATE = sql21`(
@@ -57317,6 +57901,16 @@ async function technicalAeoRoutes(app, opts) {
57317
57901
  eq56(siteCrawlPages.attemptId, scope.attemptId),
57318
57902
  eq56(siteCrawlPages.depth, 0)
57319
57903
  )).orderBy(asc10(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
57904
+ const isSurfaceableAuditRun = (projectId, runId) => Boolean(app.db.select({ id: runs.id }).from(runs).where(and44(
57905
+ eq56(runs.id, runId),
57906
+ eq56(runs.projectId, projectId),
57907
+ eq56(runs.kind, RunKinds["site-audit"]),
57908
+ inArray19(runs.status, SURFACEABLE_STATUSES),
57909
+ notProbeRun()
57910
+ )).limit(1).get());
57911
+ const assertKnownAuditRun = (projectId, runId) => {
57912
+ if (runId && !isSurfaceableAuditRun(projectId, runId)) throw notFound("Site crawl run", runId);
57913
+ };
57320
57914
  const hasLegacyAudit = (projectId) => Boolean(app.db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).innerJoin(runs, eq56(siteAuditSnapshots.runId, runs.id)).where(and44(
57321
57915
  eq56(siteAuditSnapshots.projectId, projectId),
57322
57916
  eq56(runs.projectId, projectId),
@@ -57437,7 +58031,7 @@ async function technicalAeoRoutes(app, opts) {
57437
58031
  const target = resolveCrawl(project.id, request.query.runId);
57438
58032
  const legacyAuditAvailable = hasLegacyAudit(project.id);
57439
58033
  if (!target) {
57440
- if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
58034
+ assertKnownAuditRun(project.id, request.query.runId);
57441
58035
  return emptyCrawlSummary(project.name, legacyAuditAvailable);
57442
58036
  }
57443
58037
  const snapshot = target.snapshot;
@@ -57471,16 +58065,23 @@ async function technicalAeoRoutes(app, opts) {
57471
58065
  });
57472
58066
  app.get("/projects/:name/technical-aeo/graph", async (request) => {
57473
58067
  const project = resolveProject(app.db, request.params.name);
58068
+ const linkKind = parseLinkKind(request.query.linkKind);
57474
58069
  const target = resolveCrawl(project.id, request.query.runId);
57475
58070
  if (!target) {
57476
- if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
58071
+ assertKnownAuditRun(project.id, request.query.runId);
57477
58072
  return {
57478
58073
  project: project.name,
57479
58074
  hasCrawlData: false,
57480
58075
  runId: null,
58076
+ rootNodeKey: null,
57481
58077
  layout: { state: "unavailable", version: null, reason: "no-crawl" },
58078
+ // Nothing was classified because nothing was crawled.
58079
+ templateDetection: SiteHealthTemplateDetections["unavailable-legacy-scan"],
58080
+ linkKind,
57482
58081
  totalNodes: 0,
57483
58082
  totalEdges: 0,
58083
+ totalTemplateEdges: 0,
58084
+ totalContentEdges: 0,
57484
58085
  nodes: [],
57485
58086
  edges: [],
57486
58087
  omittedNodes: 0,
@@ -57489,14 +58090,20 @@ async function technicalAeoRoutes(app, opts) {
57489
58090
  };
57490
58091
  }
57491
58092
  const snapshot = target.snapshot;
58093
+ const templateDetection = templateDetectionOf(snapshot.templateDetection);
57492
58094
  if (!snapshot.detailsAvailable || !snapshot.attemptId) {
57493
58095
  return {
57494
58096
  project: project.name,
57495
58097
  hasCrawlData: true,
57496
58098
  runId: snapshot.runId,
58099
+ rootNodeKey: null,
57497
58100
  layout: { state: "unavailable", version: null, reason: "details-unavailable" },
58101
+ templateDetection,
58102
+ linkKind,
57498
58103
  totalNodes: 0,
57499
58104
  totalEdges: 0,
58105
+ totalTemplateEdges: 0,
58106
+ totalContentEdges: 0,
57500
58107
  nodes: [],
57501
58108
  edges: [],
57502
58109
  omittedNodes: 0,
@@ -57504,6 +58111,10 @@ async function technicalAeoRoutes(app, opts) {
57504
58111
  sampled: false
57505
58112
  };
57506
58113
  }
58114
+ const rootNodeKey = rootPageInScope(
58115
+ { projectId: project.id, runId: snapshot.runId, attemptId: snapshot.attemptId },
58116
+ snapshot.rootUrl
58117
+ )?.nodeKey ?? null;
57507
58118
  const persistedLayout = app.db.select().from(siteCrawlGraphLayouts).where(and44(
57508
58119
  eq56(siteCrawlGraphLayouts.projectId, project.id),
57509
58120
  eq56(siteCrawlGraphLayouts.runId, snapshot.runId),
@@ -57514,9 +58125,14 @@ async function technicalAeoRoutes(app, opts) {
57514
58125
  project: project.name,
57515
58126
  hasCrawlData: true,
57516
58127
  runId: snapshot.runId,
58128
+ rootNodeKey,
57517
58129
  layout: { state: "unavailable", version: null, reason: "legacy-snapshot" },
58130
+ templateDetection,
58131
+ linkKind,
57518
58132
  totalNodes: 0,
57519
58133
  totalEdges: 0,
58134
+ totalTemplateEdges: 0,
58135
+ totalContentEdges: 0,
57520
58136
  nodes: [],
57521
58137
  edges: [],
57522
58138
  omittedNodes: 0,
@@ -57531,13 +58147,18 @@ async function technicalAeoRoutes(app, opts) {
57531
58147
  project: project.name,
57532
58148
  hasCrawlData: true,
57533
58149
  runId: snapshot.runId,
58150
+ rootNodeKey,
57534
58151
  layout: {
57535
58152
  state: "unavailable",
57536
58153
  version: null,
57537
58154
  reason: persistedLayout.failureCode === "empty-crawl" ? "empty-crawl" : "layout-failed"
57538
58155
  },
58156
+ templateDetection,
58157
+ linkKind,
57539
58158
  totalNodes: totalNodes2,
57540
58159
  totalEdges: totalEdges2,
58160
+ totalTemplateEdges: persistedLayout.totalTemplateEdges,
58161
+ totalContentEdges: Math.max(0, totalEdges2 - persistedLayout.totalTemplateEdges),
57541
58162
  nodes: [],
57542
58163
  edges: [],
57543
58164
  omittedNodes: totalNodes2,
@@ -57579,12 +58200,14 @@ async function technicalAeoRoutes(app, opts) {
57579
58200
  ...row,
57580
58201
  healthState: deriveSiteHealthState({ ...row, indexabilityReasons, canonicalNodeKey })
57581
58202
  }));
58203
+ const graphLinkKindFilter = linkKind === SiteHealthLinkKinds.all ? void 0 : eq56(siteCrawlGraphEdges.isTemplate, linkKind === SiteHealthLinkKinds.template);
57582
58204
  const edges = app.db.select({
57583
58205
  edgeKey: siteCrawlGraphEdges.edgeKey,
57584
58206
  sourceNodeKey: siteCrawlGraphEdges.sourceNodeKey,
57585
58207
  targetNodeKey: siteCrawlGraphEdges.targetNodeKey,
57586
58208
  followable: siteCrawlGraphEdges.followable,
57587
- occurrences: siteCrawlGraphEdges.occurrences
58209
+ occurrences: siteCrawlGraphEdges.occurrences,
58210
+ isTemplate: siteCrawlGraphEdges.isTemplate
57588
58211
  }).from(siteCrawlGraphEdges).innerJoin(graphSourceNode, and44(
57589
58212
  eq56(graphSourceNode.projectId, siteCrawlGraphEdges.projectId),
57590
58213
  eq56(graphSourceNode.runId, siteCrawlGraphEdges.runId),
@@ -57601,19 +58224,34 @@ async function technicalAeoRoutes(app, opts) {
57601
58224
  eq56(siteCrawlGraphEdges.projectId, project.id),
57602
58225
  eq56(siteCrawlGraphEdges.runId, snapshot.runId),
57603
58226
  eq56(siteCrawlGraphEdges.attemptId, snapshot.attemptId),
57604
- lt8(siteCrawlGraphEdges.sampleRank, maxEdges)
58227
+ lt8(siteCrawlGraphEdges.sampleRank, maxEdges),
58228
+ graphLinkKindFilter
57605
58229
  )).orderBy(asc10(siteCrawlGraphEdges.sampleRank)).all();
57606
58230
  const totalNodes = persistedLayout.totalNodes;
57607
58231
  const totalEdges = persistedLayout.totalEdges;
58232
+ const totalTemplateEdges = persistedLayout.totalTemplateEdges;
57608
58233
  const omittedNodes = Math.max(0, totalNodes - nodes.length);
57609
58234
  const omittedEdges = Math.max(0, totalEdges - edges.length);
57610
58235
  return {
57611
58236
  project: project.name,
57612
58237
  hasCrawlData: true,
57613
58238
  runId: snapshot.runId,
57614
- layout: { state: "ready", version: persistedLayout.layoutVersion, computedAt: persistedLayout.createdAt },
58239
+ rootNodeKey,
58240
+ layout: {
58241
+ state: "ready",
58242
+ version: persistedLayout.layoutVersion,
58243
+ computedAt: persistedLayout.createdAt,
58244
+ templateLinksExcluded: persistedLayout.templateLinksExcluded
58245
+ },
58246
+ templateDetection,
58247
+ linkKind,
57615
58248
  totalNodes,
58249
+ // Deliberately still every graph-compatible link, whatever `linkKind`
58250
+ // asked for. The two counts below split it; none of the three moves
58251
+ // because a caller narrowed the payload.
57616
58252
  totalEdges,
58253
+ totalTemplateEdges,
58254
+ totalContentEdges: Math.max(0, totalEdges - totalTemplateEdges),
57617
58255
  nodes,
57618
58256
  edges,
57619
58257
  omittedNodes,
@@ -57655,7 +58293,7 @@ async function technicalAeoRoutes(app, opts) {
57655
58293
  truncated: false
57656
58294
  });
57657
58295
  if (!target) {
57658
- if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
58296
+ assertKnownAuditRun(project.id, request.query.runId);
57659
58297
  return empty("no-crawl", null, false, false, null);
57660
58298
  }
57661
58299
  const snapshot = target.snapshot;
@@ -57781,7 +58419,7 @@ async function technicalAeoRoutes(app, opts) {
57781
58419
  }
57782
58420
  const target = resolveCrawl(project.id, request.query.runId);
57783
58421
  if (!target) {
57784
- if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
58422
+ assertKnownAuditRun(project.id, request.query.runId);
57785
58423
  return {
57786
58424
  project: project.name,
57787
58425
  runId: null,
@@ -58253,7 +58891,7 @@ async function technicalAeoRoutes(app, opts) {
58253
58891
  }
58254
58892
  const target = resolveCrawl(project.id, runId);
58255
58893
  if (!target) {
58256
- if (runId) throw notFound("Site crawl run", runId);
58894
+ assertKnownAuditRun(project.id, runId);
58257
58895
  return { state: "no-crawl", project: project.name, runId: null };
58258
58896
  }
58259
58897
  const snapshot = target.snapshot;
@@ -58294,28 +58932,52 @@ async function technicalAeoRoutes(app, opts) {
58294
58932
  const project = resolveProject(app.db, request.params.name);
58295
58933
  const target = resolveCrawl(project.id, request.query.runId);
58296
58934
  if (!target) {
58297
- if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
58298
- return { project: project.name, hasCrawlData: false, runId: null, total: 0, nextCursor: null, pages: [] };
58935
+ assertKnownAuditRun(project.id, request.query.runId);
58936
+ return { project: project.name, hasCrawlData: false, runId: null, total: 0, nextCursor: null, healthStateFilter: null, pages: [] };
58299
58937
  }
58300
58938
  const snapshot = target.snapshot;
58301
58939
  if (!snapshot.detailsAvailable || !snapshot.attemptId) {
58302
- return { project: project.name, hasCrawlData: true, runId: snapshot.runId, total: 0, nextCursor: null, pages: [] };
58940
+ return { project: project.name, hasCrawlData: true, runId: snapshot.runId, total: 0, nextCursor: null, healthStateFilter: null, pages: [] };
58303
58941
  }
58304
58942
  const filters = [
58305
58943
  eq56(siteCrawlPages.projectId, project.id),
58306
58944
  eq56(siteCrawlPages.runId, snapshot.runId),
58307
58945
  eq56(siteCrawlPages.attemptId, snapshot.attemptId)
58308
58946
  ];
58947
+ if (request.query.nodeKey) filters.push(eq56(siteCrawlPages.nodeKey, request.query.nodeKey));
58309
58948
  const inventoryEligible = parseBoolean2(request.query.inventoryEligible);
58310
58949
  if (inventoryEligible != null) filters.push(eq56(siteCrawlPages.inventoryEligible, inventoryEligible));
58311
58950
  if (request.query.fetchState) filters.push(eq56(siteCrawlPages.fetchState, request.query.fetchState));
58312
58951
  if (request.query.indexabilityState) filters.push(eq56(siteCrawlPages.indexabilityState, request.query.indexabilityState));
58313
58952
  if (request.query.auditState) filters.push(eq56(siteCrawlPages.auditState, request.query.auditState));
58314
- const where = and44(...filters);
58315
- const total = app.db.select({ value: count() }).from(siteCrawlPages).where(where).get()?.value ?? 0;
58953
+ const healthState = parseSiteHealthState(request.query.healthState);
58316
58954
  const limit = parseBoundedLimit(request.query.limit, 100, 200);
58317
58955
  const offset = decodeCursor(request.query.cursor);
58318
58956
  const orderBy = request.query.sort === "score-desc" ? [desc24(siteCrawlPages.auditScore), asc10(siteCrawlPages.nodeKey)] : request.query.sort === "score-asc" ? [asc10(siteCrawlPages.auditScore), asc10(siteCrawlPages.nodeKey)] : request.query.sort === "path" ? [asc10(siteCrawlPages.path), asc10(siteCrawlPages.nodeKey)] : [asc10(siteCrawlPages.url), asc10(siteCrawlPages.nodeKey)];
58957
+ let healthStateFilter = null;
58958
+ if (healthState) {
58959
+ const legacyRow = app.db.select({ id: siteCrawlPages.id }).from(siteCrawlPages).where(and44(
58960
+ eq56(siteCrawlPages.projectId, project.id),
58961
+ eq56(siteCrawlPages.runId, snapshot.runId),
58962
+ eq56(siteCrawlPages.attemptId, snapshot.attemptId),
58963
+ isNull5(siteCrawlPages.healthState)
58964
+ )).limit(1).get();
58965
+ healthStateFilter = legacyRow ? "unavailable-legacy-scan" : "applied";
58966
+ if (healthStateFilter === "applied") filters.push(eq56(siteCrawlPages.healthState, healthState));
58967
+ }
58968
+ if (healthStateFilter === "unavailable-legacy-scan") {
58969
+ return {
58970
+ project: project.name,
58971
+ hasCrawlData: true,
58972
+ runId: snapshot.runId,
58973
+ total: 0,
58974
+ nextCursor: null,
58975
+ healthStateFilter,
58976
+ pages: []
58977
+ };
58978
+ }
58979
+ const where = and44(...filters);
58980
+ const total = app.db.select({ value: count() }).from(siteCrawlPages).where(where).get()?.value ?? 0;
58319
58981
  const rows = app.db.select().from(siteCrawlPages).where(where).orderBy(...orderBy).limit(limit).offset(offset).all();
58320
58982
  const nextOffset = offset + rows.length;
58321
58983
  return {
@@ -58324,6 +58986,7 @@ async function technicalAeoRoutes(app, opts) {
58324
58986
  runId: snapshot.runId,
58325
58987
  total,
58326
58988
  nextCursor: nextOffset < total ? encodeCursor2(nextOffset) : null,
58989
+ healthStateFilter,
58327
58990
  pages: rows.map(mapCrawlPage)
58328
58991
  };
58329
58992
  });
@@ -58332,7 +58995,7 @@ async function technicalAeoRoutes(app, opts) {
58332
58995
  const parentPath = normalizeParentPath(request.query.parentPath);
58333
58996
  const target = resolveCrawl(project.id, request.query.runId);
58334
58997
  if (!target) {
58335
- if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
58998
+ assertKnownAuditRun(project.id, request.query.runId);
58336
58999
  return { project: project.name, hasCrawlData: false, runId: null, parentPath, nextCursor: null, children: [] };
58337
59000
  }
58338
59001
  const snapshot = target.snapshot;
@@ -58396,20 +59059,41 @@ async function technicalAeoRoutes(app, opts) {
58396
59059
  });
58397
59060
  app.get("/projects/:name/technical-aeo/internal-links", async (request) => {
58398
59061
  const project = resolveProject(app.db, request.params.name);
59062
+ const linkKind = parseLinkKind(request.query.linkKind);
58399
59063
  const target = resolveCrawl(project.id, request.query.runId);
58400
59064
  if (!target) {
58401
- if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
58402
- return { project: project.name, hasCrawlData: false, runId: null, total: 0, nextCursor: null, edges: [] };
59065
+ assertKnownAuditRun(project.id, request.query.runId);
59066
+ return {
59067
+ project: project.name,
59068
+ hasCrawlData: false,
59069
+ runId: null,
59070
+ total: 0,
59071
+ nextCursor: null,
59072
+ templateDetection: SiteHealthTemplateDetections["unavailable-legacy-scan"],
59073
+ linkKind,
59074
+ edges: []
59075
+ };
58403
59076
  }
58404
59077
  const snapshot = target.snapshot;
59078
+ const templateDetection = templateDetectionOf(snapshot.templateDetection);
58405
59079
  if (!snapshot.detailsAvailable || !snapshot.attemptId) {
58406
- return { project: project.name, hasCrawlData: true, runId: snapshot.runId, total: 0, nextCursor: null, edges: [] };
59080
+ return {
59081
+ project: project.name,
59082
+ hasCrawlData: true,
59083
+ runId: snapshot.runId,
59084
+ total: 0,
59085
+ nextCursor: null,
59086
+ templateDetection,
59087
+ linkKind,
59088
+ edges: []
59089
+ };
58407
59090
  }
58408
59091
  const filters = [
58409
59092
  eq56(siteCrawlEdges.projectId, project.id),
58410
59093
  eq56(siteCrawlEdges.runId, snapshot.runId),
58411
59094
  eq56(siteCrawlEdges.attemptId, snapshot.attemptId),
58412
- eq56(siteCrawlEdges.internal, true)
59095
+ eq56(siteCrawlEdges.internal, true),
59096
+ linkKindFilter(linkKind)
58413
59097
  ];
58414
59098
  if (request.query.sourceUrl) filters.push(eq56(siteCrawlEdges.sourceUrl, request.query.sourceUrl));
58415
59099
  if (request.query.targetUrl) filters.push(eq56(siteCrawlEdges.targetUrl, request.query.targetUrl));
@@ -58427,23 +59111,28 @@ async function technicalAeoRoutes(app, opts) {
58427
59111
  runId: snapshot.runId,
58428
59112
  total,
58429
59113
  nextCursor: nextOffset < total ? encodeCursor2(nextOffset) : null,
59114
+ templateDetection,
59115
+ linkKind,
58430
59116
  edges: rows.map(mapCrawlEdge)
58431
59117
  };
58432
59118
  });
58433
59119
  app.get("/projects/:name/technical-aeo/internal-links/neighbors", async (request) => {
58434
59120
  const project = resolveProject(app.db, request.params.name);
59121
+ const linkKind = parseLinkKind(request.query.linkKind);
58435
59122
  if (!request.query.nodeKey && !request.query.url) {
58436
59123
  throw validationError("Provide nodeKey or url");
58437
59124
  }
58438
59125
  const target = resolveCrawl(project.id, request.query.runId);
58439
59126
  if (!target) {
58440
- if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
59127
+ assertKnownAuditRun(project.id, request.query.runId);
58441
59128
  return {
58442
59129
  project: project.name,
58443
59130
  hasCrawlData: false,
58444
59131
  runId: null,
58445
59132
  nodeKey: request.query.nodeKey ?? null,
58446
59133
  url: request.query.url ?? null,
59134
+ templateDetection: SiteHealthTemplateDetections["unavailable-legacy-scan"],
59135
+ linkKind,
58447
59136
  inbound: [],
58448
59137
  outbound: [],
58449
59138
  inboundTruncated: false,
@@ -58451,6 +59140,7 @@ async function technicalAeoRoutes(app, opts) {
58451
59140
  };
58452
59141
  }
58453
59142
  const snapshot = target.snapshot;
59143
+ const templateDetection = templateDetectionOf(snapshot.templateDetection);
58454
59144
  if (!snapshot.detailsAvailable || !snapshot.attemptId) {
58455
59145
  return {
58456
59146
  project: project.name,
@@ -58458,6 +59148,8 @@ async function technicalAeoRoutes(app, opts) {
58458
59148
  runId: snapshot.runId,
58459
59149
  nodeKey: request.query.nodeKey ?? null,
58460
59150
  url: request.query.url ?? null,
59151
+ templateDetection,
59152
+ linkKind,
58461
59153
  inbound: [],
58462
59154
  outbound: [],
58463
59155
  inboundTruncated: false,
@@ -58468,7 +59160,8 @@ async function technicalAeoRoutes(app, opts) {
58468
59160
  eq56(siteCrawlEdges.projectId, project.id),
58469
59161
  eq56(siteCrawlEdges.runId, snapshot.runId),
58470
59162
  eq56(siteCrawlEdges.attemptId, snapshot.attemptId),
58471
- eq56(siteCrawlEdges.internal, true)
59163
+ eq56(siteCrawlEdges.internal, true),
59164
+ linkKindFilter(linkKind)
58472
59165
  ];
58473
59166
  const inboundMatch = request.query.nodeKey && request.query.url ? or11(eq56(siteCrawlEdges.targetNodeKey, request.query.nodeKey), eq56(siteCrawlEdges.targetUrl, request.query.url)) : request.query.nodeKey ? eq56(siteCrawlEdges.targetNodeKey, request.query.nodeKey) : eq56(siteCrawlEdges.targetUrl, request.query.url);
58474
59167
  const outboundMatch = request.query.nodeKey && request.query.url ? or11(eq56(siteCrawlEdges.sourceNodeKey, request.query.nodeKey), eq56(siteCrawlEdges.sourceUrl, request.query.url)) : request.query.nodeKey ? eq56(siteCrawlEdges.sourceNodeKey, request.query.nodeKey) : eq56(siteCrawlEdges.sourceUrl, request.query.url);
@@ -58481,6 +59174,8 @@ async function technicalAeoRoutes(app, opts) {
58481
59174
  runId: snapshot.runId,
58482
59175
  nodeKey: request.query.nodeKey ?? null,
58483
59176
  url: request.query.url ?? null,
59177
+ templateDetection,
59178
+ linkKind,
58484
59179
  inbound: inboundRows.slice(0, limit).map(mapCrawlEdge),
58485
59180
  outbound: outboundRows.slice(0, limit).map(mapCrawlEdge),
58486
59181
  inboundTruncated: inboundRows.length > limit,
@@ -58492,7 +59187,7 @@ async function technicalAeoRoutes(app, opts) {
58492
59187
  const target = resolveCrawl(project.id, request.query.runId);
58493
59188
  const legacyAuditAvailable = hasLegacyAudit(project.id);
58494
59189
  if (!target) {
58495
- if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
59190
+ assertKnownAuditRun(project.id, request.query.runId);
58496
59191
  return { project: project.name, runId: null, state: "unavailable", legacyAuditAvailable };
58497
59192
  }
58498
59193
  const snapshot = target.snapshot;
@@ -58538,12 +59233,153 @@ async function technicalAeoRoutes(app, opts) {
58538
59233
  deadLinks
58539
59234
  };
58540
59235
  });
59236
+ app.get("/projects/:name/technical-aeo/runs", async (request) => {
59237
+ const project = resolveProject(app.db, request.params.name);
59238
+ const limit = parseBoundedLimit(request.query.limit, SITE_HEALTH_SCANS_DEFAULT_LIMIT, SITE_HEALTH_SCANS_MAX_LIMIT);
59239
+ const rows = app.db.select({
59240
+ runId: runs.id,
59241
+ status: runs.status,
59242
+ createdAt: runs.createdAt,
59243
+ startedAt: runs.startedAt,
59244
+ finishedAt: runs.finishedAt
59245
+ }).from(runs).where(and44(
59246
+ eq56(runs.projectId, project.id),
59247
+ eq56(runs.kind, RunKinds["site-audit"]),
59248
+ notProbeRun()
59249
+ )).orderBy(desc24(runs.createdAt), desc24(runs.id)).limit(limit).all();
59250
+ const runIds = rows.map((row) => row.runId);
59251
+ const crawlRunIds = new Set(runIds.length === 0 ? [] : app.db.select({ runId: siteCrawlSnapshots.runId }).from(siteCrawlSnapshots).innerJoin(runs, eq56(siteCrawlSnapshots.runId, runs.id)).where(and44(
59252
+ eq56(siteCrawlSnapshots.projectId, project.id),
59253
+ eq56(runs.projectId, project.id),
59254
+ eq56(runs.kind, RunKinds["site-audit"]),
59255
+ inArray19(runs.status, SURFACEABLE_STATUSES),
59256
+ notProbeRun(),
59257
+ inArray19(siteCrawlSnapshots.runId, runIds)
59258
+ )).all().map((row) => row.runId));
59259
+ return {
59260
+ project: project.name,
59261
+ scans: rows.map((row) => ({
59262
+ ...row,
59263
+ status: row.status,
59264
+ hasCrawlData: crawlRunIds.has(row.runId)
59265
+ }))
59266
+ };
59267
+ });
59268
+ app.get("/projects/:name/technical-aeo/runs/:runId/progress", async (request) => {
59269
+ const project = resolveProject(app.db, request.params.name);
59270
+ const run = app.db.select().from(runs).where(and44(
59271
+ eq56(runs.id, request.params.runId),
59272
+ eq56(runs.projectId, project.id),
59273
+ eq56(runs.kind, RunKinds["site-audit"]),
59274
+ notProbeRun()
59275
+ )).get();
59276
+ if (!run) throw notFound("Site audit run", request.params.runId);
59277
+ const attempt = app.db.select().from(siteCrawlAttempts).where(and44(
59278
+ eq56(siteCrawlAttempts.projectId, project.id),
59279
+ eq56(siteCrawlAttempts.runId, run.id)
59280
+ )).orderBy(desc24(siteCrawlAttempts.attemptNumber), desc24(siteCrawlAttempts.updatedAt)).limit(1).get();
59281
+ const persistedLayout = attempt ? app.db.select().from(siteCrawlGraphLayouts).where(and44(
59282
+ eq56(siteCrawlGraphLayouts.projectId, project.id),
59283
+ eq56(siteCrawlGraphLayouts.runId, run.id),
59284
+ eq56(siteCrawlGraphLayouts.attemptId, attempt.id)
59285
+ )).get() : void 0;
59286
+ const layoutState = persistedLayout?.state === "ready" || persistedLayout?.state === "unavailable" ? persistedLayout.state : run.status === RunStatuses.completed || run.status === RunStatuses.partial ? "unavailable" : "pending";
59287
+ return {
59288
+ project: project.name,
59289
+ runId: run.id,
59290
+ status: run.status,
59291
+ phase: siteAuditRunPhase(run.status, attempt, layoutState),
59292
+ attempt: attempt ? {
59293
+ id: attempt.id,
59294
+ state: attempt.state,
59295
+ pagesDiscovered: attempt.pagesDiscovered,
59296
+ pagesFetched: attempt.pagesFetched,
59297
+ pagesEligible: attempt.pagesEligible,
59298
+ pagesErrored: attempt.pagesErrored,
59299
+ edgesDiscovered: attempt.edgesDiscovered,
59300
+ lastUpdatedAt: attempt.updatedAt,
59301
+ startedAt: attempt.startedAt,
59302
+ finishedAt: attempt.finishedAt,
59303
+ error: attempt.error
59304
+ } : null,
59305
+ layout: {
59306
+ state: layoutState,
59307
+ layoutVersion: persistedLayout?.layoutVersion ?? null,
59308
+ failureCode: persistedLayout?.failureCode ?? null,
59309
+ updatedAt: persistedLayout?.updatedAt ?? null
59310
+ },
59311
+ error: run.error ?? attempt?.error ?? null
59312
+ };
59313
+ });
59314
+ app.get("/projects/:name/technical-aeo/runs/:runId/page-health-preview", async (request) => app.db.transaction((tx) => {
59315
+ const project = tx.select().from(projects).where(eq56(projects.name, request.params.name)).get();
59316
+ if (!project) throw notFound("Project", request.params.name);
59317
+ const run = tx.select().from(runs).where(and44(
59318
+ eq56(runs.id, request.params.runId),
59319
+ eq56(runs.projectId, project.id),
59320
+ eq56(runs.kind, RunKinds["site-audit"]),
59321
+ notProbeRun()
59322
+ )).get();
59323
+ if (!run) throw notFound("Site audit run", request.params.runId);
59324
+ const attempt = tx.select().from(siteCrawlAttempts).where(and44(
59325
+ eq56(siteCrawlAttempts.projectId, project.id),
59326
+ eq56(siteCrawlAttempts.runId, run.id)
59327
+ )).orderBy(desc24(siteCrawlAttempts.attemptNumber), desc24(siteCrawlAttempts.updatedAt)).limit(1).get();
59328
+ const state = run.status === RunStatuses.queued ? "waiting" : run.status === RunStatuses.running ? "collecting" : "terminal";
59329
+ const base = {
59330
+ project: project.name,
59331
+ runId: run.id,
59332
+ status: run.status,
59333
+ state,
59334
+ attemptId: attempt?.id ?? null,
59335
+ updatedAt: attempt?.updatedAt ?? null
59336
+ };
59337
+ if (!attempt) {
59338
+ return { ...base, pagesAudited: 0, examples: [] };
59339
+ }
59340
+ const auditedWhere = and44(
59341
+ eq56(siteCrawlPages.projectId, project.id),
59342
+ eq56(siteCrawlPages.runId, run.id),
59343
+ eq56(siteCrawlPages.attemptId, attempt.id),
59344
+ eq56(siteCrawlPages.auditState, "success")
59345
+ );
59346
+ const pagesAudited = tx.select({ value: count() }).from(siteCrawlPages).where(auditedWhere).get()?.value ?? 0;
59347
+ if (state !== "collecting") {
59348
+ return { ...base, pagesAudited, examples: [] };
59349
+ }
59350
+ const candidates = tx.select({
59351
+ nodeKey: siteCrawlPages.nodeKey,
59352
+ url: siteCrawlPages.url,
59353
+ auditScore: siteCrawlPages.auditScore,
59354
+ auditFields: siteCrawlPages.auditFields
59355
+ }).from(siteCrawlPages).where(and44(
59356
+ auditedWhere,
59357
+ isNotNull3(siteCrawlPages.auditScore),
59358
+ lt8(siteCrawlPages.auditScore, 70)
59359
+ )).orderBy(asc10(siteCrawlPages.auditScore), asc10(siteCrawlPages.nodeKey)).limit(LIVE_PAGE_HEALTH_CANDIDATE_LIMIT).all();
59360
+ const examples = candidates.flatMap((candidate) => {
59361
+ if (candidate.auditScore == null) return [];
59362
+ const checksNeedingAttention = countLivePageHealthChecks(candidate.auditFields);
59363
+ return checksNeedingAttention > 0 ? [{
59364
+ nodeKey: candidate.nodeKey,
59365
+ url: candidate.url,
59366
+ auditScore: candidate.auditScore,
59367
+ checksNeedingAttention
59368
+ }] : [];
59369
+ }).slice(0, LIVE_PAGE_HEALTH_EXAMPLE_LIMIT);
59370
+ return { ...base, pagesAudited, examples };
59371
+ }));
58541
59372
  app.post("/projects/:name/technical-aeo/runs", async (request) => {
58542
59373
  const project = resolveProject(app.db, request.params.name);
58543
59374
  const parsed = siteAuditRunRequestSchema.safeParse(request.body ?? {});
58544
59375
  if (!parsed.success) {
58545
59376
  throw validationError(parsed.error.issues[0]?.message ?? "Invalid site-audit request");
58546
59377
  }
59378
+ if (!opts.onSiteAuditRequested) {
59379
+ throw missingDependency("Site Health execution is not available on this deployment.", {
59380
+ reason: "no-site-audit-handler"
59381
+ });
59382
+ }
58547
59383
  const effectiveRequest = normalizeSiteAuditRunRequest(parsed.data);
58548
59384
  const identityKey = siteAuditRequestIdentity(effectiveRequest);
58549
59385
  const result = app.db.transaction((tx) => {
@@ -58594,7 +59430,7 @@ async function technicalAeoRoutes(app, opts) {
58594
59430
  return { runId, status: RunStatuses.queued, created: true };
58595
59431
  });
58596
59432
  if (result.created) {
58597
- opts.onSiteAuditRequested?.(result.runId, project.id, {
59433
+ opts.onSiteAuditRequested(result.runId, project.id, {
58598
59434
  sitemapUrl: effectiveRequest.sitemapUrl ?? void 0,
58599
59435
  limit: parsed.data.limit,
58600
59436
  maxPages: effectiveRequest.maxPages,
@@ -59999,6 +60835,7 @@ export {
59999
60835
  latestQueryBasketRevision,
60000
60836
  ensureCurrentQueryBasketRevision,
60001
60837
  queueRunIfProjectIdle,
60838
+ resolveMeasurementSitemapTarget,
60002
60839
  measurementRunCompleteness,
60003
60840
  nextRunFromCron,
60004
60841
  resolveWebhookTarget,