@canonry/canonry 4.154.1 → 4.156.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agent-workspace/AGENTS.md +2 -2
- package/assets/agent-workspace/skills/aero/references/orchestration.md +3 -3
- package/assets/agent-workspace/skills/canonry/SKILL.md +2 -2
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +27 -7
- package/assets/assets/{AuditHistoryPanel-WKf-vvmg.js → AuditHistoryPanel-gmJMVlXX.js} +1 -1
- package/assets/assets/{BacklinksPage-By06DbHN.js → BacklinksPage-Bv2sTCyN.js} +1 -1
- package/assets/assets/{HistoryPage-C_PcPiyX.js → HistoryPage-13q7GkiI.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-BHnNegEa.js → MeasurementPropertyPage-BgBqXT1p.js} +1 -1
- package/assets/assets/ProjectPage-EsGwKhot.js +15 -0
- package/assets/assets/{RunRow-9qn5Eer_.js → RunRow-bPzapIGJ.js} +1 -1
- package/assets/assets/{RunsPage-FxMTprK1.js → RunsPage-Dqfz18lH.js} +1 -1
- package/assets/assets/{SettingsPage-Cr7_PrNW.js → SettingsPage-_wMecn-V.js} +1 -1
- package/assets/assets/{TrafficPage-D0qk9uMN.js → TrafficPage-RYBdsOLB.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-CXa_3_Gy.js → TrafficSourceDetailPage-DpeMMyKU.js} +1 -1
- package/assets/assets/{extract-error-message-RkTEH1Vb.js → extract-error-message-DUarAlyD.js} +1 -1
- package/assets/assets/index-BxegEwic.css +1 -0
- package/assets/assets/index-DP-fT2uI.js +83 -0
- package/assets/assets/react-sigma_core.esm.min-DnypJl5w.js +312 -0
- package/assets/assets/vendor-lucide-BtYdosYk.js +1 -0
- package/assets/assets/{vendor-markdown-CpDJch4G.js → vendor-markdown-D274_wsI.js} +1 -1
- package/assets/assets/{vendor-radix-DnzzbllM.js → vendor-radix-BxMXSwbC.js} +1 -1
- package/assets/assets/{vendor-recharts-33ACLxsD.js → vendor-recharts-C4ktw_aE.js} +1 -1
- package/assets/assets/{vendor-tanstack-BjS7iY3g.js → vendor-tanstack-pFxw0eGP.js} +1 -1
- package/assets/index.html +7 -7
- package/dist/{chunk-KRMLGFY3.js → chunk-ABNN2ZVR.js} +2438 -114
- package/dist/{chunk-NNW7TJUZ.js → chunk-AMPT2EP7.js} +527 -5
- package/dist/{chunk-ANSQ6JNO.js → chunk-CPASKGUE.js} +563 -4
- package/dist/{chunk-JOP2DGXL.js → chunk-ZWESOFA5.js} +1230 -310
- package/dist/cli.js +742 -8
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-ZY5KOSZV.js → intelligence-service-WHC7B32V.js} +2 -2
- package/dist/mcp.js +4 -4
- package/package.json +13 -11
- package/assets/assets/ProjectPage-VTbW21dz.js +0 -12
- package/assets/assets/index-DyL54s8x.css +0 -1
- package/assets/assets/index-nSWJ9yka.js +0 -83
- package/assets/assets/vendor-lucide-ByU7vBHt.js +0 -1
|
@@ -72,9 +72,23 @@ import {
|
|
|
72
72
|
RunKinds,
|
|
73
73
|
RunStatuses,
|
|
74
74
|
RunTriggers,
|
|
75
|
+
SITE_CRAWL_GRAPH_DEFAULT_MAX_EDGES,
|
|
76
|
+
SITE_CRAWL_GRAPH_DEFAULT_MAX_NODES,
|
|
77
|
+
SITE_CRAWL_GRAPH_MAX_EDGES,
|
|
78
|
+
SITE_CRAWL_GRAPH_MAX_NODES,
|
|
79
|
+
SITE_HEALTH_CHANGES_DEFAULT_LIMIT,
|
|
80
|
+
SITE_HEALTH_CHANGES_MAX_LIMIT,
|
|
81
|
+
SITE_HEALTH_PATH_DEFAULT_MAX_DEPTH,
|
|
82
|
+
SITE_HEALTH_PATH_MAX_DEPTH,
|
|
83
|
+
SITE_HEALTH_PATH_MAX_VISITED_NODES,
|
|
84
|
+
SITE_HEALTH_SUBGRAPH_DEFAULT_MAX_EDGES,
|
|
85
|
+
SITE_HEALTH_SUBGRAPH_DEFAULT_MAX_NODES,
|
|
86
|
+
SITE_HEALTH_SUBGRAPH_MAX_EDGES,
|
|
87
|
+
SITE_HEALTH_SUBGRAPH_MAX_NODES,
|
|
75
88
|
SKILL_MANIFEST_FILENAME,
|
|
76
89
|
SchedulableRunKinds,
|
|
77
90
|
SiteAuditTrendDirections,
|
|
91
|
+
SiteCrawlFetchedStates,
|
|
78
92
|
TrafficEventConfidences,
|
|
79
93
|
TrafficEventKinds,
|
|
80
94
|
TrafficEvidenceKinds,
|
|
@@ -209,6 +223,7 @@ import {
|
|
|
209
223
|
deltaPercent,
|
|
210
224
|
deltaTone,
|
|
211
225
|
deriveCitedUrlCandidates,
|
|
226
|
+
deriveSiteHealthState,
|
|
212
227
|
deriveWinnabilityClass,
|
|
213
228
|
determineAnswerMentioned,
|
|
214
229
|
discoveryBucketSchema,
|
|
@@ -228,6 +243,7 @@ import {
|
|
|
228
243
|
escapeLikePattern,
|
|
229
244
|
evaluateModelPointerExposure,
|
|
230
245
|
extractAnswerMentions,
|
|
246
|
+
factorStatusFromScore,
|
|
231
247
|
filterBrandedSeedCandidates,
|
|
232
248
|
filterCapturedCitedUrls,
|
|
233
249
|
findDuplicateLocationLabels,
|
|
@@ -403,6 +419,7 @@ import {
|
|
|
403
419
|
normalizeProjectAliases,
|
|
404
420
|
normalizeProjectDomain,
|
|
405
421
|
normalizeQueryText,
|
|
422
|
+
normalizeSiteAuditRunRequest,
|
|
406
423
|
normalizeUrlPath,
|
|
407
424
|
normalizeUserName,
|
|
408
425
|
notFound,
|
|
@@ -468,11 +485,26 @@ import {
|
|
|
468
485
|
segmentCrawlerHits,
|
|
469
486
|
serializeRunError,
|
|
470
487
|
settingsDtoSchema,
|
|
488
|
+
siteAuditPageFactorSchema,
|
|
471
489
|
siteAuditPagesResponseSchema,
|
|
490
|
+
siteAuditRequestIdentity,
|
|
472
491
|
siteAuditRunRequestSchema,
|
|
473
492
|
siteAuditRunResponseSchema,
|
|
474
493
|
siteAuditScoreSchema,
|
|
475
494
|
siteAuditTrendResponseSchema,
|
|
495
|
+
siteCrawlAuditFactorSchema,
|
|
496
|
+
siteCrawlCriticalDefectSchema,
|
|
497
|
+
siteCrawlDeadLinksResponseSchema,
|
|
498
|
+
siteCrawlGraphResponseSchema,
|
|
499
|
+
siteCrawlInternalLinksResponseSchema,
|
|
500
|
+
siteCrawlNeighborsResponseSchema,
|
|
501
|
+
siteCrawlPageAuditSchema,
|
|
502
|
+
siteCrawlPagesResponseSchema,
|
|
503
|
+
siteCrawlStructureResponseSchema,
|
|
504
|
+
siteCrawlSummarySchema,
|
|
505
|
+
siteHealthChangesResponseSchema,
|
|
506
|
+
siteHealthPathResponseSchema,
|
|
507
|
+
siteHealthSubgraphResponseSchema,
|
|
476
508
|
snapshotDiffResponseSchema,
|
|
477
509
|
snapshotListResponseSchema,
|
|
478
510
|
snapshotReportSchema,
|
|
@@ -521,10 +553,10 @@ import {
|
|
|
521
553
|
wordpressSchemaDeployResultDtoSchema,
|
|
522
554
|
wordpressSchemaStatusResultDtoSchema,
|
|
523
555
|
wordpressStatusDtoSchema
|
|
524
|
-
} from "./chunk-
|
|
556
|
+
} from "./chunk-AMPT2EP7.js";
|
|
525
557
|
|
|
526
558
|
// src/intelligence-service.ts
|
|
527
|
-
import { eq as eq57, desc as desc26, asc as asc11, and as and45, ne as ne7, or as
|
|
559
|
+
import { eq as eq57, desc as desc26, asc as asc11, and as and45, ne as ne7, or as or12, inArray as inArray20, gte as gte13, lte as lte10 } from "drizzle-orm";
|
|
528
560
|
|
|
529
561
|
// ../db/src/client.ts
|
|
530
562
|
import { mkdirSync } from "fs";
|
|
@@ -615,6 +647,16 @@ __export(schema_exports, {
|
|
|
615
647
|
schedules: () => schedules,
|
|
616
648
|
siteAuditPages: () => siteAuditPages,
|
|
617
649
|
siteAuditSnapshots: () => siteAuditSnapshots,
|
|
650
|
+
siteCrawlAttempts: () => siteCrawlAttempts,
|
|
651
|
+
siteCrawlEdges: () => siteCrawlEdges,
|
|
652
|
+
siteCrawlEventReceipts: () => siteCrawlEventReceipts,
|
|
653
|
+
siteCrawlFindings: () => siteCrawlFindings,
|
|
654
|
+
siteCrawlGraphEdges: () => siteCrawlGraphEdges,
|
|
655
|
+
siteCrawlGraphLayouts: () => siteCrawlGraphLayouts,
|
|
656
|
+
siteCrawlGraphNodes: () => siteCrawlGraphNodes,
|
|
657
|
+
siteCrawlPages: () => siteCrawlPages,
|
|
658
|
+
siteCrawlRunRequests: () => siteCrawlRunRequests,
|
|
659
|
+
siteCrawlSnapshots: () => siteCrawlSnapshots,
|
|
618
660
|
trafficSources: () => trafficSources,
|
|
619
661
|
usageCounters: () => usageCounters,
|
|
620
662
|
userSessions: () => userSessions,
|
|
@@ -839,6 +881,9 @@ var runs = sqliteTable("runs", {
|
|
|
839
881
|
createdAt: text("created_at").notNull()
|
|
840
882
|
}, (table) => [
|
|
841
883
|
index("idx_runs_project").on(table.projectId),
|
|
884
|
+
// Enables child tables that carry project + run to enforce that the two IDs
|
|
885
|
+
// belong together, rather than trusting a caller-provided project ID.
|
|
886
|
+
uniqueIndex("idx_runs_project_id").on(table.projectId, table.id),
|
|
842
887
|
index("idx_runs_status").on(table.status),
|
|
843
888
|
index("idx_runs_source").on(table.sourceId),
|
|
844
889
|
index("idx_runs_measurement_plan").on(table.projectId, table.measurementPlanVersionId, table.createdAt),
|
|
@@ -1185,6 +1230,319 @@ var siteAuditPages = sqliteTable("site_audit_pages", {
|
|
|
1185
1230
|
index("idx_site_audit_pages_run").on(table.runId),
|
|
1186
1231
|
index("idx_site_audit_pages_project_score").on(table.projectId, table.overallScore)
|
|
1187
1232
|
]);
|
|
1233
|
+
var siteCrawlRunRequests = sqliteTable("site_crawl_run_requests", {
|
|
1234
|
+
runId: text("run_id").primaryKey(),
|
|
1235
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1236
|
+
identityKey: text("identity_key").notNull(),
|
|
1237
|
+
effectiveOptions: text("effective_options", { mode: "json" }).$type().notNull(),
|
|
1238
|
+
createdAt: text("created_at").notNull()
|
|
1239
|
+
}, (table) => [
|
|
1240
|
+
index("idx_site_crawl_run_requests_project").on(table.projectId, table.runId),
|
|
1241
|
+
foreignKey({
|
|
1242
|
+
name: "site_crawl_run_requests_project_run_fk",
|
|
1243
|
+
columns: [table.projectId, table.runId],
|
|
1244
|
+
foreignColumns: [runs.projectId, runs.id]
|
|
1245
|
+
}).onDelete("cascade")
|
|
1246
|
+
]);
|
|
1247
|
+
var siteCrawlAttempts = sqliteTable("site_crawl_attempts", {
|
|
1248
|
+
id: text("id").primaryKey(),
|
|
1249
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1250
|
+
runId: text("run_id").notNull(),
|
|
1251
|
+
attemptNumber: integer("attempt_number").notNull(),
|
|
1252
|
+
state: text("state").notNull().default("queued"),
|
|
1253
|
+
lastEventSequence: integer("last_event_sequence").notNull().default(0),
|
|
1254
|
+
lastEventChecksum: text("last_event_checksum"),
|
|
1255
|
+
pagesDiscovered: integer("pages_discovered").notNull().default(0),
|
|
1256
|
+
pagesFetched: integer("pages_fetched").notNull().default(0),
|
|
1257
|
+
pagesEligible: integer("pages_eligible").notNull().default(0),
|
|
1258
|
+
pagesErrored: integer("pages_errored").notNull().default(0),
|
|
1259
|
+
edgesDiscovered: integer("edges_discovered").notNull().default(0),
|
|
1260
|
+
startedAt: text("started_at"),
|
|
1261
|
+
finishedAt: text("finished_at"),
|
|
1262
|
+
error: text("error"),
|
|
1263
|
+
createdAt: text("created_at").notNull(),
|
|
1264
|
+
updatedAt: text("updated_at").notNull()
|
|
1265
|
+
}, (table) => [
|
|
1266
|
+
uniqueIndex("idx_site_crawl_attempts_run_number").on(table.runId, table.attemptNumber),
|
|
1267
|
+
uniqueIndex("idx_site_crawl_attempts_project_run_id").on(table.projectId, table.runId, table.id),
|
|
1268
|
+
index("idx_site_crawl_attempts_project_run").on(table.projectId, table.runId),
|
|
1269
|
+
foreignKey({
|
|
1270
|
+
name: "site_crawl_attempts_project_run_fk",
|
|
1271
|
+
columns: [table.projectId, table.runId],
|
|
1272
|
+
foreignColumns: [runs.projectId, runs.id]
|
|
1273
|
+
}).onDelete("cascade")
|
|
1274
|
+
]);
|
|
1275
|
+
var siteCrawlSnapshots = sqliteTable("site_crawl_snapshots", {
|
|
1276
|
+
id: text("id").primaryKey(),
|
|
1277
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1278
|
+
runId: text("run_id").notNull(),
|
|
1279
|
+
attemptId: text("attempt_id"),
|
|
1280
|
+
rootUrl: text("root_url").notNull(),
|
|
1281
|
+
/** Original requested crawl root; null for snapshots written before v128. */
|
|
1282
|
+
requestedRootUrl: text("requested_root_url"),
|
|
1283
|
+
crawlSchemaVersion: text("crawl_schema_version").notNull().default("1.0"),
|
|
1284
|
+
engineVersion: text("engine_version").notNull().default(""),
|
|
1285
|
+
normalizationVersion: text("normalization_version").notNull().default(""),
|
|
1286
|
+
indexabilityVersion: text("indexability_version").notNull().default(""),
|
|
1287
|
+
linkScoreVersion: text("link_score_version").notNull().default(""),
|
|
1288
|
+
effectiveOptions: text("effective_options", { mode: "json" }).$type().notNull().default({}),
|
|
1289
|
+
pageBudget: integer("page_budget"),
|
|
1290
|
+
edgeBudget: integer("edge_budget"),
|
|
1291
|
+
maxDepth: integer("max_depth"),
|
|
1292
|
+
checkDeadLinks: integer("check_dead_links", { mode: "boolean" }).notNull().default(false),
|
|
1293
|
+
complete: integer("complete", { mode: "boolean" }).notNull().default(false),
|
|
1294
|
+
termination: text("termination").notNull().default("unknown"),
|
|
1295
|
+
detailsAvailable: integer("details_available", { mode: "boolean" }).notNull().default(false),
|
|
1296
|
+
pagesDiscovered: integer("pages_discovered").notNull().default(0),
|
|
1297
|
+
pagesFetched: integer("pages_fetched").notNull().default(0),
|
|
1298
|
+
pagesEligible: integer("pages_eligible").notNull().default(0),
|
|
1299
|
+
pagesErrored: integer("pages_errored").notNull().default(0),
|
|
1300
|
+
edgesDiscovered: integer("edges_discovered").notNull().default(0),
|
|
1301
|
+
findingsCount: integer("findings_count").notNull().default(0),
|
|
1302
|
+
deadLinkState: text("dead_link_state").notNull().default("disabled"),
|
|
1303
|
+
deadLinksChecked: integer("dead_links_checked").notNull().default(0),
|
|
1304
|
+
deadLinksFound: integer("dead_links_found").notNull().default(0),
|
|
1305
|
+
createdAt: text("created_at").notNull(),
|
|
1306
|
+
updatedAt: text("updated_at").notNull()
|
|
1307
|
+
}, (table) => [
|
|
1308
|
+
uniqueIndex("idx_site_crawl_snapshots_run").on(table.runId),
|
|
1309
|
+
index("idx_site_crawl_snapshots_project_created").on(table.projectId, table.createdAt),
|
|
1310
|
+
foreignKey({
|
|
1311
|
+
name: "site_crawl_snapshots_project_run_fk",
|
|
1312
|
+
columns: [table.projectId, table.runId],
|
|
1313
|
+
foreignColumns: [runs.projectId, runs.id]
|
|
1314
|
+
}).onDelete("cascade"),
|
|
1315
|
+
foreignKey({
|
|
1316
|
+
name: "site_crawl_snapshots_attempt_fk",
|
|
1317
|
+
columns: [table.projectId, table.runId, table.attemptId],
|
|
1318
|
+
foreignColumns: [siteCrawlAttempts.projectId, siteCrawlAttempts.runId, siteCrawlAttempts.id]
|
|
1319
|
+
})
|
|
1320
|
+
]);
|
|
1321
|
+
var siteCrawlPages = sqliteTable("site_crawl_pages", {
|
|
1322
|
+
id: text("id").primaryKey(),
|
|
1323
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1324
|
+
runId: text("run_id").notNull(),
|
|
1325
|
+
attemptId: text("attempt_id").notNull(),
|
|
1326
|
+
nodeKey: text("node_key").notNull(),
|
|
1327
|
+
url: text("url").notNull(),
|
|
1328
|
+
path: text("path").notNull(),
|
|
1329
|
+
parentPath: text("parent_path").notNull(),
|
|
1330
|
+
discoverySource: text("discovery_source").notNull().default("crawl"),
|
|
1331
|
+
discoveryProvenance: text("discovery_provenance", { mode: "json" }).$type().notNull().default([]),
|
|
1332
|
+
sitemapMetadata: text("sitemap_metadata", { mode: "json" }).$type().notNull().default({}),
|
|
1333
|
+
fetchState: text("fetch_state").notNull().default("queued"),
|
|
1334
|
+
fetchedAt: text("fetched_at"),
|
|
1335
|
+
httpStatus: integer("http_status"),
|
|
1336
|
+
contentType: text("content_type"),
|
|
1337
|
+
finalUrl: text("final_url"),
|
|
1338
|
+
redirectChain: text("redirect_chain", { mode: "json" }).$type().notNull().default([]),
|
|
1339
|
+
directives: text("directives", { mode: "json" }).$type().notNull().default({}),
|
|
1340
|
+
canonicalUrl: text("canonical_url"),
|
|
1341
|
+
canonicalNodeKey: text("canonical_node_key"),
|
|
1342
|
+
indexabilityState: text("indexability_state").notNull().default("unknown"),
|
|
1343
|
+
indexabilityReasons: text("indexability_reasons", { mode: "json" }).$type().notNull().default([]),
|
|
1344
|
+
auditState: text("audit_state").notNull().default("pending"),
|
|
1345
|
+
auditScore: real("audit_score"),
|
|
1346
|
+
auditFields: text("audit_fields", { mode: "json" }).$type().notNull().default({}),
|
|
1347
|
+
inventoryEligible: integer("inventory_eligible", { mode: "boolean" }).notNull().default(false),
|
|
1348
|
+
depth: integer("depth"),
|
|
1349
|
+
inboundUniqueEdges: integer("inbound_unique_edges").notNull().default(0),
|
|
1350
|
+
outboundUniqueEdges: integer("outbound_unique_edges").notNull().default(0),
|
|
1351
|
+
inboundOccurrences: integer("inbound_occurrences").notNull().default(0),
|
|
1352
|
+
outboundOccurrences: integer("outbound_occurrences").notNull().default(0),
|
|
1353
|
+
linkScoreRaw: real("link_score_raw"),
|
|
1354
|
+
linkScoreNormalized: real("link_score_normalized"),
|
|
1355
|
+
createdAt: text("created_at").notNull(),
|
|
1356
|
+
updatedAt: text("updated_at").notNull()
|
|
1357
|
+
}, (table) => [
|
|
1358
|
+
uniqueIndex("idx_site_crawl_pages_attempt_node").on(table.projectId, table.runId, table.attemptId, table.nodeKey),
|
|
1359
|
+
index("idx_site_crawl_pages_read").on(table.projectId, table.runId, table.attemptId, table.inventoryEligible, table.auditScore, table.url),
|
|
1360
|
+
index("idx_site_crawl_pages_parent").on(table.projectId, table.runId, table.attemptId, table.parentPath, table.path),
|
|
1361
|
+
index("idx_site_crawl_pages_url").on(table.projectId, table.runId, table.attemptId, table.url),
|
|
1362
|
+
foreignKey({
|
|
1363
|
+
name: "site_crawl_pages_attempt_fk",
|
|
1364
|
+
columns: [table.projectId, table.runId, table.attemptId],
|
|
1365
|
+
foreignColumns: [siteCrawlAttempts.projectId, siteCrawlAttempts.runId, siteCrawlAttempts.id]
|
|
1366
|
+
}).onDelete("cascade")
|
|
1367
|
+
]);
|
|
1368
|
+
var siteCrawlEdges = sqliteTable("site_crawl_edges", {
|
|
1369
|
+
id: text("id").primaryKey(),
|
|
1370
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1371
|
+
runId: text("run_id").notNull(),
|
|
1372
|
+
attemptId: text("attempt_id").notNull(),
|
|
1373
|
+
edgeKey: text("edge_key").notNull(),
|
|
1374
|
+
sourceNodeKey: text("source_node_key").notNull(),
|
|
1375
|
+
sourceUrl: text("source_url").notNull(),
|
|
1376
|
+
targetNodeKey: text("target_node_key"),
|
|
1377
|
+
targetUrl: text("target_url").notNull(),
|
|
1378
|
+
relation: text("relation").notNull().default("link"),
|
|
1379
|
+
internal: integer("internal", { mode: "boolean" }).notNull().default(true),
|
|
1380
|
+
followable: integer("followable", { mode: "boolean" }).notNull().default(true),
|
|
1381
|
+
occurrences: integer("occurrences").notNull().default(1),
|
|
1382
|
+
followableOccurrences: integer("followable_occurrences").notNull().default(1),
|
|
1383
|
+
nofollowOccurrences: integer("nofollow_occurrences").notNull().default(0),
|
|
1384
|
+
anchors: text("anchors", { mode: "json" }).$type().notNull().default([]),
|
|
1385
|
+
createdAt: text("created_at").notNull(),
|
|
1386
|
+
updatedAt: text("updated_at").notNull()
|
|
1387
|
+
}, (table) => [
|
|
1388
|
+
uniqueIndex("idx_site_crawl_edges_attempt_key").on(table.projectId, table.runId, table.attemptId, table.edgeKey),
|
|
1389
|
+
index("idx_site_crawl_edges_outbound").on(table.projectId, table.runId, table.attemptId, table.sourceNodeKey, table.edgeKey),
|
|
1390
|
+
index("idx_site_crawl_edges_inbound").on(table.projectId, table.runId, table.attemptId, table.targetNodeKey, table.edgeKey),
|
|
1391
|
+
index("idx_site_crawl_edges_source_url").on(table.projectId, table.runId, table.attemptId, table.sourceUrl),
|
|
1392
|
+
index("idx_site_crawl_edges_target_url").on(table.projectId, table.runId, table.attemptId, table.targetUrl),
|
|
1393
|
+
// Publish-time graph sampling filters this exact scope and consumes the
|
|
1394
|
+
// highest-occurrence internal anchors first. The mixed-direction suffix
|
|
1395
|
+
// avoids a million-row temporary ORDER BY at the maximum crawl budget.
|
|
1396
|
+
index("idx_site_crawl_edges_graph_sample").on(
|
|
1397
|
+
table.projectId,
|
|
1398
|
+
table.runId,
|
|
1399
|
+
table.attemptId,
|
|
1400
|
+
table.internal,
|
|
1401
|
+
table.relation,
|
|
1402
|
+
sql`${table.occurrences} desc`,
|
|
1403
|
+
table.edgeKey
|
|
1404
|
+
),
|
|
1405
|
+
foreignKey({
|
|
1406
|
+
name: "site_crawl_edges_attempt_fk",
|
|
1407
|
+
columns: [table.projectId, table.runId, table.attemptId],
|
|
1408
|
+
foreignColumns: [siteCrawlAttempts.projectId, siteCrawlAttempts.runId, siteCrawlAttempts.id]
|
|
1409
|
+
}).onDelete("cascade")
|
|
1410
|
+
]);
|
|
1411
|
+
var siteCrawlGraphLayouts = sqliteTable("site_crawl_graph_layouts", {
|
|
1412
|
+
id: text("id").primaryKey(),
|
|
1413
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1414
|
+
runId: text("run_id").notNull(),
|
|
1415
|
+
attemptId: text("attempt_id").notNull(),
|
|
1416
|
+
state: text("state").notNull(),
|
|
1417
|
+
layoutVersion: text("layout_version"),
|
|
1418
|
+
failureCode: text("failure_code"),
|
|
1419
|
+
totalNodes: integer("total_nodes").notNull().default(0),
|
|
1420
|
+
totalEdges: integer("total_edges").notNull().default(0),
|
|
1421
|
+
nodeCount: integer("node_count").notNull().default(0),
|
|
1422
|
+
edgeCount: integer("edge_count").notNull().default(0),
|
|
1423
|
+
createdAt: text("created_at").notNull(),
|
|
1424
|
+
updatedAt: text("updated_at").notNull()
|
|
1425
|
+
}, (table) => [
|
|
1426
|
+
uniqueIndex("idx_site_crawl_graph_layouts_attempt").on(table.projectId, table.runId, table.attemptId),
|
|
1427
|
+
index("idx_site_crawl_graph_layouts_run").on(table.projectId, table.runId),
|
|
1428
|
+
check("site_crawl_graph_layouts_state_check", sql`${table.state} in ('ready', 'unavailable')`),
|
|
1429
|
+
foreignKey({
|
|
1430
|
+
name: "site_crawl_graph_layouts_attempt_fk",
|
|
1431
|
+
columns: [table.projectId, table.runId, table.attemptId],
|
|
1432
|
+
foreignColumns: [siteCrawlAttempts.projectId, siteCrawlAttempts.runId, siteCrawlAttempts.id]
|
|
1433
|
+
}).onDelete("cascade")
|
|
1434
|
+
]);
|
|
1435
|
+
var siteCrawlGraphNodes = sqliteTable("site_crawl_graph_nodes", {
|
|
1436
|
+
id: text("id").primaryKey(),
|
|
1437
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1438
|
+
runId: text("run_id").notNull(),
|
|
1439
|
+
attemptId: text("attempt_id").notNull(),
|
|
1440
|
+
nodeKey: text("node_key").notNull(),
|
|
1441
|
+
sampleRank: integer("sample_rank").notNull(),
|
|
1442
|
+
x: real("x").notNull(),
|
|
1443
|
+
y: real("y").notNull(),
|
|
1444
|
+
createdAt: text("created_at").notNull()
|
|
1445
|
+
}, (table) => [
|
|
1446
|
+
uniqueIndex("idx_site_crawl_graph_nodes_attempt_node").on(table.projectId, table.runId, table.attemptId, table.nodeKey),
|
|
1447
|
+
uniqueIndex("idx_site_crawl_graph_nodes_attempt_rank").on(table.projectId, table.runId, table.attemptId, table.sampleRank),
|
|
1448
|
+
index("idx_site_crawl_graph_nodes_read").on(table.projectId, table.runId, table.attemptId, table.sampleRank),
|
|
1449
|
+
check("site_crawl_graph_nodes_rank_check", sql`${table.sampleRank} >= 0`),
|
|
1450
|
+
foreignKey({
|
|
1451
|
+
name: "site_crawl_graph_nodes_layout_fk",
|
|
1452
|
+
columns: [table.projectId, table.runId, table.attemptId],
|
|
1453
|
+
foreignColumns: [siteCrawlGraphLayouts.projectId, siteCrawlGraphLayouts.runId, siteCrawlGraphLayouts.attemptId]
|
|
1454
|
+
}).onDelete("cascade"),
|
|
1455
|
+
foreignKey({
|
|
1456
|
+
name: "site_crawl_graph_nodes_page_fk",
|
|
1457
|
+
columns: [table.projectId, table.runId, table.attemptId, table.nodeKey],
|
|
1458
|
+
foreignColumns: [siteCrawlPages.projectId, siteCrawlPages.runId, siteCrawlPages.attemptId, siteCrawlPages.nodeKey]
|
|
1459
|
+
}).onDelete("cascade")
|
|
1460
|
+
]);
|
|
1461
|
+
var siteCrawlGraphEdges = sqliteTable("site_crawl_graph_edges", {
|
|
1462
|
+
id: text("id").primaryKey(),
|
|
1463
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1464
|
+
runId: text("run_id").notNull(),
|
|
1465
|
+
attemptId: text("attempt_id").notNull(),
|
|
1466
|
+
edgeKey: text("edge_key").notNull(),
|
|
1467
|
+
sampleRank: integer("sample_rank").notNull(),
|
|
1468
|
+
sourceNodeKey: text("source_node_key").notNull(),
|
|
1469
|
+
targetNodeKey: text("target_node_key").notNull(),
|
|
1470
|
+
followable: integer("followable", { mode: "boolean" }).notNull(),
|
|
1471
|
+
occurrences: integer("occurrences").notNull(),
|
|
1472
|
+
createdAt: text("created_at").notNull()
|
|
1473
|
+
}, (table) => [
|
|
1474
|
+
uniqueIndex("idx_site_crawl_graph_edges_attempt_edge").on(table.projectId, table.runId, table.attemptId, table.edgeKey),
|
|
1475
|
+
uniqueIndex("idx_site_crawl_graph_edges_attempt_rank").on(table.projectId, table.runId, table.attemptId, table.sampleRank),
|
|
1476
|
+
index("idx_site_crawl_graph_edges_read").on(table.projectId, table.runId, table.attemptId, table.sampleRank),
|
|
1477
|
+
// These FKs otherwise make a layout cascade scan graph edges once per node.
|
|
1478
|
+
index("idx_site_crawl_graph_edges_source_node").on(table.projectId, table.runId, table.attemptId, table.sourceNodeKey),
|
|
1479
|
+
index("idx_site_crawl_graph_edges_target_node").on(table.projectId, table.runId, table.attemptId, table.targetNodeKey),
|
|
1480
|
+
check("site_crawl_graph_edges_rank_check", sql`${table.sampleRank} >= 0`),
|
|
1481
|
+
check("site_crawl_graph_edges_occurrences_check", sql`${table.occurrences} > 0`),
|
|
1482
|
+
foreignKey({
|
|
1483
|
+
name: "site_crawl_graph_edges_layout_fk",
|
|
1484
|
+
columns: [table.projectId, table.runId, table.attemptId],
|
|
1485
|
+
foreignColumns: [siteCrawlGraphLayouts.projectId, siteCrawlGraphLayouts.runId, siteCrawlGraphLayouts.attemptId]
|
|
1486
|
+
}).onDelete("cascade"),
|
|
1487
|
+
foreignKey({
|
|
1488
|
+
name: "site_crawl_graph_edges_source_node_fk",
|
|
1489
|
+
columns: [table.projectId, table.runId, table.attemptId, table.sourceNodeKey],
|
|
1490
|
+
foreignColumns: [siteCrawlGraphNodes.projectId, siteCrawlGraphNodes.runId, siteCrawlGraphNodes.attemptId, siteCrawlGraphNodes.nodeKey]
|
|
1491
|
+
}).onDelete("cascade"),
|
|
1492
|
+
foreignKey({
|
|
1493
|
+
name: "site_crawl_graph_edges_target_node_fk",
|
|
1494
|
+
columns: [table.projectId, table.runId, table.attemptId, table.targetNodeKey],
|
|
1495
|
+
foreignColumns: [siteCrawlGraphNodes.projectId, siteCrawlGraphNodes.runId, siteCrawlGraphNodes.attemptId, siteCrawlGraphNodes.nodeKey]
|
|
1496
|
+
}).onDelete("cascade"),
|
|
1497
|
+
foreignKey({
|
|
1498
|
+
name: "site_crawl_graph_edges_canonical_edge_fk",
|
|
1499
|
+
columns: [table.projectId, table.runId, table.attemptId, table.edgeKey],
|
|
1500
|
+
foreignColumns: [siteCrawlEdges.projectId, siteCrawlEdges.runId, siteCrawlEdges.attemptId, siteCrawlEdges.edgeKey]
|
|
1501
|
+
}).onDelete("cascade")
|
|
1502
|
+
]);
|
|
1503
|
+
var siteCrawlFindings = sqliteTable("site_crawl_findings", {
|
|
1504
|
+
id: text("id").primaryKey(),
|
|
1505
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1506
|
+
runId: text("run_id").notNull(),
|
|
1507
|
+
attemptId: text("attempt_id").notNull(),
|
|
1508
|
+
findingKey: text("finding_key").notNull(),
|
|
1509
|
+
findingType: text("finding_type").notNull(),
|
|
1510
|
+
severity: text("severity").notNull().default("info"),
|
|
1511
|
+
sourceNodeKey: text("source_node_key"),
|
|
1512
|
+
sourceUrl: text("source_url"),
|
|
1513
|
+
targetNodeKey: text("target_node_key"),
|
|
1514
|
+
targetUrl: text("target_url"),
|
|
1515
|
+
evidence: text("evidence", { mode: "json" }).$type().notNull().default({}),
|
|
1516
|
+
createdAt: text("created_at").notNull(),
|
|
1517
|
+
updatedAt: text("updated_at").notNull()
|
|
1518
|
+
}, (table) => [
|
|
1519
|
+
uniqueIndex("idx_site_crawl_findings_attempt_key").on(table.projectId, table.runId, table.attemptId, table.findingKey),
|
|
1520
|
+
index("idx_site_crawl_findings_type").on(table.projectId, table.runId, table.attemptId, table.findingType, table.findingKey),
|
|
1521
|
+
foreignKey({
|
|
1522
|
+
name: "site_crawl_findings_attempt_fk",
|
|
1523
|
+
columns: [table.projectId, table.runId, table.attemptId],
|
|
1524
|
+
foreignColumns: [siteCrawlAttempts.projectId, siteCrawlAttempts.runId, siteCrawlAttempts.id]
|
|
1525
|
+
}).onDelete("cascade")
|
|
1526
|
+
]);
|
|
1527
|
+
var siteCrawlEventReceipts = sqliteTable("site_crawl_event_receipts", {
|
|
1528
|
+
id: text("id").primaryKey(),
|
|
1529
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1530
|
+
runId: text("run_id").notNull(),
|
|
1531
|
+
attemptId: text("attempt_id").notNull(),
|
|
1532
|
+
sequence: integer("sequence").notNull(),
|
|
1533
|
+
batchId: text("batch_id").notNull(),
|
|
1534
|
+
checksum: text("checksum").notNull(),
|
|
1535
|
+
receipt: text("receipt", { mode: "json" }).$type().notNull().default({}),
|
|
1536
|
+
createdAt: text("created_at").notNull()
|
|
1537
|
+
}, (table) => [
|
|
1538
|
+
uniqueIndex("idx_site_crawl_receipts_attempt_event").on(table.attemptId, table.sequence, table.batchId),
|
|
1539
|
+
index("idx_site_crawl_receipts_project_run").on(table.projectId, table.runId, table.attemptId),
|
|
1540
|
+
foreignKey({
|
|
1541
|
+
name: "site_crawl_receipts_attempt_fk",
|
|
1542
|
+
columns: [table.projectId, table.runId, table.attemptId],
|
|
1543
|
+
foreignColumns: [siteCrawlAttempts.projectId, siteCrawlAttempts.runId, siteCrawlAttempts.id]
|
|
1544
|
+
}).onDelete("cascade")
|
|
1545
|
+
]);
|
|
1188
1546
|
var bingCoverageSnapshots = sqliteTable("bing_coverage_snapshots", {
|
|
1189
1547
|
id: text("id").primaryKey(),
|
|
1190
1548
|
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
@@ -5096,6 +5454,305 @@ var MIGRATION_VERSIONS = [
|
|
|
5096
5454
|
`ALTER TABLE gsc_coverage_snapshots ADD COLUMN verified_by_inspection INTEGER NOT NULL DEFAULT 0`,
|
|
5097
5455
|
`ALTER TABLE gsc_coverage_snapshots ADD COLUMN derived_from_impressions INTEGER NOT NULL DEFAULT 0`
|
|
5098
5456
|
]
|
|
5457
|
+
},
|
|
5458
|
+
{
|
|
5459
|
+
// Crawl persistence is intentionally separate from the legacy
|
|
5460
|
+
// site_audit_snapshots/site_audit_pages scorecard model. Older engines can
|
|
5461
|
+
// keep reading and writing that legacy surface after a rollback; new tables
|
|
5462
|
+
// only enrich an audit run when the crawler writes them.
|
|
5463
|
+
version: 126,
|
|
5464
|
+
name: "site-crawl-persistence",
|
|
5465
|
+
statements: [
|
|
5466
|
+
// Composite child FKs below enforce that project_id and run_id are a
|
|
5467
|
+
// matched pair. `runs.id` is globally unique already; this named unique
|
|
5468
|
+
// index provides SQLite's required parent key for the composite check.
|
|
5469
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_runs_project_id ON runs(project_id, id)`,
|
|
5470
|
+
`CREATE TABLE IF NOT EXISTS site_crawl_run_requests (
|
|
5471
|
+
run_id TEXT PRIMARY KEY,
|
|
5472
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
5473
|
+
identity_key TEXT NOT NULL,
|
|
5474
|
+
effective_options TEXT NOT NULL,
|
|
5475
|
+
created_at TEXT NOT NULL,
|
|
5476
|
+
FOREIGN KEY (project_id, run_id) REFERENCES runs(project_id, id) ON DELETE CASCADE
|
|
5477
|
+
)`,
|
|
5478
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_run_requests_project
|
|
5479
|
+
ON site_crawl_run_requests(project_id, run_id)`,
|
|
5480
|
+
`CREATE TABLE IF NOT EXISTS site_crawl_attempts (
|
|
5481
|
+
id TEXT PRIMARY KEY,
|
|
5482
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
5483
|
+
run_id TEXT NOT NULL,
|
|
5484
|
+
attempt_number INTEGER NOT NULL,
|
|
5485
|
+
state TEXT NOT NULL DEFAULT 'queued',
|
|
5486
|
+
last_event_sequence INTEGER NOT NULL DEFAULT 0,
|
|
5487
|
+
last_event_checksum TEXT,
|
|
5488
|
+
pages_discovered INTEGER NOT NULL DEFAULT 0,
|
|
5489
|
+
pages_fetched INTEGER NOT NULL DEFAULT 0,
|
|
5490
|
+
pages_eligible INTEGER NOT NULL DEFAULT 0,
|
|
5491
|
+
pages_errored INTEGER NOT NULL DEFAULT 0,
|
|
5492
|
+
edges_discovered INTEGER NOT NULL DEFAULT 0,
|
|
5493
|
+
started_at TEXT,
|
|
5494
|
+
finished_at TEXT,
|
|
5495
|
+
error TEXT,
|
|
5496
|
+
created_at TEXT NOT NULL,
|
|
5497
|
+
updated_at TEXT NOT NULL,
|
|
5498
|
+
FOREIGN KEY (project_id, run_id) REFERENCES runs(project_id, id) ON DELETE CASCADE
|
|
5499
|
+
)`,
|
|
5500
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_attempts_run_number
|
|
5501
|
+
ON site_crawl_attempts(run_id, attempt_number)`,
|
|
5502
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_attempts_project_run_id
|
|
5503
|
+
ON site_crawl_attempts(project_id, run_id, id)`,
|
|
5504
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_attempts_project_run
|
|
5505
|
+
ON site_crawl_attempts(project_id, run_id)`,
|
|
5506
|
+
`CREATE TABLE IF NOT EXISTS site_crawl_snapshots (
|
|
5507
|
+
id TEXT PRIMARY KEY,
|
|
5508
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
5509
|
+
run_id TEXT NOT NULL,
|
|
5510
|
+
attempt_id TEXT,
|
|
5511
|
+
root_url TEXT NOT NULL,
|
|
5512
|
+
crawl_schema_version TEXT NOT NULL DEFAULT '1.0',
|
|
5513
|
+
engine_version TEXT NOT NULL DEFAULT '',
|
|
5514
|
+
normalization_version TEXT NOT NULL DEFAULT '',
|
|
5515
|
+
indexability_version TEXT NOT NULL DEFAULT '',
|
|
5516
|
+
link_score_version TEXT NOT NULL DEFAULT '',
|
|
5517
|
+
effective_options TEXT NOT NULL DEFAULT '{}',
|
|
5518
|
+
page_budget INTEGER,
|
|
5519
|
+
edge_budget INTEGER,
|
|
5520
|
+
max_depth INTEGER,
|
|
5521
|
+
check_dead_links INTEGER NOT NULL DEFAULT 0,
|
|
5522
|
+
complete INTEGER NOT NULL DEFAULT 0,
|
|
5523
|
+
termination TEXT NOT NULL DEFAULT 'unknown',
|
|
5524
|
+
details_available INTEGER NOT NULL DEFAULT 0,
|
|
5525
|
+
pages_discovered INTEGER NOT NULL DEFAULT 0,
|
|
5526
|
+
pages_fetched INTEGER NOT NULL DEFAULT 0,
|
|
5527
|
+
pages_eligible INTEGER NOT NULL DEFAULT 0,
|
|
5528
|
+
pages_errored INTEGER NOT NULL DEFAULT 0,
|
|
5529
|
+
edges_discovered INTEGER NOT NULL DEFAULT 0,
|
|
5530
|
+
findings_count INTEGER NOT NULL DEFAULT 0,
|
|
5531
|
+
dead_link_state TEXT NOT NULL DEFAULT 'disabled',
|
|
5532
|
+
dead_links_checked INTEGER NOT NULL DEFAULT 0,
|
|
5533
|
+
dead_links_found INTEGER NOT NULL DEFAULT 0,
|
|
5534
|
+
created_at TEXT NOT NULL,
|
|
5535
|
+
updated_at TEXT NOT NULL,
|
|
5536
|
+
FOREIGN KEY (project_id, run_id) REFERENCES runs(project_id, id) ON DELETE CASCADE,
|
|
5537
|
+
FOREIGN KEY (project_id, run_id, attempt_id)
|
|
5538
|
+
REFERENCES site_crawl_attempts(project_id, run_id, id)
|
|
5539
|
+
)`,
|
|
5540
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_snapshots_run ON site_crawl_snapshots(run_id)`,
|
|
5541
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_snapshots_project_created
|
|
5542
|
+
ON site_crawl_snapshots(project_id, created_at)`,
|
|
5543
|
+
`CREATE TABLE IF NOT EXISTS site_crawl_pages (
|
|
5544
|
+
id TEXT PRIMARY KEY,
|
|
5545
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
5546
|
+
run_id TEXT NOT NULL,
|
|
5547
|
+
attempt_id TEXT NOT NULL,
|
|
5548
|
+
node_key TEXT NOT NULL,
|
|
5549
|
+
url TEXT NOT NULL,
|
|
5550
|
+
path TEXT NOT NULL,
|
|
5551
|
+
parent_path TEXT NOT NULL,
|
|
5552
|
+
discovery_source TEXT NOT NULL DEFAULT 'crawl',
|
|
5553
|
+
discovery_provenance TEXT NOT NULL DEFAULT '[]',
|
|
5554
|
+
sitemap_metadata TEXT NOT NULL DEFAULT '{}',
|
|
5555
|
+
fetch_state TEXT NOT NULL DEFAULT 'queued',
|
|
5556
|
+
fetched_at TEXT,
|
|
5557
|
+
http_status INTEGER,
|
|
5558
|
+
content_type TEXT,
|
|
5559
|
+
final_url TEXT,
|
|
5560
|
+
redirect_chain TEXT NOT NULL DEFAULT '[]',
|
|
5561
|
+
directives TEXT NOT NULL DEFAULT '{}',
|
|
5562
|
+
canonical_url TEXT,
|
|
5563
|
+
canonical_node_key TEXT,
|
|
5564
|
+
indexability_state TEXT NOT NULL DEFAULT 'unknown',
|
|
5565
|
+
indexability_reasons TEXT NOT NULL DEFAULT '[]',
|
|
5566
|
+
audit_state TEXT NOT NULL DEFAULT 'pending',
|
|
5567
|
+
audit_score REAL,
|
|
5568
|
+
audit_fields TEXT NOT NULL DEFAULT '{}',
|
|
5569
|
+
inventory_eligible INTEGER NOT NULL DEFAULT 0,
|
|
5570
|
+
depth INTEGER,
|
|
5571
|
+
inbound_unique_edges INTEGER NOT NULL DEFAULT 0,
|
|
5572
|
+
outbound_unique_edges INTEGER NOT NULL DEFAULT 0,
|
|
5573
|
+
inbound_occurrences INTEGER NOT NULL DEFAULT 0,
|
|
5574
|
+
outbound_occurrences INTEGER NOT NULL DEFAULT 0,
|
|
5575
|
+
link_score_raw REAL,
|
|
5576
|
+
link_score_normalized REAL,
|
|
5577
|
+
created_at TEXT NOT NULL,
|
|
5578
|
+
updated_at TEXT NOT NULL,
|
|
5579
|
+
FOREIGN KEY (project_id, run_id, attempt_id)
|
|
5580
|
+
REFERENCES site_crawl_attempts(project_id, run_id, id) ON DELETE CASCADE
|
|
5581
|
+
)`,
|
|
5582
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_pages_attempt_node
|
|
5583
|
+
ON site_crawl_pages(project_id, run_id, attempt_id, node_key)`,
|
|
5584
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_pages_read
|
|
5585
|
+
ON site_crawl_pages(project_id, run_id, attempt_id, inventory_eligible, audit_score, url)`,
|
|
5586
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_pages_parent
|
|
5587
|
+
ON site_crawl_pages(project_id, run_id, attempt_id, parent_path, path)`,
|
|
5588
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_pages_url
|
|
5589
|
+
ON site_crawl_pages(project_id, run_id, attempt_id, url)`,
|
|
5590
|
+
`CREATE TABLE IF NOT EXISTS site_crawl_edges (
|
|
5591
|
+
id TEXT PRIMARY KEY,
|
|
5592
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
5593
|
+
run_id TEXT NOT NULL,
|
|
5594
|
+
attempt_id TEXT NOT NULL,
|
|
5595
|
+
edge_key TEXT NOT NULL,
|
|
5596
|
+
source_node_key TEXT NOT NULL,
|
|
5597
|
+
source_url TEXT NOT NULL,
|
|
5598
|
+
target_node_key TEXT,
|
|
5599
|
+
target_url TEXT NOT NULL,
|
|
5600
|
+
relation TEXT NOT NULL DEFAULT 'link',
|
|
5601
|
+
internal INTEGER NOT NULL DEFAULT 1,
|
|
5602
|
+
followable INTEGER NOT NULL DEFAULT 1,
|
|
5603
|
+
occurrences INTEGER NOT NULL DEFAULT 1,
|
|
5604
|
+
followable_occurrences INTEGER NOT NULL DEFAULT 1,
|
|
5605
|
+
nofollow_occurrences INTEGER NOT NULL DEFAULT 0,
|
|
5606
|
+
anchors TEXT NOT NULL DEFAULT '[]',
|
|
5607
|
+
created_at TEXT NOT NULL,
|
|
5608
|
+
updated_at TEXT NOT NULL,
|
|
5609
|
+
FOREIGN KEY (project_id, run_id, attempt_id)
|
|
5610
|
+
REFERENCES site_crawl_attempts(project_id, run_id, id) ON DELETE CASCADE
|
|
5611
|
+
)`,
|
|
5612
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_edges_attempt_key
|
|
5613
|
+
ON site_crawl_edges(project_id, run_id, attempt_id, edge_key)`,
|
|
5614
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_edges_outbound
|
|
5615
|
+
ON site_crawl_edges(project_id, run_id, attempt_id, source_node_key, edge_key)`,
|
|
5616
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_edges_inbound
|
|
5617
|
+
ON site_crawl_edges(project_id, run_id, attempt_id, target_node_key, edge_key)`,
|
|
5618
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_edges_source_url
|
|
5619
|
+
ON site_crawl_edges(project_id, run_id, attempt_id, source_url)`,
|
|
5620
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_edges_target_url
|
|
5621
|
+
ON site_crawl_edges(project_id, run_id, attempt_id, target_url)`,
|
|
5622
|
+
`CREATE TABLE IF NOT EXISTS site_crawl_findings (
|
|
5623
|
+
id TEXT PRIMARY KEY,
|
|
5624
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
5625
|
+
run_id TEXT NOT NULL,
|
|
5626
|
+
attempt_id TEXT NOT NULL,
|
|
5627
|
+
finding_key TEXT NOT NULL,
|
|
5628
|
+
finding_type TEXT NOT NULL,
|
|
5629
|
+
severity TEXT NOT NULL DEFAULT 'info',
|
|
5630
|
+
source_node_key TEXT,
|
|
5631
|
+
source_url TEXT,
|
|
5632
|
+
target_node_key TEXT,
|
|
5633
|
+
target_url TEXT,
|
|
5634
|
+
evidence TEXT NOT NULL DEFAULT '{}',
|
|
5635
|
+
created_at TEXT NOT NULL,
|
|
5636
|
+
updated_at TEXT NOT NULL,
|
|
5637
|
+
FOREIGN KEY (project_id, run_id, attempt_id)
|
|
5638
|
+
REFERENCES site_crawl_attempts(project_id, run_id, id) ON DELETE CASCADE
|
|
5639
|
+
)`,
|
|
5640
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_findings_attempt_key
|
|
5641
|
+
ON site_crawl_findings(project_id, run_id, attempt_id, finding_key)`,
|
|
5642
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_findings_type
|
|
5643
|
+
ON site_crawl_findings(project_id, run_id, attempt_id, finding_type, finding_key)`,
|
|
5644
|
+
`CREATE TABLE IF NOT EXISTS site_crawl_event_receipts (
|
|
5645
|
+
id TEXT PRIMARY KEY,
|
|
5646
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
5647
|
+
run_id TEXT NOT NULL,
|
|
5648
|
+
attempt_id TEXT NOT NULL,
|
|
5649
|
+
sequence INTEGER NOT NULL,
|
|
5650
|
+
batch_id TEXT NOT NULL,
|
|
5651
|
+
checksum TEXT NOT NULL,
|
|
5652
|
+
receipt TEXT NOT NULL DEFAULT '{}',
|
|
5653
|
+
created_at TEXT NOT NULL,
|
|
5654
|
+
FOREIGN KEY (project_id, run_id, attempt_id)
|
|
5655
|
+
REFERENCES site_crawl_attempts(project_id, run_id, id) ON DELETE CASCADE
|
|
5656
|
+
)`,
|
|
5657
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_receipts_attempt_event
|
|
5658
|
+
ON site_crawl_event_receipts(attempt_id, sequence, batch_id)`,
|
|
5659
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_receipts_project_run
|
|
5660
|
+
ON site_crawl_event_receipts(project_id, run_id, attempt_id)`
|
|
5661
|
+
]
|
|
5662
|
+
},
|
|
5663
|
+
{
|
|
5664
|
+
// Graph coordinates are optional derived snapshot data. Separate tables
|
|
5665
|
+
// preserve the canonical crawl-page shape and make a missing row the
|
|
5666
|
+
// truthful compatibility signal for snapshots published before v127.
|
|
5667
|
+
version: 127,
|
|
5668
|
+
name: "site-crawl-persisted-graph-layout",
|
|
5669
|
+
statements: [
|
|
5670
|
+
`CREATE TABLE IF NOT EXISTS site_crawl_graph_layouts (
|
|
5671
|
+
id TEXT PRIMARY KEY,
|
|
5672
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
5673
|
+
run_id TEXT NOT NULL,
|
|
5674
|
+
attempt_id TEXT NOT NULL,
|
|
5675
|
+
state TEXT NOT NULL CHECK (state IN ('ready', 'unavailable')),
|
|
5676
|
+
layout_version TEXT,
|
|
5677
|
+
failure_code TEXT,
|
|
5678
|
+
total_nodes INTEGER NOT NULL DEFAULT 0,
|
|
5679
|
+
total_edges INTEGER NOT NULL DEFAULT 0,
|
|
5680
|
+
node_count INTEGER NOT NULL DEFAULT 0,
|
|
5681
|
+
edge_count INTEGER NOT NULL DEFAULT 0,
|
|
5682
|
+
created_at TEXT NOT NULL,
|
|
5683
|
+
updated_at TEXT NOT NULL,
|
|
5684
|
+
FOREIGN KEY (project_id, run_id, attempt_id)
|
|
5685
|
+
REFERENCES site_crawl_attempts(project_id, run_id, id) ON DELETE CASCADE
|
|
5686
|
+
)`,
|
|
5687
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_graph_layouts_attempt
|
|
5688
|
+
ON site_crawl_graph_layouts(project_id, run_id, attempt_id)`,
|
|
5689
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_graph_layouts_run
|
|
5690
|
+
ON site_crawl_graph_layouts(project_id, run_id)`,
|
|
5691
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_edges_graph_sample
|
|
5692
|
+
ON site_crawl_edges(project_id, run_id, attempt_id, internal, relation, occurrences DESC, edge_key ASC)`,
|
|
5693
|
+
`CREATE TABLE IF NOT EXISTS site_crawl_graph_nodes (
|
|
5694
|
+
id TEXT PRIMARY KEY,
|
|
5695
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
5696
|
+
run_id TEXT NOT NULL,
|
|
5697
|
+
attempt_id TEXT NOT NULL,
|
|
5698
|
+
node_key TEXT NOT NULL,
|
|
5699
|
+
sample_rank INTEGER NOT NULL CHECK (sample_rank >= 0),
|
|
5700
|
+
x REAL NOT NULL,
|
|
5701
|
+
y REAL NOT NULL,
|
|
5702
|
+
created_at TEXT NOT NULL,
|
|
5703
|
+
FOREIGN KEY (project_id, run_id, attempt_id)
|
|
5704
|
+
REFERENCES site_crawl_graph_layouts(project_id, run_id, attempt_id) ON DELETE CASCADE,
|
|
5705
|
+
FOREIGN KEY (project_id, run_id, attempt_id, node_key)
|
|
5706
|
+
REFERENCES site_crawl_pages(project_id, run_id, attempt_id, node_key) ON DELETE CASCADE
|
|
5707
|
+
)`,
|
|
5708
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_graph_nodes_attempt_node
|
|
5709
|
+
ON site_crawl_graph_nodes(project_id, run_id, attempt_id, node_key)`,
|
|
5710
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_graph_nodes_attempt_rank
|
|
5711
|
+
ON site_crawl_graph_nodes(project_id, run_id, attempt_id, sample_rank)`,
|
|
5712
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_graph_nodes_read
|
|
5713
|
+
ON site_crawl_graph_nodes(project_id, run_id, attempt_id, sample_rank)`,
|
|
5714
|
+
`CREATE TABLE IF NOT EXISTS site_crawl_graph_edges (
|
|
5715
|
+
id TEXT PRIMARY KEY,
|
|
5716
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
5717
|
+
run_id TEXT NOT NULL,
|
|
5718
|
+
attempt_id TEXT NOT NULL,
|
|
5719
|
+
edge_key TEXT NOT NULL,
|
|
5720
|
+
sample_rank INTEGER NOT NULL CHECK (sample_rank >= 0),
|
|
5721
|
+
source_node_key TEXT NOT NULL,
|
|
5722
|
+
target_node_key TEXT NOT NULL,
|
|
5723
|
+
followable INTEGER NOT NULL,
|
|
5724
|
+
occurrences INTEGER NOT NULL CHECK (occurrences > 0),
|
|
5725
|
+
created_at TEXT NOT NULL,
|
|
5726
|
+
FOREIGN KEY (project_id, run_id, attempt_id)
|
|
5727
|
+
REFERENCES site_crawl_graph_layouts(project_id, run_id, attempt_id) ON DELETE CASCADE,
|
|
5728
|
+
FOREIGN KEY (project_id, run_id, attempt_id, source_node_key)
|
|
5729
|
+
REFERENCES site_crawl_graph_nodes(project_id, run_id, attempt_id, node_key) ON DELETE CASCADE,
|
|
5730
|
+
FOREIGN KEY (project_id, run_id, attempt_id, target_node_key)
|
|
5731
|
+
REFERENCES site_crawl_graph_nodes(project_id, run_id, attempt_id, node_key) ON DELETE CASCADE,
|
|
5732
|
+
FOREIGN KEY (project_id, run_id, attempt_id, edge_key)
|
|
5733
|
+
REFERENCES site_crawl_edges(project_id, run_id, attempt_id, edge_key) ON DELETE CASCADE
|
|
5734
|
+
)`,
|
|
5735
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_graph_edges_attempt_edge
|
|
5736
|
+
ON site_crawl_graph_edges(project_id, run_id, attempt_id, edge_key)`,
|
|
5737
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_site_crawl_graph_edges_attempt_rank
|
|
5738
|
+
ON site_crawl_graph_edges(project_id, run_id, attempt_id, sample_rank)`,
|
|
5739
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_graph_edges_read
|
|
5740
|
+
ON site_crawl_graph_edges(project_id, run_id, attempt_id, sample_rank)`
|
|
5741
|
+
]
|
|
5742
|
+
},
|
|
5743
|
+
{
|
|
5744
|
+
// Keep the operator-requested root beside the terminal root without
|
|
5745
|
+
// rewriting immutable historical snapshots. The edge indexes make layout
|
|
5746
|
+
// cleanup/cascades linear at the graph's 20k/50k publish cap.
|
|
5747
|
+
version: 128,
|
|
5748
|
+
name: "site-crawl-requested-root-and-graph-edge-cleanup-indexes",
|
|
5749
|
+
statements: [
|
|
5750
|
+
`ALTER TABLE site_crawl_snapshots ADD COLUMN requested_root_url TEXT`,
|
|
5751
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_graph_edges_source_node
|
|
5752
|
+
ON site_crawl_graph_edges(project_id, run_id, attempt_id, source_node_key)`,
|
|
5753
|
+
`CREATE INDEX IF NOT EXISTS idx_site_crawl_graph_edges_target_node
|
|
5754
|
+
ON site_crawl_graph_edges(project_id, run_id, attempt_id, target_node_key)`
|
|
5755
|
+
]
|
|
5099
5756
|
}
|
|
5100
5757
|
];
|
|
5101
5758
|
function addRunsMeasurementPlanVersionForeignKey(tx) {
|
|
@@ -7035,7 +7692,7 @@ function buildMentionShare(snapshots, options) {
|
|
|
7035
7692
|
competitorCounts.set(competitor.domain, 0);
|
|
7036
7693
|
competitorAliases.set(
|
|
7037
7694
|
competitor.domain,
|
|
7038
|
-
competitor.brandTokens.filter((
|
|
7695
|
+
competitor.brandTokens.filter((alias2) => brandKeyFromText(alias2).length >= 3)
|
|
7039
7696
|
);
|
|
7040
7697
|
}
|
|
7041
7698
|
for (const snap of snapshots) {
|
|
@@ -7968,7 +8625,7 @@ function isOverviewRead(url) {
|
|
|
7968
8625
|
}
|
|
7969
8626
|
function isTechnicalAeoRead(url) {
|
|
7970
8627
|
const rest = projectRouteRest(url);
|
|
7971
|
-
return rest === "technical-aeo" || rest === "technical-aeo/pages" || rest === "technical-aeo/trend";
|
|
8628
|
+
return rest === "technical-aeo" || rest === "technical-aeo/pages" || rest === "technical-aeo/trend" || rest === "technical-aeo/crawl" || rest === "technical-aeo/graph" || rest === "technical-aeo/subgraph" || rest === "technical-aeo/path" || rest === "technical-aeo/changes" || rest === "technical-aeo/crawl/pages/audit" || rest === "technical-aeo/crawl/pages" || rest === "technical-aeo/structure" || rest === "technical-aeo/internal-links" || rest === "technical-aeo/internal-links/neighbors" || rest === "technical-aeo/dead-links";
|
|
7972
8629
|
}
|
|
7973
8630
|
function isReportRead(url) {
|
|
7974
8631
|
const rest = projectRouteRest(url);
|
|
@@ -9206,7 +9863,7 @@ function parseCompetitorBatch(value) {
|
|
|
9206
9863
|
|
|
9207
9864
|
// ../api-routes/src/runs.ts
|
|
9208
9865
|
import crypto12 from "crypto";
|
|
9209
|
-
import { and as and6, eq as eq11, asc, desc as desc3, or as or2, sql as sql6 } from "drizzle-orm";
|
|
9866
|
+
import { and as and6, eq as eq11, asc, desc as desc3, inArray as inArray4, or as or2, sql as sql6 } from "drizzle-orm";
|
|
9210
9867
|
import { gte as gte2 } from "drizzle-orm";
|
|
9211
9868
|
|
|
9212
9869
|
// ../api-routes/src/run-queue.ts
|
|
@@ -9336,9 +9993,9 @@ function aliasMatchesAt(textWords, aliasWords, start) {
|
|
|
9336
9993
|
}
|
|
9337
9994
|
function containsAnyAlias(answerText, aliases) {
|
|
9338
9995
|
const textWords = words(answerText);
|
|
9339
|
-
const candidates = aliases.map(words).filter((
|
|
9996
|
+
const candidates = aliases.map(words).filter((alias2) => alias2.length > 0);
|
|
9340
9997
|
for (let start = 0; start < textWords.length; start++) {
|
|
9341
|
-
if (candidates.some((
|
|
9998
|
+
if (candidates.some((alias2) => aliasMatchesAt(textWords, alias2, start))) return true;
|
|
9342
9999
|
}
|
|
9343
10000
|
return false;
|
|
9344
10001
|
}
|
|
@@ -9346,9 +10003,9 @@ function mentionedTargets(answerText, targets) {
|
|
|
9346
10003
|
const result = /* @__PURE__ */ new Set();
|
|
9347
10004
|
if (answerText === null) return result;
|
|
9348
10005
|
const textWords = words(answerText);
|
|
9349
|
-
const aliases = targets.flatMap((target) => target.aliases.map((
|
|
10006
|
+
const aliases = targets.flatMap((target) => target.aliases.map((alias2) => ({ targetId: target.id, words: words(alias2) }))).filter((alias2) => alias2.words.length > 0);
|
|
9350
10007
|
for (let start = 0; start < textWords.length; ) {
|
|
9351
|
-
const matches = aliases.filter((
|
|
10008
|
+
const matches = aliases.filter((alias2) => aliasMatchesAt(textWords, alias2.words, start));
|
|
9352
10009
|
if (matches.length === 0) {
|
|
9353
10010
|
start++;
|
|
9354
10011
|
continue;
|
|
@@ -9432,7 +10089,7 @@ function prepareReport(input) {
|
|
|
9432
10089
|
}
|
|
9433
10090
|
const answers = [];
|
|
9434
10091
|
const mentionableIds = new Set(
|
|
9435
|
-
input.targets.filter((target) => target.aliases.some((
|
|
10092
|
+
input.targets.filter((target) => target.aliases.some((alias2) => words(alias2).length > 0)).map((target) => target.id)
|
|
9436
10093
|
);
|
|
9437
10094
|
for (const observation of observationsBySlot.values()) {
|
|
9438
10095
|
const edges = [...edgesByExecution.get(observation.slot.executionId) ?? []].sort((left, right) => compareText(left.id, right.id));
|
|
@@ -9555,7 +10212,7 @@ function targetCoverageRate(targetIds, slots, edges, prepared) {
|
|
|
9555
10212
|
}
|
|
9556
10213
|
function mentionRate(target, slots, prepared) {
|
|
9557
10214
|
const numerator = slots.filter((slot) => prepared.observationsBySlot.get(slot.id)?.mentionedTargetIds.has(target.id)).length;
|
|
9558
|
-
if (target.aliases.every((
|
|
10215
|
+
if (target.aliases.every((alias2) => words(alias2).length === 0)) {
|
|
9559
10216
|
return { numerator: null, denominator: null, rate: null, reason: "aliasless" };
|
|
9560
10217
|
}
|
|
9561
10218
|
if (slots.length === 0) return { numerator: null, denominator: null, rate: null, reason: "no-population" };
|
|
@@ -9583,11 +10240,11 @@ function buildSovForSlots(slots, competitors2, projectBrandNames, projectDomain,
|
|
|
9583
10240
|
let reason = null;
|
|
9584
10241
|
if (selected.length === 0) reason = "no-population";
|
|
9585
10242
|
else if (!status.complete || !status.answerComplete) reason = "incomplete";
|
|
9586
|
-
else if (projectBrandNames.every((
|
|
10243
|
+
else if (projectBrandNames.every((alias2) => words(alias2).length === 0)) reason = "no-project-aliases";
|
|
9587
10244
|
else if (sortedCompetitors.length === 0) reason = "no-competitors";
|
|
9588
10245
|
return rows.map((row) => {
|
|
9589
10246
|
if (reason !== null) return { domain: row.domain, own: row.own, presentIn: null, of: null, reason };
|
|
9590
|
-
if (row.aliases.every((
|
|
10247
|
+
if (row.aliases.every((alias2) => words(alias2).length === 0)) {
|
|
9591
10248
|
return { domain: row.domain, own: row.own, presentIn: null, of: null, reason: "aliasless" };
|
|
9592
10249
|
}
|
|
9593
10250
|
const presentIn = selected.filter((slot) => {
|
|
@@ -9737,7 +10394,7 @@ function mentionedForEdge(observation, edge, mentionableIds) {
|
|
|
9737
10394
|
return observation.mentionedTargetIds.has(edge.targetId);
|
|
9738
10395
|
}
|
|
9739
10396
|
function mentionableTargets(input, targetIds) {
|
|
9740
|
-
return input.targets.filter((target) => targetIds.has(target.id) && target.aliases.some((
|
|
10397
|
+
return input.targets.filter((target) => targetIds.has(target.id) && target.aliases.some((alias2) => words(alias2).length > 0));
|
|
9741
10398
|
}
|
|
9742
10399
|
function scopeMentionRate(input, targetIds, slots, answered, prepared) {
|
|
9743
10400
|
const mentionable = mentionableTargets(input, targetIds);
|
|
@@ -9763,7 +10420,7 @@ function indexedScopeCitationRate(slots, edges, indexes) {
|
|
|
9763
10420
|
return { numerator, denominator: basis.length, rate: numerator / basis.length };
|
|
9764
10421
|
}
|
|
9765
10422
|
function targetMentionRate(targets, slots, answered, indexes) {
|
|
9766
|
-
const mentionable = targets?.filter((target) => target.aliases.some((
|
|
10423
|
+
const mentionable = targets?.filter((target) => target.aliases.some((alias2) => words(alias2).length > 0)) ?? [];
|
|
9767
10424
|
if (mentionable.length === 0) return unavailable("aliasless");
|
|
9768
10425
|
if (slots.length === 0) return unavailable("no-population");
|
|
9769
10426
|
if (answered.length === 0) return unavailable("evidence-incomplete");
|
|
@@ -9779,7 +10436,7 @@ function presenceIn(slots, aliases, prepared) {
|
|
|
9779
10436
|
}
|
|
9780
10437
|
function scopeBrandPresence(input, slots, answered, prepared) {
|
|
9781
10438
|
if (slots.length === 0) return unavailable("no-population");
|
|
9782
|
-
if (input.projectBrandNames.every((
|
|
10439
|
+
if (input.projectBrandNames.every((alias2) => words(alias2).length === 0)) return unavailable("no-project-aliases");
|
|
9783
10440
|
if (answered.length === 0) return unavailable("evidence-incomplete");
|
|
9784
10441
|
const numerator = presenceIn(answered, input.projectBrandNames, prepared);
|
|
9785
10442
|
return { numerator, denominator: answered.length, rate: numerator / answered.length };
|
|
@@ -10078,8 +10735,8 @@ function reportInput(revision, plan, manifest, snapshots, legacy) {
|
|
|
10078
10735
|
label: group.label,
|
|
10079
10736
|
targetIds: group.targetKeys,
|
|
10080
10737
|
competitors: (group.competitors ?? []).map((domain) => {
|
|
10081
|
-
const
|
|
10082
|
-
return { domain, aliases: brandKeyFromText(
|
|
10738
|
+
const alias2 = brandLabelFromDomain(domain);
|
|
10739
|
+
return { domain, aliases: brandKeyFromText(alias2).length >= 4 ? [alias2] : [] };
|
|
10083
10740
|
})
|
|
10084
10741
|
})),
|
|
10085
10742
|
expectedSlots: slots,
|
|
@@ -10954,7 +11611,12 @@ async function runRoutes(app, opts) {
|
|
|
10954
11611
|
const terminalStatuses = /* @__PURE__ */ new Set(["completed", "partial", "failed", "cancelled"]);
|
|
10955
11612
|
if (terminalStatuses.has(run.status)) throw runNotCancellable(run.id, run.status);
|
|
10956
11613
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
10957
|
-
app.db.update(runs).set({ status: "cancelled", finishedAt: now, error: serializeRunError({ message: "Cancelled by user" }) }).where(eq11(runs.id, run.id)).run();
|
|
11614
|
+
const cancelled = app.db.update(runs).set({ status: "cancelled", finishedAt: now, error: serializeRunError({ message: "Cancelled by user" }) }).where(and6(eq11(runs.id, run.id), inArray4(runs.status, ["queued", "running"]))).run();
|
|
11615
|
+
if (cancelled.changes === 0) {
|
|
11616
|
+
const current = app.db.select().from(runs).where(eq11(runs.id, run.id)).get();
|
|
11617
|
+
if (!current) throw notFound("Run", run.id);
|
|
11618
|
+
throw runNotCancellable(run.id, current.status);
|
|
11619
|
+
}
|
|
10958
11620
|
writeAuditLog(app.db, {
|
|
10959
11621
|
projectId: run.projectId,
|
|
10960
11622
|
actor: "api",
|
|
@@ -10962,6 +11624,7 @@ async function runRoutes(app, opts) {
|
|
|
10962
11624
|
entityType: "run",
|
|
10963
11625
|
entityId: run.id
|
|
10964
11626
|
});
|
|
11627
|
+
opts.onRunCancelled?.(run.id, run.projectId);
|
|
10965
11628
|
const updated = app.db.select().from(runs).where(eq11(runs.id, run.id)).get();
|
|
10966
11629
|
return reply.send(formatRun(updated));
|
|
10967
11630
|
});
|
|
@@ -11593,7 +12256,7 @@ function compileRules(input) {
|
|
|
11593
12256
|
const aliases = (input.rules.aliases ?? []).map(
|
|
11594
12257
|
(rule, index2) => compileRouteRule(rule, `rules.aliases[${index2}]`, ownedHosts)
|
|
11595
12258
|
);
|
|
11596
|
-
if (aliases.some((
|
|
12259
|
+
if (aliases.some((alias2) => alias2.host === primary.host)) {
|
|
11597
12260
|
throw new MeasurementDiscoveryConfigurationError("alias rules must use another owned host");
|
|
11598
12261
|
}
|
|
11599
12262
|
const excludedSlugSuffixes = sortedUnique3(
|
|
@@ -12231,7 +12894,7 @@ async function measurementServiceRoutes(app, opts) {
|
|
|
12231
12894
|
|
|
12232
12895
|
// ../api-routes/src/measurement-draft.ts
|
|
12233
12896
|
import crypto17 from "crypto";
|
|
12234
|
-
import { and as and9, asc as asc2, desc as desc5, eq as eq14, inArray as
|
|
12897
|
+
import { and as and9, asc as asc2, desc as desc5, eq as eq14, inArray as inArray5 } from "drizzle-orm";
|
|
12235
12898
|
|
|
12236
12899
|
// ../api-routes/src/measurement-draft-repo.ts
|
|
12237
12900
|
import crypto15 from "crypto";
|
|
@@ -12511,14 +13174,14 @@ function compileMeasurementDraft(authoring, context) {
|
|
|
12511
13174
|
matcherClaims.set(identity, target.stableKey);
|
|
12512
13175
|
matchers.push(matcher);
|
|
12513
13176
|
});
|
|
12514
|
-
target.aliases.forEach((
|
|
12515
|
-
const identity = measurementMentionAliasKey(
|
|
13177
|
+
target.aliases.forEach((alias2, aliasIndex) => {
|
|
13178
|
+
const identity = measurementMentionAliasKey(alias2);
|
|
12516
13179
|
if (!identity) return;
|
|
12517
13180
|
const claimedBy = aliasClaims.get(identity);
|
|
12518
13181
|
if (claimedBy !== void 0 && claimedBy !== target.stableKey) {
|
|
12519
13182
|
sink.fail(
|
|
12520
13183
|
"target-alias-ambiguous",
|
|
12521
|
-
`Target alias normalizes to the same mention identity claimed by Target ${claimedBy}: ${
|
|
13184
|
+
`Target alias normalizes to the same mention identity claimed by Target ${claimedBy}: ${alias2}`,
|
|
12522
13185
|
["targets", targetIndex, "aliases", aliasIndex]
|
|
12523
13186
|
);
|
|
12524
13187
|
return;
|
|
@@ -14753,7 +15416,7 @@ async function measurementDraftRoutes(app, opts = {}) {
|
|
|
14753
15416
|
const sets = app.db.select().from(measurementQuerySets).where(eq14(measurementQuerySets.projectId, project.id)).orderBy(asc2(measurementQuerySets.name)).all();
|
|
14754
15417
|
const counts2 = /* @__PURE__ */ new Map();
|
|
14755
15418
|
if (sets.length) {
|
|
14756
|
-
for (const item2 of app.db.select({ querySetId: measurementQuerySetItems.querySetId }).from(measurementQuerySetItems).where(
|
|
15419
|
+
for (const item2 of app.db.select({ querySetId: measurementQuerySetItems.querySetId }).from(measurementQuerySetItems).where(inArray5(measurementQuerySetItems.querySetId, sets.map((set) => set.id))).all()) {
|
|
14757
15420
|
counts2.set(item2.querySetId, (counts2.get(item2.querySetId) ?? 0) + 1);
|
|
14758
15421
|
}
|
|
14759
15422
|
}
|
|
@@ -16276,10 +16939,10 @@ async function measurementPropertyEvidenceRoutes(app) {
|
|
|
16276
16939
|
}
|
|
16277
16940
|
|
|
16278
16941
|
// ../api-routes/src/measurement-portfolio-reads.ts
|
|
16279
|
-
import { and as and14, desc as desc6, eq as eq20, inArray as
|
|
16942
|
+
import { and as and14, desc as desc6, eq as eq20, inArray as inArray7, lt as lt3, or as or3 } from "drizzle-orm";
|
|
16280
16943
|
|
|
16281
16944
|
// ../api-routes/src/measurement-question-reads.ts
|
|
16282
|
-
import { and as and13, eq as eq19, getTableColumns, inArray as
|
|
16945
|
+
import { and as and13, eq as eq19, getTableColumns, inArray as inArray6, sql as sql7 } from "drizzle-orm";
|
|
16283
16946
|
var DEFAULT_LIMIT2 = 50;
|
|
16284
16947
|
var TERMINAL_STATUSES = /* @__PURE__ */ new Set([
|
|
16285
16948
|
RunStatuses.completed,
|
|
@@ -16395,7 +17058,7 @@ function selectedQuestionSnapshots(db, runId, manifest, scope) {
|
|
|
16395
17058
|
if (scope !== void 0) {
|
|
16396
17059
|
const executionIds = [...new Set(manifest.expectedSlots.map((slot) => slot.executionId))];
|
|
16397
17060
|
if (executionIds.length === 0) return [];
|
|
16398
|
-
conditions.push(
|
|
17061
|
+
conditions.push(inArray6(querySnapshots.measurementExecutionId, executionIds));
|
|
16399
17062
|
}
|
|
16400
17063
|
if (scope?.provider !== void 0) {
|
|
16401
17064
|
conditions.push(sql7`lower(trim(${querySnapshots.provider})) = ${normalizeText(scope.provider)}`);
|
|
@@ -16410,7 +17073,7 @@ function rawResponsesForLegacyQuestionSnapshots(db, runId, snapshots) {
|
|
|
16410
17073
|
rawResponse: querySnapshots.rawResponse
|
|
16411
17074
|
}).from(querySnapshots).where(and13(
|
|
16412
17075
|
eq19(querySnapshots.runId, runId),
|
|
16413
|
-
|
|
17076
|
+
inArray6(querySnapshots.id, legacyIds)
|
|
16414
17077
|
)).all().map((snapshot) => [snapshot.id, snapshot.rawResponse]));
|
|
16415
17078
|
}
|
|
16416
17079
|
function withLegacyRawEvidence(db, runId, snapshots) {
|
|
@@ -17056,7 +17719,7 @@ function previousComparableRun(db, active, current) {
|
|
|
17056
17719
|
eq20(runs.projectId, current.projectId),
|
|
17057
17720
|
eq20(runs.measurementPlanVersionId, active.version.id),
|
|
17058
17721
|
eq20(runs.kind, RunKinds["answer-visibility"]),
|
|
17059
|
-
|
|
17722
|
+
inArray7(runs.status, [RunStatuses.completed, RunStatuses.partial]),
|
|
17060
17723
|
or3(
|
|
17061
17724
|
lt3(runs.createdAt, current.createdAt),
|
|
17062
17725
|
and14(eq20(runs.createdAt, current.createdAt), lt3(runs.id, current.id))
|
|
@@ -17882,7 +18545,7 @@ function normalizeCompetitorList2(domains) {
|
|
|
17882
18545
|
}
|
|
17883
18546
|
|
|
17884
18547
|
// ../api-routes/src/history.ts
|
|
17885
|
-
import { and as and16, asc as asc3, desc as desc7, eq as eq22, gte as gte3, inArray as
|
|
18548
|
+
import { and as and16, asc as asc3, desc as desc7, eq as eq22, gte as gte3, inArray as inArray8 } from "drizzle-orm";
|
|
17886
18549
|
|
|
17887
18550
|
// ../api-routes/src/notification-redaction.ts
|
|
17888
18551
|
var REDACTED_URL = {
|
|
@@ -17968,7 +18631,7 @@ async function historyRoutes(app) {
|
|
|
17968
18631
|
recommendedCompetitors: querySnapshots.recommendedCompetitors,
|
|
17969
18632
|
location: querySnapshots.location,
|
|
17970
18633
|
createdAt: querySnapshots.createdAt
|
|
17971
|
-
}).from(querySnapshots).leftJoin(queries, eq22(querySnapshots.queryId, queries.id)).where(
|
|
18634
|
+
}).from(querySnapshots).leftJoin(queries, eq22(querySnapshots.queryId, queries.id)).where(inArray8(querySnapshots.runId, projectRuns.map((r) => r.id))).orderBy(desc7(querySnapshots.createdAt)).all();
|
|
17972
18635
|
const locationFilter = request.query.location;
|
|
17973
18636
|
const filtered = locationFilter !== void 0 ? allSnapshots.filter((s) => s.location === (locationFilter || null)) : allSnapshots;
|
|
17974
18637
|
const total = filtered.length;
|
|
@@ -18014,7 +18677,7 @@ async function historyRoutes(app) {
|
|
|
18014
18677
|
return reply.send([]);
|
|
18015
18678
|
}
|
|
18016
18679
|
const runIds = new Set(projectRuns.map((r) => r.id));
|
|
18017
|
-
const rawSnapshots = app.db.select().from(querySnapshots).where(
|
|
18680
|
+
const rawSnapshots = app.db.select().from(querySnapshots).where(inArray8(querySnapshots.runId, [...runIds])).all();
|
|
18018
18681
|
const timelineLocationFilter = request.query.location;
|
|
18019
18682
|
const filteredSnapshots = timelineLocationFilter !== void 0 ? rawSnapshots.filter((s) => s.location === (timelineLocationFilter || null)) : rawSnapshots;
|
|
18020
18683
|
const allSnapshots = filteredSnapshots.map((snapshot) => ({
|
|
@@ -18135,7 +18798,7 @@ async function historyRoutes(app) {
|
|
|
18135
18798
|
throw validationError("Both run1 and run2 query params are required");
|
|
18136
18799
|
}
|
|
18137
18800
|
const requestedRunIds = [.../* @__PURE__ */ new Set([run1, run2])];
|
|
18138
|
-
const runRows = app.db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
18801
|
+
const runRows = app.db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(inArray8(runs.id, requestedRunIds)).all();
|
|
18139
18802
|
const runsById = new Map(runRows.map((row) => [row.id, row]));
|
|
18140
18803
|
for (const runId of requestedRunIds) {
|
|
18141
18804
|
const run = runsById.get(runId);
|
|
@@ -18233,7 +18896,7 @@ function parseOptionalPositiveInt(value, max) {
|
|
|
18233
18896
|
}
|
|
18234
18897
|
|
|
18235
18898
|
// ../api-routes/src/analytics.ts
|
|
18236
|
-
import { and as and17, desc as desc8, eq as eq23, gte as gte4, inArray as
|
|
18899
|
+
import { and as and17, desc as desc8, eq as eq23, gte as gte4, inArray as inArray9, lt as lt4 } from "drizzle-orm";
|
|
18237
18900
|
|
|
18238
18901
|
// ../api-routes/src/model-evidence.ts
|
|
18239
18902
|
function classifyModelEvidence(values) {
|
|
@@ -18368,7 +19031,7 @@ async function analyticsRoutes(app) {
|
|
|
18368
19031
|
const projectRuns = app.db.select().from(runs).where(and17(
|
|
18369
19032
|
eq23(runs.projectId, project.id),
|
|
18370
19033
|
eq23(runs.kind, RunKinds["answer-visibility"]),
|
|
18371
|
-
|
|
19034
|
+
inArray9(runs.status, [RunStatuses.completed, RunStatuses.partial]),
|
|
18372
19035
|
notProbeRun(),
|
|
18373
19036
|
cutoff ? gte4(runs.createdAt, cutoff) : void 0
|
|
18374
19037
|
)).orderBy(desc8(runs.createdAt)).all().filter((r) => measuredWhole(app.db, r)).sort((a, b) => a.createdAt.localeCompare(b.createdAt));
|
|
@@ -18411,7 +19074,7 @@ async function analyticsRoutes(app) {
|
|
|
18411
19074
|
citationState: querySnapshots.citationState,
|
|
18412
19075
|
answerMentioned: querySnapshots.answerMentioned,
|
|
18413
19076
|
answerText: querySnapshots.answerText
|
|
18414
|
-
}).from(querySnapshots).where(
|
|
19077
|
+
}).from(querySnapshots).where(inArray9(querySnapshots.runId, runIds)).all();
|
|
18415
19078
|
const rawSnapshots = referenceBasket ? loadedSnapshots.filter(
|
|
18416
19079
|
(s) => s.queryId !== null || s.queryText !== null && referenceBasket.has(normalizeQueryText(s.queryText))
|
|
18417
19080
|
) : filterTrackedSnapshots(loadedSnapshots);
|
|
@@ -18453,7 +19116,7 @@ async function analyticsRoutes(app) {
|
|
|
18453
19116
|
const anchorRunPredicate = and17(
|
|
18454
19117
|
eq23(runs.projectId, project.id),
|
|
18455
19118
|
eq23(runs.kind, RunKinds["answer-visibility"]),
|
|
18456
|
-
|
|
19119
|
+
inArray9(runs.status, [RunStatuses.completed, RunStatuses.partial]),
|
|
18457
19120
|
notProbeRun(),
|
|
18458
19121
|
lt4(runs.createdAt, cutoff)
|
|
18459
19122
|
);
|
|
@@ -18483,7 +19146,7 @@ async function analyticsRoutes(app) {
|
|
|
18483
19146
|
provider: querySnapshots.provider,
|
|
18484
19147
|
model: querySnapshots.model,
|
|
18485
19148
|
servedModel: querySnapshots.servedModel
|
|
18486
|
-
}).from(querySnapshots).where(
|
|
19149
|
+
}).from(querySnapshots).where(inArray9(querySnapshots.runId, runIds2)).all())) {
|
|
18487
19150
|
if (pending.has(snapshot.provider)) {
|
|
18488
19151
|
const models = modelsByProvider.get(snapshot.provider) ?? [];
|
|
18489
19152
|
models.push(snapshot.model);
|
|
@@ -18629,7 +19292,7 @@ async function analyticsRoutes(app) {
|
|
|
18629
19292
|
citationState: querySnapshots.citationState,
|
|
18630
19293
|
answerMentioned: querySnapshots.answerMentioned,
|
|
18631
19294
|
answerText: querySnapshots.answerText
|
|
18632
|
-
}).from(querySnapshots).where(
|
|
19295
|
+
}).from(querySnapshots).where(inArray9(querySnapshots.runId, windowRunIds)).all());
|
|
18633
19296
|
for (const s of allWindowSnaps) {
|
|
18634
19297
|
const timePoint = runIdToCreatedAt.get(s.runId) ?? s.runId;
|
|
18635
19298
|
let entry = consistencyMap.get(s.queryId);
|
|
@@ -18650,7 +19313,7 @@ async function analyticsRoutes(app) {
|
|
|
18650
19313
|
answerMentioned: querySnapshots.answerMentioned,
|
|
18651
19314
|
answerText: querySnapshots.answerText,
|
|
18652
19315
|
competitorOverlap: querySnapshots.competitorOverlap
|
|
18653
|
-
}).from(querySnapshots).leftJoin(queries, eq23(querySnapshots.queryId, queries.id)).where(
|
|
19316
|
+
}).from(querySnapshots).leftJoin(queries, eq23(querySnapshots.queryId, queries.id)).where(inArray9(querySnapshots.runId, latestGroupRunIds)).all());
|
|
18654
19317
|
const snapshots = rawSnapshots.map((s) => ({
|
|
18655
19318
|
...s,
|
|
18656
19319
|
resolvedMentioned: resolveSnapshotAnswerMentioned(s, project)
|
|
@@ -18769,7 +19432,7 @@ async function analyticsRoutes(app) {
|
|
|
18769
19432
|
query: queries.query,
|
|
18770
19433
|
provider: querySnapshots.provider,
|
|
18771
19434
|
rawResponse: querySnapshots.rawResponse
|
|
18772
|
-
}).from(querySnapshots).leftJoin(queries, eq23(querySnapshots.queryId, queries.id)).where(
|
|
19435
|
+
}).from(querySnapshots).leftJoin(queries, eq23(querySnapshots.queryId, queries.id)).where(inArray9(querySnapshots.runId, windowRunIds)).all();
|
|
18773
19436
|
const overallCounts = /* @__PURE__ */ new Map();
|
|
18774
19437
|
const byQuery = {};
|
|
18775
19438
|
const overallDomains = /* @__PURE__ */ new Map();
|
|
@@ -19041,7 +19704,7 @@ function buildCategoryCounts(counts2) {
|
|
|
19041
19704
|
}
|
|
19042
19705
|
|
|
19043
19706
|
// ../api-routes/src/intelligence.ts
|
|
19044
|
-
import { eq as eq24, desc as desc9, and as and18, inArray as
|
|
19707
|
+
import { eq as eq24, desc as desc9, and as and18, inArray as inArray10, like } from "drizzle-orm";
|
|
19045
19708
|
var SEVERITY_RANK = { low: 0, medium: 1, high: 2, critical: 3 };
|
|
19046
19709
|
function severitiesAtOrAbove(min) {
|
|
19047
19710
|
const floor = SEVERITY_RANK[min];
|
|
@@ -19171,7 +19834,7 @@ async function intelligenceRoutes(app) {
|
|
|
19171
19834
|
}
|
|
19172
19835
|
const severityFilter = request.query.severity?.trim();
|
|
19173
19836
|
if (severityFilter) {
|
|
19174
|
-
conditions.push(
|
|
19837
|
+
conditions.push(inArray10(insights.severity, severitiesAtOrAbove(severityFilter)));
|
|
19175
19838
|
}
|
|
19176
19839
|
let limit;
|
|
19177
19840
|
if (request.query.limit !== void 0) {
|
|
@@ -19209,7 +19872,7 @@ async function intelligenceRoutes(app) {
|
|
|
19209
19872
|
const projectVisRuns = app.db.select({ id: runs.id, createdAt: runs.createdAt }).from(runs).where(and18(
|
|
19210
19873
|
eq24(runs.projectId, project.id),
|
|
19211
19874
|
eq24(runs.kind, RunKinds["answer-visibility"]),
|
|
19212
|
-
|
|
19875
|
+
inArray10(runs.status, [RunStatuses.completed, RunStatuses.partial]),
|
|
19213
19876
|
// Health-latest is the dashboard headline; probe runs must not
|
|
19214
19877
|
// displace the most recent real visibility sweep.
|
|
19215
19878
|
notProbeRun()
|
|
@@ -19219,7 +19882,7 @@ async function intelligenceRoutes(app) {
|
|
|
19219
19882
|
if (latestGroupRunIds.length > 0) {
|
|
19220
19883
|
const groupRows = app.db.select().from(healthSnapshots).where(and18(
|
|
19221
19884
|
eq24(healthSnapshots.projectId, project.id),
|
|
19222
|
-
|
|
19885
|
+
inArray10(healthSnapshots.runId, latestGroupRunIds)
|
|
19223
19886
|
)).all();
|
|
19224
19887
|
if (groupRows.length > 0) {
|
|
19225
19888
|
return reply.send(aggregateHealthSnapshots(project.id, groupRows));
|
|
@@ -19241,14 +19904,14 @@ async function intelligenceRoutes(app) {
|
|
|
19241
19904
|
}
|
|
19242
19905
|
|
|
19243
19906
|
// ../api-routes/src/report.ts
|
|
19244
|
-
import { and as and22, desc as desc12, eq as eq28, gte as gte5, inArray as
|
|
19907
|
+
import { and as and22, desc as desc12, eq as eq28, gte as gte5, inArray as inArray12, lt as lt5, lte as lte3, ne as ne3, or as or4, sql as sql9 } from "drizzle-orm";
|
|
19245
19908
|
|
|
19246
19909
|
// ../api-routes/src/content.ts
|
|
19247
19910
|
import crypto21 from "crypto";
|
|
19248
19911
|
import { and as and20, desc as desc11, eq as eq26 } from "drizzle-orm";
|
|
19249
19912
|
|
|
19250
19913
|
// ../api-routes/src/content-data.ts
|
|
19251
|
-
import { and as and19, eq as eq25, desc as desc10, inArray as
|
|
19914
|
+
import { and as and19, eq as eq25, desc as desc10, inArray as inArray11 } from "drizzle-orm";
|
|
19252
19915
|
var RECENT_RUNS_WINDOW = 5;
|
|
19253
19916
|
function loadOrchestratorInput(db, project, locationFilter = void 0) {
|
|
19254
19917
|
const projectId = project.id;
|
|
@@ -19384,7 +20047,7 @@ function listRecentAnswerVisibilityRunIds(db, projectId, limit, locationFilter)
|
|
|
19384
20047
|
// Queued/running/failed/cancelled runs may have partial or no
|
|
19385
20048
|
// snapshots; including them risks pointing latestRunId at a run with
|
|
19386
20049
|
// no usable evidence.
|
|
19387
|
-
|
|
20050
|
+
inArray11(runs.status, [RunStatuses.completed, RunStatuses.partial]),
|
|
19388
20051
|
// Probe runs are operator/agent test runs; they must not poison the
|
|
19389
20052
|
// recent-runs window the content engine uses to recommend actions.
|
|
19390
20053
|
notProbeRun()
|
|
@@ -19434,7 +20097,7 @@ function buildCandidateQueries(opts) {
|
|
|
19434
20097
|
const queryRows = opts.db.select({ id: queries.id, text: queries.query }).from(queries).where(eq25(queries.projectId, opts.projectId)).all();
|
|
19435
20098
|
const queryIdByText = new Map(queryRows.map((r) => [r.text, r.id]));
|
|
19436
20099
|
const candidateQueryIds = opts.candidateQueryStrings.map((q) => queryIdByText.get(q)).filter((id) => Boolean(id));
|
|
19437
|
-
const snapshotRows = filterTrackedSnapshots(opts.db.select().from(querySnapshots).where(
|
|
20100
|
+
const snapshotRows = filterTrackedSnapshots(opts.db.select().from(querySnapshots).where(inArray11(querySnapshots.runId, opts.recentRunIds)).all()).filter((r) => candidateQueryIds.includes(r.queryId));
|
|
19438
20101
|
const snapshotsByQuery = /* @__PURE__ */ new Map();
|
|
19439
20102
|
for (const row of snapshotRows) {
|
|
19440
20103
|
const list = snapshotsByQuery.get(row.queryId) ?? [];
|
|
@@ -22361,7 +23024,7 @@ function loadSnapshotsForRun(db, runId) {
|
|
|
22361
23024
|
}
|
|
22362
23025
|
function loadSnapshotsForRunIds(db, runIds) {
|
|
22363
23026
|
if (runIds.length === 0) return [];
|
|
22364
|
-
const rows = db.select().from(querySnapshots).where(
|
|
23027
|
+
const rows = db.select().from(querySnapshots).where(inArray12(querySnapshots.runId, [...runIds])).all();
|
|
22365
23028
|
return rows.filter((r) => r.queryId !== null).map((r) => ({
|
|
22366
23029
|
id: r.id,
|
|
22367
23030
|
runId: r.runId,
|
|
@@ -23110,7 +23773,7 @@ function buildInsightList(db, projectId, locationFilter) {
|
|
|
23110
23773
|
)
|
|
23111
23774
|
).orderBy(desc12(runs.createdAt)).all().filter((r) => locationFilter === void 0 || (r.location ?? null) === locationFilter).slice(0, INSIGHT_LOOKBACK_RUNS).map((r) => r.id);
|
|
23112
23775
|
if (recentRunIds.length === 0) return [];
|
|
23113
|
-
const rows = db.select().from(insights).where(and22(eq28(insights.projectId, projectId),
|
|
23776
|
+
const rows = db.select().from(insights).where(and22(eq28(insights.projectId, projectId), inArray12(insights.runId, recentRunIds))).orderBy(desc12(insights.createdAt)).all();
|
|
23114
23777
|
const severityRank = { critical: 0, high: 1, medium: 2, low: 3 };
|
|
23115
23778
|
const flat = rows.filter((r) => !r.dismissed).map((r) => {
|
|
23116
23779
|
const recommendation = r.recommendation;
|
|
@@ -24859,7 +25522,7 @@ async function organicEvidenceRoutes(app) {
|
|
|
24859
25522
|
}
|
|
24860
25523
|
|
|
24861
25524
|
// ../api-routes/src/citations.ts
|
|
24862
|
-
import { and as and25, eq as eq31, inArray as
|
|
25525
|
+
import { and as and25, eq as eq31, inArray as inArray13 } from "drizzle-orm";
|
|
24863
25526
|
async function citationRoutes(app) {
|
|
24864
25527
|
app.get("/projects/:name/citations/visibility", async (request, reply) => {
|
|
24865
25528
|
const project = resolveProject(app.db, request.params.name);
|
|
@@ -24883,7 +25546,7 @@ async function citationRoutes(app) {
|
|
|
24883
25546
|
competitorOverlap: querySnapshots.competitorOverlap,
|
|
24884
25547
|
answerMentioned: querySnapshots.answerMentioned,
|
|
24885
25548
|
createdAt: querySnapshots.createdAt
|
|
24886
|
-
}).from(querySnapshots).where(
|
|
25549
|
+
}).from(querySnapshots).where(inArray13(querySnapshots.runId, projectRuns.map((r) => r.id))).all();
|
|
24887
25550
|
if (rawSnapshots.length === 0) {
|
|
24888
25551
|
return reply.send(emptyCitationVisibility("no-runs-yet"));
|
|
24889
25552
|
}
|
|
@@ -25022,7 +25685,7 @@ function computeCitationVisibility(input) {
|
|
|
25022
25685
|
}
|
|
25023
25686
|
|
|
25024
25687
|
// ../api-routes/src/visibility-stats.ts
|
|
25025
|
-
import { and as and26, desc as desc14, eq as eq32, inArray as
|
|
25688
|
+
import { and as and26, desc as desc14, eq as eq32, inArray as inArray14 } from "drizzle-orm";
|
|
25026
25689
|
|
|
25027
25690
|
// ../api-routes/src/visibility-compare.ts
|
|
25028
25691
|
var VISIBILITY_COMPARE_MIN_RUNS = 5;
|
|
@@ -25433,7 +26096,7 @@ async function visibilityStatsRoutes(app) {
|
|
|
25433
26096
|
citationState: querySnapshots.citationState,
|
|
25434
26097
|
answerMentioned: querySnapshots.answerMentioned,
|
|
25435
26098
|
createdAt: querySnapshots.createdAt
|
|
25436
|
-
}).from(querySnapshots).where(
|
|
26099
|
+
}).from(querySnapshots).where(inArray14(querySnapshots.runId, runIds)).all() : [];
|
|
25437
26100
|
const stats = computeVisibilityStats({ queries: projectQueries, snapshots, groupBy });
|
|
25438
26101
|
const queryAttribution = buildQueryAttribution(projectQueries);
|
|
25439
26102
|
let shareOfVoice;
|
|
@@ -25448,7 +26111,7 @@ async function visibilityStatsRoutes(app) {
|
|
|
25448
26111
|
queryText: querySnapshots.queryText,
|
|
25449
26112
|
answerMentioned: querySnapshots.answerMentioned,
|
|
25450
26113
|
answerText: querySnapshots.answerText
|
|
25451
|
-
}).from(querySnapshots).where(
|
|
26114
|
+
}).from(querySnapshots).where(inArray14(querySnapshots.runId, runIds)).all() : [];
|
|
25452
26115
|
const attributedSovSnapshots = sovSnapshots.filter((s) => resolveCurrentQuery(queryAttribution, s) !== void 0);
|
|
25453
26116
|
const result = buildMentionShare(
|
|
25454
26117
|
attributedSovSnapshots.map((s) => ({ projectMentioned: s.answerMentioned === true, answerText: s.answerText })),
|
|
@@ -25529,7 +26192,7 @@ async function visibilityStatsRoutes(app) {
|
|
|
25529
26192
|
answerMentioned: querySnapshots.answerMentioned,
|
|
25530
26193
|
answerText: querySnapshots.answerText,
|
|
25531
26194
|
citedDomains: querySnapshots.citedDomains
|
|
25532
|
-
}).from(querySnapshots).where(
|
|
26195
|
+
}).from(querySnapshots).where(inArray14(querySnapshots.runId, runIds)).all() : [];
|
|
25533
26196
|
return { runCount: monthRuns.length, snapshots };
|
|
25534
26197
|
};
|
|
25535
26198
|
const fromMonth = loadMonth(fromBounds);
|
|
@@ -25784,7 +26447,7 @@ async function resultsExportRoutes(app) {
|
|
|
25784
26447
|
}
|
|
25785
26448
|
|
|
25786
26449
|
// ../api-routes/src/composites.ts
|
|
25787
|
-
import { eq as eq34, and as and28, desc as desc15, sql as sql12, like as like2, or as or7, inArray as
|
|
26450
|
+
import { eq as eq34, and as and28, desc as desc15, sql as sql12, like as like2, or as or7, inArray as inArray15 } from "drizzle-orm";
|
|
25788
26451
|
var TOP_INSIGHT_LIMIT = 5;
|
|
25789
26452
|
var SEARCH_HIT_HARD_LIMIT = 50;
|
|
25790
26453
|
var SEARCH_SNIPPET_RADIUS = 80;
|
|
@@ -26051,7 +26714,7 @@ function loadSnapshotsByRunIds(app, runIds, queryIdByText) {
|
|
|
26051
26714
|
answerText: querySnapshots.answerText,
|
|
26052
26715
|
competitorOverlap: querySnapshots.competitorOverlap,
|
|
26053
26716
|
citedDomains: querySnapshots.citedDomains
|
|
26054
|
-
}).from(querySnapshots).where(
|
|
26717
|
+
}).from(querySnapshots).where(inArray15(querySnapshots.runId, [...runIds])).all();
|
|
26055
26718
|
for (const row of rows) {
|
|
26056
26719
|
const queryText = row.queryText?.trim() || null;
|
|
26057
26720
|
let queryId;
|
|
@@ -26686,6 +27349,17 @@ var SCHEMA_TABLE = {
|
|
|
26686
27349
|
SiteAuditRunResponseDto: siteAuditRunResponseSchema,
|
|
26687
27350
|
SiteAuditScoreDto: siteAuditScoreSchema,
|
|
26688
27351
|
SiteAuditTrendResponseDto: siteAuditTrendResponseSchema,
|
|
27352
|
+
SiteCrawlDeadLinksResponseDto: siteCrawlDeadLinksResponseSchema,
|
|
27353
|
+
SiteCrawlGraphResponseDto: siteCrawlGraphResponseSchema,
|
|
27354
|
+
SiteCrawlInternalLinksResponseDto: siteCrawlInternalLinksResponseSchema,
|
|
27355
|
+
SiteCrawlNeighborsResponseDto: siteCrawlNeighborsResponseSchema,
|
|
27356
|
+
SiteCrawlPageAuditDto: siteCrawlPageAuditSchema,
|
|
27357
|
+
SiteCrawlPagesResponseDto: siteCrawlPagesResponseSchema,
|
|
27358
|
+
SiteCrawlStructureResponseDto: siteCrawlStructureResponseSchema,
|
|
27359
|
+
SiteCrawlSummaryDto: siteCrawlSummarySchema,
|
|
27360
|
+
SiteHealthChangesResponseDto: siteHealthChangesResponseSchema,
|
|
27361
|
+
SiteHealthPathResponseDto: siteHealthPathResponseSchema,
|
|
27362
|
+
SiteHealthSubgraphResponseDto: siteHealthSubgraphResponseSchema,
|
|
26689
27363
|
SnapshotDiffResponse: snapshotDiffResponseSchema,
|
|
26690
27364
|
SnapshotListResponse: snapshotListResponseSchema,
|
|
26691
27365
|
SnapshotReportDto: snapshotReportSchema,
|
|
@@ -27142,6 +27816,18 @@ var measurementCursorParameter = {
|
|
|
27142
27816
|
description: "Opaque cursor from the previous page.",
|
|
27143
27817
|
schema: stringSchema
|
|
27144
27818
|
};
|
|
27819
|
+
var crawlCursorParameter = {
|
|
27820
|
+
name: "cursor",
|
|
27821
|
+
in: "query",
|
|
27822
|
+
description: "Opaque cursor from the previous bounded crawl response.",
|
|
27823
|
+
schema: stringSchema
|
|
27824
|
+
};
|
|
27825
|
+
var crawlLimitParameter = {
|
|
27826
|
+
name: "limit",
|
|
27827
|
+
in: "query",
|
|
27828
|
+
description: "Page size. Defaults to 100 (50 for structure/neighbors); maximum 200 (100 for structure/neighbors).",
|
|
27829
|
+
schema: { type: "integer", minimum: 1, maximum: 200 }
|
|
27830
|
+
};
|
|
27145
27831
|
var measurementOverviewSortParameter = {
|
|
27146
27832
|
name: "sort",
|
|
27147
27833
|
in: "query",
|
|
@@ -32593,11 +33279,219 @@ var routeCatalog = [
|
|
|
32593
33279
|
404: errorResponse("Project not found.")
|
|
32594
33280
|
}
|
|
32595
33281
|
},
|
|
33282
|
+
{
|
|
33283
|
+
method: "get",
|
|
33284
|
+
path: "/api/v1/projects/{name}/technical-aeo/crawl",
|
|
33285
|
+
summary: "Get persisted Technical AEO crawl metadata",
|
|
33286
|
+
description: "Returns the latest complete non-probe site-audit crawl, or the selected historical run (which may be partial). `hasCrawlData=false` never synthesizes a graph from legacy scorecard rows; `legacyAuditAvailable` says that the old score/pages/trend data can still be read separately.",
|
|
33287
|
+
tags: ["technical-aeo"],
|
|
33288
|
+
parameters: [
|
|
33289
|
+
nameParameter,
|
|
33290
|
+
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest crawl.", schema: stringSchema }
|
|
33291
|
+
],
|
|
33292
|
+
responses: {
|
|
33293
|
+
200: jsonResponse("Persisted crawl summary returned.", "SiteCrawlSummaryDto"),
|
|
33294
|
+
404: errorResponse("Project or crawl-bearing site-audit run not found.")
|
|
33295
|
+
}
|
|
33296
|
+
},
|
|
33297
|
+
{
|
|
33298
|
+
method: "get",
|
|
33299
|
+
path: "/api/v1/projects/{name}/technical-aeo/graph",
|
|
33300
|
+
summary: "Get a persisted Site Health graph projection",
|
|
33301
|
+
description: "Returns the deterministic graph projection computed once when the latest complete or selected historical crawl was published. ForceAtlas2 positions and the exact internal-anchor edge sample are persisted, so reads run no layout physics and never rescan the crawl edge table. The response is bounded to 20,000 nodes / 50,000 edges; `layout`, `omittedNodes`, `omittedEdges`, and `sampled` disclose legacy/unavailable layouts and intentional truncation.",
|
|
33302
|
+
tags: ["technical-aeo"],
|
|
33303
|
+
parameters: [
|
|
33304
|
+
nameParameter,
|
|
33305
|
+
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest complete crawl.", schema: stringSchema },
|
|
33306
|
+
{ name: "maxNodes", in: "query", description: "Maximum graph nodes. Defaults to and is capped at 20,000.", schema: { type: "integer", minimum: 1, maximum: 2e4 } },
|
|
33307
|
+
{ name: "maxEdges", in: "query", description: "Maximum graph edges. Defaults to and is capped at 50,000.", schema: { type: "integer", minimum: 1, maximum: 5e4 } }
|
|
33308
|
+
],
|
|
33309
|
+
responses: {
|
|
33310
|
+
200: jsonResponse("Bounded Site Health graph projection returned.", "SiteCrawlGraphResponseDto"),
|
|
33311
|
+
404: errorResponse("Project or crawl-bearing site-audit run not found.")
|
|
33312
|
+
}
|
|
33313
|
+
},
|
|
33314
|
+
{
|
|
33315
|
+
method: "get",
|
|
33316
|
+
path: "/api/v1/projects/{name}/technical-aeo/subgraph",
|
|
33317
|
+
summary: "Get a bounded canonical Site Health neighborhood",
|
|
33318
|
+
description: "Agent-oriented canonical crawl neighborhood around one page. Reads full crawl page/link rows rather than the sampled visualization, excludes layout coordinates, and returns snapshot completeness/termination. When a hard bound is reached, countAccuracy=lower-bound makes totalNodes, totalEdges, omittedNodes, and omittedEdges explicit minimums rather than false exact totals.",
|
|
33319
|
+
tags: ["technical-aeo"],
|
|
33320
|
+
parameters: [
|
|
33321
|
+
nameParameter,
|
|
33322
|
+
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest complete crawl.", schema: stringSchema },
|
|
33323
|
+
{ name: "nodeKey", in: "query", description: "Focus page canonical node key. Omit nodeKey and url to use the crawl root.", schema: stringSchema },
|
|
33324
|
+
{ name: "url", in: "query", description: "Focus page canonical URL. Omit nodeKey and url to use the crawl root.", schema: stringSchema },
|
|
33325
|
+
{ name: "hops", in: "query", description: "Undirected neighborhood radius. Defaults to 1; maximum 3.", schema: { type: "integer", minimum: 0, maximum: 3 } },
|
|
33326
|
+
{ name: "maxNodes", in: "query", description: "Maximum canonical nodes. Defaults to 25; maximum 200.", schema: { type: "integer", minimum: 1, maximum: 200 } },
|
|
33327
|
+
{ name: "maxEdges", in: "query", description: "Maximum canonical edges. Defaults to 50; maximum 500.", schema: { type: "integer", minimum: 1, maximum: 500 } }
|
|
33328
|
+
],
|
|
33329
|
+
responses: {
|
|
33330
|
+
200: jsonResponse("Bounded canonical Site Health neighborhood returned.", "SiteHealthSubgraphResponseDto"),
|
|
33331
|
+
404: errorResponse("Project, crawl-bearing run, or focus page not found.")
|
|
33332
|
+
}
|
|
33333
|
+
},
|
|
33334
|
+
{
|
|
33335
|
+
method: "get",
|
|
33336
|
+
path: "/api/v1/projects/{name}/technical-aeo/path",
|
|
33337
|
+
summary: "Find a directed internal-link path between pages",
|
|
33338
|
+
description: "Returns the shortest directed path over followable internal anchor links. The source defaults to the crawl root; the target must be identified by toNodeKey or toUrl. Exploration is bounded to 5,000 nodes and returns an explicit unreachable or truncated state. Snapshot complete and termination provenance prevents an unreachable result from a selected partial crawl being mistaken for a site-wide claim.",
|
|
33339
|
+
tags: ["technical-aeo"],
|
|
33340
|
+
parameters: [
|
|
33341
|
+
nameParameter,
|
|
33342
|
+
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest complete crawl.", schema: stringSchema },
|
|
33343
|
+
{ name: "fromNodeKey", in: "query", description: "Source canonical node key. Omit both source selectors to start at the crawl root.", schema: stringSchema },
|
|
33344
|
+
{ name: "fromUrl", in: "query", description: "Source canonical URL. Omit both source selectors to start at the crawl root.", schema: stringSchema },
|
|
33345
|
+
{ name: "toNodeKey", in: "query", description: "Target canonical node key. Required when toUrl is omitted.", schema: stringSchema },
|
|
33346
|
+
{ name: "toUrl", in: "query", description: "Target canonical URL. Required when toNodeKey is omitted.", schema: stringSchema },
|
|
33347
|
+
{ name: "maxDepth", in: "query", description: "Maximum directed link depth. Defaults to 12; maximum 24.", schema: { type: "integer", minimum: 1, maximum: 24 } }
|
|
33348
|
+
],
|
|
33349
|
+
responses: {
|
|
33350
|
+
200: jsonResponse("Directed internal-link path result returned.", "SiteHealthPathResponseDto"),
|
|
33351
|
+
400: errorResponse("A target selector is required."),
|
|
33352
|
+
404: errorResponse("Project, crawl-bearing run, source page, or target page not found.")
|
|
33353
|
+
}
|
|
33354
|
+
},
|
|
33355
|
+
{
|
|
33356
|
+
method: "get",
|
|
33357
|
+
path: "/api/v1/projects/{name}/technical-aeo/changes",
|
|
33358
|
+
summary: "Compare canonical Site Health snapshots",
|
|
33359
|
+
description: "Returns exact page and internal-link additions, removals, and semantic field changes between immutable complete crawl snapshots. Omitted toRunId selects the latest complete crawl; omitted fromRunId selects the complete crawl immediately before the target. ForceAtlas2 positions are presentation data and never count as a change. The first page includes exact post-filter summary counts; cursor pages set summary and total to null so they never repeat full snapshot joins or trust caller-carried counts. Results use a snapshot- and filter-bound keyset cursor.",
|
|
33360
|
+
tags: ["technical-aeo"],
|
|
33361
|
+
parameters: [
|
|
33362
|
+
nameParameter,
|
|
33363
|
+
{ name: "fromRunId", in: "query", description: "Baseline complete crawl. Omit for the complete crawl immediately before the target.", schema: stringSchema },
|
|
33364
|
+
{ name: "toRunId", in: "query", description: "Target complete crawl. Omit for the latest complete crawl.", schema: stringSchema },
|
|
33365
|
+
{ name: "scope", in: "query", description: "Restrict records and first-page summary counts to pages, links, or both.", schema: { type: "string", enum: ["all", "pages", "links"], default: "all" } },
|
|
33366
|
+
{ name: "change", in: "query", description: "Restrict records and first-page summary counts to one change kind.", schema: { type: "string", enum: ["all", "added", "removed", "changed"], default: "all" } },
|
|
33367
|
+
{ name: "cursor", in: "query", description: "Opaque keyset cursor bound to both run IDs and the selected filters.", schema: stringSchema },
|
|
33368
|
+
{ name: "limit", in: "query", description: "Maximum change records. Defaults to 25; maximum 100.", schema: { type: "integer", minimum: 1, maximum: 100 } }
|
|
33369
|
+
],
|
|
33370
|
+
responses: {
|
|
33371
|
+
200: jsonResponse("Canonical Site Health snapshot comparison returned.", "SiteHealthChangesResponseDto"),
|
|
33372
|
+
400: errorResponse("Invalid filters, run direction, or cursor context."),
|
|
33373
|
+
404: errorResponse("Project or selected crawl-bearing run not found.")
|
|
33374
|
+
}
|
|
33375
|
+
},
|
|
33376
|
+
{
|
|
33377
|
+
method: "get",
|
|
33378
|
+
path: "/api/v1/projects/{name}/technical-aeo/crawl/pages/audit",
|
|
33379
|
+
summary: "Get exact audit evidence for one persisted crawl page",
|
|
33380
|
+
description: "Returns one page score with the exact weighted-factor findings, recommendations, applicability, and independent critical defects persisted for the selected crawl. Exactly one of nodeKey or url is required. The discriminated state preserves crawl completeness and termination provenance, distinguishes a page that was not audited from one that was not found, and marks legacy score-only rows without fabricating missing evidence. This one-page read keeps verbose evidence out of the bounded interactive graph projection.",
|
|
33381
|
+
tags: ["technical-aeo"],
|
|
33382
|
+
parameters: [
|
|
33383
|
+
nameParameter,
|
|
33384
|
+
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest complete crawl.", schema: stringSchema },
|
|
33385
|
+
{ name: "nodeKey", in: "query", description: "Exact canonical crawl node key. Required when url is omitted.", schema: stringSchema },
|
|
33386
|
+
{ name: "url", in: "query", description: "Exact persisted crawl URL. Required when nodeKey is omitted.", schema: stringSchema }
|
|
33387
|
+
],
|
|
33388
|
+
responses: {
|
|
33389
|
+
200: jsonResponse("Page audit evidence state returned.", "SiteCrawlPageAuditDto"),
|
|
33390
|
+
400: errorResponse("Exactly one page selector is required."),
|
|
33391
|
+
404: errorResponse("Project or explicitly selected crawl-bearing run not found.")
|
|
33392
|
+
}
|
|
33393
|
+
},
|
|
33394
|
+
{
|
|
33395
|
+
method: "get",
|
|
33396
|
+
path: "/api/v1/projects/{name}/technical-aeo/crawl/pages",
|
|
33397
|
+
summary: "List persisted Technical AEO crawl pages",
|
|
33398
|
+
description: "Cursor-paged canonical crawl nodes for the latest or selected crawl. `inventoryEligible` is Canonry technical-inventory eligibility, not a statement about actual Google index state.",
|
|
33399
|
+
tags: ["technical-aeo"],
|
|
33400
|
+
parameters: [
|
|
33401
|
+
nameParameter,
|
|
33402
|
+
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest crawl.", schema: stringSchema },
|
|
33403
|
+
{ name: "inventoryEligible", in: "query", description: "Filter Canonry technical inventory eligibility (`true` or `false`).", schema: booleanSchema },
|
|
33404
|
+
{ name: "fetchState", in: "query", description: "Filter by persisted fetch state.", schema: stringSchema },
|
|
33405
|
+
{ name: "indexabilityState", in: "query", description: "Filter by crawler-derived indexability state; this is not Google index coverage.", schema: stringSchema },
|
|
33406
|
+
{ name: "auditState", in: "query", description: "Filter by audit state.", schema: stringSchema },
|
|
33407
|
+
{ name: "sort", in: "query", description: "Sort order. Defaults to `url`.", schema: { type: "string", enum: ["url", "path", "score-asc", "score-desc"] } },
|
|
33408
|
+
crawlCursorParameter,
|
|
33409
|
+
crawlLimitParameter
|
|
33410
|
+
],
|
|
33411
|
+
responses: {
|
|
33412
|
+
200: jsonResponse("Crawl pages returned.", "SiteCrawlPagesResponseDto"),
|
|
33413
|
+
404: errorResponse("Project or crawl-bearing site-audit run not found.")
|
|
33414
|
+
}
|
|
33415
|
+
},
|
|
33416
|
+
{
|
|
33417
|
+
method: "get",
|
|
33418
|
+
path: "/api/v1/projects/{name}/technical-aeo/structure",
|
|
33419
|
+
summary: "List one level of Technical AEO crawl structure",
|
|
33420
|
+
description: "Returns immediate children under `parentPath` only. It is deliberately bounded and never emits a full site tree.",
|
|
33421
|
+
tags: ["technical-aeo"],
|
|
33422
|
+
parameters: [
|
|
33423
|
+
nameParameter,
|
|
33424
|
+
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest crawl.", schema: stringSchema },
|
|
33425
|
+
{ name: "parentPath", in: "query", description: "Parent URL path. Defaults to `/`.", schema: stringSchema },
|
|
33426
|
+
crawlCursorParameter,
|
|
33427
|
+
{ name: "limit", in: "query", description: "Maximum immediate children. Defaults to 50; maximum 100.", schema: { type: "integer", minimum: 1, maximum: 100 } }
|
|
33428
|
+
],
|
|
33429
|
+
responses: {
|
|
33430
|
+
200: jsonResponse("Immediate crawl children returned.", "SiteCrawlStructureResponseDto"),
|
|
33431
|
+
404: errorResponse("Project or crawl-bearing site-audit run not found.")
|
|
33432
|
+
}
|
|
33433
|
+
},
|
|
33434
|
+
{
|
|
33435
|
+
method: "get",
|
|
33436
|
+
path: "/api/v1/projects/{name}/technical-aeo/internal-links",
|
|
33437
|
+
summary: "List persisted internal crawl links",
|
|
33438
|
+
description: "Cursor-paged internal edges for the latest or selected crawl. Optional source/target/followability filters remain project-, run-, and attempt-scoped.",
|
|
33439
|
+
tags: ["technical-aeo"],
|
|
33440
|
+
parameters: [
|
|
33441
|
+
nameParameter,
|
|
33442
|
+
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest crawl.", schema: stringSchema },
|
|
33443
|
+
{ name: "sourceUrl", in: "query", description: "Restrict to a source URL.", schema: stringSchema },
|
|
33444
|
+
{ name: "targetUrl", in: "query", description: "Restrict to a target URL.", schema: stringSchema },
|
|
33445
|
+
{ name: "followable", in: "query", description: "Restrict to followable or nofollow link observations.", schema: booleanSchema },
|
|
33446
|
+
crawlCursorParameter,
|
|
33447
|
+
crawlLimitParameter
|
|
33448
|
+
],
|
|
33449
|
+
responses: {
|
|
33450
|
+
200: jsonResponse("Internal crawl links returned.", "SiteCrawlInternalLinksResponseDto"),
|
|
33451
|
+
404: errorResponse("Project or crawl-bearing site-audit run not found.")
|
|
33452
|
+
}
|
|
33453
|
+
},
|
|
33454
|
+
{
|
|
33455
|
+
method: "get",
|
|
33456
|
+
path: "/api/v1/projects/{name}/technical-aeo/internal-links/neighbors",
|
|
33457
|
+
summary: "Get a bounded internal-link neighborhood",
|
|
33458
|
+
description: "Returns bounded inbound and outbound internal edges for one `nodeKey` or URL; this is not a full graph traversal.",
|
|
33459
|
+
tags: ["technical-aeo"],
|
|
33460
|
+
parameters: [
|
|
33461
|
+
nameParameter,
|
|
33462
|
+
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest crawl.", schema: stringSchema },
|
|
33463
|
+
{ name: "nodeKey", in: "query", description: "Canonical crawl node key.", schema: stringSchema },
|
|
33464
|
+
{ name: "url", in: "query", description: "Canonical crawl URL.", schema: stringSchema },
|
|
33465
|
+
{ name: "limit", in: "query", description: "Maximum inbound and outbound edges independently. Defaults to 50; maximum 100.", schema: { type: "integer", minimum: 1, maximum: 100 } }
|
|
33466
|
+
],
|
|
33467
|
+
responses: {
|
|
33468
|
+
200: jsonResponse("Bounded internal-link neighborhood returned.", "SiteCrawlNeighborsResponseDto"),
|
|
33469
|
+
400: errorResponse("A nodeKey or URL is required."),
|
|
33470
|
+
404: errorResponse("Project or crawl-bearing site-audit run not found.")
|
|
33471
|
+
}
|
|
33472
|
+
},
|
|
33473
|
+
{
|
|
33474
|
+
method: "get",
|
|
33475
|
+
path: "/api/v1/projects/{name}/technical-aeo/dead-links",
|
|
33476
|
+
summary: "List dead-link findings when checks were enabled",
|
|
33477
|
+
description: "Returns a discriminated `disabled`, `complete`, `partial`, or `unavailable` state. A disabled check is never returned as an empty zero-result list.",
|
|
33478
|
+
tags: ["technical-aeo"],
|
|
33479
|
+
parameters: [
|
|
33480
|
+
nameParameter,
|
|
33481
|
+
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest crawl.", schema: stringSchema },
|
|
33482
|
+
crawlCursorParameter,
|
|
33483
|
+
crawlLimitParameter
|
|
33484
|
+
],
|
|
33485
|
+
responses: {
|
|
33486
|
+
200: jsonResponse("Dead-link status and findings returned.", "SiteCrawlDeadLinksResponseDto"),
|
|
33487
|
+
404: errorResponse("Project or crawl-bearing site-audit run not found.")
|
|
33488
|
+
}
|
|
33489
|
+
},
|
|
32596
33490
|
{
|
|
32597
33491
|
method: "post",
|
|
32598
33492
|
path: "/api/v1/projects/{name}/technical-aeo/runs",
|
|
32599
33493
|
summary: "Trigger a Technical AEO site-audit run",
|
|
32600
|
-
description: "Queues a `site-audit` run
|
|
33494
|
+
description: "Queues a `site-audit` run. The run discovers pages from the root, recursive sitemaps, and internal links. An active run is reused only when its persisted effective sitemap, budgets, depth, and dead-link option match exactly; a semantic mismatch returns 409.",
|
|
32601
33495
|
tags: ["technical-aeo"],
|
|
32602
33496
|
parameters: [nameParameter],
|
|
32603
33497
|
requestBody: {
|
|
@@ -32607,8 +33501,12 @@ var routeCatalog = [
|
|
|
32607
33501
|
schema: {
|
|
32608
33502
|
type: "object",
|
|
32609
33503
|
properties: {
|
|
32610
|
-
sitemapUrl: { ...stringSchema, description: "
|
|
32611
|
-
limit: { ...integerSchema, description: "
|
|
33504
|
+
sitemapUrl: { ...stringSchema, description: "Deprecated compatibility alias for the sitemap override." },
|
|
33505
|
+
limit: { ...integerSchema, minimum: 1, maximum: 2e3, description: "Deprecated compatibility alias for maxPages. Max 2000." },
|
|
33506
|
+
maxPages: { ...integerSchema, minimum: 1, maximum: 5e4, description: "Crawl page budget. Defaults to 1000; max 50000." },
|
|
33507
|
+
maxEdges: { ...integerSchema, minimum: 1, maximum: 1e6, description: "Internal-link observation budget. Defaults to 100000; max 1000000." },
|
|
33508
|
+
maxDepth: { ...integerSchema, minimum: 0, maximum: 100, description: "Maximum crawl depth from the root. Max 100." },
|
|
33509
|
+
checkDeadLinks: { ...booleanSchema, default: false, description: "Enable dead-link checking. Defaults to false." }
|
|
32612
33510
|
}
|
|
32613
33511
|
}
|
|
32614
33512
|
}
|
|
@@ -32617,6 +33515,7 @@ var routeCatalog = [
|
|
|
32617
33515
|
responses: {
|
|
32618
33516
|
200: jsonResponse("Site-audit run queued (or the in-flight run returned).", "SiteAuditRunResponseDto"),
|
|
32619
33517
|
400: errorResponse("Invalid site-audit request."),
|
|
33518
|
+
409: errorResponse("A crawl with different effective options is already queued or running."),
|
|
32620
33519
|
404: errorResponse("Project not found.")
|
|
32621
33520
|
}
|
|
32622
33521
|
}
|
|
@@ -33654,7 +34553,7 @@ function formatNotification(row) {
|
|
|
33654
34553
|
|
|
33655
34554
|
// ../api-routes/src/google.ts
|
|
33656
34555
|
import crypto29 from "crypto";
|
|
33657
|
-
import { eq as eq39, and as and30, desc as desc17, sql as sql13, inArray as
|
|
34556
|
+
import { eq as eq39, and as and30, desc as desc17, sql as sql13, inArray as inArray16 } from "drizzle-orm";
|
|
33658
34557
|
|
|
33659
34558
|
// ../api-routes/src/gbp-summary.ts
|
|
33660
34559
|
function computeMetricTotals(rows) {
|
|
@@ -36914,10 +37813,10 @@ async function googleRoutes(app, opts) {
|
|
|
36914
37813
|
locationProfiles: []
|
|
36915
37814
|
});
|
|
36916
37815
|
}
|
|
36917
|
-
const metricRows = app.db.select().from(gbpDailyMetrics).where(and30(eq39(gbpDailyMetrics.projectId, project.id),
|
|
36918
|
-
const keywordRows = app.db.select().from(gbpKeywordImpressions).where(and30(eq39(gbpKeywordImpressions.projectId, project.id),
|
|
36919
|
-
const placeActionRows = app.db.select().from(gbpPlaceActions).where(and30(eq39(gbpPlaceActions.projectId, project.id),
|
|
36920
|
-
const lodgingRows = app.db.select().from(gbpLodgingSnapshots).where(and30(eq39(gbpLodgingSnapshots.projectId, project.id),
|
|
37816
|
+
const metricRows = app.db.select().from(gbpDailyMetrics).where(and30(eq39(gbpDailyMetrics.projectId, project.id), inArray16(gbpDailyMetrics.locationName, locationNames))).all();
|
|
37817
|
+
const keywordRows = app.db.select().from(gbpKeywordImpressions).where(and30(eq39(gbpKeywordImpressions.projectId, project.id), inArray16(gbpKeywordImpressions.locationName, locationNames))).all();
|
|
37818
|
+
const placeActionRows = app.db.select().from(gbpPlaceActions).where(and30(eq39(gbpPlaceActions.projectId, project.id), inArray16(gbpPlaceActions.locationName, locationNames))).all();
|
|
37819
|
+
const lodgingRows = app.db.select().from(gbpLodgingSnapshots).where(and30(eq39(gbpLodgingSnapshots.projectId, project.id), inArray16(gbpLodgingSnapshots.locationName, locationNames))).orderBy(desc17(gbpLodgingSnapshots.syncedAt)).all();
|
|
36921
37820
|
const latestLodgingByLocation = /* @__PURE__ */ new Map();
|
|
36922
37821
|
for (const row of lodgingRows) {
|
|
36923
37822
|
if (!latestLodgingByLocation.has(row.locationName)) {
|
|
@@ -36931,7 +37830,7 @@ async function googleRoutes(app, opts) {
|
|
|
36931
37830
|
regularHours: gbpLocations.regularHours,
|
|
36932
37831
|
primaryPhone: gbpLocations.primaryPhone,
|
|
36933
37832
|
openStatus: gbpLocations.openStatus
|
|
36934
|
-
}).from(gbpLocations).where(and30(eq39(gbpLocations.projectId, project.id),
|
|
37833
|
+
}).from(gbpLocations).where(and30(eq39(gbpLocations.projectId, project.id), inArray16(gbpLocations.locationName, locationNames))).all();
|
|
36935
37834
|
return buildGbpSummary({
|
|
36936
37835
|
locationName,
|
|
36937
37836
|
locationCount: locationNames.length,
|
|
@@ -36947,7 +37846,7 @@ async function googleRoutes(app, opts) {
|
|
|
36947
37846
|
|
|
36948
37847
|
// ../api-routes/src/ads.ts
|
|
36949
37848
|
import crypto32 from "crypto";
|
|
36950
|
-
import { eq as eq41, and as and32, asc as asc8, desc as desc18, gt as gt2, gte as gte9, lt as lt6, lte as lte8, ne as ne4, inArray as
|
|
37849
|
+
import { eq as eq41, and as and32, asc as asc8, desc as desc18, gt as gt2, gte as gte9, lt as lt6, lte as lte8, ne as ne4, inArray as inArray17, or as or9, isNull as isNull3, isNotNull as isNotNull2, sql as sql15 } from "drizzle-orm";
|
|
36951
37850
|
|
|
36952
37851
|
// ../api-routes/src/ads-activation-routes.ts
|
|
36953
37852
|
import crypto31 from "crypto";
|
|
@@ -39894,7 +40793,7 @@ function semanticallyMatchesMaterializedEntity(app, input) {
|
|
|
39894
40793
|
eq41(adsOperations.projectId, input.projectId),
|
|
39895
40794
|
eq41(adsOperations.entityType, input.entityType),
|
|
39896
40795
|
eq41(adsOperations.entityId, input.entityId),
|
|
39897
|
-
|
|
40796
|
+
inArray17(adsOperations.kind, [createKind, updateKind])
|
|
39898
40797
|
)).orderBy(asc8(adsOperations.createdAt), asc8(adsOperations.id)).all();
|
|
39899
40798
|
if (rows.some((row) => row.state === AdsOperationStates.pending || row.state === AdsOperationStates.reconciling || row.state === AdsOperationStates.unknown || row.kind === updateKind && row.state === AdsOperationStates.succeeded)) {
|
|
39900
40799
|
return false;
|
|
@@ -40331,7 +41230,7 @@ function quarantineExhaustedReconciliation(app, row, now, policy, context) {
|
|
|
40331
41230
|
eq41(adsOperations.id, row.id),
|
|
40332
41231
|
eq41(adsOperations.projectId, row.projectId),
|
|
40333
41232
|
gte9(adsOperations.reconcileAttempts, policy.maxAttempts),
|
|
40334
|
-
|
|
41233
|
+
inArray17(adsOperations.state, [
|
|
40335
41234
|
AdsOperationStates.pending,
|
|
40336
41235
|
AdsOperationStates.unknown,
|
|
40337
41236
|
AdsOperationStates.reconciling
|
|
@@ -40555,7 +41454,7 @@ async function sweepAdsOperationsOnce(app, opts, config) {
|
|
|
40555
41454
|
eq41(adsOperations.reconcileStrategy, AdsReconcileStrategies.create_fingerprint),
|
|
40556
41455
|
or9(
|
|
40557
41456
|
isNotNull2(adsOperations.entityId),
|
|
40558
|
-
|
|
41457
|
+
inArray17(adsOperations.state, [
|
|
40559
41458
|
AdsOperationStates.pending,
|
|
40560
41459
|
AdsOperationStates.reconciling
|
|
40561
41460
|
])
|
|
@@ -40603,7 +41502,7 @@ async function sweepAdsOperationsOnce(app, opts, config) {
|
|
|
40603
41502
|
const eligibleProjectIds = [...credentialsByProjectId.keys()];
|
|
40604
41503
|
if (eligibleProjectIds.length === 0) return;
|
|
40605
41504
|
const rows = app.db.select().from(adsOperations).where(and32(
|
|
40606
|
-
|
|
41505
|
+
inArray17(adsOperations.projectId, eligibleProjectIds),
|
|
40607
41506
|
staleReconcileWhere
|
|
40608
41507
|
)).orderBy(asc8(adsOperations.updatedAt)).limit(config.batchSize).all();
|
|
40609
41508
|
for (const row of rows) {
|
|
@@ -40931,7 +41830,7 @@ async function adsRoutes(app, opts) {
|
|
|
40931
41830
|
const cursor = parsed.data.cursor ? decodeAdsOperationCursor(parsed.data.cursor, project.id, parsed.data.state) : void 0;
|
|
40932
41831
|
const rows = app.db.select().from(adsOperations).where(and32(
|
|
40933
41832
|
eq41(adsOperations.projectId, project.id),
|
|
40934
|
-
|
|
41833
|
+
inArray17(adsOperations.state, parsed.data.state),
|
|
40935
41834
|
cursor ? or9(
|
|
40936
41835
|
gt2(adsOperations.createdAt, cursor.createdAt),
|
|
40937
41836
|
and32(
|
|
@@ -41278,7 +42177,7 @@ async function adsRoutes(app, opts) {
|
|
|
41278
42177
|
const inFlight = app.db.select({ id: runs.id, status: runs.status }).from(runs).where(and32(
|
|
41279
42178
|
eq41(runs.projectId, project.id),
|
|
41280
42179
|
eq41(runs.kind, RunKinds["ads-sync"]),
|
|
41281
|
-
|
|
42180
|
+
inArray17(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
41282
42181
|
)).get();
|
|
41283
42182
|
if (inFlight) {
|
|
41284
42183
|
const existing = { runId: inFlight.id, status: inFlight.status };
|
|
@@ -45912,7 +46811,7 @@ async function queryBacklinks(opts) {
|
|
|
45912
46811
|
const reversed = opts.targets.map(reverseDomain);
|
|
45913
46812
|
const targetList = reversed.map(quote).join(", ");
|
|
45914
46813
|
const limitClause = opts.limitPerTarget ? `QUALIFY row_number() OVER (PARTITION BY t.target_rev_domain ORDER BY v.num_hosts DESC) <= ${Math.floor(opts.limitPerTarget)}` : "";
|
|
45915
|
-
const
|
|
46814
|
+
const sql21 = `
|
|
45916
46815
|
WITH vertices AS (
|
|
45917
46816
|
SELECT * FROM read_csv(
|
|
45918
46817
|
${quote(opts.vertexPath)},
|
|
@@ -45948,7 +46847,7 @@ async function queryBacklinks(opts) {
|
|
|
45948
46847
|
const conn = await instance.connect();
|
|
45949
46848
|
let rows;
|
|
45950
46849
|
try {
|
|
45951
|
-
const reader = await conn.runAndReadAll(
|
|
46850
|
+
const reader = await conn.runAndReadAll(sql21);
|
|
45952
46851
|
rows = reader.getRowObjects();
|
|
45953
46852
|
} finally {
|
|
45954
46853
|
conn.disconnectSync?.();
|
|
@@ -54135,7 +55034,7 @@ async function doctorRoutes(app, opts) {
|
|
|
54135
55034
|
|
|
54136
55035
|
// ../api-routes/src/discovery/routes.ts
|
|
54137
55036
|
import crypto41 from "crypto";
|
|
54138
|
-
import { and as and42, desc as desc23, eq as eq53, gte as gte12, inArray as
|
|
55037
|
+
import { and as and42, desc as desc23, eq as eq53, gte as gte12, inArray as inArray18, isNull as isNull4, or as or10 } from "drizzle-orm";
|
|
54139
55038
|
var MAX_INFLIGHT_DISCOVERY_AGE_MS = 2 * 60 * 60 * 1e3;
|
|
54140
55039
|
async function discoveryRoutes(app, opts) {
|
|
54141
55040
|
app.post("/projects/:name/discover/run", async (request, reply) => {
|
|
@@ -54190,7 +55089,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
54190
55089
|
// must never reuse another set's session. Null (= Gemini-only
|
|
54191
55090
|
// default, including explicit ['gemini']) matches legacy rows.
|
|
54192
55091
|
seedProviders == null ? isNull4(discoverySessions.seedProviders) : eq53(discoverySessions.seedProviders, seedProviders),
|
|
54193
|
-
|
|
55092
|
+
inArray18(discoverySessions.status, [
|
|
54194
55093
|
DiscoverySessionStatuses.queued,
|
|
54195
55094
|
DiscoverySessionStatuses.seeding,
|
|
54196
55095
|
DiscoverySessionStatuses.probing
|
|
@@ -54815,7 +55714,15 @@ function dedupeStrings(input) {
|
|
|
54815
55714
|
|
|
54816
55715
|
// ../api-routes/src/technical-aeo.ts
|
|
54817
55716
|
import crypto43 from "crypto";
|
|
54818
|
-
import { and as and43, asc as asc10, count, desc as desc24, eq as eq55, inArray as
|
|
55717
|
+
import { and as and43, asc as asc10, count, desc as desc24, eq as eq55, inArray as inArray19, isNotNull as isNotNull3, lt as lt7, or as or11, sql as sql20 } from "drizzle-orm";
|
|
55718
|
+
import { alias } from "drizzle-orm/sqlite-core";
|
|
55719
|
+
var FETCHED_SITE_CRAWL_STATES = /* @__PURE__ */ new Set([
|
|
55720
|
+
...SiteCrawlFetchedStates,
|
|
55721
|
+
// Snapshots published before the full-crawl contract used this aggregate
|
|
55722
|
+
// state. Keep historical structure counts truthful without admitting it to
|
|
55723
|
+
// the current crawler vocabulary.
|
|
55724
|
+
"fetched"
|
|
55725
|
+
]);
|
|
54819
55726
|
var SURFACEABLE_STATUSES = [RunStatuses.completed, RunStatuses.partial];
|
|
54820
55727
|
function emptyScore(projectName) {
|
|
54821
55728
|
return {
|
|
@@ -54844,13 +55751,274 @@ function parsePositiveInt(value, fallback, max) {
|
|
|
54844
55751
|
if (!Number.isFinite(n) || n < 0) return fallback;
|
|
54845
55752
|
return Math.min(max, Math.floor(n));
|
|
54846
55753
|
}
|
|
55754
|
+
function parseBoundedLimit(value, fallback, max) {
|
|
55755
|
+
const parsed = parsePositiveInt(value, fallback, max);
|
|
55756
|
+
return Math.max(1, parsed);
|
|
55757
|
+
}
|
|
55758
|
+
function parseBoolean2(value) {
|
|
55759
|
+
if (value === true || value === "true" || value === "1") return true;
|
|
55760
|
+
if (value === false || value === "false" || value === "0") return false;
|
|
55761
|
+
return null;
|
|
55762
|
+
}
|
|
55763
|
+
function encodeCursor2(offset) {
|
|
55764
|
+
return Buffer.from(JSON.stringify({ offset })).toString("base64url");
|
|
55765
|
+
}
|
|
55766
|
+
function decodeCursor(value) {
|
|
55767
|
+
if (typeof value !== "string" || value.length === 0 || value.length > 256) return 0;
|
|
55768
|
+
try {
|
|
55769
|
+
const parsed = JSON.parse(Buffer.from(value, "base64url").toString("utf8"));
|
|
55770
|
+
return typeof parsed.offset === "number" && Number.isSafeInteger(parsed.offset) && parsed.offset >= 0 ? Math.min(parsed.offset, 1e6) : 0;
|
|
55771
|
+
} catch {
|
|
55772
|
+
return 0;
|
|
55773
|
+
}
|
|
55774
|
+
}
|
|
55775
|
+
function normalizeParentPath(value) {
|
|
55776
|
+
if (typeof value !== "string" || value.trim() === "") return "/";
|
|
55777
|
+
const bare = value.trim().split(/[?#]/, 1)[0] ?? "/";
|
|
55778
|
+
const prefixed = bare.startsWith("/") ? bare : `/${bare}`;
|
|
55779
|
+
const collapsed = prefixed.replace(/\/{2,}/g, "/");
|
|
55780
|
+
return collapsed.length > 1 && collapsed.endsWith("/") ? collapsed.slice(0, -1) : collapsed;
|
|
55781
|
+
}
|
|
55782
|
+
var MAX_STRUCTURE_SOURCE_ROWS = 5e4;
|
|
55783
|
+
function structureHierarchyPath(path8) {
|
|
55784
|
+
if (path8.length <= 1) return path8;
|
|
55785
|
+
return path8.replace(/\/+$/, "") || "/";
|
|
55786
|
+
}
|
|
55787
|
+
function structureChildPath(parentPath, path8) {
|
|
55788
|
+
const hierarchyPath = structureHierarchyPath(path8);
|
|
55789
|
+
if (hierarchyPath === parentPath) return null;
|
|
55790
|
+
const relative = parentPath === "/" ? hierarchyPath.startsWith("/") ? hierarchyPath.slice(1) : "" : hierarchyPath.startsWith(`${parentPath}/`) ? hierarchyPath.slice(parentPath.length + 1) : "";
|
|
55791
|
+
if (relative.length === 0) return null;
|
|
55792
|
+
const separator = relative.indexOf("/");
|
|
55793
|
+
const segment = separator === -1 ? relative : relative.slice(0, separator);
|
|
55794
|
+
return parentPath === "/" ? `/${segment}` : `${parentPath}/${segment}`;
|
|
55795
|
+
}
|
|
55796
|
+
function mapCrawlPage(row) {
|
|
55797
|
+
return {
|
|
55798
|
+
nodeKey: row.nodeKey,
|
|
55799
|
+
url: row.url,
|
|
55800
|
+
finalUrl: row.finalUrl,
|
|
55801
|
+
path: row.path,
|
|
55802
|
+
parentPath: row.parentPath,
|
|
55803
|
+
discoverySource: row.discoverySource,
|
|
55804
|
+
fetchState: row.fetchState,
|
|
55805
|
+
httpStatus: row.httpStatus,
|
|
55806
|
+
canonicalUrl: row.canonicalUrl,
|
|
55807
|
+
indexabilityState: row.indexabilityState,
|
|
55808
|
+
indexabilityReasons: row.indexabilityReasons,
|
|
55809
|
+
auditState: row.auditState,
|
|
55810
|
+
auditScore: row.auditScore,
|
|
55811
|
+
inventoryEligible: row.inventoryEligible,
|
|
55812
|
+
depth: row.depth,
|
|
55813
|
+
inboundUniqueEdges: row.inboundUniqueEdges,
|
|
55814
|
+
outboundUniqueEdges: row.outboundUniqueEdges,
|
|
55815
|
+
inboundOccurrences: row.inboundOccurrences,
|
|
55816
|
+
outboundOccurrences: row.outboundOccurrences,
|
|
55817
|
+
linkScoreRaw: row.linkScoreRaw,
|
|
55818
|
+
linkScoreNormalized: row.linkScoreNormalized,
|
|
55819
|
+
healthState: deriveSiteHealthState(row)
|
|
55820
|
+
};
|
|
55821
|
+
}
|
|
55822
|
+
function mapCrawlPageAuditEvidence(row) {
|
|
55823
|
+
const fields = row.auditFields;
|
|
55824
|
+
const rawFactors = Array.isArray(fields.factors) ? fields.factors : [];
|
|
55825
|
+
const rawCriticalDefects = Array.isArray(fields.criticalDefects) ? fields.criticalDefects : [];
|
|
55826
|
+
const parsedFactors = rawFactors.map((factor) => siteCrawlAuditFactorSchema.safeParse(factor));
|
|
55827
|
+
const parsedCriticalDefects = rawCriticalDefects.map((defect) => siteCrawlCriticalDefectSchema.safeParse(defect));
|
|
55828
|
+
const complete = fields.schemaVersion === "1.0" && Array.isArray(fields.factors) && Array.isArray(fields.criticalDefects) && parsedFactors.every((result) => result.success) && parsedCriticalDefects.every((result) => result.success);
|
|
55829
|
+
const factors = parsedFactors.flatMap((result, index2) => {
|
|
55830
|
+
if (result.success) return [result.data];
|
|
55831
|
+
const legacy = siteAuditPageFactorSchema.safeParse(rawFactors[index2]);
|
|
55832
|
+
return legacy.success ? [{
|
|
55833
|
+
...legacy.data,
|
|
55834
|
+
status: factorStatusFromScore(legacy.data.score),
|
|
55835
|
+
applicable: null,
|
|
55836
|
+
findings: [],
|
|
55837
|
+
recommendations: []
|
|
55838
|
+
}] : [];
|
|
55839
|
+
});
|
|
55840
|
+
return {
|
|
55841
|
+
evidenceState: complete ? "complete" : "scores-only",
|
|
55842
|
+
factors,
|
|
55843
|
+
criticalDefects: parsedCriticalDefects.flatMap((result) => result.success ? [result.data] : [])
|
|
55844
|
+
};
|
|
55845
|
+
}
|
|
55846
|
+
function mapCrawlEdge(row) {
|
|
55847
|
+
return {
|
|
55848
|
+
edgeKey: row.edgeKey,
|
|
55849
|
+
sourceNodeKey: row.sourceNodeKey,
|
|
55850
|
+
sourceUrl: row.sourceUrl,
|
|
55851
|
+
targetNodeKey: row.targetNodeKey,
|
|
55852
|
+
targetUrl: row.targetUrl,
|
|
55853
|
+
relation: row.relation,
|
|
55854
|
+
internal: row.internal,
|
|
55855
|
+
followable: row.followable,
|
|
55856
|
+
occurrences: row.occurrences,
|
|
55857
|
+
followableOccurrences: row.followableOccurrences,
|
|
55858
|
+
nofollowOccurrences: row.nofollowOccurrences,
|
|
55859
|
+
anchors: row.anchors
|
|
55860
|
+
};
|
|
55861
|
+
}
|
|
55862
|
+
var graphSourceNode = alias(siteCrawlGraphNodes, "site_crawl_graph_source_node");
|
|
55863
|
+
var graphTargetNode = alias(siteCrawlGraphNodes, "site_crawl_graph_target_node");
|
|
55864
|
+
var PAGE_CHANGE_PREDICATE = sql20`(
|
|
55865
|
+
current.url IS NOT previous.url
|
|
55866
|
+
OR current.final_url IS NOT previous.final_url
|
|
55867
|
+
OR current.path IS NOT previous.path
|
|
55868
|
+
OR current.parent_path IS NOT previous.parent_path
|
|
55869
|
+
OR current.discovery_source IS NOT previous.discovery_source
|
|
55870
|
+
OR current.fetch_state IS NOT previous.fetch_state
|
|
55871
|
+
OR current.http_status IS NOT previous.http_status
|
|
55872
|
+
OR current.canonical_url IS NOT previous.canonical_url
|
|
55873
|
+
OR current.indexability_state IS NOT previous.indexability_state
|
|
55874
|
+
OR current.indexability_reasons IS NOT previous.indexability_reasons
|
|
55875
|
+
OR current.audit_state IS NOT previous.audit_state
|
|
55876
|
+
OR current.audit_score IS NOT previous.audit_score
|
|
55877
|
+
OR current.inventory_eligible IS NOT previous.inventory_eligible
|
|
55878
|
+
OR current.depth IS NOT previous.depth
|
|
55879
|
+
OR current.inbound_unique_edges IS NOT previous.inbound_unique_edges
|
|
55880
|
+
OR current.outbound_unique_edges IS NOT previous.outbound_unique_edges
|
|
55881
|
+
OR current.inbound_occurrences IS NOT previous.inbound_occurrences
|
|
55882
|
+
OR current.outbound_occurrences IS NOT previous.outbound_occurrences
|
|
55883
|
+
OR current.link_score_raw IS NOT previous.link_score_raw
|
|
55884
|
+
OR current.link_score_normalized IS NOT previous.link_score_normalized
|
|
55885
|
+
)`;
|
|
55886
|
+
var LINK_CHANGE_PREDICATE = sql20`(
|
|
55887
|
+
current.source_node_key IS NOT previous.source_node_key
|
|
55888
|
+
OR current.source_url IS NOT previous.source_url
|
|
55889
|
+
OR current.target_node_key IS NOT previous.target_node_key
|
|
55890
|
+
OR current.target_url IS NOT previous.target_url
|
|
55891
|
+
OR current.relation IS NOT previous.relation
|
|
55892
|
+
OR current.followable IS NOT previous.followable
|
|
55893
|
+
OR current.occurrences IS NOT previous.occurrences
|
|
55894
|
+
OR current.followable_occurrences IS NOT previous.followable_occurrences
|
|
55895
|
+
OR current.nofollow_occurrences IS NOT previous.nofollow_occurrences
|
|
55896
|
+
OR current.anchors IS NOT previous.anchors
|
|
55897
|
+
)`;
|
|
55898
|
+
function encodeSiteHealthChangesCursor(cursor) {
|
|
55899
|
+
return Buffer.from(JSON.stringify(cursor)).toString("base64url");
|
|
55900
|
+
}
|
|
55901
|
+
function decodeSiteHealthChangesCursor(value, context) {
|
|
55902
|
+
if (value == null || value === "") return null;
|
|
55903
|
+
if (typeof value !== "string" || value.length > 2048) throw validationError("Invalid Site Health changes cursor");
|
|
55904
|
+
try {
|
|
55905
|
+
const parsed = JSON.parse(Buffer.from(value, "base64url").toString("utf8"));
|
|
55906
|
+
const valid = parsed.v === 1 && parsed.fromRunId === context.fromRunId && parsed.toRunId === context.toRunId && parsed.scope === context.scope && parsed.change === context.change && (parsed.entity === "page" || parsed.entity === "link") && typeof parsed.key === "string" && parsed.key.length > 0 && parsed.key.length <= 2048;
|
|
55907
|
+
if (!valid) throw new Error("context mismatch");
|
|
55908
|
+
return parsed;
|
|
55909
|
+
} catch {
|
|
55910
|
+
throw validationError("Site Health changes cursor does not match this snapshot comparison and filter");
|
|
55911
|
+
}
|
|
55912
|
+
}
|
|
55913
|
+
function compareChangeKeys(left, right) {
|
|
55914
|
+
return Buffer.compare(Buffer.from(left.key), Buffer.from(right.key));
|
|
55915
|
+
}
|
|
55916
|
+
function valueChanged(before, after) {
|
|
55917
|
+
if (Array.isArray(before) || Array.isArray(after) || before && typeof before === "object" || after && typeof after === "object") {
|
|
55918
|
+
return JSON.stringify(before) !== JSON.stringify(after);
|
|
55919
|
+
}
|
|
55920
|
+
return before !== after;
|
|
55921
|
+
}
|
|
55922
|
+
var PAGE_CHANGE_FIELDS = [
|
|
55923
|
+
"url",
|
|
55924
|
+
"finalUrl",
|
|
55925
|
+
"path",
|
|
55926
|
+
"parentPath",
|
|
55927
|
+
"discoverySource",
|
|
55928
|
+
"fetchState",
|
|
55929
|
+
"httpStatus",
|
|
55930
|
+
"canonicalUrl",
|
|
55931
|
+
"indexabilityState",
|
|
55932
|
+
"indexabilityReasons",
|
|
55933
|
+
"auditState",
|
|
55934
|
+
"auditScore",
|
|
55935
|
+
"inventoryEligible",
|
|
55936
|
+
"depth",
|
|
55937
|
+
"inboundUniqueEdges",
|
|
55938
|
+
"outboundUniqueEdges",
|
|
55939
|
+
"inboundOccurrences",
|
|
55940
|
+
"outboundOccurrences",
|
|
55941
|
+
"linkScoreRaw",
|
|
55942
|
+
"linkScoreNormalized"
|
|
55943
|
+
];
|
|
55944
|
+
var LINK_CHANGE_FIELDS = [
|
|
55945
|
+
"sourceNodeKey",
|
|
55946
|
+
"sourceUrl",
|
|
55947
|
+
"targetNodeKey",
|
|
55948
|
+
"targetUrl",
|
|
55949
|
+
"relation",
|
|
55950
|
+
"internal",
|
|
55951
|
+
"followable",
|
|
55952
|
+
"occurrences",
|
|
55953
|
+
"followableOccurrences",
|
|
55954
|
+
"nofollowOccurrences",
|
|
55955
|
+
"anchors"
|
|
55956
|
+
];
|
|
55957
|
+
function changedFields(before, after, fields) {
|
|
55958
|
+
if (!before || !after) return [];
|
|
55959
|
+
return fields.filter((field) => valueChanged(before[field], after[field])).map(String);
|
|
55960
|
+
}
|
|
54847
55961
|
async function technicalAeoRoutes(app, opts) {
|
|
55962
|
+
const resolveCrawl = (projectId, runId) => {
|
|
55963
|
+
const filters = [
|
|
55964
|
+
eq55(siteCrawlSnapshots.projectId, projectId),
|
|
55965
|
+
eq55(runs.projectId, projectId),
|
|
55966
|
+
eq55(runs.kind, RunKinds["site-audit"]),
|
|
55967
|
+
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
55968
|
+
notProbeRun()
|
|
55969
|
+
];
|
|
55970
|
+
if (runId) {
|
|
55971
|
+
filters.push(eq55(siteCrawlSnapshots.runId, runId));
|
|
55972
|
+
} else {
|
|
55973
|
+
filters.push(eq55(siteCrawlSnapshots.complete, true), eq55(runs.status, RunStatuses.completed));
|
|
55974
|
+
}
|
|
55975
|
+
return app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq55(siteCrawlSnapshots.runId, runs.id)).where(and43(...filters)).orderBy(desc24(siteCrawlSnapshots.createdAt), desc24(siteCrawlSnapshots.runId)).limit(1).get();
|
|
55976
|
+
};
|
|
55977
|
+
const detailScopeFor = (projectId, snapshot) => snapshot.detailsAvailable && snapshot.attemptId ? { projectId, runId: snapshot.runId, attemptId: snapshot.attemptId } : null;
|
|
55978
|
+
const pageInScope = (scope, selector) => {
|
|
55979
|
+
if (!selector.nodeKey && !selector.url) return null;
|
|
55980
|
+
return app.db.select().from(siteCrawlPages).where(and43(
|
|
55981
|
+
eq55(siteCrawlPages.projectId, scope.projectId),
|
|
55982
|
+
eq55(siteCrawlPages.runId, scope.runId),
|
|
55983
|
+
eq55(siteCrawlPages.attemptId, scope.attemptId),
|
|
55984
|
+
selector.nodeKey ? eq55(siteCrawlPages.nodeKey, selector.nodeKey) : eq55(siteCrawlPages.url, selector.url)
|
|
55985
|
+
)).orderBy(asc10(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
55986
|
+
};
|
|
55987
|
+
const rootPageInScope = (scope, rootUrl) => pageInScope(scope, { url: rootUrl }) ?? app.db.select().from(siteCrawlPages).where(and43(
|
|
55988
|
+
eq55(siteCrawlPages.projectId, scope.projectId),
|
|
55989
|
+
eq55(siteCrawlPages.runId, scope.runId),
|
|
55990
|
+
eq55(siteCrawlPages.attemptId, scope.attemptId),
|
|
55991
|
+
eq55(siteCrawlPages.depth, 0)
|
|
55992
|
+
)).orderBy(asc10(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
55993
|
+
const hasLegacyAudit = (projectId) => Boolean(app.db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).innerJoin(runs, eq55(siteAuditSnapshots.runId, runs.id)).where(and43(
|
|
55994
|
+
eq55(siteAuditSnapshots.projectId, projectId),
|
|
55995
|
+
eq55(runs.projectId, projectId),
|
|
55996
|
+
eq55(runs.kind, RunKinds["site-audit"]),
|
|
55997
|
+
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
55998
|
+
notProbeRun()
|
|
55999
|
+
)).limit(1).get());
|
|
56000
|
+
const emptyCrawlSummary = (projectName, legacyAuditAvailable) => ({
|
|
56001
|
+
project: projectName,
|
|
56002
|
+
hasCrawlData: false,
|
|
56003
|
+
legacyAuditAvailable,
|
|
56004
|
+
runId: null,
|
|
56005
|
+
runStatus: null,
|
|
56006
|
+
requestedRootUrl: null,
|
|
56007
|
+
rootUrl: null,
|
|
56008
|
+
effectiveOptions: {},
|
|
56009
|
+
complete: false,
|
|
56010
|
+
termination: null,
|
|
56011
|
+
detailsAvailable: false,
|
|
56012
|
+
counts: { pagesDiscovered: 0, pagesFetched: 0, pagesEligible: 0, edges: 0, findings: 0 },
|
|
56013
|
+
deadLinks: { state: "unavailable" }
|
|
56014
|
+
});
|
|
54848
56015
|
app.get("/projects/:name/technical-aeo", async (request) => {
|
|
54849
56016
|
const project = resolveProject(app.db, request.params.name);
|
|
54850
56017
|
const baseFilters = [
|
|
54851
56018
|
eq55(siteAuditSnapshots.projectId, project.id),
|
|
56019
|
+
eq55(runs.projectId, project.id),
|
|
54852
56020
|
eq55(runs.kind, RunKinds["site-audit"]),
|
|
54853
|
-
|
|
56021
|
+
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
54854
56022
|
notProbeRun()
|
|
54855
56023
|
];
|
|
54856
56024
|
const targetFilters = request.query.runId ? [...baseFilters, eq55(siteAuditSnapshots.runId, request.query.runId)] : baseFilters;
|
|
@@ -54888,8 +56056,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
54888
56056
|
const project = resolveProject(app.db, request.params.name);
|
|
54889
56057
|
const targetFilters = [
|
|
54890
56058
|
eq55(siteAuditSnapshots.projectId, project.id),
|
|
56059
|
+
eq55(runs.projectId, project.id),
|
|
54891
56060
|
eq55(runs.kind, RunKinds["site-audit"]),
|
|
54892
|
-
|
|
56061
|
+
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
54893
56062
|
notProbeRun()
|
|
54894
56063
|
];
|
|
54895
56064
|
if (request.query.runId) targetFilters.push(eq55(siteAuditSnapshots.runId, request.query.runId));
|
|
@@ -54901,7 +56070,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
54901
56070
|
return { project: project.name, runId: null, auditedAt: null, total: 0, pages: [] };
|
|
54902
56071
|
}
|
|
54903
56072
|
const statusFilter = request.query.status === "success" || request.query.status === "error" ? request.query.status : null;
|
|
54904
|
-
const conds = [eq55(siteAuditPages.runId, latest.runId)];
|
|
56073
|
+
const conds = [eq55(siteAuditPages.projectId, project.id), eq55(siteAuditPages.runId, latest.runId)];
|
|
54905
56074
|
if (statusFilter) conds.push(eq55(siteAuditPages.status, statusFilter));
|
|
54906
56075
|
const where = and43(...conds);
|
|
54907
56076
|
const totalRow = app.db.select({ value: count() }).from(siteAuditPages).where(where).get();
|
|
@@ -54929,41 +56098,1185 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
54929
56098
|
pagesAudited: siteAuditSnapshots.pagesAudited
|
|
54930
56099
|
}).from(siteAuditSnapshots).innerJoin(runs, eq55(siteAuditSnapshots.runId, runs.id)).where(and43(
|
|
54931
56100
|
eq55(siteAuditSnapshots.projectId, project.id),
|
|
56101
|
+
eq55(runs.projectId, project.id),
|
|
54932
56102
|
eq55(runs.kind, RunKinds["site-audit"]),
|
|
54933
|
-
|
|
56103
|
+
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
54934
56104
|
notProbeRun()
|
|
54935
56105
|
)).orderBy(desc24(siteAuditSnapshots.createdAt)).limit(limit).all();
|
|
54936
56106
|
return { project: project.name, points: rows.reverse() };
|
|
54937
56107
|
});
|
|
56108
|
+
app.get("/projects/:name/technical-aeo/crawl", async (request) => {
|
|
56109
|
+
const project = resolveProject(app.db, request.params.name);
|
|
56110
|
+
const target = resolveCrawl(project.id, request.query.runId);
|
|
56111
|
+
const legacyAuditAvailable = hasLegacyAudit(project.id);
|
|
56112
|
+
if (!target) {
|
|
56113
|
+
if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
|
|
56114
|
+
return emptyCrawlSummary(project.name, legacyAuditAvailable);
|
|
56115
|
+
}
|
|
56116
|
+
const snapshot = target.snapshot;
|
|
56117
|
+
const deadLinks = !snapshot.checkDeadLinks || snapshot.deadLinkState === "disabled" ? { state: "disabled" } : snapshot.deadLinkState === "complete" ? { state: "complete", checked: snapshot.deadLinksChecked, found: snapshot.deadLinksFound } : snapshot.deadLinkState === "partial" ? { state: "partial", checked: snapshot.deadLinksChecked, found: snapshot.deadLinksFound } : { state: "unavailable" };
|
|
56118
|
+
return {
|
|
56119
|
+
project: project.name,
|
|
56120
|
+
hasCrawlData: true,
|
|
56121
|
+
legacyAuditAvailable,
|
|
56122
|
+
runId: snapshot.runId,
|
|
56123
|
+
runStatus: target.runStatus,
|
|
56124
|
+
requestedRootUrl: snapshot.requestedRootUrl,
|
|
56125
|
+
rootUrl: snapshot.rootUrl,
|
|
56126
|
+
crawlSchemaVersion: snapshot.crawlSchemaVersion,
|
|
56127
|
+
engineVersion: snapshot.engineVersion,
|
|
56128
|
+
normalizationVersion: snapshot.normalizationVersion,
|
|
56129
|
+
indexabilityVersion: snapshot.indexabilityVersion,
|
|
56130
|
+
linkScoreVersion: snapshot.linkScoreVersion,
|
|
56131
|
+
effectiveOptions: snapshot.effectiveOptions,
|
|
56132
|
+
complete: snapshot.complete,
|
|
56133
|
+
termination: snapshot.termination,
|
|
56134
|
+
detailsAvailable: snapshot.detailsAvailable,
|
|
56135
|
+
counts: {
|
|
56136
|
+
pagesDiscovered: snapshot.pagesDiscovered,
|
|
56137
|
+
pagesFetched: snapshot.pagesFetched,
|
|
56138
|
+
pagesEligible: snapshot.pagesEligible,
|
|
56139
|
+
edges: snapshot.edgesDiscovered,
|
|
56140
|
+
findings: snapshot.findingsCount
|
|
56141
|
+
},
|
|
56142
|
+
deadLinks
|
|
56143
|
+
};
|
|
56144
|
+
});
|
|
56145
|
+
app.get("/projects/:name/technical-aeo/graph", async (request) => {
|
|
56146
|
+
const project = resolveProject(app.db, request.params.name);
|
|
56147
|
+
const target = resolveCrawl(project.id, request.query.runId);
|
|
56148
|
+
if (!target) {
|
|
56149
|
+
if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
|
|
56150
|
+
return {
|
|
56151
|
+
project: project.name,
|
|
56152
|
+
hasCrawlData: false,
|
|
56153
|
+
runId: null,
|
|
56154
|
+
layout: { state: "unavailable", version: null, reason: "no-crawl" },
|
|
56155
|
+
totalNodes: 0,
|
|
56156
|
+
totalEdges: 0,
|
|
56157
|
+
nodes: [],
|
|
56158
|
+
edges: [],
|
|
56159
|
+
omittedNodes: 0,
|
|
56160
|
+
omittedEdges: 0,
|
|
56161
|
+
sampled: false
|
|
56162
|
+
};
|
|
56163
|
+
}
|
|
56164
|
+
const snapshot = target.snapshot;
|
|
56165
|
+
if (!snapshot.detailsAvailable || !snapshot.attemptId) {
|
|
56166
|
+
return {
|
|
56167
|
+
project: project.name,
|
|
56168
|
+
hasCrawlData: true,
|
|
56169
|
+
runId: snapshot.runId,
|
|
56170
|
+
layout: { state: "unavailable", version: null, reason: "details-unavailable" },
|
|
56171
|
+
totalNodes: 0,
|
|
56172
|
+
totalEdges: 0,
|
|
56173
|
+
nodes: [],
|
|
56174
|
+
edges: [],
|
|
56175
|
+
omittedNodes: 0,
|
|
56176
|
+
omittedEdges: 0,
|
|
56177
|
+
sampled: false
|
|
56178
|
+
};
|
|
56179
|
+
}
|
|
56180
|
+
const persistedLayout = app.db.select().from(siteCrawlGraphLayouts).where(and43(
|
|
56181
|
+
eq55(siteCrawlGraphLayouts.projectId, project.id),
|
|
56182
|
+
eq55(siteCrawlGraphLayouts.runId, snapshot.runId),
|
|
56183
|
+
eq55(siteCrawlGraphLayouts.attemptId, snapshot.attemptId)
|
|
56184
|
+
)).limit(1).get();
|
|
56185
|
+
if (!persistedLayout) {
|
|
56186
|
+
return {
|
|
56187
|
+
project: project.name,
|
|
56188
|
+
hasCrawlData: true,
|
|
56189
|
+
runId: snapshot.runId,
|
|
56190
|
+
layout: { state: "unavailable", version: null, reason: "legacy-snapshot" },
|
|
56191
|
+
totalNodes: 0,
|
|
56192
|
+
totalEdges: 0,
|
|
56193
|
+
nodes: [],
|
|
56194
|
+
edges: [],
|
|
56195
|
+
omittedNodes: 0,
|
|
56196
|
+
omittedEdges: 0,
|
|
56197
|
+
sampled: false
|
|
56198
|
+
};
|
|
56199
|
+
}
|
|
56200
|
+
if (persistedLayout.state !== "ready" || !persistedLayout.layoutVersion) {
|
|
56201
|
+
const totalNodes2 = persistedLayout.totalNodes;
|
|
56202
|
+
const totalEdges2 = persistedLayout.totalEdges;
|
|
56203
|
+
return {
|
|
56204
|
+
project: project.name,
|
|
56205
|
+
hasCrawlData: true,
|
|
56206
|
+
runId: snapshot.runId,
|
|
56207
|
+
layout: {
|
|
56208
|
+
state: "unavailable",
|
|
56209
|
+
version: null,
|
|
56210
|
+
reason: persistedLayout.failureCode === "empty-crawl" ? "empty-crawl" : "layout-failed"
|
|
56211
|
+
},
|
|
56212
|
+
totalNodes: totalNodes2,
|
|
56213
|
+
totalEdges: totalEdges2,
|
|
56214
|
+
nodes: [],
|
|
56215
|
+
edges: [],
|
|
56216
|
+
omittedNodes: totalNodes2,
|
|
56217
|
+
omittedEdges: totalEdges2,
|
|
56218
|
+
sampled: totalNodes2 > 0 || totalEdges2 > 0
|
|
56219
|
+
};
|
|
56220
|
+
}
|
|
56221
|
+
const maxNodes = parseBoundedLimit(request.query.maxNodes, SITE_CRAWL_GRAPH_DEFAULT_MAX_NODES, SITE_CRAWL_GRAPH_MAX_NODES);
|
|
56222
|
+
const maxEdges = parseBoundedLimit(request.query.maxEdges, SITE_CRAWL_GRAPH_DEFAULT_MAX_EDGES, SITE_CRAWL_GRAPH_MAX_EDGES);
|
|
56223
|
+
const graphScope = [
|
|
56224
|
+
eq55(siteCrawlGraphNodes.projectId, project.id),
|
|
56225
|
+
eq55(siteCrawlGraphNodes.runId, snapshot.runId),
|
|
56226
|
+
eq55(siteCrawlGraphNodes.attemptId, snapshot.attemptId)
|
|
56227
|
+
];
|
|
56228
|
+
const nodeRows = app.db.select({
|
|
56229
|
+
nodeKey: siteCrawlPages.nodeKey,
|
|
56230
|
+
url: siteCrawlPages.url,
|
|
56231
|
+
path: siteCrawlPages.path,
|
|
56232
|
+
depth: siteCrawlPages.depth,
|
|
56233
|
+
indexabilityState: siteCrawlPages.indexabilityState,
|
|
56234
|
+
indexabilityReasons: siteCrawlPages.indexabilityReasons,
|
|
56235
|
+
canonicalNodeKey: siteCrawlPages.canonicalNodeKey,
|
|
56236
|
+
fetchState: siteCrawlPages.fetchState,
|
|
56237
|
+
auditState: siteCrawlPages.auditState,
|
|
56238
|
+
auditScore: siteCrawlPages.auditScore,
|
|
56239
|
+
inventoryEligible: siteCrawlPages.inventoryEligible,
|
|
56240
|
+
inboundUniqueEdges: siteCrawlPages.inboundUniqueEdges,
|
|
56241
|
+
outboundUniqueEdges: siteCrawlPages.outboundUniqueEdges,
|
|
56242
|
+
linkScoreNormalized: siteCrawlPages.linkScoreNormalized,
|
|
56243
|
+
x: siteCrawlGraphNodes.x,
|
|
56244
|
+
y: siteCrawlGraphNodes.y
|
|
56245
|
+
}).from(siteCrawlGraphNodes).innerJoin(siteCrawlPages, and43(
|
|
56246
|
+
eq55(siteCrawlPages.projectId, siteCrawlGraphNodes.projectId),
|
|
56247
|
+
eq55(siteCrawlPages.runId, siteCrawlGraphNodes.runId),
|
|
56248
|
+
eq55(siteCrawlPages.attemptId, siteCrawlGraphNodes.attemptId),
|
|
56249
|
+
eq55(siteCrawlPages.nodeKey, siteCrawlGraphNodes.nodeKey)
|
|
56250
|
+
)).where(and43(...graphScope, lt7(siteCrawlGraphNodes.sampleRank, maxNodes))).orderBy(asc10(siteCrawlGraphNodes.sampleRank)).all();
|
|
56251
|
+
const nodes = nodeRows.map(({ indexabilityReasons, canonicalNodeKey, ...row }) => ({
|
|
56252
|
+
...row,
|
|
56253
|
+
healthState: deriveSiteHealthState({ ...row, indexabilityReasons, canonicalNodeKey })
|
|
56254
|
+
}));
|
|
56255
|
+
const edges = app.db.select({
|
|
56256
|
+
edgeKey: siteCrawlGraphEdges.edgeKey,
|
|
56257
|
+
sourceNodeKey: siteCrawlGraphEdges.sourceNodeKey,
|
|
56258
|
+
targetNodeKey: siteCrawlGraphEdges.targetNodeKey,
|
|
56259
|
+
followable: siteCrawlGraphEdges.followable,
|
|
56260
|
+
occurrences: siteCrawlGraphEdges.occurrences
|
|
56261
|
+
}).from(siteCrawlGraphEdges).innerJoin(graphSourceNode, and43(
|
|
56262
|
+
eq55(graphSourceNode.projectId, siteCrawlGraphEdges.projectId),
|
|
56263
|
+
eq55(graphSourceNode.runId, siteCrawlGraphEdges.runId),
|
|
56264
|
+
eq55(graphSourceNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
56265
|
+
eq55(graphSourceNode.nodeKey, siteCrawlGraphEdges.sourceNodeKey),
|
|
56266
|
+
lt7(graphSourceNode.sampleRank, maxNodes)
|
|
56267
|
+
)).innerJoin(graphTargetNode, and43(
|
|
56268
|
+
eq55(graphTargetNode.projectId, siteCrawlGraphEdges.projectId),
|
|
56269
|
+
eq55(graphTargetNode.runId, siteCrawlGraphEdges.runId),
|
|
56270
|
+
eq55(graphTargetNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
56271
|
+
eq55(graphTargetNode.nodeKey, siteCrawlGraphEdges.targetNodeKey),
|
|
56272
|
+
lt7(graphTargetNode.sampleRank, maxNodes)
|
|
56273
|
+
)).where(and43(
|
|
56274
|
+
eq55(siteCrawlGraphEdges.projectId, project.id),
|
|
56275
|
+
eq55(siteCrawlGraphEdges.runId, snapshot.runId),
|
|
56276
|
+
eq55(siteCrawlGraphEdges.attemptId, snapshot.attemptId),
|
|
56277
|
+
lt7(siteCrawlGraphEdges.sampleRank, maxEdges)
|
|
56278
|
+
)).orderBy(asc10(siteCrawlGraphEdges.sampleRank)).all();
|
|
56279
|
+
const totalNodes = persistedLayout.totalNodes;
|
|
56280
|
+
const totalEdges = persistedLayout.totalEdges;
|
|
56281
|
+
const omittedNodes = Math.max(0, totalNodes - nodes.length);
|
|
56282
|
+
const omittedEdges = Math.max(0, totalEdges - edges.length);
|
|
56283
|
+
return {
|
|
56284
|
+
project: project.name,
|
|
56285
|
+
hasCrawlData: true,
|
|
56286
|
+
runId: snapshot.runId,
|
|
56287
|
+
layout: { state: "ready", version: persistedLayout.layoutVersion, computedAt: persistedLayout.createdAt },
|
|
56288
|
+
totalNodes,
|
|
56289
|
+
totalEdges,
|
|
56290
|
+
nodes,
|
|
56291
|
+
edges,
|
|
56292
|
+
omittedNodes,
|
|
56293
|
+
omittedEdges,
|
|
56294
|
+
sampled: omittedNodes > 0 || omittedEdges > 0
|
|
56295
|
+
};
|
|
56296
|
+
});
|
|
56297
|
+
app.get("/projects/:name/technical-aeo/subgraph", async (request) => {
|
|
56298
|
+
const project = resolveProject(app.db, request.params.name);
|
|
56299
|
+
const hops = Math.min(3, parsePositiveInt(request.query.hops, 1, 3));
|
|
56300
|
+
const maxNodes = parseBoundedLimit(
|
|
56301
|
+
request.query.maxNodes,
|
|
56302
|
+
SITE_HEALTH_SUBGRAPH_DEFAULT_MAX_NODES,
|
|
56303
|
+
SITE_HEALTH_SUBGRAPH_MAX_NODES
|
|
56304
|
+
);
|
|
56305
|
+
const maxEdges = parseBoundedLimit(
|
|
56306
|
+
request.query.maxEdges,
|
|
56307
|
+
SITE_HEALTH_SUBGRAPH_DEFAULT_MAX_EDGES,
|
|
56308
|
+
SITE_HEALTH_SUBGRAPH_MAX_EDGES
|
|
56309
|
+
);
|
|
56310
|
+
const target = resolveCrawl(project.id, request.query.runId);
|
|
56311
|
+
const empty = (state, runId, hasCrawlData, complete, termination) => ({
|
|
56312
|
+
project: project.name,
|
|
56313
|
+
hasCrawlData,
|
|
56314
|
+
runId,
|
|
56315
|
+
complete,
|
|
56316
|
+
termination,
|
|
56317
|
+
state,
|
|
56318
|
+
focusNodeKey: null,
|
|
56319
|
+
focusUrl: null,
|
|
56320
|
+
hops,
|
|
56321
|
+
totalNodes: 0,
|
|
56322
|
+
totalEdges: 0,
|
|
56323
|
+
nodes: [],
|
|
56324
|
+
edges: [],
|
|
56325
|
+
omittedNodes: 0,
|
|
56326
|
+
omittedEdges: 0,
|
|
56327
|
+
countAccuracy: state === "no-crawl" ? "exact" : "lower-bound",
|
|
56328
|
+
truncated: false
|
|
56329
|
+
});
|
|
56330
|
+
if (!target) {
|
|
56331
|
+
if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
|
|
56332
|
+
return empty("no-crawl", null, false, false, null);
|
|
56333
|
+
}
|
|
56334
|
+
const snapshot = target.snapshot;
|
|
56335
|
+
const scope = detailScopeFor(project.id, snapshot);
|
|
56336
|
+
if (!scope) return empty("details-unavailable", snapshot.runId, true, snapshot.complete, snapshot.termination);
|
|
56337
|
+
const explicitFocus = Boolean(request.query.nodeKey || request.query.url);
|
|
56338
|
+
const focus = explicitFocus ? pageInScope(scope, { nodeKey: request.query.nodeKey, url: request.query.url }) : rootPageInScope(scope, snapshot.rootUrl);
|
|
56339
|
+
if (!focus) {
|
|
56340
|
+
if (explicitFocus) throw notFound("Site crawl page", request.query.nodeKey ?? request.query.url);
|
|
56341
|
+
return {
|
|
56342
|
+
...empty("details-unavailable", snapshot.runId, true, snapshot.complete, snapshot.termination),
|
|
56343
|
+
truncated: false
|
|
56344
|
+
};
|
|
56345
|
+
}
|
|
56346
|
+
const distances = /* @__PURE__ */ new Map([[focus.nodeKey, 0]]);
|
|
56347
|
+
const directRelations = /* @__PURE__ */ new Map();
|
|
56348
|
+
const edgeRows = /* @__PURE__ */ new Map();
|
|
56349
|
+
const omittedNodeKeys = /* @__PURE__ */ new Set();
|
|
56350
|
+
const omittedEdgeKeys = /* @__PURE__ */ new Set();
|
|
56351
|
+
const seenEdgeKeys = /* @__PURE__ */ new Set();
|
|
56352
|
+
let frontier = [focus.nodeKey];
|
|
56353
|
+
let queryTruncated = false;
|
|
56354
|
+
for (let distance = 0; distance < hops && frontier.length > 0; distance += 1) {
|
|
56355
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and43(
|
|
56356
|
+
eq55(siteCrawlEdges.projectId, scope.projectId),
|
|
56357
|
+
eq55(siteCrawlEdges.runId, scope.runId),
|
|
56358
|
+
eq55(siteCrawlEdges.attemptId, scope.attemptId),
|
|
56359
|
+
eq55(siteCrawlEdges.internal, true),
|
|
56360
|
+
isNotNull3(siteCrawlEdges.targetNodeKey),
|
|
56361
|
+
or11(
|
|
56362
|
+
inArray19(siteCrawlEdges.sourceNodeKey, frontier),
|
|
56363
|
+
inArray19(siteCrawlEdges.targetNodeKey, frontier)
|
|
56364
|
+
)
|
|
56365
|
+
)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(maxEdges + maxNodes + 1).all();
|
|
56366
|
+
if (candidates.length > maxEdges + maxNodes) queryTruncated = true;
|
|
56367
|
+
const next = /* @__PURE__ */ new Set();
|
|
56368
|
+
for (const edge of candidates.slice(0, maxEdges + maxNodes)) {
|
|
56369
|
+
if (!edge.targetNodeKey || seenEdgeKeys.has(edge.edgeKey)) continue;
|
|
56370
|
+
seenEdgeKeys.add(edge.edgeKey);
|
|
56371
|
+
const sourceInFrontier = frontier.includes(edge.sourceNodeKey);
|
|
56372
|
+
const neighborKey = sourceInFrontier ? edge.targetNodeKey : edge.sourceNodeKey;
|
|
56373
|
+
if (!distances.has(neighborKey)) {
|
|
56374
|
+
if (distances.size < maxNodes) {
|
|
56375
|
+
distances.set(neighborKey, distance + 1);
|
|
56376
|
+
next.add(neighborKey);
|
|
56377
|
+
} else {
|
|
56378
|
+
omittedNodeKeys.add(neighborKey);
|
|
56379
|
+
}
|
|
56380
|
+
}
|
|
56381
|
+
if (distance === 0 && neighborKey !== focus.nodeKey) {
|
|
56382
|
+
const direction = edge.sourceNodeKey === focus.nodeKey ? "outbound" : "inbound";
|
|
56383
|
+
const prior = directRelations.get(neighborKey);
|
|
56384
|
+
directRelations.set(neighborKey, prior && prior !== direction ? "both" : direction);
|
|
56385
|
+
}
|
|
56386
|
+
if (distances.has(edge.sourceNodeKey) && distances.has(edge.targetNodeKey)) {
|
|
56387
|
+
if (edgeRows.size < maxEdges) edgeRows.set(edge.edgeKey, edge);
|
|
56388
|
+
else omittedEdgeKeys.add(edge.edgeKey);
|
|
56389
|
+
} else {
|
|
56390
|
+
omittedEdgeKeys.add(edge.edgeKey);
|
|
56391
|
+
}
|
|
56392
|
+
}
|
|
56393
|
+
frontier = [...next].sort();
|
|
56394
|
+
}
|
|
56395
|
+
const nodeKeys = [...distances.keys()];
|
|
56396
|
+
const pageRows = nodeKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and43(
|
|
56397
|
+
eq55(siteCrawlPages.projectId, scope.projectId),
|
|
56398
|
+
eq55(siteCrawlPages.runId, scope.runId),
|
|
56399
|
+
eq55(siteCrawlPages.attemptId, scope.attemptId),
|
|
56400
|
+
inArray19(siteCrawlPages.nodeKey, nodeKeys)
|
|
56401
|
+
)).all();
|
|
56402
|
+
const persistedNodeKeys = new Set(pageRows.map((row) => row.nodeKey));
|
|
56403
|
+
const outermostNodeKeys = pageRows.filter((row) => distances.get(row.nodeKey) === hops).map((row) => row.nodeKey);
|
|
56404
|
+
const omittedOutermostEdge = outermostNodeKeys.length === 0 ? void 0 : app.db.select({ edgeKey: siteCrawlEdges.edgeKey }).from(siteCrawlEdges).where(and43(
|
|
56405
|
+
eq55(siteCrawlEdges.projectId, scope.projectId),
|
|
56406
|
+
eq55(siteCrawlEdges.runId, scope.runId),
|
|
56407
|
+
eq55(siteCrawlEdges.attemptId, scope.attemptId),
|
|
56408
|
+
eq55(siteCrawlEdges.internal, true),
|
|
56409
|
+
isNotNull3(siteCrawlEdges.targetNodeKey),
|
|
56410
|
+
inArray19(siteCrawlEdges.sourceNodeKey, outermostNodeKeys),
|
|
56411
|
+
inArray19(siteCrawlEdges.targetNodeKey, outermostNodeKeys)
|
|
56412
|
+
)).limit(1).get();
|
|
56413
|
+
if (omittedOutermostEdge && !seenEdgeKeys.has(omittedOutermostEdge.edgeKey)) {
|
|
56414
|
+
omittedEdgeKeys.add(omittedOutermostEdge.edgeKey);
|
|
56415
|
+
}
|
|
56416
|
+
const nodes = pageRows.map((row) => ({
|
|
56417
|
+
...mapCrawlPage(row),
|
|
56418
|
+
distance: distances.get(row.nodeKey) ?? 0,
|
|
56419
|
+
relationToFocus: row.nodeKey === focus.nodeKey ? "focus" : directRelations.get(row.nodeKey) ?? "transitive"
|
|
56420
|
+
})).sort((a, b) => a.distance - b.distance || a.nodeKey.localeCompare(b.nodeKey));
|
|
56421
|
+
const edges = [...edgeRows.values()].filter((edge) => edge.targetNodeKey && persistedNodeKeys.has(edge.sourceNodeKey) && persistedNodeKeys.has(edge.targetNodeKey)).sort((a, b) => a.edgeKey.localeCompare(b.edgeKey)).map(mapCrawlEdge);
|
|
56422
|
+
const omittedNodes = omittedNodeKeys.size + Math.max(0, distances.size - pageRows.length);
|
|
56423
|
+
const omittedEdges = omittedEdgeKeys.size + (queryTruncated ? 1 : 0) + Math.max(0, edgeRows.size - edges.length);
|
|
56424
|
+
const truncated = omittedNodes > 0 || omittedEdges > 0;
|
|
56425
|
+
return {
|
|
56426
|
+
project: project.name,
|
|
56427
|
+
hasCrawlData: true,
|
|
56428
|
+
runId: snapshot.runId,
|
|
56429
|
+
complete: snapshot.complete,
|
|
56430
|
+
termination: snapshot.termination,
|
|
56431
|
+
state: "ready",
|
|
56432
|
+
focusNodeKey: focus.nodeKey,
|
|
56433
|
+
focusUrl: focus.url,
|
|
56434
|
+
hops,
|
|
56435
|
+
totalNodes: nodes.length + omittedNodes,
|
|
56436
|
+
totalEdges: edges.length + omittedEdges,
|
|
56437
|
+
nodes,
|
|
56438
|
+
edges,
|
|
56439
|
+
omittedNodes,
|
|
56440
|
+
omittedEdges,
|
|
56441
|
+
countAccuracy: truncated ? "lower-bound" : "exact",
|
|
56442
|
+
truncated
|
|
56443
|
+
};
|
|
56444
|
+
});
|
|
56445
|
+
app.get("/projects/:name/technical-aeo/path", async (request) => {
|
|
56446
|
+
const project = resolveProject(app.db, request.params.name);
|
|
56447
|
+
const maxDepth = parseBoundedLimit(
|
|
56448
|
+
request.query.maxDepth,
|
|
56449
|
+
SITE_HEALTH_PATH_DEFAULT_MAX_DEPTH,
|
|
56450
|
+
SITE_HEALTH_PATH_MAX_DEPTH
|
|
56451
|
+
);
|
|
56452
|
+
if (!request.query.toNodeKey && !request.query.toUrl) {
|
|
56453
|
+
throw validationError("toNodeKey or toUrl is required");
|
|
56454
|
+
}
|
|
56455
|
+
const target = resolveCrawl(project.id, request.query.runId);
|
|
56456
|
+
if (!target) {
|
|
56457
|
+
if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
|
|
56458
|
+
return {
|
|
56459
|
+
project: project.name,
|
|
56460
|
+
runId: null,
|
|
56461
|
+
state: "no-crawl",
|
|
56462
|
+
from: null,
|
|
56463
|
+
to: null,
|
|
56464
|
+
complete: false,
|
|
56465
|
+
termination: null,
|
|
56466
|
+
maxDepth,
|
|
56467
|
+
visitedNodes: 0,
|
|
56468
|
+
nodes: [],
|
|
56469
|
+
edges: []
|
|
56470
|
+
};
|
|
56471
|
+
}
|
|
56472
|
+
const snapshot = target.snapshot;
|
|
56473
|
+
const scope = detailScopeFor(project.id, snapshot);
|
|
56474
|
+
if (!scope) {
|
|
56475
|
+
return {
|
|
56476
|
+
project: project.name,
|
|
56477
|
+
runId: snapshot.runId,
|
|
56478
|
+
state: "details-unavailable",
|
|
56479
|
+
from: null,
|
|
56480
|
+
to: null,
|
|
56481
|
+
complete: snapshot.complete,
|
|
56482
|
+
termination: snapshot.termination,
|
|
56483
|
+
maxDepth,
|
|
56484
|
+
visitedNodes: 0,
|
|
56485
|
+
nodes: [],
|
|
56486
|
+
edges: []
|
|
56487
|
+
};
|
|
56488
|
+
}
|
|
56489
|
+
const explicitFrom = Boolean(request.query.fromNodeKey || request.query.fromUrl);
|
|
56490
|
+
const fromPage = explicitFrom ? pageInScope(scope, { nodeKey: request.query.fromNodeKey, url: request.query.fromUrl }) : rootPageInScope(scope, snapshot.rootUrl);
|
|
56491
|
+
if (!fromPage) throw notFound("Site crawl page", request.query.fromNodeKey ?? request.query.fromUrl ?? snapshot.rootUrl);
|
|
56492
|
+
const toPage = pageInScope(scope, { nodeKey: request.query.toNodeKey, url: request.query.toUrl });
|
|
56493
|
+
if (!toPage) throw notFound("Site crawl page", request.query.toNodeKey ?? request.query.toUrl);
|
|
56494
|
+
const reference = (row) => ({
|
|
56495
|
+
nodeKey: row.nodeKey,
|
|
56496
|
+
url: row.url,
|
|
56497
|
+
path: row.path
|
|
56498
|
+
});
|
|
56499
|
+
const predecessor = /* @__PURE__ */ new Map();
|
|
56500
|
+
const visited = /* @__PURE__ */ new Set([fromPage.nodeKey]);
|
|
56501
|
+
let frontier = [fromPage.nodeKey];
|
|
56502
|
+
let found = fromPage.nodeKey === toPage.nodeKey;
|
|
56503
|
+
let truncated = false;
|
|
56504
|
+
for (let depth = 0; depth < maxDepth && frontier.length > 0 && !found; depth += 1) {
|
|
56505
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and43(
|
|
56506
|
+
eq55(siteCrawlEdges.projectId, scope.projectId),
|
|
56507
|
+
eq55(siteCrawlEdges.runId, scope.runId),
|
|
56508
|
+
eq55(siteCrawlEdges.attemptId, scope.attemptId),
|
|
56509
|
+
eq55(siteCrawlEdges.internal, true),
|
|
56510
|
+
eq55(siteCrawlEdges.followable, true),
|
|
56511
|
+
eq55(siteCrawlEdges.relation, "anchor"),
|
|
56512
|
+
isNotNull3(siteCrawlEdges.targetNodeKey),
|
|
56513
|
+
inArray19(siteCrawlEdges.sourceNodeKey, frontier)
|
|
56514
|
+
)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(SITE_HEALTH_PATH_MAX_VISITED_NODES + 1).all();
|
|
56515
|
+
if (candidates.length > SITE_HEALTH_PATH_MAX_VISITED_NODES) truncated = true;
|
|
56516
|
+
const candidateTargetKeys = [...new Set(candidates.map((edge) => edge.targetNodeKey).filter((nodeKey) => nodeKey != null))];
|
|
56517
|
+
const persistedTargetKeys = new Set(candidateTargetKeys.length === 0 ? [] : app.db.select({ nodeKey: siteCrawlPages.nodeKey }).from(siteCrawlPages).where(and43(
|
|
56518
|
+
eq55(siteCrawlPages.projectId, scope.projectId),
|
|
56519
|
+
eq55(siteCrawlPages.runId, scope.runId),
|
|
56520
|
+
eq55(siteCrawlPages.attemptId, scope.attemptId),
|
|
56521
|
+
inArray19(siteCrawlPages.nodeKey, candidateTargetKeys)
|
|
56522
|
+
)).all().map((row) => row.nodeKey));
|
|
56523
|
+
const next = /* @__PURE__ */ new Set();
|
|
56524
|
+
for (const edge of candidates.slice(0, SITE_HEALTH_PATH_MAX_VISITED_NODES)) {
|
|
56525
|
+
if (!edge.targetNodeKey || !persistedTargetKeys.has(edge.targetNodeKey) || visited.has(edge.targetNodeKey)) continue;
|
|
56526
|
+
if (visited.size >= SITE_HEALTH_PATH_MAX_VISITED_NODES) {
|
|
56527
|
+
truncated = true;
|
|
56528
|
+
break;
|
|
56529
|
+
}
|
|
56530
|
+
visited.add(edge.targetNodeKey);
|
|
56531
|
+
predecessor.set(edge.targetNodeKey, { nodeKey: edge.sourceNodeKey, edge });
|
|
56532
|
+
next.add(edge.targetNodeKey);
|
|
56533
|
+
if (edge.targetNodeKey === toPage.nodeKey) {
|
|
56534
|
+
found = true;
|
|
56535
|
+
break;
|
|
56536
|
+
}
|
|
56537
|
+
}
|
|
56538
|
+
frontier = [...next].sort();
|
|
56539
|
+
}
|
|
56540
|
+
if (!found) {
|
|
56541
|
+
return {
|
|
56542
|
+
project: project.name,
|
|
56543
|
+
runId: snapshot.runId,
|
|
56544
|
+
complete: snapshot.complete,
|
|
56545
|
+
termination: snapshot.termination,
|
|
56546
|
+
state: truncated || frontier.length > 0 ? "truncated" : "unreachable",
|
|
56547
|
+
from: reference(fromPage),
|
|
56548
|
+
to: reference(toPage),
|
|
56549
|
+
maxDepth,
|
|
56550
|
+
visitedNodes: visited.size,
|
|
56551
|
+
nodes: [],
|
|
56552
|
+
edges: []
|
|
56553
|
+
};
|
|
56554
|
+
}
|
|
56555
|
+
const pathKeys = [toPage.nodeKey];
|
|
56556
|
+
const pathEdges = [];
|
|
56557
|
+
while (pathKeys[0] !== fromPage.nodeKey) {
|
|
56558
|
+
const step = predecessor.get(pathKeys[0]);
|
|
56559
|
+
if (!step) break;
|
|
56560
|
+
pathEdges.unshift(step.edge);
|
|
56561
|
+
pathKeys.unshift(step.nodeKey);
|
|
56562
|
+
}
|
|
56563
|
+
const pathRows = app.db.select().from(siteCrawlPages).where(and43(
|
|
56564
|
+
eq55(siteCrawlPages.projectId, scope.projectId),
|
|
56565
|
+
eq55(siteCrawlPages.runId, scope.runId),
|
|
56566
|
+
eq55(siteCrawlPages.attemptId, scope.attemptId),
|
|
56567
|
+
inArray19(siteCrawlPages.nodeKey, pathKeys)
|
|
56568
|
+
)).all();
|
|
56569
|
+
const pageByKey = new Map(pathRows.map((row) => [row.nodeKey, row]));
|
|
56570
|
+
return {
|
|
56571
|
+
project: project.name,
|
|
56572
|
+
runId: snapshot.runId,
|
|
56573
|
+
complete: snapshot.complete,
|
|
56574
|
+
termination: snapshot.termination,
|
|
56575
|
+
state: "found",
|
|
56576
|
+
from: reference(fromPage),
|
|
56577
|
+
to: reference(toPage),
|
|
56578
|
+
maxDepth,
|
|
56579
|
+
visitedNodes: visited.size,
|
|
56580
|
+
nodes: pathKeys.map((key) => mapCrawlPage(pageByKey.get(key))),
|
|
56581
|
+
edges: pathEdges.map(mapCrawlEdge)
|
|
56582
|
+
};
|
|
56583
|
+
});
|
|
56584
|
+
app.get("/projects/:name/technical-aeo/changes", async (request) => {
|
|
56585
|
+
const project = resolveProject(app.db, request.params.name);
|
|
56586
|
+
const scopeFilter = request.query.scope ?? "all";
|
|
56587
|
+
if (scopeFilter !== "all" && scopeFilter !== "pages" && scopeFilter !== "links") {
|
|
56588
|
+
throw validationError("scope must be all, pages, or links");
|
|
56589
|
+
}
|
|
56590
|
+
const changeFilter = request.query.change ?? "all";
|
|
56591
|
+
if (changeFilter !== "all" && changeFilter !== "added" && changeFilter !== "removed" && changeFilter !== "changed") {
|
|
56592
|
+
throw validationError("change must be all, added, removed, or changed");
|
|
56593
|
+
}
|
|
56594
|
+
const completeSnapshotFilters = [
|
|
56595
|
+
eq55(siteCrawlSnapshots.projectId, project.id),
|
|
56596
|
+
eq55(runs.projectId, project.id),
|
|
56597
|
+
eq55(runs.kind, RunKinds["site-audit"]),
|
|
56598
|
+
eq55(runs.status, RunStatuses.completed),
|
|
56599
|
+
eq55(siteCrawlSnapshots.complete, true),
|
|
56600
|
+
notProbeRun()
|
|
56601
|
+
];
|
|
56602
|
+
const afterTarget = request.query.toRunId ? resolveCrawl(project.id, request.query.toRunId) : app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq55(siteCrawlSnapshots.runId, runs.id)).where(and43(...completeSnapshotFilters)).orderBy(desc24(siteCrawlSnapshots.createdAt), desc24(siteCrawlSnapshots.runId)).limit(1).get();
|
|
56603
|
+
if (request.query.toRunId && !afterTarget) throw notFound("Site crawl run", request.query.toRunId);
|
|
56604
|
+
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq55(siteCrawlSnapshots.runId, runs.id)).where(and43(
|
|
56605
|
+
...completeSnapshotFilters,
|
|
56606
|
+
or11(
|
|
56607
|
+
lt7(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
56608
|
+
and43(
|
|
56609
|
+
eq55(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
56610
|
+
lt7(siteCrawlSnapshots.runId, afterTarget.snapshot.runId)
|
|
56611
|
+
)
|
|
56612
|
+
)
|
|
56613
|
+
)).orderBy(desc24(siteCrawlSnapshots.createdAt), desc24(siteCrawlSnapshots.runId)).limit(1).get() : void 0;
|
|
56614
|
+
if (request.query.fromRunId && !beforeTarget) throw notFound("Site crawl run", request.query.fromRunId);
|
|
56615
|
+
if (!afterTarget) {
|
|
56616
|
+
return { project: project.name, state: "unavailable", reason: "no-crawl", fromRunId: null, toRunId: null };
|
|
56617
|
+
}
|
|
56618
|
+
if (!beforeTarget) {
|
|
56619
|
+
return {
|
|
56620
|
+
project: project.name,
|
|
56621
|
+
state: "unavailable",
|
|
56622
|
+
reason: "insufficient-history",
|
|
56623
|
+
fromRunId: null,
|
|
56624
|
+
toRunId: afterTarget.snapshot.runId
|
|
56625
|
+
};
|
|
56626
|
+
}
|
|
56627
|
+
const beforeSnapshot = beforeTarget.snapshot;
|
|
56628
|
+
const afterSnapshot = afterTarget.snapshot;
|
|
56629
|
+
if (beforeSnapshot.runId === afterSnapshot.runId) {
|
|
56630
|
+
throw validationError("fromRunId and toRunId must identify different crawls");
|
|
56631
|
+
}
|
|
56632
|
+
const crawlOrder = beforeSnapshot.createdAt.localeCompare(afterSnapshot.createdAt);
|
|
56633
|
+
if (crawlOrder > 0 || crawlOrder === 0 && beforeSnapshot.runId >= afterSnapshot.runId) {
|
|
56634
|
+
throw validationError("fromRunId must identify a crawl earlier than toRunId");
|
|
56635
|
+
}
|
|
56636
|
+
if (!beforeSnapshot.complete || !afterSnapshot.complete || beforeTarget.runStatus !== RunStatuses.completed || afterTarget.runStatus !== RunStatuses.completed) {
|
|
56637
|
+
return {
|
|
56638
|
+
project: project.name,
|
|
56639
|
+
state: "unavailable",
|
|
56640
|
+
reason: "partial-not-comparable",
|
|
56641
|
+
fromRunId: beforeSnapshot.runId,
|
|
56642
|
+
toRunId: afterSnapshot.runId
|
|
56643
|
+
};
|
|
56644
|
+
}
|
|
56645
|
+
const beforeScope = detailScopeFor(project.id, beforeSnapshot);
|
|
56646
|
+
const afterScope = detailScopeFor(project.id, afterSnapshot);
|
|
56647
|
+
if (!beforeScope || !afterScope) {
|
|
56648
|
+
return {
|
|
56649
|
+
project: project.name,
|
|
56650
|
+
state: "unavailable",
|
|
56651
|
+
reason: "details-unavailable",
|
|
56652
|
+
fromRunId: beforeSnapshot.runId,
|
|
56653
|
+
toRunId: afterSnapshot.runId
|
|
56654
|
+
};
|
|
56655
|
+
}
|
|
56656
|
+
const versions = {
|
|
56657
|
+
crawlSchema: [beforeSnapshot.crawlSchemaVersion, afterSnapshot.crawlSchemaVersion],
|
|
56658
|
+
normalization: [beforeSnapshot.normalizationVersion, afterSnapshot.normalizationVersion],
|
|
56659
|
+
indexability: [beforeSnapshot.indexabilityVersion, afterSnapshot.indexabilityVersion],
|
|
56660
|
+
linkScore: [beforeSnapshot.linkScoreVersion, afterSnapshot.linkScoreVersion]
|
|
56661
|
+
};
|
|
56662
|
+
const mismatchedVersions = Object.entries(versions).filter(([, pair]) => pair[0] !== pair[1]).map(([name]) => name);
|
|
56663
|
+
if (mismatchedVersions.length > 0) {
|
|
56664
|
+
return {
|
|
56665
|
+
project: project.name,
|
|
56666
|
+
state: "incompatible",
|
|
56667
|
+
reason: "incompatible-versions",
|
|
56668
|
+
fromRunId: beforeSnapshot.runId,
|
|
56669
|
+
toRunId: afterSnapshot.runId,
|
|
56670
|
+
mismatchedVersions
|
|
56671
|
+
};
|
|
56672
|
+
}
|
|
56673
|
+
const cursor = decodeSiteHealthChangesCursor(request.query.cursor, {
|
|
56674
|
+
fromRunId: beforeSnapshot.runId,
|
|
56675
|
+
toRunId: afterSnapshot.runId,
|
|
56676
|
+
scope: scopeFilter,
|
|
56677
|
+
change: changeFilter
|
|
56678
|
+
});
|
|
56679
|
+
const loadSummary = () => {
|
|
56680
|
+
const includeAdded = changeFilter === "all" || changeFilter === "added";
|
|
56681
|
+
const includeRemoved = changeFilter === "all" || changeFilter === "removed";
|
|
56682
|
+
const includeChanged = changeFilter === "all" || changeFilter === "changed";
|
|
56683
|
+
const zero2 = () => ({ added: 0, removed: 0, changed: 0 });
|
|
56684
|
+
const summary2 = { pages: zero2(), links: zero2() };
|
|
56685
|
+
if (scopeFilter !== "links") {
|
|
56686
|
+
if (includeAdded || includeChanged) {
|
|
56687
|
+
const addedExpression = includeAdded ? sql20`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql20`0`;
|
|
56688
|
+
const changedExpression = includeChanged ? sql20`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${PAGE_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql20`0`;
|
|
56689
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql20`
|
|
56690
|
+
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
56691
|
+
FROM site_crawl_pages AS current
|
|
56692
|
+
LEFT JOIN site_crawl_pages AS previous
|
|
56693
|
+
ON previous.project_id = ${beforeScope.projectId}
|
|
56694
|
+
AND previous.run_id = ${beforeScope.runId}
|
|
56695
|
+
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
56696
|
+
AND previous.node_key = current.node_key
|
|
56697
|
+
WHERE current.project_id = ${afterScope.projectId}
|
|
56698
|
+
AND current.run_id = ${afterScope.runId}
|
|
56699
|
+
AND current.attempt_id = ${afterScope.attemptId}
|
|
56700
|
+
`);
|
|
56701
|
+
summary2.pages.added = Number(counts2.added);
|
|
56702
|
+
summary2.pages.changed = Number(counts2.changed);
|
|
56703
|
+
}
|
|
56704
|
+
if (includeRemoved) {
|
|
56705
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql20`
|
|
56706
|
+
SELECT COUNT(*) AS removed
|
|
56707
|
+
FROM site_crawl_pages AS previous
|
|
56708
|
+
LEFT JOIN site_crawl_pages AS current
|
|
56709
|
+
ON current.project_id = ${afterScope.projectId}
|
|
56710
|
+
AND current.run_id = ${afterScope.runId}
|
|
56711
|
+
AND current.attempt_id = ${afterScope.attemptId}
|
|
56712
|
+
AND current.node_key = previous.node_key
|
|
56713
|
+
WHERE previous.project_id = ${beforeScope.projectId}
|
|
56714
|
+
AND previous.run_id = ${beforeScope.runId}
|
|
56715
|
+
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
56716
|
+
AND current.id IS NULL
|
|
56717
|
+
`);
|
|
56718
|
+
summary2.pages.removed = Number(counts2.removed);
|
|
56719
|
+
}
|
|
56720
|
+
}
|
|
56721
|
+
if (scopeFilter !== "pages") {
|
|
56722
|
+
if (includeAdded || includeChanged) {
|
|
56723
|
+
const addedExpression = includeAdded ? sql20`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql20`0`;
|
|
56724
|
+
const changedExpression = includeChanged ? sql20`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${LINK_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql20`0`;
|
|
56725
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql20`
|
|
56726
|
+
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
56727
|
+
FROM site_crawl_edges AS current
|
|
56728
|
+
LEFT JOIN site_crawl_edges AS previous
|
|
56729
|
+
ON previous.project_id = ${beforeScope.projectId}
|
|
56730
|
+
AND previous.run_id = ${beforeScope.runId}
|
|
56731
|
+
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
56732
|
+
AND previous.internal = 1
|
|
56733
|
+
AND previous.edge_key = current.edge_key
|
|
56734
|
+
WHERE current.project_id = ${afterScope.projectId}
|
|
56735
|
+
AND current.run_id = ${afterScope.runId}
|
|
56736
|
+
AND current.attempt_id = ${afterScope.attemptId}
|
|
56737
|
+
AND current.internal = 1
|
|
56738
|
+
`);
|
|
56739
|
+
summary2.links.added = Number(counts2.added);
|
|
56740
|
+
summary2.links.changed = Number(counts2.changed);
|
|
56741
|
+
}
|
|
56742
|
+
if (includeRemoved) {
|
|
56743
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql20`
|
|
56744
|
+
SELECT COUNT(*) AS removed
|
|
56745
|
+
FROM site_crawl_edges AS previous
|
|
56746
|
+
LEFT JOIN site_crawl_edges AS current
|
|
56747
|
+
ON current.project_id = ${afterScope.projectId}
|
|
56748
|
+
AND current.run_id = ${afterScope.runId}
|
|
56749
|
+
AND current.attempt_id = ${afterScope.attemptId}
|
|
56750
|
+
AND current.internal = 1
|
|
56751
|
+
AND current.edge_key = previous.edge_key
|
|
56752
|
+
WHERE previous.project_id = ${beforeScope.projectId}
|
|
56753
|
+
AND previous.run_id = ${beforeScope.runId}
|
|
56754
|
+
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
56755
|
+
AND previous.internal = 1
|
|
56756
|
+
AND current.id IS NULL
|
|
56757
|
+
`);
|
|
56758
|
+
summary2.links.removed = Number(counts2.removed);
|
|
56759
|
+
}
|
|
56760
|
+
}
|
|
56761
|
+
return summary2;
|
|
56762
|
+
};
|
|
56763
|
+
const summary = cursor ? null : loadSummary();
|
|
56764
|
+
const limit = parseBoundedLimit(request.query.limit, SITE_HEALTH_CHANGES_DEFAULT_LIMIT, SITE_HEALTH_CHANGES_MAX_LIMIT);
|
|
56765
|
+
const sumCounts = (counts2) => counts2.added + counts2.removed + counts2.changed;
|
|
56766
|
+
const total = summary ? sumCounts(summary.pages) + sumCounts(summary.links) : null;
|
|
56767
|
+
const collectEntity = (entity, afterKey, wanted2) => {
|
|
56768
|
+
if (wanted2 <= 0) return [];
|
|
56769
|
+
const includeAfter = changeFilter === "all" || changeFilter === "added" || changeFilter === "changed";
|
|
56770
|
+
const includeRemoved = changeFilter === "all" || changeFilter === "removed";
|
|
56771
|
+
const table = entity === "page" ? "site_crawl_pages" : "site_crawl_edges";
|
|
56772
|
+
const keyColumn = entity === "page" ? "node_key" : "edge_key";
|
|
56773
|
+
const internalCurrent = entity === "link" ? sql20`AND current.internal = 1` : sql20``;
|
|
56774
|
+
const internalPreviousJoin = entity === "link" ? sql20`AND previous.internal = 1` : sql20``;
|
|
56775
|
+
const internalPrevious = entity === "link" ? sql20`AND previous.internal = 1` : sql20``;
|
|
56776
|
+
const internalCurrentJoin = entity === "link" ? sql20`AND current.internal = 1` : sql20``;
|
|
56777
|
+
const difference2 = entity === "page" ? PAGE_CHANGE_PREDICATE : LINK_CHANGE_PREDICATE;
|
|
56778
|
+
const afterKind = changeFilter === "added" ? sql20`previous.id IS NULL` : changeFilter === "changed" ? sql20`previous.id IS NOT NULL AND ${difference2}` : sql20`previous.id IS NULL OR (previous.id IS NOT NULL AND ${difference2})`;
|
|
56779
|
+
const afterRows = includeAfter ? app.db.all(sql20`
|
|
56780
|
+
SELECT ${entity} AS entity,
|
|
56781
|
+
CASE WHEN previous.id IS NULL THEN 'added' ELSE 'changed' END AS change,
|
|
56782
|
+
current.${sql20.raw(keyColumn)} AS key
|
|
56783
|
+
FROM ${sql20.raw(table)} AS current
|
|
56784
|
+
LEFT JOIN ${sql20.raw(table)} AS previous
|
|
56785
|
+
ON previous.project_id = ${beforeScope.projectId}
|
|
56786
|
+
AND previous.run_id = ${beforeScope.runId}
|
|
56787
|
+
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
56788
|
+
${internalPreviousJoin}
|
|
56789
|
+
AND previous.${sql20.raw(keyColumn)} = current.${sql20.raw(keyColumn)}
|
|
56790
|
+
WHERE current.project_id = ${afterScope.projectId}
|
|
56791
|
+
AND current.run_id = ${afterScope.runId}
|
|
56792
|
+
AND current.attempt_id = ${afterScope.attemptId}
|
|
56793
|
+
${internalCurrent}
|
|
56794
|
+
AND current.${sql20.raw(keyColumn)} > ${afterKey}
|
|
56795
|
+
AND (${afterKind})
|
|
56796
|
+
ORDER BY current.${sql20.raw(keyColumn)} ASC
|
|
56797
|
+
LIMIT ${wanted2}
|
|
56798
|
+
`) : [];
|
|
56799
|
+
const removedRows = includeRemoved ? app.db.all(sql20`
|
|
56800
|
+
SELECT ${entity} AS entity, 'removed' AS change, previous.${sql20.raw(keyColumn)} AS key
|
|
56801
|
+
FROM ${sql20.raw(table)} AS previous
|
|
56802
|
+
LEFT JOIN ${sql20.raw(table)} AS current
|
|
56803
|
+
ON current.project_id = ${afterScope.projectId}
|
|
56804
|
+
AND current.run_id = ${afterScope.runId}
|
|
56805
|
+
AND current.attempt_id = ${afterScope.attemptId}
|
|
56806
|
+
${internalCurrentJoin}
|
|
56807
|
+
AND current.${sql20.raw(keyColumn)} = previous.${sql20.raw(keyColumn)}
|
|
56808
|
+
WHERE previous.project_id = ${beforeScope.projectId}
|
|
56809
|
+
AND previous.run_id = ${beforeScope.runId}
|
|
56810
|
+
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
56811
|
+
${internalPrevious}
|
|
56812
|
+
AND previous.${sql20.raw(keyColumn)} > ${afterKey}
|
|
56813
|
+
AND current.id IS NULL
|
|
56814
|
+
ORDER BY previous.${sql20.raw(keyColumn)} ASC
|
|
56815
|
+
LIMIT ${wanted2}
|
|
56816
|
+
`) : [];
|
|
56817
|
+
const merged = [];
|
|
56818
|
+
let afterIndex = 0;
|
|
56819
|
+
let removedIndex = 0;
|
|
56820
|
+
while (merged.length < wanted2 && (afterIndex < afterRows.length || removedIndex < removedRows.length)) {
|
|
56821
|
+
const afterRow = afterRows.at(afterIndex);
|
|
56822
|
+
const removedRow = removedRows.at(removedIndex);
|
|
56823
|
+
if (!removedRow || afterRow && compareChangeKeys(afterRow, removedRow) <= 0) {
|
|
56824
|
+
merged.push(afterRow);
|
|
56825
|
+
afterIndex += 1;
|
|
56826
|
+
} else {
|
|
56827
|
+
merged.push(removedRow);
|
|
56828
|
+
removedIndex += 1;
|
|
56829
|
+
}
|
|
56830
|
+
}
|
|
56831
|
+
return merged;
|
|
56832
|
+
};
|
|
56833
|
+
const keyRows = [];
|
|
56834
|
+
const wanted = limit + 1;
|
|
56835
|
+
if (scopeFilter !== "links" && cursor?.entity !== "link") {
|
|
56836
|
+
keyRows.push(...collectEntity("page", cursor?.entity === "page" ? cursor.key : "", wanted));
|
|
56837
|
+
}
|
|
56838
|
+
if (keyRows.length < wanted && scopeFilter !== "pages") {
|
|
56839
|
+
keyRows.push(...collectEntity("link", cursor?.entity === "link" ? cursor.key : "", wanted - keyRows.length));
|
|
56840
|
+
}
|
|
56841
|
+
const visibleKeys = keyRows.slice(0, limit);
|
|
56842
|
+
const pageKeys = visibleKeys.filter((row) => row.entity === "page").map((row) => row.key);
|
|
56843
|
+
const linkKeys = visibleKeys.filter((row) => row.entity === "link").map((row) => row.key);
|
|
56844
|
+
const pageRowsFor = (scope) => pageKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and43(
|
|
56845
|
+
eq55(siteCrawlPages.projectId, scope.projectId),
|
|
56846
|
+
eq55(siteCrawlPages.runId, scope.runId),
|
|
56847
|
+
eq55(siteCrawlPages.attemptId, scope.attemptId),
|
|
56848
|
+
inArray19(siteCrawlPages.nodeKey, pageKeys)
|
|
56849
|
+
)).all();
|
|
56850
|
+
const edgeRowsFor = (scope) => linkKeys.length === 0 ? [] : app.db.select().from(siteCrawlEdges).where(and43(
|
|
56851
|
+
eq55(siteCrawlEdges.projectId, scope.projectId),
|
|
56852
|
+
eq55(siteCrawlEdges.runId, scope.runId),
|
|
56853
|
+
eq55(siteCrawlEdges.attemptId, scope.attemptId),
|
|
56854
|
+
eq55(siteCrawlEdges.internal, true),
|
|
56855
|
+
inArray19(siteCrawlEdges.edgeKey, linkKeys)
|
|
56856
|
+
)).all();
|
|
56857
|
+
const beforePages = new Map(pageRowsFor(beforeScope).map((row) => [row.nodeKey, mapCrawlPage(row)]));
|
|
56858
|
+
const afterPages = new Map(pageRowsFor(afterScope).map((row) => [row.nodeKey, mapCrawlPage(row)]));
|
|
56859
|
+
const beforeLinks = new Map(edgeRowsFor(beforeScope).map((row) => [row.edgeKey, mapCrawlEdge(row)]));
|
|
56860
|
+
const afterLinks = new Map(edgeRowsFor(afterScope).map((row) => [row.edgeKey, mapCrawlEdge(row)]));
|
|
56861
|
+
const changes = visibleKeys.map((row) => {
|
|
56862
|
+
if (row.entity === "page") {
|
|
56863
|
+
const before2 = beforePages.get(row.key) ?? null;
|
|
56864
|
+
const after2 = afterPages.get(row.key) ?? null;
|
|
56865
|
+
const pageChangedFields = changedFields(before2, after2, PAGE_CHANGE_FIELDS);
|
|
56866
|
+
if (before2 && after2 && before2.healthState !== after2.healthState) pageChangedFields.push("healthState");
|
|
56867
|
+
return {
|
|
56868
|
+
entity: "page",
|
|
56869
|
+
change: row.change,
|
|
56870
|
+
key: row.key,
|
|
56871
|
+
changedFields: pageChangedFields,
|
|
56872
|
+
before: before2,
|
|
56873
|
+
after: after2
|
|
56874
|
+
};
|
|
56875
|
+
}
|
|
56876
|
+
const before = beforeLinks.get(row.key) ?? null;
|
|
56877
|
+
const after = afterLinks.get(row.key) ?? null;
|
|
56878
|
+
return {
|
|
56879
|
+
entity: "link",
|
|
56880
|
+
change: row.change,
|
|
56881
|
+
key: row.key,
|
|
56882
|
+
changedFields: changedFields(before, after, LINK_CHANGE_FIELDS),
|
|
56883
|
+
before,
|
|
56884
|
+
after
|
|
56885
|
+
};
|
|
56886
|
+
});
|
|
56887
|
+
return {
|
|
56888
|
+
project: project.name,
|
|
56889
|
+
state: "ready",
|
|
56890
|
+
fromRunId: beforeSnapshot.runId,
|
|
56891
|
+
toRunId: afterSnapshot.runId,
|
|
56892
|
+
versions: {
|
|
56893
|
+
crawlSchema: afterSnapshot.crawlSchemaVersion,
|
|
56894
|
+
normalization: afterSnapshot.normalizationVersion,
|
|
56895
|
+
indexability: afterSnapshot.indexabilityVersion,
|
|
56896
|
+
linkScore: afterSnapshot.linkScoreVersion
|
|
56897
|
+
},
|
|
56898
|
+
filters: { scope: scopeFilter, change: changeFilter },
|
|
56899
|
+
summaryState: cursor ? "omitted-on-continuation" : "exact",
|
|
56900
|
+
summary,
|
|
56901
|
+
total,
|
|
56902
|
+
nextCursor: keyRows.length > limit && visibleKeys.length > 0 ? encodeSiteHealthChangesCursor({
|
|
56903
|
+
v: 1,
|
|
56904
|
+
fromRunId: beforeSnapshot.runId,
|
|
56905
|
+
toRunId: afterSnapshot.runId,
|
|
56906
|
+
scope: scopeFilter,
|
|
56907
|
+
change: changeFilter,
|
|
56908
|
+
entity: visibleKeys.at(-1).entity,
|
|
56909
|
+
key: visibleKeys.at(-1).key
|
|
56910
|
+
}) : null,
|
|
56911
|
+
changes
|
|
56912
|
+
};
|
|
56913
|
+
});
|
|
56914
|
+
app.get("/projects/:name/technical-aeo/crawl/pages/audit", async (request) => {
|
|
56915
|
+
const project = resolveProject(app.db, request.params.name);
|
|
56916
|
+
const scalarQueryString = (value, name) => {
|
|
56917
|
+
if (value === void 0) return void 0;
|
|
56918
|
+
if (typeof value !== "string") throw validationError(`${name} must be provided once`);
|
|
56919
|
+
return value;
|
|
56920
|
+
};
|
|
56921
|
+
const runId = scalarQueryString(request.query.runId, "runId");
|
|
56922
|
+
const nodeKey = scalarQueryString(request.query.nodeKey, "nodeKey")?.trim();
|
|
56923
|
+
const url = scalarQueryString(request.query.url, "url")?.trim();
|
|
56924
|
+
if (Number(Boolean(nodeKey)) + Number(Boolean(url)) !== 1) {
|
|
56925
|
+
throw validationError("Provide exactly one of nodeKey or url");
|
|
56926
|
+
}
|
|
56927
|
+
const target = resolveCrawl(project.id, runId);
|
|
56928
|
+
if (!target) {
|
|
56929
|
+
if (runId) throw notFound("Site crawl run", runId);
|
|
56930
|
+
return { state: "no-crawl", project: project.name, runId: null };
|
|
56931
|
+
}
|
|
56932
|
+
const snapshot = target.snapshot;
|
|
56933
|
+
const provenance = {
|
|
56934
|
+
project: project.name,
|
|
56935
|
+
runId: snapshot.runId,
|
|
56936
|
+
complete: snapshot.complete,
|
|
56937
|
+
termination: snapshot.termination
|
|
56938
|
+
};
|
|
56939
|
+
const scope = detailScopeFor(project.id, snapshot);
|
|
56940
|
+
if (!scope) return { state: "details-unavailable", ...provenance };
|
|
56941
|
+
const page = pageInScope(scope, nodeKey ? { nodeKey } : { url });
|
|
56942
|
+
if (!page) return { state: "not-found", ...provenance };
|
|
56943
|
+
const identity = {
|
|
56944
|
+
nodeKey: page.nodeKey,
|
|
56945
|
+
url: page.url,
|
|
56946
|
+
auditState: page.auditState
|
|
56947
|
+
};
|
|
56948
|
+
if (page.auditScore == null) {
|
|
56949
|
+
return {
|
|
56950
|
+
state: "not-audited",
|
|
56951
|
+
...provenance,
|
|
56952
|
+
...identity,
|
|
56953
|
+
auditScore: null,
|
|
56954
|
+
factors: [],
|
|
56955
|
+
criticalDefects: []
|
|
56956
|
+
};
|
|
56957
|
+
}
|
|
56958
|
+
return {
|
|
56959
|
+
state: "ready",
|
|
56960
|
+
...provenance,
|
|
56961
|
+
...identity,
|
|
56962
|
+
auditScore: page.auditScore,
|
|
56963
|
+
...mapCrawlPageAuditEvidence(page)
|
|
56964
|
+
};
|
|
56965
|
+
});
|
|
56966
|
+
app.get("/projects/:name/technical-aeo/crawl/pages", async (request) => {
|
|
56967
|
+
const project = resolveProject(app.db, request.params.name);
|
|
56968
|
+
const target = resolveCrawl(project.id, request.query.runId);
|
|
56969
|
+
if (!target) {
|
|
56970
|
+
if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
|
|
56971
|
+
return { project: project.name, hasCrawlData: false, runId: null, total: 0, nextCursor: null, pages: [] };
|
|
56972
|
+
}
|
|
56973
|
+
const snapshot = target.snapshot;
|
|
56974
|
+
if (!snapshot.detailsAvailable || !snapshot.attemptId) {
|
|
56975
|
+
return { project: project.name, hasCrawlData: true, runId: snapshot.runId, total: 0, nextCursor: null, pages: [] };
|
|
56976
|
+
}
|
|
56977
|
+
const filters = [
|
|
56978
|
+
eq55(siteCrawlPages.projectId, project.id),
|
|
56979
|
+
eq55(siteCrawlPages.runId, snapshot.runId),
|
|
56980
|
+
eq55(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
56981
|
+
];
|
|
56982
|
+
const inventoryEligible = parseBoolean2(request.query.inventoryEligible);
|
|
56983
|
+
if (inventoryEligible != null) filters.push(eq55(siteCrawlPages.inventoryEligible, inventoryEligible));
|
|
56984
|
+
if (request.query.fetchState) filters.push(eq55(siteCrawlPages.fetchState, request.query.fetchState));
|
|
56985
|
+
if (request.query.indexabilityState) filters.push(eq55(siteCrawlPages.indexabilityState, request.query.indexabilityState));
|
|
56986
|
+
if (request.query.auditState) filters.push(eq55(siteCrawlPages.auditState, request.query.auditState));
|
|
56987
|
+
const where = and43(...filters);
|
|
56988
|
+
const total = app.db.select({ value: count() }).from(siteCrawlPages).where(where).get()?.value ?? 0;
|
|
56989
|
+
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
56990
|
+
const offset = decodeCursor(request.query.cursor);
|
|
56991
|
+
const orderBy = request.query.sort === "score-desc" ? [desc24(siteCrawlPages.auditScore), asc10(siteCrawlPages.nodeKey)] : request.query.sort === "score-asc" ? [asc10(siteCrawlPages.auditScore), asc10(siteCrawlPages.nodeKey)] : request.query.sort === "path" ? [asc10(siteCrawlPages.path), asc10(siteCrawlPages.nodeKey)] : [asc10(siteCrawlPages.url), asc10(siteCrawlPages.nodeKey)];
|
|
56992
|
+
const rows = app.db.select().from(siteCrawlPages).where(where).orderBy(...orderBy).limit(limit).offset(offset).all();
|
|
56993
|
+
const nextOffset = offset + rows.length;
|
|
56994
|
+
return {
|
|
56995
|
+
project: project.name,
|
|
56996
|
+
hasCrawlData: true,
|
|
56997
|
+
runId: snapshot.runId,
|
|
56998
|
+
total,
|
|
56999
|
+
nextCursor: nextOffset < total ? encodeCursor2(nextOffset) : null,
|
|
57000
|
+
pages: rows.map(mapCrawlPage)
|
|
57001
|
+
};
|
|
57002
|
+
});
|
|
57003
|
+
app.get("/projects/:name/technical-aeo/structure", async (request) => {
|
|
57004
|
+
const project = resolveProject(app.db, request.params.name);
|
|
57005
|
+
const parentPath = normalizeParentPath(request.query.parentPath);
|
|
57006
|
+
const target = resolveCrawl(project.id, request.query.runId);
|
|
57007
|
+
if (!target) {
|
|
57008
|
+
if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
|
|
57009
|
+
return { project: project.name, hasCrawlData: false, runId: null, parentPath, nextCursor: null, children: [] };
|
|
57010
|
+
}
|
|
57011
|
+
const snapshot = target.snapshot;
|
|
57012
|
+
if (!snapshot.detailsAvailable || !snapshot.attemptId) {
|
|
57013
|
+
return { project: project.name, hasCrawlData: true, runId: snapshot.runId, parentPath, nextCursor: null, children: [] };
|
|
57014
|
+
}
|
|
57015
|
+
const sourceRows = app.db.select({
|
|
57016
|
+
path: siteCrawlPages.path,
|
|
57017
|
+
url: siteCrawlPages.url,
|
|
57018
|
+
inventoryEligible: siteCrawlPages.inventoryEligible,
|
|
57019
|
+
fetchState: siteCrawlPages.fetchState
|
|
57020
|
+
}).from(siteCrawlPages).where(and43(
|
|
57021
|
+
eq55(siteCrawlPages.projectId, project.id),
|
|
57022
|
+
eq55(siteCrawlPages.runId, snapshot.runId),
|
|
57023
|
+
eq55(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
57024
|
+
)).limit(MAX_STRUCTURE_SOURCE_ROWS + 1).all();
|
|
57025
|
+
if (sourceRows.length > MAX_STRUCTURE_SOURCE_ROWS) {
|
|
57026
|
+
throw validationError(`Persisted crawl exceeds the ${MAX_STRUCTURE_SOURCE_ROWS}-page structure limit`);
|
|
57027
|
+
}
|
|
57028
|
+
const children = /* @__PURE__ */ new Map();
|
|
57029
|
+
for (const row of sourceRows) {
|
|
57030
|
+
const childPath = structureChildPath(parentPath, row.path);
|
|
57031
|
+
if (!childPath) continue;
|
|
57032
|
+
const child = children.get(childPath) ?? {
|
|
57033
|
+
url: null,
|
|
57034
|
+
hasPage: false,
|
|
57035
|
+
pageCount: 0,
|
|
57036
|
+
inventoryEligibleCount: 0,
|
|
57037
|
+
fetchedCount: 0
|
|
57038
|
+
};
|
|
57039
|
+
child.pageCount++;
|
|
57040
|
+
if (row.inventoryEligible) child.inventoryEligibleCount++;
|
|
57041
|
+
if (FETCHED_SITE_CRAWL_STATES.has(row.fetchState)) child.fetchedCount++;
|
|
57042
|
+
if (structureHierarchyPath(row.path) === childPath) {
|
|
57043
|
+
child.hasPage = true;
|
|
57044
|
+
if (child.url == null || row.url > child.url) child.url = row.url;
|
|
57045
|
+
}
|
|
57046
|
+
children.set(childPath, child);
|
|
57047
|
+
}
|
|
57048
|
+
const limit = parseBoundedLimit(request.query.limit, 50, 100);
|
|
57049
|
+
const offset = decodeCursor(request.query.cursor);
|
|
57050
|
+
const sortedChildren = [...children.entries()].sort(([left], [right]) => left.localeCompare(right));
|
|
57051
|
+
const page = sortedChildren.slice(offset, offset + limit + 1);
|
|
57052
|
+
const hasMore = page.length > limit;
|
|
57053
|
+
const visible = hasMore ? page.slice(0, limit) : page;
|
|
57054
|
+
return {
|
|
57055
|
+
project: project.name,
|
|
57056
|
+
hasCrawlData: true,
|
|
57057
|
+
runId: snapshot.runId,
|
|
57058
|
+
parentPath,
|
|
57059
|
+
nextCursor: hasMore ? encodeCursor2(offset + visible.length) : null,
|
|
57060
|
+
children: visible.map(([path8, child]) => ({
|
|
57061
|
+
path: path8,
|
|
57062
|
+
url: child.url,
|
|
57063
|
+
hasPage: child.hasPage,
|
|
57064
|
+
pageCount: child.pageCount,
|
|
57065
|
+
inventoryEligibleCount: child.inventoryEligibleCount,
|
|
57066
|
+
fetchedCount: child.fetchedCount
|
|
57067
|
+
}))
|
|
57068
|
+
};
|
|
57069
|
+
});
|
|
57070
|
+
app.get("/projects/:name/technical-aeo/internal-links", async (request) => {
|
|
57071
|
+
const project = resolveProject(app.db, request.params.name);
|
|
57072
|
+
const target = resolveCrawl(project.id, request.query.runId);
|
|
57073
|
+
if (!target) {
|
|
57074
|
+
if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
|
|
57075
|
+
return { project: project.name, hasCrawlData: false, runId: null, total: 0, nextCursor: null, edges: [] };
|
|
57076
|
+
}
|
|
57077
|
+
const snapshot = target.snapshot;
|
|
57078
|
+
if (!snapshot.detailsAvailable || !snapshot.attemptId) {
|
|
57079
|
+
return { project: project.name, hasCrawlData: true, runId: snapshot.runId, total: 0, nextCursor: null, edges: [] };
|
|
57080
|
+
}
|
|
57081
|
+
const filters = [
|
|
57082
|
+
eq55(siteCrawlEdges.projectId, project.id),
|
|
57083
|
+
eq55(siteCrawlEdges.runId, snapshot.runId),
|
|
57084
|
+
eq55(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
57085
|
+
eq55(siteCrawlEdges.internal, true)
|
|
57086
|
+
];
|
|
57087
|
+
if (request.query.sourceUrl) filters.push(eq55(siteCrawlEdges.sourceUrl, request.query.sourceUrl));
|
|
57088
|
+
if (request.query.targetUrl) filters.push(eq55(siteCrawlEdges.targetUrl, request.query.targetUrl));
|
|
57089
|
+
const followable = parseBoolean2(request.query.followable);
|
|
57090
|
+
if (followable != null) filters.push(eq55(siteCrawlEdges.followable, followable));
|
|
57091
|
+
const where = and43(...filters);
|
|
57092
|
+
const total = app.db.select({ value: count() }).from(siteCrawlEdges).where(where).get()?.value ?? 0;
|
|
57093
|
+
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
57094
|
+
const offset = decodeCursor(request.query.cursor);
|
|
57095
|
+
const rows = app.db.select().from(siteCrawlEdges).where(where).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(limit).offset(offset).all();
|
|
57096
|
+
const nextOffset = offset + rows.length;
|
|
57097
|
+
return {
|
|
57098
|
+
project: project.name,
|
|
57099
|
+
hasCrawlData: true,
|
|
57100
|
+
runId: snapshot.runId,
|
|
57101
|
+
total,
|
|
57102
|
+
nextCursor: nextOffset < total ? encodeCursor2(nextOffset) : null,
|
|
57103
|
+
edges: rows.map(mapCrawlEdge)
|
|
57104
|
+
};
|
|
57105
|
+
});
|
|
57106
|
+
app.get("/projects/:name/technical-aeo/internal-links/neighbors", async (request) => {
|
|
57107
|
+
const project = resolveProject(app.db, request.params.name);
|
|
57108
|
+
if (!request.query.nodeKey && !request.query.url) {
|
|
57109
|
+
throw validationError("Provide nodeKey or url");
|
|
57110
|
+
}
|
|
57111
|
+
const target = resolveCrawl(project.id, request.query.runId);
|
|
57112
|
+
if (!target) {
|
|
57113
|
+
if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
|
|
57114
|
+
return {
|
|
57115
|
+
project: project.name,
|
|
57116
|
+
hasCrawlData: false,
|
|
57117
|
+
runId: null,
|
|
57118
|
+
nodeKey: request.query.nodeKey ?? null,
|
|
57119
|
+
url: request.query.url ?? null,
|
|
57120
|
+
inbound: [],
|
|
57121
|
+
outbound: [],
|
|
57122
|
+
inboundTruncated: false,
|
|
57123
|
+
outboundTruncated: false
|
|
57124
|
+
};
|
|
57125
|
+
}
|
|
57126
|
+
const snapshot = target.snapshot;
|
|
57127
|
+
if (!snapshot.detailsAvailable || !snapshot.attemptId) {
|
|
57128
|
+
return {
|
|
57129
|
+
project: project.name,
|
|
57130
|
+
hasCrawlData: true,
|
|
57131
|
+
runId: snapshot.runId,
|
|
57132
|
+
nodeKey: request.query.nodeKey ?? null,
|
|
57133
|
+
url: request.query.url ?? null,
|
|
57134
|
+
inbound: [],
|
|
57135
|
+
outbound: [],
|
|
57136
|
+
inboundTruncated: false,
|
|
57137
|
+
outboundTruncated: false
|
|
57138
|
+
};
|
|
57139
|
+
}
|
|
57140
|
+
const scope = [
|
|
57141
|
+
eq55(siteCrawlEdges.projectId, project.id),
|
|
57142
|
+
eq55(siteCrawlEdges.runId, snapshot.runId),
|
|
57143
|
+
eq55(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
57144
|
+
eq55(siteCrawlEdges.internal, true)
|
|
57145
|
+
];
|
|
57146
|
+
const inboundMatch = request.query.nodeKey && request.query.url ? or11(eq55(siteCrawlEdges.targetNodeKey, request.query.nodeKey), eq55(siteCrawlEdges.targetUrl, request.query.url)) : request.query.nodeKey ? eq55(siteCrawlEdges.targetNodeKey, request.query.nodeKey) : eq55(siteCrawlEdges.targetUrl, request.query.url);
|
|
57147
|
+
const outboundMatch = request.query.nodeKey && request.query.url ? or11(eq55(siteCrawlEdges.sourceNodeKey, request.query.nodeKey), eq55(siteCrawlEdges.sourceUrl, request.query.url)) : request.query.nodeKey ? eq55(siteCrawlEdges.sourceNodeKey, request.query.nodeKey) : eq55(siteCrawlEdges.sourceUrl, request.query.url);
|
|
57148
|
+
const limit = parseBoundedLimit(request.query.limit, 50, 100);
|
|
57149
|
+
const inboundRows = app.db.select().from(siteCrawlEdges).where(and43(...scope, inboundMatch)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
57150
|
+
const outboundRows = app.db.select().from(siteCrawlEdges).where(and43(...scope, outboundMatch)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
57151
|
+
return {
|
|
57152
|
+
project: project.name,
|
|
57153
|
+
hasCrawlData: true,
|
|
57154
|
+
runId: snapshot.runId,
|
|
57155
|
+
nodeKey: request.query.nodeKey ?? null,
|
|
57156
|
+
url: request.query.url ?? null,
|
|
57157
|
+
inbound: inboundRows.slice(0, limit).map(mapCrawlEdge),
|
|
57158
|
+
outbound: outboundRows.slice(0, limit).map(mapCrawlEdge),
|
|
57159
|
+
inboundTruncated: inboundRows.length > limit,
|
|
57160
|
+
outboundTruncated: outboundRows.length > limit
|
|
57161
|
+
};
|
|
57162
|
+
});
|
|
57163
|
+
app.get("/projects/:name/technical-aeo/dead-links", async (request) => {
|
|
57164
|
+
const project = resolveProject(app.db, request.params.name);
|
|
57165
|
+
const target = resolveCrawl(project.id, request.query.runId);
|
|
57166
|
+
const legacyAuditAvailable = hasLegacyAudit(project.id);
|
|
57167
|
+
if (!target) {
|
|
57168
|
+
if (request.query.runId) throw notFound("Site crawl run", request.query.runId);
|
|
57169
|
+
return { project: project.name, runId: null, state: "unavailable", legacyAuditAvailable };
|
|
57170
|
+
}
|
|
57171
|
+
const snapshot = target.snapshot;
|
|
57172
|
+
if (!snapshot.checkDeadLinks || snapshot.deadLinkState === "disabled") {
|
|
57173
|
+
return { project: project.name, runId: snapshot.runId, state: "disabled", checkDeadLinks: false };
|
|
57174
|
+
}
|
|
57175
|
+
if (snapshot.deadLinkState === "unavailable") {
|
|
57176
|
+
return { project: project.name, runId: snapshot.runId, state: "unavailable", legacyAuditAvailable };
|
|
57177
|
+
}
|
|
57178
|
+
if (!snapshot.attemptId) {
|
|
57179
|
+
return { project: project.name, runId: snapshot.runId, state: "partial", checkDeadLinks: true, checked: snapshot.deadLinksChecked, found: snapshot.deadLinksFound, total: 0, nextCursor: null, deadLinks: [] };
|
|
57180
|
+
}
|
|
57181
|
+
const where = and43(
|
|
57182
|
+
eq55(siteCrawlFindings.projectId, project.id),
|
|
57183
|
+
eq55(siteCrawlFindings.runId, snapshot.runId),
|
|
57184
|
+
eq55(siteCrawlFindings.attemptId, snapshot.attemptId),
|
|
57185
|
+
eq55(siteCrawlFindings.findingType, "dead-link")
|
|
57186
|
+
);
|
|
57187
|
+
const total = app.db.select({ value: count() }).from(siteCrawlFindings).where(where).get()?.value ?? 0;
|
|
57188
|
+
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
57189
|
+
const offset = decodeCursor(request.query.cursor);
|
|
57190
|
+
const rows = app.db.select().from(siteCrawlFindings).where(where).orderBy(asc10(siteCrawlFindings.findingKey)).limit(limit).offset(offset).all();
|
|
57191
|
+
const nextOffset = offset + rows.length;
|
|
57192
|
+
const deadLinks = rows.map((row) => ({
|
|
57193
|
+
findingKey: row.findingKey,
|
|
57194
|
+
severity: row.severity,
|
|
57195
|
+
sourceNodeKey: row.sourceNodeKey,
|
|
57196
|
+
sourceUrl: row.sourceUrl,
|
|
57197
|
+
targetNodeKey: row.targetNodeKey,
|
|
57198
|
+
targetUrl: row.targetUrl,
|
|
57199
|
+
evidence: row.evidence
|
|
57200
|
+
}));
|
|
57201
|
+
const state = snapshot.deadLinkState === "complete" ? "complete" : "partial";
|
|
57202
|
+
return {
|
|
57203
|
+
project: project.name,
|
|
57204
|
+
runId: snapshot.runId,
|
|
57205
|
+
state,
|
|
57206
|
+
checkDeadLinks: true,
|
|
57207
|
+
checked: snapshot.deadLinksChecked,
|
|
57208
|
+
found: snapshot.deadLinksFound,
|
|
57209
|
+
total,
|
|
57210
|
+
nextCursor: nextOffset < total ? encodeCursor2(nextOffset) : null,
|
|
57211
|
+
deadLinks
|
|
57212
|
+
};
|
|
57213
|
+
});
|
|
54938
57214
|
app.post("/projects/:name/technical-aeo/runs", async (request) => {
|
|
54939
57215
|
const project = resolveProject(app.db, request.params.name);
|
|
54940
57216
|
const parsed = siteAuditRunRequestSchema.safeParse(request.body ?? {});
|
|
54941
57217
|
if (!parsed.success) {
|
|
54942
57218
|
throw validationError(parsed.error.issues[0]?.message ?? "Invalid site-audit request");
|
|
54943
57219
|
}
|
|
54944
|
-
const
|
|
54945
|
-
|
|
54946
|
-
|
|
54947
|
-
|
|
54948
|
-
|
|
54949
|
-
|
|
54950
|
-
|
|
57220
|
+
const effectiveRequest = normalizeSiteAuditRunRequest(parsed.data);
|
|
57221
|
+
const identityKey = siteAuditRequestIdentity(effectiveRequest);
|
|
57222
|
+
const result = app.db.transaction((tx) => {
|
|
57223
|
+
const existing = tx.select({
|
|
57224
|
+
id: runs.id,
|
|
57225
|
+
status: runs.status,
|
|
57226
|
+
identityKey: siteCrawlRunRequests.identityKey,
|
|
57227
|
+
effectiveOptions: siteCrawlRunRequests.effectiveOptions
|
|
57228
|
+
}).from(runs).leftJoin(siteCrawlRunRequests, and43(
|
|
57229
|
+
eq55(siteCrawlRunRequests.projectId, runs.projectId),
|
|
57230
|
+
eq55(siteCrawlRunRequests.runId, runs.id)
|
|
57231
|
+
)).where(and43(
|
|
57232
|
+
eq55(runs.projectId, project.id),
|
|
57233
|
+
eq55(runs.kind, RunKinds["site-audit"]),
|
|
57234
|
+
inArray19(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
57235
|
+
)).get();
|
|
57236
|
+
if (existing) {
|
|
57237
|
+
if (existing.identityKey === identityKey) {
|
|
57238
|
+
return { runId: existing.id, status: existing.status, created: false };
|
|
57239
|
+
}
|
|
57240
|
+
throw operationInProgress(
|
|
57241
|
+
`A Technical AEO crawl is already ${existing.status} with different request options.`,
|
|
57242
|
+
{
|
|
57243
|
+
activeRunId: existing.id,
|
|
57244
|
+
activeStatus: existing.status,
|
|
57245
|
+
activeOptions: existing.effectiveOptions,
|
|
57246
|
+
requestedOptions: effectiveRequest
|
|
57247
|
+
}
|
|
57248
|
+
);
|
|
57249
|
+
}
|
|
57250
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
57251
|
+
const runId = crypto43.randomUUID();
|
|
57252
|
+
tx.insert(runs).values({
|
|
57253
|
+
id: runId,
|
|
57254
|
+
projectId: project.id,
|
|
57255
|
+
kind: RunKinds["site-audit"],
|
|
57256
|
+
status: RunStatuses.queued,
|
|
57257
|
+
trigger: RunTriggers.manual,
|
|
57258
|
+
createdAt: now
|
|
57259
|
+
}).run();
|
|
57260
|
+
tx.insert(siteCrawlRunRequests).values({
|
|
57261
|
+
runId,
|
|
57262
|
+
projectId: project.id,
|
|
57263
|
+
identityKey,
|
|
57264
|
+
effectiveOptions: effectiveRequest,
|
|
57265
|
+
createdAt: now
|
|
57266
|
+
}).run();
|
|
57267
|
+
return { runId, status: RunStatuses.queued, created: true };
|
|
57268
|
+
});
|
|
57269
|
+
if (result.created) {
|
|
57270
|
+
opts.onSiteAuditRequested?.(result.runId, project.id, {
|
|
57271
|
+
sitemapUrl: effectiveRequest.sitemapUrl ?? void 0,
|
|
57272
|
+
limit: parsed.data.limit,
|
|
57273
|
+
maxPages: effectiveRequest.maxPages,
|
|
57274
|
+
maxEdges: effectiveRequest.maxEdges,
|
|
57275
|
+
maxDepth: effectiveRequest.maxDepth ?? void 0,
|
|
57276
|
+
checkDeadLinks: effectiveRequest.checkDeadLinks
|
|
57277
|
+
});
|
|
54951
57278
|
}
|
|
54952
|
-
|
|
54953
|
-
const runId = crypto43.randomUUID();
|
|
54954
|
-
app.db.insert(runs).values({
|
|
54955
|
-
id: runId,
|
|
54956
|
-
projectId: project.id,
|
|
54957
|
-
kind: RunKinds["site-audit"],
|
|
54958
|
-
status: RunStatuses.queued,
|
|
54959
|
-
trigger: RunTriggers.manual,
|
|
54960
|
-
createdAt: now
|
|
54961
|
-
}).run();
|
|
54962
|
-
opts.onSiteAuditRequested?.(runId, project.id, {
|
|
54963
|
-
sitemapUrl: parsed.data.sitemapUrl,
|
|
54964
|
-
limit: parsed.data.limit
|
|
54965
|
-
});
|
|
54966
|
-
return { runId, status: RunStatuses.queued };
|
|
57279
|
+
return { runId: result.runId, status: result.status };
|
|
54967
57280
|
});
|
|
54968
57281
|
}
|
|
54969
57282
|
|
|
@@ -55113,6 +57426,7 @@ async function apiRoutes(app, opts) {
|
|
|
55113
57426
|
await api.register(competitorRoutes);
|
|
55114
57427
|
await api.register(runRoutes, {
|
|
55115
57428
|
onRunCreated: opts.onRunCreated,
|
|
57429
|
+
onRunCancelled: opts.onRunCancelled,
|
|
55116
57430
|
validProviderNames: opts.providerAdapters?.map((a) => a.name),
|
|
55117
57431
|
getRunnableProviderNames: opts.getRunnableProviderNames,
|
|
55118
57432
|
getEffectiveProviderModels: opts.getEffectiveProviderModels
|
|
@@ -55530,7 +57844,7 @@ function extractRecommendedCompetitors(answerText, ownDomains, citedDomains, com
|
|
|
55530
57844
|
}
|
|
55531
57845
|
const ownKeys = new Set([...ownBrandAliases].map(brandKeyFromText));
|
|
55532
57846
|
const knownCompetitorAliases = new Set(
|
|
55533
|
-
[...citedDomains, ...competitorDomains].flatMap((domain) => collectBrandAliasesFromDomain(domain)).filter((
|
|
57847
|
+
[...citedDomains, ...competitorDomains].flatMap((domain) => collectBrandAliasesFromDomain(domain)).filter((alias2) => !ownKeys.has(brandKeyFromText(alias2)))
|
|
55534
57848
|
);
|
|
55535
57849
|
if (knownCompetitorAliases.size === 0) return [];
|
|
55536
57850
|
const candidatePatterns = [
|
|
@@ -55602,7 +57916,7 @@ function collectBrandAliasesFromDomain(domain) {
|
|
|
55602
57916
|
return [...aliases];
|
|
55603
57917
|
}
|
|
55604
57918
|
function matchesBrandAlias(candidate, aliases) {
|
|
55605
|
-
return [...aliases].some((
|
|
57919
|
+
return [...aliases].some((alias2) => textContainsBrandAlias(candidate, alias2));
|
|
55606
57920
|
}
|
|
55607
57921
|
|
|
55608
57922
|
// src/intelligence-service.ts
|
|
@@ -55634,7 +57948,7 @@ var IntelligenceService = class {
|
|
|
55634
57948
|
const recentRuns = this.db.select().from(runs).where(
|
|
55635
57949
|
and45(
|
|
55636
57950
|
eq57(runs.projectId, projectId),
|
|
55637
|
-
|
|
57951
|
+
or12(eq57(runs.status, "completed"), eq57(runs.status, "partial")),
|
|
55638
57952
|
// Defensive: RunCoordinator already skips probes before this is
|
|
55639
57953
|
// called, but if a future call site invokes analyzeAndPersist
|
|
55640
57954
|
// directly for a probe, probes still must not pollute the
|
|
@@ -55937,7 +58251,7 @@ var IntelligenceService = class {
|
|
|
55937
58251
|
const allRuns = this.db.select().from(runs).where(
|
|
55938
58252
|
and45(
|
|
55939
58253
|
eq57(runs.projectId, project.id),
|
|
55940
|
-
|
|
58254
|
+
or12(eq57(runs.status, "completed"), eq57(runs.status, "partial")),
|
|
55941
58255
|
// Backfill must not replay probe runs as if they were real sweeps.
|
|
55942
58256
|
ne7(runs.trigger, RunTriggers.probe)
|
|
55943
58257
|
)
|
|
@@ -55970,7 +58284,7 @@ var IntelligenceService = class {
|
|
|
55970
58284
|
let wouldDeleteTotal = 0;
|
|
55971
58285
|
const existingByRunId = /* @__PURE__ */ new Map();
|
|
55972
58286
|
if (isDryRun && targetRuns.length > 0) {
|
|
55973
|
-
const rows = this.db.select({ runId: insights.runId }).from(insights).where(
|
|
58287
|
+
const rows = this.db.select({ runId: insights.runId }).from(insights).where(inArray20(insights.runId, targetRuns.map((r) => r.id))).all();
|
|
55974
58288
|
for (const r of rows) {
|
|
55975
58289
|
if (r.runId == null) continue;
|
|
55976
58290
|
existingByRunId.set(r.runId, (existingByRunId.get(r.runId) ?? 0) + 1);
|
|
@@ -56115,7 +58429,7 @@ var IntelligenceService = class {
|
|
|
56115
58429
|
and45(
|
|
56116
58430
|
eq57(runs.projectId, projectId),
|
|
56117
58431
|
eq57(runs.kind, RunKinds["answer-visibility"]),
|
|
56118
|
-
|
|
58432
|
+
or12(eq57(runs.status, "completed"), eq57(runs.status, "partial")),
|
|
56119
58433
|
// Defensive — see top of file.
|
|
56120
58434
|
ne7(runs.trigger, RunTriggers.probe)
|
|
56121
58435
|
)
|
|
@@ -56135,7 +58449,7 @@ var IntelligenceService = class {
|
|
|
56135
58449
|
const haveHistory = recentRunIds.length > 0;
|
|
56136
58450
|
const priorRegressionsByPair = /* @__PURE__ */ new Map();
|
|
56137
58451
|
if (haveHistory) {
|
|
56138
|
-
const priorRows = this.db.select({ query: insights.query, provider: insights.provider, runId: insights.runId }).from(insights).where(and45(eq57(insights.type, "regression"),
|
|
58452
|
+
const priorRows = this.db.select({ query: insights.query, provider: insights.provider, runId: insights.runId }).from(insights).where(and45(eq57(insights.type, "regression"), inArray20(insights.runId, recentRunIds))).all();
|
|
56139
58453
|
const regressionGroups = /* @__PURE__ */ new Map();
|
|
56140
58454
|
for (const row of priorRows) {
|
|
56141
58455
|
if (!row.runId) continue;
|
|
@@ -56241,6 +58555,16 @@ export {
|
|
|
56241
58555
|
gscCoverageSnapshots,
|
|
56242
58556
|
siteAuditSnapshots,
|
|
56243
58557
|
siteAuditPages,
|
|
58558
|
+
siteCrawlRunRequests,
|
|
58559
|
+
siteCrawlAttempts,
|
|
58560
|
+
siteCrawlSnapshots,
|
|
58561
|
+
siteCrawlPages,
|
|
58562
|
+
siteCrawlEdges,
|
|
58563
|
+
siteCrawlGraphLayouts,
|
|
58564
|
+
siteCrawlGraphNodes,
|
|
58565
|
+
siteCrawlGraphEdges,
|
|
58566
|
+
siteCrawlFindings,
|
|
58567
|
+
siteCrawlEventReceipts,
|
|
56244
58568
|
bingCoverageSnapshots,
|
|
56245
58569
|
bingUrlInspections,
|
|
56246
58570
|
gaTrafficSnapshots,
|