@canonry/canonry 4.154.1 → 4.156.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agent-workspace/AGENTS.md +2 -2
- package/assets/agent-workspace/skills/aero/references/orchestration.md +3 -3
- package/assets/agent-workspace/skills/canonry/SKILL.md +2 -2
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +27 -7
- package/assets/assets/{AuditHistoryPanel-WKf-vvmg.js → AuditHistoryPanel-gmJMVlXX.js} +1 -1
- package/assets/assets/{BacklinksPage-By06DbHN.js → BacklinksPage-Bv2sTCyN.js} +1 -1
- package/assets/assets/{HistoryPage-C_PcPiyX.js → HistoryPage-13q7GkiI.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-BHnNegEa.js → MeasurementPropertyPage-BgBqXT1p.js} +1 -1
- package/assets/assets/ProjectPage-EsGwKhot.js +15 -0
- package/assets/assets/{RunRow-9qn5Eer_.js → RunRow-bPzapIGJ.js} +1 -1
- package/assets/assets/{RunsPage-FxMTprK1.js → RunsPage-Dqfz18lH.js} +1 -1
- package/assets/assets/{SettingsPage-Cr7_PrNW.js → SettingsPage-_wMecn-V.js} +1 -1
- package/assets/assets/{TrafficPage-D0qk9uMN.js → TrafficPage-RYBdsOLB.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-CXa_3_Gy.js → TrafficSourceDetailPage-DpeMMyKU.js} +1 -1
- package/assets/assets/{extract-error-message-RkTEH1Vb.js → extract-error-message-DUarAlyD.js} +1 -1
- package/assets/assets/index-BxegEwic.css +1 -0
- package/assets/assets/index-DP-fT2uI.js +83 -0
- package/assets/assets/react-sigma_core.esm.min-DnypJl5w.js +312 -0
- package/assets/assets/vendor-lucide-BtYdosYk.js +1 -0
- package/assets/assets/{vendor-markdown-CpDJch4G.js → vendor-markdown-D274_wsI.js} +1 -1
- package/assets/assets/{vendor-radix-DnzzbllM.js → vendor-radix-BxMXSwbC.js} +1 -1
- package/assets/assets/{vendor-recharts-33ACLxsD.js → vendor-recharts-C4ktw_aE.js} +1 -1
- package/assets/assets/{vendor-tanstack-BjS7iY3g.js → vendor-tanstack-pFxw0eGP.js} +1 -1
- package/assets/index.html +7 -7
- package/dist/{chunk-KRMLGFY3.js → chunk-ABNN2ZVR.js} +2438 -114
- package/dist/{chunk-NNW7TJUZ.js → chunk-AMPT2EP7.js} +527 -5
- package/dist/{chunk-ANSQ6JNO.js → chunk-CPASKGUE.js} +563 -4
- package/dist/{chunk-JOP2DGXL.js → chunk-ZWESOFA5.js} +1230 -310
- package/dist/cli.js +742 -8
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-ZY5KOSZV.js → intelligence-service-WHC7B32V.js} +2 -2
- package/dist/mcp.js +4 -4
- package/package.json +13 -11
- package/assets/assets/ProjectPage-VTbW21dz.js +0 -12
- package/assets/assets/index-DyL54s8x.css +0 -1
- package/assets/assets/index-nSWJ9yka.js +0 -83
- package/assets/assets/vendor-lucide-ByU7vBHt.js +0 -1
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
loadConfig,
|
|
11
11
|
loadConfigRaw,
|
|
12
12
|
saveConfigPatch
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-CPASKGUE.js";
|
|
14
14
|
import {
|
|
15
15
|
CC_CACHE_DIR,
|
|
16
16
|
DUCKDB_SPEC,
|
|
@@ -124,9 +124,19 @@ import {
|
|
|
124
124
|
schedules,
|
|
125
125
|
siteAuditPages,
|
|
126
126
|
siteAuditSnapshots,
|
|
127
|
+
siteCrawlAttempts,
|
|
128
|
+
siteCrawlEdges,
|
|
129
|
+
siteCrawlEventReceipts,
|
|
130
|
+
siteCrawlFindings,
|
|
131
|
+
siteCrawlGraphEdges,
|
|
132
|
+
siteCrawlGraphLayouts,
|
|
133
|
+
siteCrawlGraphNodes,
|
|
134
|
+
siteCrawlPages,
|
|
135
|
+
siteCrawlRunRequests,
|
|
136
|
+
siteCrawlSnapshots,
|
|
127
137
|
toAlertView,
|
|
128
138
|
usageCounters
|
|
129
|
-
} from "./chunk-
|
|
139
|
+
} from "./chunk-ABNN2ZVR.js";
|
|
130
140
|
import {
|
|
131
141
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
132
142
|
AGENT_PROVIDER_IDS,
|
|
@@ -151,6 +161,12 @@ import {
|
|
|
151
161
|
RunKinds,
|
|
152
162
|
RunStatuses,
|
|
153
163
|
RunTriggers,
|
|
164
|
+
SITE_AUDIT_DEFAULT_EDGE_LIMIT,
|
|
165
|
+
SITE_AUDIT_DEFAULT_PAGE_LIMIT,
|
|
166
|
+
SITE_AUDIT_MAX_EDGE_LIMIT,
|
|
167
|
+
SITE_AUDIT_MAX_PAGE_LIMIT,
|
|
168
|
+
SITE_CRAWL_GRAPH_MAX_EDGES,
|
|
169
|
+
SITE_CRAWL_GRAPH_MAX_NODES,
|
|
154
170
|
SKILL_MANIFEST_FILENAME,
|
|
155
171
|
SchedulableRunKinds,
|
|
156
172
|
SkillsClients,
|
|
@@ -200,6 +216,7 @@ import {
|
|
|
200
216
|
normalizeIdTokens,
|
|
201
217
|
normalizeMeasurementExecutionQueryText,
|
|
202
218
|
normalizeServedModel,
|
|
219
|
+
normalizeSiteAuditRunRequest,
|
|
203
220
|
normalizeUrlPath,
|
|
204
221
|
notFound,
|
|
205
222
|
parseMeasurementRunManifestV1,
|
|
@@ -210,6 +227,7 @@ import {
|
|
|
210
227
|
resolveSnapshotRequestQueries,
|
|
211
228
|
serializeForInlineScript,
|
|
212
229
|
serializeRunError,
|
|
230
|
+
siteAuditRequestIdentity,
|
|
213
231
|
skillsClientSchema,
|
|
214
232
|
splitList,
|
|
215
233
|
startOfDayHourInTimeZone,
|
|
@@ -218,7 +236,7 @@ import {
|
|
|
218
236
|
validationError,
|
|
219
237
|
winnabilityClassLabel,
|
|
220
238
|
withRetry
|
|
221
|
-
} from "./chunk-
|
|
239
|
+
} from "./chunk-AMPT2EP7.js";
|
|
222
240
|
|
|
223
241
|
// src/telemetry.ts
|
|
224
242
|
import crypto from "crypto";
|
|
@@ -548,12 +566,12 @@ function checkLatestVersionForServer(opts) {
|
|
|
548
566
|
}
|
|
549
567
|
|
|
550
568
|
// src/server.ts
|
|
551
|
-
import { createRequire as
|
|
552
|
-
import
|
|
569
|
+
import { createRequire as createRequire4 } from "module";
|
|
570
|
+
import crypto25 from "crypto";
|
|
553
571
|
import fs8 from "fs";
|
|
554
572
|
import path9 from "path";
|
|
555
573
|
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
556
|
-
import { and as
|
|
574
|
+
import { and as and17, eq as eq23 } from "drizzle-orm";
|
|
557
575
|
import Fastify from "fastify";
|
|
558
576
|
import os5 from "os";
|
|
559
577
|
|
|
@@ -3547,11 +3565,23 @@ var JobRunner = class {
|
|
|
3547
3565
|
this.registry = registry;
|
|
3548
3566
|
}
|
|
3549
3567
|
recoverStaleRuns() {
|
|
3550
|
-
const stale = this.db.select({ id: runs.id, status: runs.status }).from(runs).where(inArray(runs.status, ["running", "queued"])).all();
|
|
3568
|
+
const stale = this.db.select({ id: runs.id, projectId: runs.projectId, kind: runs.kind, status: runs.status }).from(runs).where(inArray(runs.status, ["running", "queued"])).all();
|
|
3551
3569
|
if (stale.length === 0) return;
|
|
3552
3570
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
3553
3571
|
for (const run of stale) {
|
|
3554
|
-
this.db.
|
|
3572
|
+
const recovered = this.db.transaction((tx) => {
|
|
3573
|
+
const claim = tx.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(and2(eq2(runs.id, run.id), eq2(runs.status, run.status))).run();
|
|
3574
|
+
if (claim.changes === 0) return false;
|
|
3575
|
+
if (run.kind === RunKinds["site-audit"]) {
|
|
3576
|
+
tx.update(siteCrawlAttempts).set({ state: "failed", finishedAt: now, updatedAt: now, error: "Server restarted while run was in progress" }).where(and2(
|
|
3577
|
+
eq2(siteCrawlAttempts.projectId, run.projectId),
|
|
3578
|
+
eq2(siteCrawlAttempts.runId, run.id),
|
|
3579
|
+
inArray(siteCrawlAttempts.state, ["queued", "running"])
|
|
3580
|
+
)).run();
|
|
3581
|
+
}
|
|
3582
|
+
return true;
|
|
3583
|
+
});
|
|
3584
|
+
if (!recovered) continue;
|
|
3555
3585
|
log.warn("run.recovered-stale", { runId: run.id, previousStatus: run.status });
|
|
3556
3586
|
}
|
|
3557
3587
|
}
|
|
@@ -4119,7 +4149,7 @@ var JobRunner = class {
|
|
|
4119
4149
|
}
|
|
4120
4150
|
}
|
|
4121
4151
|
}
|
|
4122
|
-
incrementUsage(scope, metric,
|
|
4152
|
+
incrementUsage(scope, metric, count2) {
|
|
4123
4153
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4124
4154
|
const period = now.slice(0, 10);
|
|
4125
4155
|
this.db.insert(usageCounters).values({
|
|
@@ -4127,11 +4157,11 @@ var JobRunner = class {
|
|
|
4127
4157
|
scope,
|
|
4128
4158
|
period,
|
|
4129
4159
|
metric,
|
|
4130
|
-
count,
|
|
4160
|
+
count: count2,
|
|
4131
4161
|
updatedAt: now
|
|
4132
4162
|
}).onConflictDoUpdate({
|
|
4133
4163
|
target: [usageCounters.scope, usageCounters.period, usageCounters.metric],
|
|
4134
|
-
set: { count: sql2`${usageCounters.count} + ${
|
|
4164
|
+
set: { count: sql2`${usageCounters.count} + ${count2}`, updatedAt: now }
|
|
4135
4165
|
}).run();
|
|
4136
4166
|
}
|
|
4137
4167
|
flushProviderUsage(providerDispatchCounts, providerReservations) {
|
|
@@ -4250,8 +4280,19 @@ function writeCoverageSnapshot(db, projectId, runId, opts = {}) {
|
|
|
4250
4280
|
const since = isoDaysAgo(windowDays);
|
|
4251
4281
|
const pageRows = db.select({ page: gscSearchData.page, impressions: gscSearchData.impressions }).from(gscSearchData).where(and3(eq3(gscSearchData.projectId, projectId), gte(gscSearchData.date, since))).all();
|
|
4252
4282
|
const allInspections = db.select().from(gscUrlInspections).where(eq3(gscUrlInspections.projectId, projectId)).all();
|
|
4283
|
+
const latestCrawl = db.select({ runId: siteCrawlSnapshots.runId, attemptId: siteCrawlSnapshots.attemptId }).from(siteCrawlSnapshots).innerJoin(runs, eq3(siteCrawlSnapshots.runId, runs.id)).where(and3(
|
|
4284
|
+
eq3(siteCrawlSnapshots.projectId, projectId),
|
|
4285
|
+
eq3(siteCrawlSnapshots.complete, true),
|
|
4286
|
+
eq3(runs.status, "completed")
|
|
4287
|
+
)).orderBy(desc(siteCrawlSnapshots.createdAt)).limit(1).get();
|
|
4288
|
+
const hasCrawlHistory = Boolean(db.select({ id: siteCrawlSnapshots.id }).from(siteCrawlSnapshots).where(eq3(siteCrawlSnapshots.projectId, projectId)).limit(1).get());
|
|
4253
4289
|
const latestAudit = db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).where(eq3(siteAuditSnapshots.projectId, projectId)).orderBy(desc(siteAuditSnapshots.createdAt)).limit(1).get();
|
|
4254
|
-
const sitemapUrls =
|
|
4290
|
+
const sitemapUrls = latestCrawl?.attemptId ? db.select({ url: siteCrawlPages.url }).from(siteCrawlPages).where(and3(
|
|
4291
|
+
eq3(siteCrawlPages.projectId, projectId),
|
|
4292
|
+
eq3(siteCrawlPages.runId, latestCrawl.runId),
|
|
4293
|
+
eq3(siteCrawlPages.attemptId, latestCrawl.attemptId),
|
|
4294
|
+
eq3(siteCrawlPages.inventoryEligible, true)
|
|
4295
|
+
)).all() : !hasCrawlHistory && latestAudit ? db.select({ url: siteAuditPages.url }).from(siteAuditPages).where(eq3(siteAuditPages.runId, latestAudit.runId)).all() : [];
|
|
4255
4296
|
const impressionsByKey = /* @__PURE__ */ new Map();
|
|
4256
4297
|
for (const row of pageRows) {
|
|
4257
4298
|
const key = pageKey(row.page);
|
|
@@ -6894,165 +6935,1021 @@ function buildDiscoveryInsightTitle(input) {
|
|
|
6894
6935
|
}
|
|
6895
6936
|
|
|
6896
6937
|
// src/execute-site-audit.ts
|
|
6938
|
+
import crypto17 from "crypto";
|
|
6939
|
+
import { and as and11, eq as eq14, sql as sql6 } from "drizzle-orm";
|
|
6940
|
+
import { runSiteCrawl } from "@canonry/aeo-audit";
|
|
6941
|
+
|
|
6942
|
+
// src/site-crawl-graph-layout.ts
|
|
6897
6943
|
import crypto16 from "crypto";
|
|
6898
|
-
import {
|
|
6899
|
-
import {
|
|
6944
|
+
import { createRequire as createRequire3 } from "module";
|
|
6945
|
+
import { Worker } from "worker_threads";
|
|
6946
|
+
import { and as and10, asc, count, desc as desc6, eq as eq13, isNull, ne as ne2, or, sql as sql5 } from "drizzle-orm";
|
|
6947
|
+
import { alias } from "drizzle-orm/sqlite-core";
|
|
6948
|
+
var SITE_CRAWL_GRAPH_LAYOUT_VERSION = "site-health-fa2-v1";
|
|
6949
|
+
var SITE_CRAWL_GRAPH_LAYOUT_TIMEOUT_MS = 15e3;
|
|
6950
|
+
var SITE_CRAWL_GRAPH_SEED_SCALE = 100;
|
|
6951
|
+
var graphSourcePage = alias(siteCrawlPages, "layout_graph_source_page");
|
|
6952
|
+
var graphTargetPage = alias(siteCrawlPages, "layout_graph_target_page");
|
|
6953
|
+
var FORCE_ATLAS_SETTINGS = Object.freeze({
|
|
6954
|
+
adjustSizes: false,
|
|
6955
|
+
barnesHutOptimize: true,
|
|
6956
|
+
barnesHutTheta: 0.5,
|
|
6957
|
+
edgeWeightInfluence: 0.5,
|
|
6958
|
+
gravity: 1,
|
|
6959
|
+
linLogMode: false,
|
|
6960
|
+
outboundAttractionDistribution: false,
|
|
6961
|
+
scalingRatio: 10,
|
|
6962
|
+
slowDown: 10,
|
|
6963
|
+
strongGravityMode: false
|
|
6964
|
+
});
|
|
6965
|
+
var WORKER_SOURCE = String.raw`
|
|
6966
|
+
const {parentPort, workerData} = require('node:worker_threads');
|
|
6967
|
+
try {
|
|
6968
|
+
const graphologyModule = require(workerData.graphologyPath);
|
|
6969
|
+
const forceAtlasModule = require(workerData.forceAtlasPath);
|
|
6970
|
+
const MultiDirectedGraph = graphologyModule.MultiDirectedGraph;
|
|
6971
|
+
const forceAtlas2 = forceAtlasModule.default || forceAtlasModule;
|
|
6972
|
+
const graph = new MultiDirectedGraph({allowSelfLoops: true});
|
|
6973
|
+
for (const node of workerData.input.nodes) {
|
|
6974
|
+
graph.addNode(node.nodeKey, {x: node.x, y: node.y});
|
|
6975
|
+
}
|
|
6976
|
+
for (const edge of workerData.input.edges) {
|
|
6977
|
+
graph.addDirectedEdgeWithKey(edge.edgeKey, edge.sourceNodeKey, edge.targetNodeKey, {
|
|
6978
|
+
weight: 1 + Math.log2(Math.max(1, edge.occurrences)),
|
|
6979
|
+
});
|
|
6980
|
+
}
|
|
6981
|
+
if (graph.order > 1 && graph.size > 0) {
|
|
6982
|
+
forceAtlas2.assign(graph, {
|
|
6983
|
+
iterations: workerData.input.iterations,
|
|
6984
|
+
settings: workerData.settings,
|
|
6985
|
+
});
|
|
6986
|
+
}
|
|
6987
|
+
const positions = [];
|
|
6988
|
+
graph.forEachNode((nodeKey, attributes) => {
|
|
6989
|
+
positions.push({nodeKey, x: attributes.x, y: attributes.y});
|
|
6990
|
+
});
|
|
6991
|
+
parentPort.postMessage({ok: true, positions});
|
|
6992
|
+
} catch (error) {
|
|
6993
|
+
parentPort.postMessage({
|
|
6994
|
+
ok: false,
|
|
6995
|
+
error: error && typeof error.message === 'string' ? error.message : String(error),
|
|
6996
|
+
});
|
|
6997
|
+
}
|
|
6998
|
+
`;
|
|
6999
|
+
var GraphLayoutTimeoutError = class extends Error {
|
|
7000
|
+
name = "GraphLayoutTimeoutError";
|
|
7001
|
+
};
|
|
7002
|
+
function hashUnit(value) {
|
|
7003
|
+
let hash = 2166136261;
|
|
7004
|
+
for (let i = 0; i < value.length; i++) {
|
|
7005
|
+
hash ^= value.charCodeAt(i);
|
|
7006
|
+
hash = Math.imul(hash, 16777619);
|
|
7007
|
+
}
|
|
7008
|
+
return (hash >>> 0) / 4294967296;
|
|
7009
|
+
}
|
|
7010
|
+
function pathSection(path10) {
|
|
7011
|
+
return path10.split("/").filter(Boolean)[0] ?? "__root__";
|
|
7012
|
+
}
|
|
7013
|
+
function seedSiteCrawlGraphNodes(nodes, rootNodeKey, priorPositions) {
|
|
7014
|
+
return [...nodes].sort((a, b) => a.sampleRank - b.sampleRank || a.nodeKey.localeCompare(b.nodeKey)).map((node) => {
|
|
7015
|
+
const prior = priorPositions?.get(node.nodeKey);
|
|
7016
|
+
if (prior && Number.isFinite(prior.x) && Number.isFinite(prior.y)) {
|
|
7017
|
+
return {
|
|
7018
|
+
...node,
|
|
7019
|
+
x: prior.x * SITE_CRAWL_GRAPH_SEED_SCALE,
|
|
7020
|
+
y: prior.y * SITE_CRAWL_GRAPH_SEED_SCALE
|
|
7021
|
+
};
|
|
7022
|
+
}
|
|
7023
|
+
if (node.nodeKey === rootNodeKey) return { ...node, x: 0, y: 0 };
|
|
7024
|
+
const section = pathSection(node.path);
|
|
7025
|
+
const sectionAngle = hashUnit(`section:${section}`) * Math.PI * 2;
|
|
7026
|
+
const sectionRadius = section === "__root__" ? 24 : 100;
|
|
7027
|
+
const clusterX = Math.cos(sectionAngle) * sectionRadius;
|
|
7028
|
+
const clusterY = Math.sin(sectionAngle) * sectionRadius;
|
|
7029
|
+
const localAngle = hashUnit(`node-angle:${node.nodeKey}`) * Math.PI * 2;
|
|
7030
|
+
const depth = Math.max(0, Math.min(12, node.depth ?? 4));
|
|
7031
|
+
const localRadius = 4 + depth * 2.5 + Math.sqrt(hashUnit(`node-radius:${node.nodeKey}`)) * 14;
|
|
7032
|
+
return {
|
|
7033
|
+
...node,
|
|
7034
|
+
x: clusterX + Math.cos(localAngle) * localRadius,
|
|
7035
|
+
y: clusterY + Math.sin(localAngle) * localRadius
|
|
7036
|
+
};
|
|
7037
|
+
});
|
|
7038
|
+
}
|
|
7039
|
+
function adaptiveForceAtlasIterations(nodeCount, edgeCount) {
|
|
7040
|
+
if (nodeCount <= 100 && edgeCount <= 500) return 100;
|
|
7041
|
+
if (nodeCount <= 500 && edgeCount <= 2500) return 70;
|
|
7042
|
+
if (nodeCount <= 2e3 && edgeCount <= 1e4) return 40;
|
|
7043
|
+
if (nodeCount <= 5e3 && edgeCount <= 2e4) return 24;
|
|
7044
|
+
if (nodeCount <= 1e4 && edgeCount <= 35e3) return 12;
|
|
7045
|
+
return 6;
|
|
7046
|
+
}
|
|
7047
|
+
async function computeForceAtlasPositions(input, options = {}) {
|
|
7048
|
+
if (input.nodes.length <= 1 || input.edges.length === 0) {
|
|
7049
|
+
return input.nodes.map(({ nodeKey, x, y }) => ({ nodeKey, x, y }));
|
|
7050
|
+
}
|
|
7051
|
+
const requireFromPackage = createRequire3(import.meta.url);
|
|
7052
|
+
const worker = new Worker(WORKER_SOURCE, {
|
|
7053
|
+
eval: true,
|
|
7054
|
+
workerData: {
|
|
7055
|
+
graphologyPath: requireFromPackage.resolve("graphology"),
|
|
7056
|
+
forceAtlasPath: requireFromPackage.resolve("graphology-layout-forceatlas2"),
|
|
7057
|
+
input,
|
|
7058
|
+
settings: FORCE_ATLAS_SETTINGS
|
|
7059
|
+
}
|
|
7060
|
+
});
|
|
7061
|
+
const timeoutMs = options.timeoutMs ?? SITE_CRAWL_GRAPH_LAYOUT_TIMEOUT_MS;
|
|
7062
|
+
return await new Promise((resolve, reject) => {
|
|
7063
|
+
let settled = false;
|
|
7064
|
+
const finish = (result) => {
|
|
7065
|
+
if (settled) return;
|
|
7066
|
+
settled = true;
|
|
7067
|
+
clearTimeout(timer);
|
|
7068
|
+
options.signal?.removeEventListener("abort", onAbort);
|
|
7069
|
+
void worker.terminate();
|
|
7070
|
+
if (result.error) reject(result.error instanceof Error ? result.error : new Error("Graph layout failed with a non-Error reason", { cause: result.error }));
|
|
7071
|
+
else resolve(result.positions ?? []);
|
|
7072
|
+
};
|
|
7073
|
+
const onAbort = () => finish({ error: options.signal?.reason ?? new DOMException("Layout aborted", "AbortError") });
|
|
7074
|
+
const timer = setTimeout(() => finish({ error: new GraphLayoutTimeoutError(`Graph layout exceeded ${timeoutMs}ms`) }), timeoutMs);
|
|
7075
|
+
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
7076
|
+
if (options.signal?.aborted) return onAbort();
|
|
7077
|
+
worker.once("message", (message) => {
|
|
7078
|
+
if (!message.ok) finish({ error: new Error(message.error ?? "Graph layout worker failed") });
|
|
7079
|
+
else finish({ positions: message.positions });
|
|
7080
|
+
});
|
|
7081
|
+
worker.once("error", (error) => finish({ error }));
|
|
7082
|
+
worker.once("exit", (code) => {
|
|
7083
|
+
if (code !== 0 && !settled) finish({ error: new Error(`Graph layout worker exited with code ${code}`) });
|
|
7084
|
+
});
|
|
7085
|
+
});
|
|
7086
|
+
}
|
|
7087
|
+
function normalizePositions(seededNodes, computed, rootNodeKey) {
|
|
7088
|
+
const byKey = new Map(computed.map((position) => [position.nodeKey, position]));
|
|
7089
|
+
if (byKey.size !== seededNodes.length) throw new Error("Graph layout returned an incomplete node set");
|
|
7090
|
+
const root = rootNodeKey ? byKey.get(rootNodeKey) : null;
|
|
7091
|
+
const centerX = root?.x ?? computed.reduce((sum, position) => sum + position.x, 0) / Math.max(1, computed.length);
|
|
7092
|
+
const centerY = root?.y ?? computed.reduce((sum, position) => sum + position.y, 0) / Math.max(1, computed.length);
|
|
7093
|
+
if (!Number.isFinite(centerX) || !Number.isFinite(centerY)) throw new Error("Graph layout returned a non-finite center");
|
|
7094
|
+
let scale = 0;
|
|
7095
|
+
for (const position of computed) {
|
|
7096
|
+
if (!Number.isFinite(position.x) || !Number.isFinite(position.y)) throw new Error("Graph layout returned non-finite coordinates");
|
|
7097
|
+
scale = Math.max(scale, Math.abs(position.x - centerX), Math.abs(position.y - centerY));
|
|
7098
|
+
}
|
|
7099
|
+
scale = scale > 0 ? scale : 1;
|
|
7100
|
+
return seededNodes.map((node) => {
|
|
7101
|
+
const position = byKey.get(node.nodeKey);
|
|
7102
|
+
if (!position) throw new Error(`Graph layout omitted ${node.nodeKey}`);
|
|
7103
|
+
return {
|
|
7104
|
+
nodeKey: node.nodeKey,
|
|
7105
|
+
sampleRank: node.sampleRank,
|
|
7106
|
+
x: Math.round((position.x - centerX) / scale * 1e6) / 1e6,
|
|
7107
|
+
y: Math.round((position.y - centerY) / scale * 1e6) / 1e6
|
|
7108
|
+
};
|
|
7109
|
+
});
|
|
7110
|
+
}
|
|
7111
|
+
async function layoutSiteCrawlGraphInput(input, options = {}) {
|
|
7112
|
+
options.signal?.throwIfAborted();
|
|
7113
|
+
if (input.nodes.length === 0) {
|
|
7114
|
+
return {
|
|
7115
|
+
state: "unavailable",
|
|
7116
|
+
failureCode: "empty-crawl",
|
|
7117
|
+
totalNodes: input.totalNodes,
|
|
7118
|
+
totalEdges: input.totalEdges,
|
|
7119
|
+
nodeCount: 0,
|
|
7120
|
+
edgeCount: 0,
|
|
7121
|
+
nodes: [],
|
|
7122
|
+
edges: []
|
|
7123
|
+
};
|
|
7124
|
+
}
|
|
7125
|
+
const seeded = seedSiteCrawlGraphNodes(input.nodes, input.rootNodeKey, input.priorPositions);
|
|
7126
|
+
try {
|
|
7127
|
+
const compute = options.computePositions ?? computeForceAtlasPositions;
|
|
7128
|
+
const positions = await compute({
|
|
7129
|
+
nodes: seeded,
|
|
7130
|
+
edges: input.edges,
|
|
7131
|
+
iterations: adaptiveForceAtlasIterations(input.nodes.length, input.edges.length)
|
|
7132
|
+
}, { signal: options.signal, timeoutMs: options.timeoutMs });
|
|
7133
|
+
options.signal?.throwIfAborted();
|
|
7134
|
+
return {
|
|
7135
|
+
state: "ready",
|
|
7136
|
+
layoutVersion: SITE_CRAWL_GRAPH_LAYOUT_VERSION,
|
|
7137
|
+
totalNodes: input.totalNodes,
|
|
7138
|
+
totalEdges: input.totalEdges,
|
|
7139
|
+
nodeCount: input.nodes.length,
|
|
7140
|
+
edgeCount: input.edges.length,
|
|
7141
|
+
nodes: normalizePositions(seeded, positions, input.rootNodeKey),
|
|
7142
|
+
edges: input.edges.map((edge, sampleRank) => ({ ...edge, sampleRank }))
|
|
7143
|
+
};
|
|
7144
|
+
} catch (error) {
|
|
7145
|
+
if (options.signal?.aborted) options.signal.throwIfAborted();
|
|
7146
|
+
if (error instanceof DOMException && error.name === "AbortError") throw error;
|
|
7147
|
+
return {
|
|
7148
|
+
state: "unavailable",
|
|
7149
|
+
failureCode: error instanceof GraphLayoutTimeoutError ? "layout-timeout" : "layout-error",
|
|
7150
|
+
totalNodes: input.totalNodes,
|
|
7151
|
+
totalEdges: input.totalEdges,
|
|
7152
|
+
nodeCount: 0,
|
|
7153
|
+
edgeCount: 0,
|
|
7154
|
+
nodes: [],
|
|
7155
|
+
edges: []
|
|
7156
|
+
};
|
|
7157
|
+
}
|
|
7158
|
+
}
|
|
7159
|
+
function loadPriorSiteCrawlGraphPositions(db, scope) {
|
|
7160
|
+
const prior = db.select({
|
|
7161
|
+
runId: siteCrawlSnapshots.runId,
|
|
7162
|
+
attemptId: siteCrawlSnapshots.attemptId
|
|
7163
|
+
}).from(siteCrawlSnapshots).innerJoin(siteCrawlGraphLayouts, and10(
|
|
7164
|
+
eq13(siteCrawlGraphLayouts.projectId, siteCrawlSnapshots.projectId),
|
|
7165
|
+
eq13(siteCrawlGraphLayouts.runId, siteCrawlSnapshots.runId),
|
|
7166
|
+
eq13(siteCrawlGraphLayouts.attemptId, siteCrawlSnapshots.attemptId)
|
|
7167
|
+
)).where(and10(
|
|
7168
|
+
eq13(siteCrawlSnapshots.projectId, scope.projectId),
|
|
7169
|
+
ne2(siteCrawlSnapshots.runId, scope.runId),
|
|
7170
|
+
eq13(siteCrawlSnapshots.complete, true),
|
|
7171
|
+
eq13(siteCrawlGraphLayouts.state, "ready"),
|
|
7172
|
+
eq13(siteCrawlGraphLayouts.layoutVersion, SITE_CRAWL_GRAPH_LAYOUT_VERSION)
|
|
7173
|
+
)).orderBy(desc6(siteCrawlSnapshots.createdAt), desc6(siteCrawlSnapshots.id)).limit(1).get();
|
|
7174
|
+
if (!prior?.attemptId) return /* @__PURE__ */ new Map();
|
|
7175
|
+
return new Map(db.select({
|
|
7176
|
+
nodeKey: siteCrawlGraphNodes.nodeKey,
|
|
7177
|
+
x: siteCrawlGraphNodes.x,
|
|
7178
|
+
y: siteCrawlGraphNodes.y
|
|
7179
|
+
}).from(siteCrawlGraphNodes).where(and10(
|
|
7180
|
+
eq13(siteCrawlGraphNodes.projectId, scope.projectId),
|
|
7181
|
+
eq13(siteCrawlGraphNodes.runId, prior.runId),
|
|
7182
|
+
eq13(siteCrawlGraphNodes.attemptId, prior.attemptId)
|
|
7183
|
+
)).all().map((node) => [node.nodeKey, { x: node.x, y: node.y }]));
|
|
7184
|
+
}
|
|
7185
|
+
async function prepareSiteCrawlGraphLayout(db, scope, options = {}) {
|
|
7186
|
+
let totalNodes = 0;
|
|
7187
|
+
let totalEdges = 0;
|
|
7188
|
+
try {
|
|
7189
|
+
const assertNotAborted = () => options.signal?.throwIfAborted();
|
|
7190
|
+
assertNotAborted();
|
|
7191
|
+
const pageScope = [
|
|
7192
|
+
eq13(siteCrawlPages.projectId, scope.projectId),
|
|
7193
|
+
eq13(siteCrawlPages.runId, scope.runId),
|
|
7194
|
+
eq13(siteCrawlPages.attemptId, scope.attemptId)
|
|
7195
|
+
];
|
|
7196
|
+
const edgeScope = [
|
|
7197
|
+
eq13(siteCrawlEdges.projectId, scope.projectId),
|
|
7198
|
+
eq13(siteCrawlEdges.runId, scope.runId),
|
|
7199
|
+
eq13(siteCrawlEdges.attemptId, scope.attemptId),
|
|
7200
|
+
eq13(siteCrawlEdges.internal, true),
|
|
7201
|
+
eq13(siteCrawlEdges.relation, "anchor")
|
|
7202
|
+
];
|
|
7203
|
+
totalNodes = db.select({ value: count() }).from(siteCrawlPages).where(and10(...pageScope)).get()?.value ?? 0;
|
|
7204
|
+
assertNotAborted();
|
|
7205
|
+
totalEdges = db.select({ value: count() }).from(siteCrawlEdges).innerJoin(graphSourcePage, and10(
|
|
7206
|
+
eq13(graphSourcePage.projectId, scope.projectId),
|
|
7207
|
+
eq13(graphSourcePage.runId, scope.runId),
|
|
7208
|
+
eq13(graphSourcePage.attemptId, scope.attemptId),
|
|
7209
|
+
eq13(graphSourcePage.nodeKey, siteCrawlEdges.sourceNodeKey)
|
|
7210
|
+
)).innerJoin(graphTargetPage, and10(
|
|
7211
|
+
eq13(graphTargetPage.projectId, scope.projectId),
|
|
7212
|
+
eq13(graphTargetPage.runId, scope.runId),
|
|
7213
|
+
eq13(graphTargetPage.attemptId, scope.attemptId),
|
|
7214
|
+
eq13(graphTargetPage.nodeKey, siteCrawlEdges.targetNodeKey)
|
|
7215
|
+
)).where(and10(...edgeScope)).get()?.value ?? 0;
|
|
7216
|
+
assertNotAborted();
|
|
7217
|
+
const graphSeedColumns = {
|
|
7218
|
+
nodeKey: siteCrawlPages.nodeKey,
|
|
7219
|
+
path: siteCrawlPages.path,
|
|
7220
|
+
depth: siteCrawlPages.depth
|
|
7221
|
+
};
|
|
7222
|
+
const rootRows = db.select(graphSeedColumns).from(siteCrawlPages).where(and10(...pageScope, eq13(siteCrawlPages.url, scope.rootUrl))).orderBy(asc(siteCrawlPages.nodeKey)).limit(1).all();
|
|
7223
|
+
assertNotAborted();
|
|
7224
|
+
const retainedRows = [...rootRows];
|
|
7225
|
+
if (retainedRows.length < SITE_CRAWL_GRAPH_MAX_NODES) {
|
|
7226
|
+
retainedRows.push(...db.select(graphSeedColumns).from(siteCrawlPages).where(and10(...pageScope, eq13(siteCrawlPages.depth, 0), ne2(siteCrawlPages.url, scope.rootUrl))).orderBy(asc(siteCrawlPages.nodeKey)).limit(SITE_CRAWL_GRAPH_MAX_NODES - retainedRows.length).all());
|
|
7227
|
+
}
|
|
7228
|
+
assertNotAborted();
|
|
7229
|
+
if (retainedRows.length < SITE_CRAWL_GRAPH_MAX_NODES) {
|
|
7230
|
+
retainedRows.push(...db.select(graphSeedColumns).from(siteCrawlPages).where(and10(
|
|
7231
|
+
...pageScope,
|
|
7232
|
+
or(isNull(siteCrawlPages.depth), ne2(siteCrawlPages.depth, 0)),
|
|
7233
|
+
ne2(siteCrawlPages.url, scope.rootUrl)
|
|
7234
|
+
)).orderBy(desc6(siteCrawlPages.linkScoreNormalized), asc(siteCrawlPages.nodeKey)).limit(SITE_CRAWL_GRAPH_MAX_NODES - retainedRows.length).all());
|
|
7235
|
+
}
|
|
7236
|
+
assertNotAborted();
|
|
7237
|
+
const nodes = retainedRows.map((row, sampleRank) => ({
|
|
7238
|
+
nodeKey: row.nodeKey,
|
|
7239
|
+
path: row.path,
|
|
7240
|
+
depth: row.depth,
|
|
7241
|
+
sampleRank
|
|
7242
|
+
}));
|
|
7243
|
+
const rootNodeKey = rootRows[0]?.nodeKey ?? null;
|
|
7244
|
+
const priorPositions = loadPriorSiteCrawlGraphPositions(db, scope);
|
|
7245
|
+
assertNotAborted();
|
|
7246
|
+
const selectedKeys = JSON.stringify(nodes.map((node) => node.nodeKey));
|
|
7247
|
+
const edgeRows = nodes.length === 0 ? [] : db.all(sql5`
|
|
7248
|
+
WITH selected(node_key) AS (SELECT value FROM json_each(${selectedKeys}))
|
|
7249
|
+
SELECT
|
|
7250
|
+
edge.edge_key AS edgeKey,
|
|
7251
|
+
edge.source_node_key AS sourceNodeKey,
|
|
7252
|
+
edge.target_node_key AS targetNodeKey,
|
|
7253
|
+
edge.followable AS followable,
|
|
7254
|
+
edge.occurrences AS occurrences
|
|
7255
|
+
FROM site_crawl_edges AS edge
|
|
7256
|
+
INNER JOIN selected AS source ON source.node_key = edge.source_node_key
|
|
7257
|
+
INNER JOIN selected AS target ON target.node_key = edge.target_node_key
|
|
7258
|
+
WHERE edge.project_id = ${scope.projectId}
|
|
7259
|
+
AND edge.run_id = ${scope.runId}
|
|
7260
|
+
AND edge.attempt_id = ${scope.attemptId}
|
|
7261
|
+
AND edge.internal = 1
|
|
7262
|
+
AND edge.relation = 'anchor'
|
|
7263
|
+
ORDER BY edge.occurrences DESC, edge.edge_key ASC
|
|
7264
|
+
LIMIT ${SITE_CRAWL_GRAPH_MAX_EDGES}
|
|
7265
|
+
`);
|
|
7266
|
+
assertNotAborted();
|
|
7267
|
+
return await layoutSiteCrawlGraphInput({
|
|
7268
|
+
totalNodes,
|
|
7269
|
+
totalEdges,
|
|
7270
|
+
nodes,
|
|
7271
|
+
edges: edgeRows,
|
|
7272
|
+
rootNodeKey,
|
|
7273
|
+
priorPositions
|
|
7274
|
+
}, options);
|
|
7275
|
+
} catch (error) {
|
|
7276
|
+
if (options.signal?.aborted) options.signal.throwIfAborted();
|
|
7277
|
+
if (error instanceof DOMException && error.name === "AbortError") throw error;
|
|
7278
|
+
return {
|
|
7279
|
+
state: "unavailable",
|
|
7280
|
+
failureCode: "layout-error",
|
|
7281
|
+
totalNodes,
|
|
7282
|
+
totalEdges,
|
|
7283
|
+
nodeCount: 0,
|
|
7284
|
+
edgeCount: 0,
|
|
7285
|
+
nodes: [],
|
|
7286
|
+
edges: []
|
|
7287
|
+
};
|
|
7288
|
+
}
|
|
7289
|
+
}
|
|
7290
|
+
function persistSiteCrawlGraphLayout(db, scope, layout, now = (/* @__PURE__ */ new Date()).toISOString()) {
|
|
7291
|
+
db.transaction((tx) => {
|
|
7292
|
+
tx.delete(siteCrawlGraphLayouts).where(and10(
|
|
7293
|
+
eq13(siteCrawlGraphLayouts.projectId, scope.projectId),
|
|
7294
|
+
eq13(siteCrawlGraphLayouts.runId, scope.runId),
|
|
7295
|
+
eq13(siteCrawlGraphLayouts.attemptId, scope.attemptId)
|
|
7296
|
+
)).run();
|
|
7297
|
+
tx.insert(siteCrawlGraphLayouts).values({
|
|
7298
|
+
id: crypto16.randomUUID(),
|
|
7299
|
+
projectId: scope.projectId,
|
|
7300
|
+
runId: scope.runId,
|
|
7301
|
+
attemptId: scope.attemptId,
|
|
7302
|
+
state: layout.state,
|
|
7303
|
+
layoutVersion: layout.state === "ready" ? layout.layoutVersion : null,
|
|
7304
|
+
failureCode: layout.state === "unavailable" ? layout.failureCode : null,
|
|
7305
|
+
totalNodes: layout.totalNodes,
|
|
7306
|
+
totalEdges: layout.totalEdges,
|
|
7307
|
+
nodeCount: layout.nodeCount,
|
|
7308
|
+
edgeCount: layout.edgeCount,
|
|
7309
|
+
createdAt: now,
|
|
7310
|
+
updatedAt: now
|
|
7311
|
+
}).run();
|
|
7312
|
+
if (layout.state !== "ready") return;
|
|
7313
|
+
const batchSize = 250;
|
|
7314
|
+
for (let offset = 0; offset < layout.nodes.length; offset += batchSize) {
|
|
7315
|
+
tx.insert(siteCrawlGraphNodes).values(layout.nodes.slice(offset, offset + batchSize).map((node) => ({
|
|
7316
|
+
id: crypto16.randomUUID(),
|
|
7317
|
+
projectId: scope.projectId,
|
|
7318
|
+
runId: scope.runId,
|
|
7319
|
+
attemptId: scope.attemptId,
|
|
7320
|
+
nodeKey: node.nodeKey,
|
|
7321
|
+
sampleRank: node.sampleRank,
|
|
7322
|
+
x: node.x,
|
|
7323
|
+
y: node.y,
|
|
7324
|
+
createdAt: now
|
|
7325
|
+
}))).run();
|
|
7326
|
+
}
|
|
7327
|
+
for (let offset = 0; offset < layout.edges.length; offset += batchSize) {
|
|
7328
|
+
tx.insert(siteCrawlGraphEdges).values(layout.edges.slice(offset, offset + batchSize).map((edge) => ({
|
|
7329
|
+
id: crypto16.randomUUID(),
|
|
7330
|
+
projectId: scope.projectId,
|
|
7331
|
+
runId: scope.runId,
|
|
7332
|
+
attemptId: scope.attemptId,
|
|
7333
|
+
edgeKey: edge.edgeKey,
|
|
7334
|
+
sampleRank: edge.sampleRank,
|
|
7335
|
+
sourceNodeKey: edge.sourceNodeKey,
|
|
7336
|
+
targetNodeKey: edge.targetNodeKey,
|
|
7337
|
+
followable: edge.followable,
|
|
7338
|
+
occurrences: edge.occurrences,
|
|
7339
|
+
createdAt: now
|
|
7340
|
+
}))).run();
|
|
7341
|
+
}
|
|
7342
|
+
});
|
|
7343
|
+
}
|
|
7344
|
+
function deleteSiteCrawlGraphLayout(db, scope) {
|
|
7345
|
+
db.delete(siteCrawlGraphLayouts).where(and10(
|
|
7346
|
+
eq13(siteCrawlGraphLayouts.projectId, scope.projectId),
|
|
7347
|
+
eq13(siteCrawlGraphLayouts.runId, scope.runId),
|
|
7348
|
+
eq13(siteCrawlGraphLayouts.attemptId, scope.attemptId)
|
|
7349
|
+
)).run();
|
|
7350
|
+
}
|
|
7351
|
+
|
|
7352
|
+
// src/execute-site-audit.ts
|
|
6900
7353
|
var log12 = createLogger("SiteAudit");
|
|
6901
|
-
var
|
|
6902
|
-
|
|
7354
|
+
var SiteAuditCancelledError = class extends Error {
|
|
7355
|
+
name = "SiteAuditCancelledError";
|
|
7356
|
+
};
|
|
6903
7357
|
function toHomepageUrl(canonicalDomain) {
|
|
6904
7358
|
const trimmed = canonicalDomain.trim().replace(/\/+$/, "");
|
|
6905
7359
|
return /^https?:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`;
|
|
6906
7360
|
}
|
|
6907
7361
|
async function assertSiteAuditUrlAllowed(rawUrl, field) {
|
|
6908
7362
|
const check = await resolveWebhookTarget(rawUrl);
|
|
6909
|
-
if (!check.ok) {
|
|
6910
|
-
throw new Error(`${field} ${check.message.replace(/^"url" /, "")}`);
|
|
6911
|
-
}
|
|
7363
|
+
if (!check.ok) throw new Error(`${field} ${check.message.replace(/^"url" /, "")}`);
|
|
6912
7364
|
}
|
|
6913
7365
|
function clampSiteAuditLimit(limit) {
|
|
6914
7366
|
if (limit == null || !Number.isFinite(limit)) return SITE_AUDIT_DEFAULT_PAGE_LIMIT;
|
|
6915
7367
|
return Math.max(1, Math.min(SITE_AUDIT_MAX_PAGE_LIMIT, Math.floor(limit)));
|
|
6916
7368
|
}
|
|
7369
|
+
function clampSiteAuditEdgeLimit(limit) {
|
|
7370
|
+
if (limit == null || !Number.isFinite(limit)) return SITE_AUDIT_DEFAULT_EDGE_LIMIT;
|
|
7371
|
+
return Math.max(1, Math.min(SITE_AUDIT_MAX_EDGE_LIMIT, Math.floor(limit)));
|
|
7372
|
+
}
|
|
6917
7373
|
function toPageFactor(factor) {
|
|
7374
|
+
return { id: factor.id, name: factor.name, weight: factor.weight, score: factor.score };
|
|
7375
|
+
}
|
|
7376
|
+
function toCrawlAuditFactor(factor) {
|
|
6918
7377
|
return {
|
|
6919
|
-
|
|
6920
|
-
|
|
6921
|
-
|
|
6922
|
-
|
|
7378
|
+
...toPageFactor(factor),
|
|
7379
|
+
status: factorStatusFromScore(factor.score),
|
|
7380
|
+
applicable: factor.applicable ?? null,
|
|
7381
|
+
findings: factor.findings,
|
|
7382
|
+
recommendations: factor.recommendations
|
|
7383
|
+
};
|
|
7384
|
+
}
|
|
7385
|
+
function crawlAuditFields(audit) {
|
|
7386
|
+
return {
|
|
7387
|
+
schemaVersion: "1.0",
|
|
7388
|
+
factors: audit.factors.map(toCrawlAuditFactor),
|
|
7389
|
+
criticalDefects: audit.criticalDefects
|
|
6923
7390
|
};
|
|
6924
7391
|
}
|
|
6925
7392
|
function computeFactorAverages(pages) {
|
|
6926
7393
|
const byId = /* @__PURE__ */ new Map();
|
|
6927
7394
|
for (const page of pages) {
|
|
6928
|
-
if (
|
|
6929
|
-
for (const factor of page.factors) {
|
|
6930
|
-
|
|
6931
|
-
|
|
6932
|
-
|
|
6933
|
-
byId.set(factor.id, entry);
|
|
6934
|
-
}
|
|
6935
|
-
entry.scores.push(factor.score);
|
|
7395
|
+
if (!page.audit) continue;
|
|
7396
|
+
for (const factor of page.audit.factors) {
|
|
7397
|
+
const current = byId.get(factor.id) ?? { name: factor.name, weight: factor.weight, scores: [], pass: 0, partial: 0, fail: 0 };
|
|
7398
|
+
byId.set(factor.id, current);
|
|
7399
|
+
current.scores.push(factor.score);
|
|
6936
7400
|
const status = factorStatusFromScore(factor.score);
|
|
6937
|
-
if (status === "pass")
|
|
6938
|
-
else if (status === "partial")
|
|
6939
|
-
else
|
|
7401
|
+
if (status === "pass") current.pass++;
|
|
7402
|
+
else if (status === "partial") current.partial++;
|
|
7403
|
+
else current.fail++;
|
|
6940
7404
|
}
|
|
6941
7405
|
}
|
|
6942
|
-
|
|
6943
|
-
|
|
6944
|
-
|
|
6945
|
-
summaries.push({
|
|
7406
|
+
return [...byId.entries()].map(([id, value]) => {
|
|
7407
|
+
const avgScore = Math.round(value.scores.reduce((sum, score) => sum + score, 0) / value.scores.length);
|
|
7408
|
+
return {
|
|
6946
7409
|
id,
|
|
6947
|
-
name:
|
|
6948
|
-
weight:
|
|
7410
|
+
name: value.name,
|
|
7411
|
+
weight: value.weight,
|
|
6949
7412
|
avgScore,
|
|
6950
7413
|
status: factorStatusFromScore(avgScore),
|
|
6951
|
-
pagesPassing:
|
|
6952
|
-
pagesPartial:
|
|
6953
|
-
pagesFailing:
|
|
6954
|
-
}
|
|
7414
|
+
pagesPassing: value.pass,
|
|
7415
|
+
pagesPartial: value.partial,
|
|
7416
|
+
pagesFailing: value.fail
|
|
7417
|
+
};
|
|
7418
|
+
}).sort((a, b) => b.weight - a.weight || a.name.localeCompare(b.name));
|
|
7419
|
+
}
|
|
7420
|
+
function pagePath(page) {
|
|
7421
|
+
if (page.path) return page.path;
|
|
7422
|
+
try {
|
|
7423
|
+
return new URL(page.finalUrl ?? page.requestedUrl).pathname || "/";
|
|
7424
|
+
} catch {
|
|
7425
|
+
return "/";
|
|
7426
|
+
}
|
|
7427
|
+
}
|
|
7428
|
+
function parentPath(page) {
|
|
7429
|
+
const path10 = pagePath(page);
|
|
7430
|
+
if (path10 === "/") return "/";
|
|
7431
|
+
const trimmed = path10.endsWith("/") ? path10.slice(0, -1) : path10;
|
|
7432
|
+
const slash = trimmed.lastIndexOf("/");
|
|
7433
|
+
return slash <= 0 ? "/" : trimmed.slice(0, slash) || "/";
|
|
7434
|
+
}
|
|
7435
|
+
function pageDiscoverySource(page) {
|
|
7436
|
+
if (page.provenance.root) return "root";
|
|
7437
|
+
if (page.provenance.sitemapSources.length > 0) return "sitemap";
|
|
7438
|
+
return "link";
|
|
7439
|
+
}
|
|
7440
|
+
function pageAuditState(page) {
|
|
7441
|
+
if (page.audit) return "success";
|
|
7442
|
+
if (page.state === "fetch-error") return "error";
|
|
7443
|
+
return "not-applicable";
|
|
7444
|
+
}
|
|
7445
|
+
function isInventoryEligible(page) {
|
|
7446
|
+
return page.state === "html" && page.indexability.state === "indexable";
|
|
7447
|
+
}
|
|
7448
|
+
function isCancelled(error, signal) {
|
|
7449
|
+
if (error instanceof SiteAuditCancelledError) return true;
|
|
7450
|
+
if (signal?.aborted && signal.reason !== void 0 && error === signal.reason) return true;
|
|
7451
|
+
return error instanceof DOMException && error.name === "AbortError";
|
|
7452
|
+
}
|
|
7453
|
+
function isLegacyScorecardPage(page) {
|
|
7454
|
+
return page.audit != null || page.state === "fetch-error";
|
|
7455
|
+
}
|
|
7456
|
+
function emptyCompleteCrawlError(rootUrl, finalRootUrl, pagesObserved) {
|
|
7457
|
+
if (!finalRootUrl) return new Error(`Site audit could not successfully audit any of ${pagesObserved} observed page(s).`);
|
|
7458
|
+
try {
|
|
7459
|
+
const rootHost = new URL(rootUrl).hostname;
|
|
7460
|
+
const finalRootHost = new URL(finalRootUrl).hostname;
|
|
7461
|
+
if (rootHost !== finalRootHost) {
|
|
7462
|
+
return new Error(
|
|
7463
|
+
`Site audit could not successfully audit any of ${pagesObserved} observed page(s): root URL redirected off-host from ${rootHost} to ${finalRootHost} (${finalRootUrl}).`
|
|
7464
|
+
);
|
|
7465
|
+
}
|
|
7466
|
+
} catch {
|
|
6955
7467
|
}
|
|
6956
|
-
|
|
6957
|
-
|
|
7468
|
+
return new Error(`Site audit could not successfully audit any of ${pagesObserved} observed page(s).`);
|
|
7469
|
+
}
|
|
7470
|
+
function toLegacyIssue(summary, totalPages) {
|
|
7471
|
+
const affected = summary.pagesPartial + summary.pagesFailing;
|
|
7472
|
+
if (affected === 0 || summary.avgScore >= 70) return null;
|
|
7473
|
+
return {
|
|
7474
|
+
factorId: summary.id,
|
|
7475
|
+
factorName: summary.name,
|
|
7476
|
+
avgScore: summary.avgScore,
|
|
7477
|
+
affectedPages: affected,
|
|
7478
|
+
totalPages,
|
|
7479
|
+
affectedPct: totalPages > 0 ? Math.round(affected / totalPages * 100) : 0,
|
|
7480
|
+
topRecommendations: []
|
|
7481
|
+
};
|
|
7482
|
+
}
|
|
7483
|
+
function observedErrorCount(pages) {
|
|
7484
|
+
let total = 0;
|
|
7485
|
+
for (const page of pages) if (page.state === "fetch-error") total++;
|
|
7486
|
+
return total;
|
|
7487
|
+
}
|
|
7488
|
+
function deadLinkCheckedCount(edges, pages) {
|
|
7489
|
+
const attemptedUrls = /* @__PURE__ */ new Set();
|
|
7490
|
+
for (const page of pages) {
|
|
7491
|
+
if (page.state === "discovered" || page.state === "robots-blocked") continue;
|
|
7492
|
+
attemptedUrls.add(page.requestedUrl);
|
|
7493
|
+
if (page.finalUrl) attemptedUrls.add(page.finalUrl);
|
|
7494
|
+
}
|
|
7495
|
+
return new Set([...edges].filter((edge) => edge.type === "anchor" && edge.classification === "internal" && attemptedUrls.has(edge.to)).map((edge) => edge.to)).size;
|
|
6958
7496
|
}
|
|
6959
7497
|
async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
6960
7498
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
6961
|
-
db.update(runs).set({ status: "running", startedAt }).where(
|
|
6962
|
-
|
|
6963
|
-
|
|
6964
|
-
|
|
6965
|
-
|
|
7499
|
+
const claim = db.update(runs).set({ status: "running", startedAt }).where(and11(eq14(runs.id, runId), eq14(runs.projectId, projectId), eq14(runs.status, "queued"))).run();
|
|
7500
|
+
if (claim.changes === 0) return;
|
|
7501
|
+
const attemptNumber = (db.select({ value: sql6`coalesce(max(${siteCrawlAttempts.attemptNumber}), 0)` }).from(siteCrawlAttempts).where(eq14(siteCrawlAttempts.runId, runId)).get()?.value ?? 0) + 1;
|
|
7502
|
+
const attemptId = crypto17.randomUUID();
|
|
7503
|
+
db.insert(siteCrawlAttempts).values({
|
|
7504
|
+
id: attemptId,
|
|
7505
|
+
projectId,
|
|
7506
|
+
runId,
|
|
7507
|
+
attemptNumber,
|
|
7508
|
+
state: "running",
|
|
7509
|
+
startedAt,
|
|
7510
|
+
createdAt: startedAt,
|
|
7511
|
+
updatedAt: startedAt
|
|
7512
|
+
}).run();
|
|
7513
|
+
const observedPages = /* @__PURE__ */ new Map();
|
|
7514
|
+
const observedEdges = /* @__PURE__ */ new Map();
|
|
7515
|
+
const nodeKeyByUrl = /* @__PURE__ */ new Map();
|
|
7516
|
+
const nodePriorityByUrl = /* @__PURE__ */ new Map();
|
|
7517
|
+
const registerNode = (url, nodeKey, priority, changedUrls) => {
|
|
7518
|
+
const currentPriority = nodePriorityByUrl.get(url);
|
|
7519
|
+
const currentNodeKey = nodeKeyByUrl.get(url);
|
|
7520
|
+
if (currentPriority == null || priority > currentPriority || priority === currentPriority && currentNodeKey != null && nodeKey.localeCompare(currentNodeKey) < 0) {
|
|
7521
|
+
nodePriorityByUrl.set(url, priority);
|
|
7522
|
+
nodeKeyByUrl.set(url, nodeKey);
|
|
7523
|
+
changedUrls.add(url);
|
|
6966
7524
|
}
|
|
7525
|
+
};
|
|
7526
|
+
const registerPageNode = (page, changedUrls) => {
|
|
7527
|
+
observedPages.set(page.key, page);
|
|
7528
|
+
registerNode(page.requestedUrl, page.key, 2, changedUrls);
|
|
7529
|
+
if (page.finalUrl && page.finalUrl !== page.requestedUrl) registerNode(page.finalUrl, page.key, 1, changedUrls);
|
|
7530
|
+
};
|
|
7531
|
+
const bindReferences = (tx, changedUrls, now) => {
|
|
7532
|
+
for (const url of [...new Set(changedUrls)].sort()) {
|
|
7533
|
+
const nodeKey = nodeKeyByUrl.get(url);
|
|
7534
|
+
if (!nodeKey) continue;
|
|
7535
|
+
const scope = [
|
|
7536
|
+
eq14(siteCrawlEdges.projectId, projectId),
|
|
7537
|
+
eq14(siteCrawlEdges.runId, runId),
|
|
7538
|
+
eq14(siteCrawlEdges.attemptId, attemptId)
|
|
7539
|
+
];
|
|
7540
|
+
tx.update(siteCrawlEdges).set({ sourceNodeKey: nodeKey, updatedAt: now }).where(and11(
|
|
7541
|
+
...scope,
|
|
7542
|
+
eq14(siteCrawlEdges.sourceUrl, url)
|
|
7543
|
+
)).run();
|
|
7544
|
+
tx.update(siteCrawlEdges).set({ targetNodeKey: nodeKey, updatedAt: now }).where(and11(
|
|
7545
|
+
...scope,
|
|
7546
|
+
eq14(siteCrawlEdges.targetUrl, url)
|
|
7547
|
+
)).run();
|
|
7548
|
+
tx.update(siteCrawlPages).set({ canonicalNodeKey: nodeKey, updatedAt: now }).where(and11(
|
|
7549
|
+
eq14(siteCrawlPages.projectId, projectId),
|
|
7550
|
+
eq14(siteCrawlPages.runId, runId),
|
|
7551
|
+
eq14(siteCrawlPages.attemptId, attemptId),
|
|
7552
|
+
eq14(siteCrawlPages.canonicalUrl, url)
|
|
7553
|
+
)).run();
|
|
7554
|
+
}
|
|
7555
|
+
};
|
|
7556
|
+
const persistPage = (tx, page, now) => {
|
|
7557
|
+
const audit = page.audit;
|
|
7558
|
+
tx.insert(siteCrawlPages).values({
|
|
7559
|
+
id: crypto17.randomUUID(),
|
|
7560
|
+
projectId,
|
|
7561
|
+
runId,
|
|
7562
|
+
attemptId,
|
|
7563
|
+
nodeKey: page.key,
|
|
7564
|
+
url: page.requestedUrl,
|
|
7565
|
+
path: pagePath(page),
|
|
7566
|
+
parentPath: parentPath(page),
|
|
7567
|
+
discoverySource: pageDiscoverySource(page),
|
|
7568
|
+
discoveryProvenance: [{
|
|
7569
|
+
discoveredFrom: page.provenance.discoveredFrom,
|
|
7570
|
+
sitemapSources: page.provenance.sitemapSources,
|
|
7571
|
+
root: page.provenance.root
|
|
7572
|
+
}],
|
|
7573
|
+
sitemapMetadata: { sources: page.provenance.sitemapSources },
|
|
7574
|
+
fetchState: page.state,
|
|
7575
|
+
fetchedAt: page.state === "discovered" || page.state === "robots-blocked" ? null : now,
|
|
7576
|
+
httpStatus: page.statusCode,
|
|
7577
|
+
contentType: page.contentType,
|
|
7578
|
+
finalUrl: page.finalUrl,
|
|
7579
|
+
redirectChain: page.redirectChain.map((hop) => hop.to),
|
|
7580
|
+
directives: { metaRobots: page.metaRobots, xRobots: page.xRobots },
|
|
7581
|
+
canonicalUrl: page.canonicalUrl,
|
|
7582
|
+
canonicalNodeKey: page.canonicalUrl ? nodeKeyByUrl.get(page.canonicalUrl) ?? null : null,
|
|
7583
|
+
indexabilityState: page.indexability.state,
|
|
7584
|
+
indexabilityReasons: page.indexability.reasons,
|
|
7585
|
+
auditState: pageAuditState(page),
|
|
7586
|
+
auditScore: audit?.overallScore ?? null,
|
|
7587
|
+
auditFields: audit ? crawlAuditFields(audit) : {},
|
|
7588
|
+
inventoryEligible: isInventoryEligible(page),
|
|
7589
|
+
depth: page.depth,
|
|
7590
|
+
inboundUniqueEdges: page.metrics.inbound.uniqueEdges,
|
|
7591
|
+
outboundUniqueEdges: page.metrics.outbound.uniqueEdges,
|
|
7592
|
+
inboundOccurrences: page.metrics.inbound.totalOccurrences,
|
|
7593
|
+
outboundOccurrences: page.metrics.outbound.totalOccurrences,
|
|
7594
|
+
linkScoreRaw: page.metrics.linkScoreRaw,
|
|
7595
|
+
linkScoreNormalized: page.metrics.linkScore,
|
|
7596
|
+
createdAt: now,
|
|
7597
|
+
updatedAt: now
|
|
7598
|
+
}).onConflictDoUpdate({
|
|
7599
|
+
target: [siteCrawlPages.projectId, siteCrawlPages.runId, siteCrawlPages.attemptId, siteCrawlPages.nodeKey],
|
|
7600
|
+
set: {
|
|
7601
|
+
url: page.requestedUrl,
|
|
7602
|
+
path: pagePath(page),
|
|
7603
|
+
parentPath: parentPath(page),
|
|
7604
|
+
discoverySource: pageDiscoverySource(page),
|
|
7605
|
+
discoveryProvenance: [{ discoveredFrom: page.provenance.discoveredFrom, sitemapSources: page.provenance.sitemapSources, root: page.provenance.root }],
|
|
7606
|
+
sitemapMetadata: { sources: page.provenance.sitemapSources },
|
|
7607
|
+
fetchState: page.state,
|
|
7608
|
+
fetchedAt: page.state === "discovered" || page.state === "robots-blocked" ? null : sql6`coalesce(${siteCrawlPages.fetchedAt}, ${now})`,
|
|
7609
|
+
httpStatus: page.statusCode,
|
|
7610
|
+
contentType: page.contentType,
|
|
7611
|
+
finalUrl: page.finalUrl,
|
|
7612
|
+
redirectChain: page.redirectChain.map((hop) => hop.to),
|
|
7613
|
+
directives: { metaRobots: page.metaRobots, xRobots: page.xRobots },
|
|
7614
|
+
canonicalUrl: page.canonicalUrl,
|
|
7615
|
+
canonicalNodeKey: page.canonicalUrl ? nodeKeyByUrl.get(page.canonicalUrl) ?? null : null,
|
|
7616
|
+
indexabilityState: page.indexability.state,
|
|
7617
|
+
indexabilityReasons: page.indexability.reasons,
|
|
7618
|
+
auditState: pageAuditState(page),
|
|
7619
|
+
auditScore: audit?.overallScore ?? null,
|
|
7620
|
+
auditFields: audit ? crawlAuditFields(audit) : {},
|
|
7621
|
+
inventoryEligible: isInventoryEligible(page),
|
|
7622
|
+
depth: page.depth,
|
|
7623
|
+
inboundUniqueEdges: page.metrics.inbound.uniqueEdges,
|
|
7624
|
+
outboundUniqueEdges: page.metrics.outbound.uniqueEdges,
|
|
7625
|
+
inboundOccurrences: page.metrics.inbound.totalOccurrences,
|
|
7626
|
+
outboundOccurrences: page.metrics.outbound.totalOccurrences,
|
|
7627
|
+
linkScoreRaw: page.metrics.linkScoreRaw,
|
|
7628
|
+
linkScoreNormalized: page.metrics.linkScore,
|
|
7629
|
+
updatedAt: now
|
|
7630
|
+
}
|
|
7631
|
+
}).run();
|
|
7632
|
+
};
|
|
7633
|
+
const persistPages = (tx, pages, now) => {
|
|
7634
|
+
const changedUrls = /* @__PURE__ */ new Set();
|
|
7635
|
+
for (const page of pages) registerPageNode(page, changedUrls);
|
|
7636
|
+
for (const page of pages) persistPage(tx, page, now);
|
|
7637
|
+
bindReferences(tx, changedUrls, now);
|
|
7638
|
+
};
|
|
7639
|
+
const persistEdge = (tx, edge, now) => {
|
|
7640
|
+
observedEdges.set(edge.key, edge);
|
|
7641
|
+
tx.insert(siteCrawlEdges).values({
|
|
7642
|
+
id: crypto17.randomUUID(),
|
|
7643
|
+
projectId,
|
|
7644
|
+
runId,
|
|
7645
|
+
attemptId,
|
|
7646
|
+
edgeKey: edge.key,
|
|
7647
|
+
sourceNodeKey: nodeKeyByUrl.get(edge.from) ?? edge.from,
|
|
7648
|
+
sourceUrl: edge.from,
|
|
7649
|
+
targetNodeKey: nodeKeyByUrl.get(edge.to) ?? null,
|
|
7650
|
+
targetUrl: edge.to,
|
|
7651
|
+
relation: edge.type,
|
|
7652
|
+
internal: edge.classification === "internal",
|
|
7653
|
+
followable: edge.followableOccurrences > 0,
|
|
7654
|
+
occurrences: edge.totalOccurrences,
|
|
7655
|
+
followableOccurrences: edge.followableOccurrences,
|
|
7656
|
+
nofollowOccurrences: edge.nofollowOccurrences,
|
|
7657
|
+
anchors: edge.anchorSummaries.map((anchor) => anchor.text),
|
|
7658
|
+
createdAt: now,
|
|
7659
|
+
updatedAt: now
|
|
7660
|
+
}).onConflictDoUpdate({
|
|
7661
|
+
target: [siteCrawlEdges.projectId, siteCrawlEdges.runId, siteCrawlEdges.attemptId, siteCrawlEdges.edgeKey],
|
|
7662
|
+
set: {
|
|
7663
|
+
sourceNodeKey: nodeKeyByUrl.get(edge.from) ?? edge.from,
|
|
7664
|
+
sourceUrl: edge.from,
|
|
7665
|
+
targetNodeKey: nodeKeyByUrl.get(edge.to) ?? null,
|
|
7666
|
+
targetUrl: edge.to,
|
|
7667
|
+
relation: edge.type,
|
|
7668
|
+
internal: edge.classification === "internal",
|
|
7669
|
+
followable: edge.followableOccurrences > 0,
|
|
7670
|
+
occurrences: edge.totalOccurrences,
|
|
7671
|
+
followableOccurrences: edge.followableOccurrences,
|
|
7672
|
+
nofollowOccurrences: edge.nofollowOccurrences,
|
|
7673
|
+
anchors: edge.anchorSummaries.map((anchor) => anchor.text),
|
|
7674
|
+
updatedAt: now
|
|
7675
|
+
}
|
|
7676
|
+
}).run();
|
|
7677
|
+
};
|
|
7678
|
+
const persistMetrics = (tx, key, metrics, now) => {
|
|
7679
|
+
tx.update(siteCrawlPages).set({
|
|
7680
|
+
inboundUniqueEdges: metrics.inbound.uniqueEdges,
|
|
7681
|
+
outboundUniqueEdges: metrics.outbound.uniqueEdges,
|
|
7682
|
+
inboundOccurrences: metrics.inbound.totalOccurrences,
|
|
7683
|
+
outboundOccurrences: metrics.outbound.totalOccurrences,
|
|
7684
|
+
depth: metrics.shortestFollowableAnchorDepth,
|
|
7685
|
+
linkScoreRaw: metrics.linkScoreRaw,
|
|
7686
|
+
linkScoreNormalized: metrics.linkScore,
|
|
7687
|
+
updatedAt: now
|
|
7688
|
+
}).where(and11(
|
|
7689
|
+
eq14(siteCrawlPages.projectId, projectId),
|
|
7690
|
+
eq14(siteCrawlPages.runId, runId),
|
|
7691
|
+
eq14(siteCrawlPages.attemptId, attemptId),
|
|
7692
|
+
eq14(siteCrawlPages.nodeKey, key)
|
|
7693
|
+
)).run();
|
|
7694
|
+
};
|
|
7695
|
+
const persistEvent = async (event) => {
|
|
7696
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7697
|
+
db.transaction((tx) => {
|
|
7698
|
+
const receipt = tx.select({ checksum: siteCrawlEventReceipts.checksum }).from(siteCrawlEventReceipts).where(and11(
|
|
7699
|
+
eq14(siteCrawlEventReceipts.attemptId, attemptId),
|
|
7700
|
+
eq14(siteCrawlEventReceipts.sequence, event.sequence),
|
|
7701
|
+
eq14(siteCrawlEventReceipts.batchId, event.batchId)
|
|
7702
|
+
)).get();
|
|
7703
|
+
if (receipt) {
|
|
7704
|
+
if (receipt.checksum !== event.checksum) {
|
|
7705
|
+
throw new Error(`Crawl checkpoint checksum mismatch for ${event.batchId} (${event.sequence})`);
|
|
7706
|
+
}
|
|
7707
|
+
return;
|
|
7708
|
+
}
|
|
7709
|
+
tx.insert(siteCrawlEventReceipts).values({
|
|
7710
|
+
id: crypto17.randomUUID(),
|
|
7711
|
+
projectId,
|
|
7712
|
+
runId,
|
|
7713
|
+
attemptId,
|
|
7714
|
+
sequence: event.sequence,
|
|
7715
|
+
batchId: event.batchId,
|
|
7716
|
+
checksum: event.checksum,
|
|
7717
|
+
receipt: { type: event.type },
|
|
7718
|
+
createdAt: now
|
|
7719
|
+
}).run();
|
|
7720
|
+
if (event.type === "pages") persistPages(tx, event.rows, now);
|
|
7721
|
+
if (event.type === "edges") for (const edge of event.rows) persistEdge(tx, edge, now);
|
|
7722
|
+
if (event.type === "metrics") for (const metric of event.rows) persistMetrics(tx, metric.key, metric.metrics, now);
|
|
7723
|
+
const progress = event.type === "progress" ? event.progress : event.type === "summary" ? event.summary : null;
|
|
7724
|
+
if (progress) {
|
|
7725
|
+
tx.update(siteCrawlAttempts).set({
|
|
7726
|
+
lastEventSequence: event.sequence,
|
|
7727
|
+
lastEventChecksum: event.checksum,
|
|
7728
|
+
pagesDiscovered: progress.pagesDiscovered,
|
|
7729
|
+
pagesFetched: progress.pagesFetched,
|
|
7730
|
+
pagesEligible: event.type === "summary" ? observedPages.size === 0 ? 0 : [...observedPages.values()].filter(isInventoryEligible).length : sql6`${siteCrawlAttempts.pagesEligible}`,
|
|
7731
|
+
pagesErrored: event.type === "summary" ? observedErrorCount(observedPages.values()) : sql6`${siteCrawlAttempts.pagesErrored}`,
|
|
7732
|
+
edgesDiscovered: progress.edgesObserved,
|
|
7733
|
+
updatedAt: now
|
|
7734
|
+
}).where(eq14(siteCrawlAttempts.id, attemptId)).run();
|
|
7735
|
+
} else {
|
|
7736
|
+
tx.update(siteCrawlAttempts).set({
|
|
7737
|
+
lastEventSequence: event.sequence,
|
|
7738
|
+
lastEventChecksum: event.checksum,
|
|
7739
|
+
updatedAt: now
|
|
7740
|
+
}).where(eq14(siteCrawlAttempts.id, attemptId)).run();
|
|
7741
|
+
}
|
|
7742
|
+
});
|
|
7743
|
+
};
|
|
7744
|
+
try {
|
|
7745
|
+
const project = db.select().from(projects).where(eq14(projects.id, projectId)).get();
|
|
7746
|
+
if (!project) throw new Error(`Project not found: ${projectId}`);
|
|
6967
7747
|
const homepageUrl = toHomepageUrl(project.canonicalDomain);
|
|
6968
|
-
const
|
|
6969
|
-
|
|
7748
|
+
const maxPages = clampSiteAuditLimit(opts.maxPages ?? opts.limit);
|
|
7749
|
+
const maxEdges = clampSiteAuditEdgeLimit(opts.maxEdges);
|
|
6970
7750
|
await assertSiteAuditUrlAllowed(homepageUrl, "canonicalDomain");
|
|
6971
7751
|
if (opts.sitemapUrl) await assertSiteAuditUrlAllowed(opts.sitemapUrl, "sitemapUrl");
|
|
6972
|
-
|
|
6973
|
-
const
|
|
6974
|
-
|
|
6975
|
-
|
|
6976
|
-
|
|
6977
|
-
|
|
7752
|
+
log12.info("start", { runId, projectId, homepageUrl, maxPages, maxEdges, maxDepth: opts.maxDepth ?? null });
|
|
7753
|
+
const report = await runSiteCrawl(homepageUrl, {
|
|
7754
|
+
mode: "summary",
|
|
7755
|
+
sitemapUrl: opts.sitemapUrl,
|
|
7756
|
+
maxPages,
|
|
7757
|
+
maxEdges,
|
|
7758
|
+
maxDepth: opts.maxDepth,
|
|
7759
|
+
checkDeadLinks: opts.checkDeadLinks ?? false,
|
|
7760
|
+
signal: opts.signal,
|
|
7761
|
+
onEvent: persistEvent
|
|
7762
|
+
});
|
|
7763
|
+
const crawlSummary = report.summary;
|
|
7764
|
+
const hasAuditedPages = crawlSummary.auditRollup.auditedPages > 0;
|
|
7765
|
+
if (crawlSummary.complete && !hasAuditedPages) {
|
|
7766
|
+
throw emptyCompleteCrawlError(crawlSummary.rootUrl, crawlSummary.finalRootUrl, crawlSummary.pagesObserved);
|
|
7767
|
+
}
|
|
7768
|
+
const graphLayout = await prepareSiteCrawlGraphLayout(db, {
|
|
7769
|
+
projectId,
|
|
7770
|
+
runId,
|
|
7771
|
+
attemptId,
|
|
7772
|
+
rootUrl: crawlSummary.finalRootUrl ?? crawlSummary.rootUrl
|
|
7773
|
+
}, { signal: opts.signal, timeoutMs: opts.graphLayoutTimeoutMs });
|
|
7774
|
+
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
7775
|
+
const factors = computeFactorAverages([...observedPages.values()]);
|
|
7776
|
+
const errorCount = observedErrorCount(observedPages.values());
|
|
7777
|
+
const terminalStatus = crawlSummary.complete ? "completed" : "partial";
|
|
7778
|
+
const deadLinksChecked = opts.checkDeadLinks ? deadLinkCheckedCount(observedEdges.values(), observedPages.values()) : 0;
|
|
7779
|
+
const deadLinksFound = report.deadLinks.findings.length;
|
|
7780
|
+
const legacyIssues = factors.map((factor) => toLegacyIssue(factor, crawlSummary.auditRollup.auditedPages)).filter((issue) => issue !== null);
|
|
7781
|
+
try {
|
|
7782
|
+
persistSiteCrawlGraphLayout(db, { projectId, runId, attemptId }, graphLayout, finishedAt);
|
|
7783
|
+
} catch (error) {
|
|
7784
|
+
log12.warn("graph-layout.persist-failed", {
|
|
6978
7785
|
runId,
|
|
6979
7786
|
projectId,
|
|
6980
|
-
|
|
6981
|
-
audited: report.pagesAudited,
|
|
6982
|
-
dropped: auditable - report.pagesAudited,
|
|
6983
|
-
limit
|
|
7787
|
+
error: error instanceof Error ? error.message : String(error)
|
|
6984
7788
|
});
|
|
7789
|
+
try {
|
|
7790
|
+
persistSiteCrawlGraphLayout(db, { projectId, runId, attemptId }, {
|
|
7791
|
+
state: "unavailable",
|
|
7792
|
+
failureCode: "layout-error",
|
|
7793
|
+
totalNodes: graphLayout.totalNodes,
|
|
7794
|
+
totalEdges: graphLayout.totalEdges,
|
|
7795
|
+
nodeCount: 0,
|
|
7796
|
+
edgeCount: 0,
|
|
7797
|
+
nodes: [],
|
|
7798
|
+
edges: []
|
|
7799
|
+
}, finishedAt);
|
|
7800
|
+
} catch (markerError) {
|
|
7801
|
+
log12.warn("graph-layout.marker-persist-failed", {
|
|
7802
|
+
runId,
|
|
7803
|
+
projectId,
|
|
7804
|
+
error: markerError instanceof Error ? markerError.message : String(markerError)
|
|
7805
|
+
});
|
|
7806
|
+
}
|
|
6985
7807
|
}
|
|
6986
|
-
|
|
6987
|
-
|
|
6988
|
-
|
|
6989
|
-
)
|
|
6990
|
-
|
|
6991
|
-
|
|
6992
|
-
const status = pagesErrored === 0 ? "completed" : "partial";
|
|
6993
|
-
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
6994
|
-
db.transaction((tx) => {
|
|
6995
|
-
tx.insert(siteAuditSnapshots).values({
|
|
6996
|
-
id: crypto16.randomUUID(),
|
|
6997
|
-
projectId,
|
|
6998
|
-
runId,
|
|
6999
|
-
sitemapUrl: report.sitemapUrl,
|
|
7000
|
-
auditedAt: report.auditedAt,
|
|
7001
|
-
aggregateScore: report.aggregateScore,
|
|
7002
|
-
pagesDiscovered: report.pagesDiscovered,
|
|
7003
|
-
pagesAudited: report.pagesAudited,
|
|
7004
|
-
pagesSkipped: report.pagesSkipped,
|
|
7005
|
-
pagesErrored,
|
|
7006
|
-
factorAverages,
|
|
7007
|
-
// aeo-audit v3 enriches these (topIssues, avgGrade-free); keep only the
|
|
7008
|
-
// fields our DTO exposes so the stored JSON stays lean.
|
|
7009
|
-
crossCuttingIssues: report.crossCuttingIssues.map((issue) => ({
|
|
7010
|
-
factorId: issue.factorId,
|
|
7011
|
-
factorName: issue.factorName,
|
|
7012
|
-
avgScore: issue.avgScore,
|
|
7013
|
-
affectedPages: issue.affectedPages,
|
|
7014
|
-
totalPages: issue.totalPages,
|
|
7015
|
-
affectedPct: issue.totalPages > 0 ? Math.round(issue.affectedPages / issue.totalPages * 100) : 0,
|
|
7016
|
-
topRecommendations: issue.topRecommendations
|
|
7017
|
-
})),
|
|
7018
|
-
// v3 prioritizedFixes are structured PrioritizedFix objects; persist the
|
|
7019
|
-
// ready-to-display one-line summary to keep the DTO a string list.
|
|
7020
|
-
prioritizedFixes: report.prioritizedFixes.map((fix) => fix.summary),
|
|
7021
|
-
createdAt: finishedAt
|
|
7022
|
-
}).run();
|
|
7023
|
-
for (const page of report.pages) {
|
|
7024
|
-
tx.insert(siteAuditPages).values({
|
|
7025
|
-
id: crypto16.randomUUID(),
|
|
7808
|
+
const published = db.transaction((tx) => {
|
|
7809
|
+
const claim2 = tx.update(runs).set({ status: terminalStatus, finishedAt }).where(and11(eq14(runs.id, runId), eq14(runs.status, "running"))).run();
|
|
7810
|
+
if (claim2.changes === 0) return false;
|
|
7811
|
+
if (hasAuditedPages) {
|
|
7812
|
+
tx.insert(siteAuditSnapshots).values({
|
|
7813
|
+
id: crypto17.randomUUID(),
|
|
7026
7814
|
projectId,
|
|
7027
7815
|
runId,
|
|
7028
|
-
|
|
7029
|
-
|
|
7030
|
-
|
|
7031
|
-
|
|
7032
|
-
|
|
7816
|
+
sitemapUrl: opts.sitemapUrl ?? `${homepageUrl}/sitemap.xml`,
|
|
7817
|
+
auditedAt: crawlSummary.completedAt,
|
|
7818
|
+
aggregateScore: crawlSummary.auditRollup.aggregateScore ?? 0,
|
|
7819
|
+
pagesDiscovered: crawlSummary.pagesDiscovered,
|
|
7820
|
+
pagesAudited: crawlSummary.auditRollup.auditedPages,
|
|
7821
|
+
pagesSkipped: Math.max(0, crawlSummary.pagesDiscovered - crawlSummary.pagesObserved),
|
|
7822
|
+
pagesErrored: errorCount,
|
|
7823
|
+
factorAverages: factors,
|
|
7824
|
+
crossCuttingIssues: legacyIssues,
|
|
7825
|
+
prioritizedFixes: legacyIssues.map((issue) => `Improve ${issue.factorName}`),
|
|
7033
7826
|
createdAt: finishedAt
|
|
7034
7827
|
}).run();
|
|
7828
|
+
for (const page of observedPages.values()) {
|
|
7829
|
+
if (!isLegacyScorecardPage(page)) continue;
|
|
7830
|
+
tx.insert(siteAuditPages).values({
|
|
7831
|
+
id: crypto17.randomUUID(),
|
|
7832
|
+
projectId,
|
|
7833
|
+
runId,
|
|
7834
|
+
url: page.finalUrl ?? page.requestedUrl,
|
|
7835
|
+
overallScore: page.audit?.overallScore ?? 0,
|
|
7836
|
+
status: page.audit ? "success" : "error",
|
|
7837
|
+
error: page.error,
|
|
7838
|
+
factors: page.audit?.factors.map(toPageFactor) ?? [],
|
|
7839
|
+
createdAt: finishedAt
|
|
7840
|
+
}).run();
|
|
7841
|
+
}
|
|
7035
7842
|
}
|
|
7036
|
-
tx.
|
|
7843
|
+
tx.insert(siteCrawlSnapshots).values({
|
|
7844
|
+
id: crypto17.randomUUID(),
|
|
7845
|
+
projectId,
|
|
7846
|
+
runId,
|
|
7847
|
+
attemptId,
|
|
7848
|
+
requestedRootUrl: crawlSummary.rootUrl,
|
|
7849
|
+
rootUrl: crawlSummary.finalRootUrl ?? crawlSummary.rootUrl,
|
|
7850
|
+
crawlSchemaVersion: crawlSummary.crawlSchemaVersion,
|
|
7851
|
+
engineVersion: crawlSummary.engineVersion,
|
|
7852
|
+
normalizationVersion: crawlSummary.urlNormalizationVersion,
|
|
7853
|
+
indexabilityVersion: crawlSummary.indexabilityRulesetVersion,
|
|
7854
|
+
linkScoreVersion: crawlSummary.linkScoreAlgorithmVersion,
|
|
7855
|
+
effectiveOptions: {
|
|
7856
|
+
mode: "summary",
|
|
7857
|
+
sitemapUrl: opts.sitemapUrl ?? null,
|
|
7858
|
+
maxPages,
|
|
7859
|
+
maxEdges,
|
|
7860
|
+
maxDepth: opts.maxDepth ?? null,
|
|
7861
|
+
checkDeadLinks: opts.checkDeadLinks ?? false
|
|
7862
|
+
},
|
|
7863
|
+
pageBudget: maxPages,
|
|
7864
|
+
edgeBudget: maxEdges,
|
|
7865
|
+
maxDepth: opts.maxDepth ?? null,
|
|
7866
|
+
checkDeadLinks: opts.checkDeadLinks ?? false,
|
|
7867
|
+
complete: crawlSummary.complete,
|
|
7868
|
+
termination: crawlSummary.terminationReason ?? "complete",
|
|
7869
|
+
detailsAvailable: true,
|
|
7870
|
+
pagesDiscovered: crawlSummary.pagesDiscovered,
|
|
7871
|
+
pagesFetched: crawlSummary.pagesFetched,
|
|
7872
|
+
pagesEligible: [...observedPages.values()].filter(isInventoryEligible).length,
|
|
7873
|
+
pagesErrored: errorCount,
|
|
7874
|
+
edgesDiscovered: crawlSummary.edgesObserved,
|
|
7875
|
+
findingsCount: deadLinksFound,
|
|
7876
|
+
deadLinkState: report.deadLinks.state,
|
|
7877
|
+
deadLinksChecked,
|
|
7878
|
+
deadLinksFound,
|
|
7879
|
+
createdAt: finishedAt,
|
|
7880
|
+
updatedAt: finishedAt
|
|
7881
|
+
}).run();
|
|
7882
|
+
if (opts.checkDeadLinks) {
|
|
7883
|
+
for (const finding of report.deadLinks.findings) {
|
|
7884
|
+
tx.insert(siteCrawlFindings).values({
|
|
7885
|
+
id: crypto17.randomUUID(),
|
|
7886
|
+
projectId,
|
|
7887
|
+
runId,
|
|
7888
|
+
attemptId,
|
|
7889
|
+
findingKey: finding.key,
|
|
7890
|
+
findingType: "dead-link",
|
|
7891
|
+
severity: "error",
|
|
7892
|
+
sourceNodeKey: nodeKeyByUrl.get(finding.from) ?? null,
|
|
7893
|
+
sourceUrl: finding.from,
|
|
7894
|
+
targetNodeKey: nodeKeyByUrl.get(finding.to) ?? null,
|
|
7895
|
+
targetUrl: finding.to,
|
|
7896
|
+
evidence: { statusCode: finding.statusCode, reason: finding.reason },
|
|
7897
|
+
createdAt: finishedAt,
|
|
7898
|
+
updatedAt: finishedAt
|
|
7899
|
+
}).run();
|
|
7900
|
+
}
|
|
7901
|
+
}
|
|
7902
|
+
tx.update(siteCrawlAttempts).set({
|
|
7903
|
+
state: terminalStatus,
|
|
7904
|
+
pagesDiscovered: crawlSummary.pagesDiscovered,
|
|
7905
|
+
pagesFetched: crawlSummary.pagesFetched,
|
|
7906
|
+
pagesEligible: [...observedPages.values()].filter(isInventoryEligible).length,
|
|
7907
|
+
pagesErrored: errorCount,
|
|
7908
|
+
edgesDiscovered: crawlSummary.edgesObserved,
|
|
7909
|
+
finishedAt,
|
|
7910
|
+
updatedAt: finishedAt
|
|
7911
|
+
}).where(eq14(siteCrawlAttempts.id, attemptId)).run();
|
|
7912
|
+
return true;
|
|
7037
7913
|
});
|
|
7038
|
-
|
|
7039
|
-
|
|
7040
|
-
|
|
7041
|
-
|
|
7042
|
-
|
|
7043
|
-
|
|
7044
|
-
|
|
7914
|
+
if (!published) {
|
|
7915
|
+
try {
|
|
7916
|
+
deleteSiteCrawlGraphLayout(db, { projectId, runId, attemptId });
|
|
7917
|
+
} catch (cleanupError) {
|
|
7918
|
+
log12.warn("graph-layout.cleanup-failed", {
|
|
7919
|
+
runId,
|
|
7920
|
+
projectId,
|
|
7921
|
+
error: cleanupError instanceof Error ? cleanupError.message : String(cleanupError)
|
|
7922
|
+
});
|
|
7923
|
+
}
|
|
7924
|
+
const current = db.select({ status: runs.status }).from(runs).where(eq14(runs.id, runId)).get();
|
|
7925
|
+
if (current?.status === "cancelled") throw new SiteAuditCancelledError(`Site audit cancelled before publication: ${runId}`);
|
|
7926
|
+
throw new Error(`Site audit run was no longer running before publication: ${runId}`);
|
|
7927
|
+
}
|
|
7928
|
+
log12.info("completed", { runId, projectId, status: terminalStatus, complete: crawlSummary.complete, pages: crawlSummary.pagesObserved });
|
|
7929
|
+
} catch (error) {
|
|
7930
|
+
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
7931
|
+
const cancelled = isCancelled(error, opts.signal);
|
|
7932
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
7933
|
+
db.transaction((tx) => {
|
|
7934
|
+
tx.update(siteCrawlAttempts).set({
|
|
7935
|
+
state: cancelled ? "cancelled" : "failed",
|
|
7936
|
+
error: message,
|
|
7937
|
+
finishedAt,
|
|
7938
|
+
updatedAt: finishedAt
|
|
7939
|
+
}).where(eq14(siteCrawlAttempts.id, attemptId)).run();
|
|
7940
|
+
tx.update(runs).set({
|
|
7941
|
+
status: cancelled ? "cancelled" : "failed",
|
|
7942
|
+
error: message,
|
|
7943
|
+
finishedAt
|
|
7944
|
+
}).where(and11(eq14(runs.id, runId), eq14(runs.status, "running"))).run();
|
|
7045
7945
|
});
|
|
7046
|
-
|
|
7047
|
-
|
|
7048
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq13(runs.id, runId)).run();
|
|
7049
|
-
log12.error("failed", { runId, projectId, error: errorMsg });
|
|
7050
|
-
throw err;
|
|
7946
|
+
log12.error("failed", { runId, projectId, cancelled, error: message });
|
|
7947
|
+
throw error;
|
|
7051
7948
|
}
|
|
7052
7949
|
}
|
|
7053
7950
|
|
|
7054
7951
|
// src/commands/backfill.ts
|
|
7055
|
-
import { and as
|
|
7952
|
+
import { and as and12, eq as eq15, inArray as inArray4, isNull as isNull2, sql as sql7 } from "drizzle-orm";
|
|
7056
7953
|
var SNAPSHOT_BATCH_SIZE = 500;
|
|
7057
7954
|
async function backfillAnswerVisibilityCommand(opts) {
|
|
7058
7955
|
const config = loadConfig();
|
|
@@ -7060,7 +7957,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
7060
7957
|
migrate(db);
|
|
7061
7958
|
const projectFilter = opts?.project?.trim();
|
|
7062
7959
|
const isDryRun = opts?.dryRun === true;
|
|
7063
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
7960
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq15(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
7064
7961
|
let examined = 0;
|
|
7065
7962
|
let updated = 0;
|
|
7066
7963
|
let wouldUpdate = 0;
|
|
@@ -7068,10 +7965,10 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
7068
7965
|
let reparsed = 0;
|
|
7069
7966
|
let providerErrors = 0;
|
|
7070
7967
|
if (scopedProjects.length > 0) {
|
|
7071
|
-
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
7072
|
-
|
|
7968
|
+
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(and12(
|
|
7969
|
+
eq15(runs.kind, RunKinds["answer-visibility"]),
|
|
7073
7970
|
inArray4(runs.projectId, scopedProjects.map((project) => project.id))
|
|
7074
|
-
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
7971
|
+
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(eq15(runs.kind, RunKinds["answer-visibility"])).all();
|
|
7075
7972
|
const runIdsByProject = /* @__PURE__ */ new Map();
|
|
7076
7973
|
for (const run of runRows) {
|
|
7077
7974
|
const existing = runIdsByProject.get(run.projectId);
|
|
@@ -7079,7 +7976,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
7079
7976
|
else runIdsByProject.set(run.projectId, [run.id]);
|
|
7080
7977
|
}
|
|
7081
7978
|
for (const project of scopedProjects) {
|
|
7082
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
7979
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq15(competitors.projectId, project.id)).all().map((row) => row.domain);
|
|
7083
7980
|
const runIds = runIdsByProject.get(project.id) ?? [];
|
|
7084
7981
|
if (runIds.length === 0) continue;
|
|
7085
7982
|
const projectDomains = effectiveDomains({
|
|
@@ -7172,7 +8069,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
7172
8069
|
} else {
|
|
7173
8070
|
db.transaction((tx) => {
|
|
7174
8071
|
for (const update of pendingUpdates) {
|
|
7175
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
8072
|
+
tx.update(querySnapshots).set(update.patch).where(eq15(querySnapshots.id, update.id)).run();
|
|
7176
8073
|
}
|
|
7177
8074
|
});
|
|
7178
8075
|
updated += pendingUpdates.length;
|
|
@@ -7221,13 +8118,13 @@ No DB writes performed. Re-run without --dry-run to apply.`);
|
|
|
7221
8118
|
function backfillNormalizedPaths(db, opts) {
|
|
7222
8119
|
const baseConditions = [];
|
|
7223
8120
|
if (opts?.projectId) {
|
|
7224
|
-
baseConditions.push(
|
|
8121
|
+
baseConditions.push(eq15(gaTrafficSnapshots.projectId, opts.projectId));
|
|
7225
8122
|
}
|
|
7226
8123
|
const rows = db.select({
|
|
7227
8124
|
id: gaTrafficSnapshots.id,
|
|
7228
8125
|
landingPage: gaTrafficSnapshots.landingPage,
|
|
7229
8126
|
landingPageNormalized: gaTrafficSnapshots.landingPageNormalized
|
|
7230
|
-
}).from(gaTrafficSnapshots).where(baseConditions.length > 0 ?
|
|
8127
|
+
}).from(gaTrafficSnapshots).where(baseConditions.length > 0 ? and12(...baseConditions) : void 0).all();
|
|
7231
8128
|
let updated = 0;
|
|
7232
8129
|
let unchanged = 0;
|
|
7233
8130
|
if (rows.length > 0) {
|
|
@@ -7242,7 +8139,7 @@ function backfillNormalizedPaths(db, opts) {
|
|
|
7242
8139
|
unchanged++;
|
|
7243
8140
|
continue;
|
|
7244
8141
|
}
|
|
7245
|
-
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(
|
|
8142
|
+
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(eq15(gaTrafficSnapshots.id, row.id)).run();
|
|
7246
8143
|
updated++;
|
|
7247
8144
|
}
|
|
7248
8145
|
});
|
|
@@ -7256,7 +8153,7 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
7256
8153
|
const projectFilter = opts?.project?.trim();
|
|
7257
8154
|
let projectId;
|
|
7258
8155
|
if (projectFilter) {
|
|
7259
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
8156
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq15(projects.name, projectFilter)).get();
|
|
7260
8157
|
if (!project) {
|
|
7261
8158
|
const result2 = {
|
|
7262
8159
|
project: projectFilter,
|
|
@@ -7293,13 +8190,13 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
7293
8190
|
function backfillAiReferralPaths(db, opts) {
|
|
7294
8191
|
const baseConditions = [];
|
|
7295
8192
|
if (opts?.projectId) {
|
|
7296
|
-
baseConditions.push(
|
|
8193
|
+
baseConditions.push(eq15(gaAiReferrals.projectId, opts.projectId));
|
|
7297
8194
|
}
|
|
7298
8195
|
const rows = db.select({
|
|
7299
8196
|
id: gaAiReferrals.id,
|
|
7300
8197
|
landingPage: gaAiReferrals.landingPage,
|
|
7301
8198
|
landingPageNormalized: gaAiReferrals.landingPageNormalized
|
|
7302
|
-
}).from(gaAiReferrals).where(baseConditions.length > 0 ?
|
|
8199
|
+
}).from(gaAiReferrals).where(baseConditions.length > 0 ? and12(...baseConditions) : void 0).all();
|
|
7303
8200
|
let updated = 0;
|
|
7304
8201
|
let unchanged = 0;
|
|
7305
8202
|
if (rows.length > 0) {
|
|
@@ -7314,7 +8211,7 @@ function backfillAiReferralPaths(db, opts) {
|
|
|
7314
8211
|
unchanged++;
|
|
7315
8212
|
continue;
|
|
7316
8213
|
}
|
|
7317
|
-
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(
|
|
8214
|
+
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(eq15(gaAiReferrals.id, row.id)).run();
|
|
7318
8215
|
updated++;
|
|
7319
8216
|
}
|
|
7320
8217
|
});
|
|
@@ -7328,7 +8225,7 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
7328
8225
|
const projectFilter = opts?.project?.trim();
|
|
7329
8226
|
let projectId;
|
|
7330
8227
|
if (projectFilter) {
|
|
7331
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
8228
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq15(projects.name, projectFilter)).get();
|
|
7332
8229
|
if (!project) {
|
|
7333
8230
|
const result2 = {
|
|
7334
8231
|
project: projectFilter,
|
|
@@ -7364,10 +8261,10 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
7364
8261
|
}
|
|
7365
8262
|
function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
7366
8263
|
const isDryRun = opts?.dryRun === true;
|
|
7367
|
-
const project = db.select().from(projects).where(
|
|
8264
|
+
const project = db.select().from(projects).where(eq15(projects.id, projectId)).get();
|
|
7368
8265
|
if (!project) return { examined: 0, updated: 0, mentioned: 0 };
|
|
7369
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
7370
|
-
const runRows = db.select({ id: runs.id }).from(runs).where(
|
|
8266
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq15(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
8267
|
+
const runRows = db.select({ id: runs.id }).from(runs).where(and12(eq15(runs.kind, RunKinds["answer-visibility"]), eq15(runs.projectId, projectId))).all();
|
|
7371
8268
|
const runIds = runRows.map((r) => r.id);
|
|
7372
8269
|
let examined = 0;
|
|
7373
8270
|
let updated = 0;
|
|
@@ -7441,7 +8338,7 @@ function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
|
7441
8338
|
} else {
|
|
7442
8339
|
db.transaction((tx) => {
|
|
7443
8340
|
for (const update of pendingUpdates) {
|
|
7444
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
8341
|
+
tx.update(querySnapshots).set(update.patch).where(eq15(querySnapshots.id, update.id)).run();
|
|
7445
8342
|
}
|
|
7446
8343
|
});
|
|
7447
8344
|
updated += pendingUpdates.length;
|
|
@@ -7456,7 +8353,7 @@ async function backfillAnswerMentionsCommand(opts) {
|
|
|
7456
8353
|
migrate(db);
|
|
7457
8354
|
const projectFilter = opts?.project?.trim();
|
|
7458
8355
|
const isDryRun = opts?.dryRun === true;
|
|
7459
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
8356
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq15(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
7460
8357
|
let examined = 0;
|
|
7461
8358
|
let updated = 0;
|
|
7462
8359
|
let wouldUpdate = 0;
|
|
@@ -7516,7 +8413,7 @@ function readStoredGroundingSources(rawResponse) {
|
|
|
7516
8413
|
return result;
|
|
7517
8414
|
}
|
|
7518
8415
|
async function backfillInsightsCommand(project, opts) {
|
|
7519
|
-
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-
|
|
8416
|
+
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-WHC7B32V.js");
|
|
7520
8417
|
const config = loadConfig();
|
|
7521
8418
|
const db = createClient(config.database);
|
|
7522
8419
|
migrate(db);
|
|
@@ -7675,7 +8572,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
7675
8572
|
const config = loadConfig();
|
|
7676
8573
|
const db = createClient(config.database);
|
|
7677
8574
|
migrate(db);
|
|
7678
|
-
const project = db.select().from(projects).where(
|
|
8575
|
+
const project = db.select().from(projects).where(eq15(projects.name, opts.project)).get();
|
|
7679
8576
|
if (!project) {
|
|
7680
8577
|
throw new Error(`Project "${opts.project}" not found`);
|
|
7681
8578
|
}
|
|
@@ -7686,8 +8583,8 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
7686
8583
|
process.stderr.write(`Recovering orphan snapshot attribution for "${project.name}"${mode}...
|
|
7687
8584
|
`);
|
|
7688
8585
|
}
|
|
7689
|
-
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(
|
|
7690
|
-
|
|
8586
|
+
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(and12(
|
|
8587
|
+
eq15(auditLog.projectId, project.id),
|
|
7691
8588
|
inArray4(auditLog.action, ["keywords.appended", "keywords.deleted", "queries.appended", "queries.deleted", "queries.replaced"])
|
|
7692
8589
|
)).orderBy(auditLog.createdAt).all();
|
|
7693
8590
|
const history = replayQueryAuditLog(events);
|
|
@@ -7695,10 +8592,10 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
7695
8592
|
runId: runs.id,
|
|
7696
8593
|
createdAt: runs.createdAt,
|
|
7697
8594
|
location: runs.location
|
|
7698
|
-
}).from(runs).innerJoin(querySnapshots,
|
|
7699
|
-
|
|
7700
|
-
|
|
7701
|
-
|
|
8595
|
+
}).from(runs).innerJoin(querySnapshots, eq15(querySnapshots.runId, runs.id)).where(and12(
|
|
8596
|
+
eq15(runs.projectId, project.id),
|
|
8597
|
+
isNull2(querySnapshots.queryId),
|
|
8598
|
+
isNull2(querySnapshots.queryText)
|
|
7702
8599
|
)).groupBy(runs.id).orderBy(runs.createdAt).all();
|
|
7703
8600
|
const result = {
|
|
7704
8601
|
project: project.name,
|
|
@@ -7718,10 +8615,10 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
7718
8615
|
provider: querySnapshots.provider,
|
|
7719
8616
|
createdAt: querySnapshots.createdAt,
|
|
7720
8617
|
answerText: querySnapshots.answerText
|
|
7721
|
-
}).from(querySnapshots).where(
|
|
7722
|
-
|
|
7723
|
-
|
|
7724
|
-
|
|
8618
|
+
}).from(querySnapshots).where(and12(
|
|
8619
|
+
eq15(querySnapshots.runId, run.runId),
|
|
8620
|
+
isNull2(querySnapshots.queryId),
|
|
8621
|
+
isNull2(querySnapshots.queryText)
|
|
7725
8622
|
)).orderBy(querySnapshots.provider, querySnapshots.createdAt).all();
|
|
7726
8623
|
const byProvider = /* @__PURE__ */ new Map();
|
|
7727
8624
|
for (const s of orphanSnaps) {
|
|
@@ -7785,7 +8682,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
7785
8682
|
if (!isDryRun && updates.length > 0) {
|
|
7786
8683
|
db.transaction((tx) => {
|
|
7787
8684
|
for (const u of updates) {
|
|
7788
|
-
tx.update(querySnapshots).set({ queryText: u.queryText }).where(
|
|
8685
|
+
tx.update(querySnapshots).set({ queryText: u.queryText }).where(eq15(querySnapshots.id, u.id)).run();
|
|
7789
8686
|
}
|
|
7790
8687
|
});
|
|
7791
8688
|
}
|
|
@@ -7859,7 +8756,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7859
8756
|
const projectFilter = opts?.project?.trim();
|
|
7860
8757
|
const isDryRun = opts?.dryRun === true;
|
|
7861
8758
|
const isJson = isMachineFormat(opts?.format);
|
|
7862
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
8759
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq15(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
7863
8760
|
if (scopedProjects.length === 0) {
|
|
7864
8761
|
if (projectFilter && !isJson) {
|
|
7865
8762
|
process.stderr.write(`No project named "${projectFilter}".
|
|
@@ -7884,8 +8781,8 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7884
8781
|
dryRun: isDryRun,
|
|
7885
8782
|
byBot: {}
|
|
7886
8783
|
};
|
|
7887
|
-
const unknownCountRow = db.select({ n:
|
|
7888
|
-
|
|
8784
|
+
const unknownCountRow = db.select({ n: sql7`count(*)` }).from(rawEventSamples).where(and12(
|
|
8785
|
+
eq15(rawEventSamples.eventType, "unknown"),
|
|
7889
8786
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
7890
8787
|
)).get();
|
|
7891
8788
|
result.unknownBefore = Number(unknownCountRow?.n ?? 0);
|
|
@@ -7897,8 +8794,8 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7897
8794
|
userAgent: rawEventSamples.userAgent,
|
|
7898
8795
|
pathNormalized: rawEventSamples.pathNormalized,
|
|
7899
8796
|
status: rawEventSamples.status
|
|
7900
|
-
}).from(rawEventSamples).where(
|
|
7901
|
-
|
|
8797
|
+
}).from(rawEventSamples).where(and12(
|
|
8798
|
+
eq15(rawEventSamples.eventType, "unknown"),
|
|
7902
8799
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
7903
8800
|
)).all();
|
|
7904
8801
|
result.examined = unknownSamples.length;
|
|
@@ -7937,7 +8834,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7937
8834
|
result.reclassified++;
|
|
7938
8835
|
result.byBot[classified.botId] = (result.byBot[classified.botId] ?? 0) + 1;
|
|
7939
8836
|
if (isDryRun) continue;
|
|
7940
|
-
db.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(
|
|
8837
|
+
db.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(eq15(rawEventSamples.id, snap.id)).run();
|
|
7941
8838
|
const tsHour = new Date(snap.ts);
|
|
7942
8839
|
tsHour.setUTCMinutes(0, 0, 0);
|
|
7943
8840
|
if (userFetch) {
|
|
@@ -7965,7 +8862,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7965
8862
|
aiUserFetchEventsHourly.status
|
|
7966
8863
|
],
|
|
7967
8864
|
set: {
|
|
7968
|
-
hits:
|
|
8865
|
+
hits: sql7`${aiUserFetchEventsHourly.hits} + 1`,
|
|
7969
8866
|
updatedAt: now
|
|
7970
8867
|
}
|
|
7971
8868
|
}).run();
|
|
@@ -7994,15 +8891,15 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7994
8891
|
crawlerEventsHourly.status
|
|
7995
8892
|
],
|
|
7996
8893
|
set: {
|
|
7997
|
-
hits:
|
|
8894
|
+
hits: sql7`${crawlerEventsHourly.hits} + 1`,
|
|
7998
8895
|
updatedAt: now
|
|
7999
8896
|
}
|
|
8000
8897
|
}).run();
|
|
8001
8898
|
}
|
|
8002
8899
|
}
|
|
8003
8900
|
if (!isDryRun) {
|
|
8004
|
-
const afterRow = db.select({ n:
|
|
8005
|
-
|
|
8901
|
+
const afterRow = db.select({ n: sql7`count(*)` }).from(rawEventSamples).where(and12(
|
|
8902
|
+
eq15(rawEventSamples.eventType, "unknown"),
|
|
8006
8903
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
8007
8904
|
)).get();
|
|
8008
8905
|
result.unknownAfter = Number(afterRow?.n ?? 0);
|
|
@@ -8026,8 +8923,8 @@ Traffic classification ${result.dryRun ? "preview" : "recovery"} complete.`);
|
|
|
8026
8923
|
if (result.reclassified > 0) {
|
|
8027
8924
|
console.log(` By bot:`);
|
|
8028
8925
|
const sorted = Object.entries(result.byBot).sort(([, a], [, b]) => b - a);
|
|
8029
|
-
for (const [bot,
|
|
8030
|
-
console.log(` ${
|
|
8926
|
+
for (const [bot, count2] of sorted) {
|
|
8927
|
+
console.log(` ${count2.toString().padStart(5)} ${bot}`);
|
|
8031
8928
|
}
|
|
8032
8929
|
}
|
|
8033
8930
|
if (result.dryRun) {
|
|
@@ -8037,7 +8934,7 @@ No DB writes performed. Re-run without --dry-run to apply.`);
|
|
|
8037
8934
|
}
|
|
8038
8935
|
|
|
8039
8936
|
// src/commands/skills.ts
|
|
8040
|
-
import
|
|
8937
|
+
import crypto18 from "crypto";
|
|
8041
8938
|
import fs4 from "fs";
|
|
8042
8939
|
import os4 from "os";
|
|
8043
8940
|
import path5 from "path";
|
|
@@ -8092,7 +8989,7 @@ function walkRelative(dir, prefix = "") {
|
|
|
8092
8989
|
return out.sort();
|
|
8093
8990
|
}
|
|
8094
8991
|
function sha256File(filePath) {
|
|
8095
|
-
return
|
|
8992
|
+
return crypto18.createHash("sha256").update(fs4.readFileSync(filePath)).digest("hex");
|
|
8096
8993
|
}
|
|
8097
8994
|
function readSkillManifest(skillDir) {
|
|
8098
8995
|
try {
|
|
@@ -8434,9 +9331,9 @@ var ProviderRegistry = class {
|
|
|
8434
9331
|
};
|
|
8435
9332
|
|
|
8436
9333
|
// src/scheduler.ts
|
|
8437
|
-
import
|
|
9334
|
+
import crypto19 from "crypto";
|
|
8438
9335
|
import cron from "node-cron";
|
|
8439
|
-
import { and as
|
|
9336
|
+
import { and as and13, eq as eq16, inArray as inArray5, notExists, sql as sql8 } from "drizzle-orm";
|
|
8440
9337
|
var log13 = createLogger("Scheduler");
|
|
8441
9338
|
var DEFAULT_HEALTH_CRON = "0 */6 * * *";
|
|
8442
9339
|
function taskKey(projectId, kind) {
|
|
@@ -8468,16 +9365,16 @@ var Scheduler = class {
|
|
|
8468
9365
|
*/
|
|
8469
9366
|
ensureHealthSchedules() {
|
|
8470
9367
|
const projectsWithoutHealth = this.db.select({ id: projects.id }).from(projects).where(notExists(
|
|
8471
|
-
this.db.select({ one:
|
|
8472
|
-
|
|
8473
|
-
|
|
9368
|
+
this.db.select({ one: sql8`1` }).from(schedules).where(and13(
|
|
9369
|
+
eq16(schedules.projectId, projects.id),
|
|
9370
|
+
eq16(schedules.kind, SchedulableRunKinds.doctor)
|
|
8474
9371
|
))
|
|
8475
9372
|
)).all();
|
|
8476
9373
|
if (projectsWithoutHealth.length === 0) return;
|
|
8477
9374
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8478
9375
|
for (const project of projectsWithoutHealth) {
|
|
8479
9376
|
this.db.insert(schedules).values({
|
|
8480
|
-
id:
|
|
9377
|
+
id: crypto19.randomUUID(),
|
|
8481
9378
|
projectId: project.id,
|
|
8482
9379
|
kind: SchedulableRunKinds.doctor,
|
|
8483
9380
|
cronExpr: DEFAULT_HEALTH_CRON,
|
|
@@ -8521,7 +9418,7 @@ var Scheduler = class {
|
|
|
8521
9418
|
start() {
|
|
8522
9419
|
this.ensureHealthSchedules();
|
|
8523
9420
|
this.ensureQueryBaskets();
|
|
8524
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
9421
|
+
const allSchedules = this.db.select().from(schedules).where(eq16(schedules.enabled, true)).all();
|
|
8525
9422
|
for (const schedule of allSchedules) {
|
|
8526
9423
|
const missedRunAt = schedule.nextRunAt;
|
|
8527
9424
|
this.registerCronTask(schedule);
|
|
@@ -8551,7 +9448,7 @@ var Scheduler = class {
|
|
|
8551
9448
|
this.stopTask(key, existing, "Stopped");
|
|
8552
9449
|
this.tasks.delete(key);
|
|
8553
9450
|
}
|
|
8554
|
-
const schedule = this.db.select().from(schedules).where(
|
|
9451
|
+
const schedule = this.db.select().from(schedules).where(and13(eq16(schedules.projectId, projectId), eq16(schedules.kind, kind))).get();
|
|
8555
9452
|
if (schedule && schedule.enabled) {
|
|
8556
9453
|
this.registerCronTask(schedule);
|
|
8557
9454
|
}
|
|
@@ -8592,21 +9489,21 @@ var Scheduler = class {
|
|
|
8592
9489
|
this.db.update(schedules).set({
|
|
8593
9490
|
nextRunAt: nextRunFromCron(cronExpr, timezone),
|
|
8594
9491
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
8595
|
-
}).where(
|
|
9492
|
+
}).where(eq16(schedules.id, scheduleId)).run();
|
|
8596
9493
|
const label = schedule.preset ?? cronExpr;
|
|
8597
9494
|
log13.info("cron.registered", { projectId, kind, schedule: label, timezone });
|
|
8598
9495
|
}
|
|
8599
9496
|
triggerRun(scheduleId, projectId, kind) {
|
|
8600
9497
|
try {
|
|
8601
9498
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8602
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
9499
|
+
const currentSchedule = this.db.select().from(schedules).where(eq16(schedules.id, scheduleId)).get();
|
|
8603
9500
|
if (!currentSchedule || !currentSchedule.enabled) {
|
|
8604
9501
|
log13.warn("schedule.stale", { scheduleId, projectId, kind, msg: "schedule no longer exists or is disabled" });
|
|
8605
9502
|
this.remove(projectId, kind);
|
|
8606
9503
|
return;
|
|
8607
9504
|
}
|
|
8608
9505
|
const nextRunAt = nextRunFromCron(currentSchedule.cronExpr, currentSchedule.timezone);
|
|
8609
|
-
const project = this.db.select().from(projects).where(
|
|
9506
|
+
const project = this.db.select().from(projects).where(eq16(projects.id, projectId)).get();
|
|
8610
9507
|
if (!project) {
|
|
8611
9508
|
log13.error("project.not-found", { projectId, kind, msg: "skipping scheduled run" });
|
|
8612
9509
|
this.remove(projectId, kind);
|
|
@@ -8626,7 +9523,7 @@ var Scheduler = class {
|
|
|
8626
9523
|
lastRunAt: now,
|
|
8627
9524
|
nextRunAt,
|
|
8628
9525
|
updatedAt: now
|
|
8629
|
-
}).where(
|
|
9526
|
+
}).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
8630
9527
|
log13.info("traffic-sync.triggered", { projectName: project.name, sourceId });
|
|
8631
9528
|
this.callbacks.onTrafficSyncRequested(project.name, sourceId);
|
|
8632
9529
|
return;
|
|
@@ -8636,7 +9533,7 @@ var Scheduler = class {
|
|
|
8636
9533
|
log13.warn("gbp-sync.no-callback", { scheduleId, projectId, msg: "host did not register onGbpSyncRequested" });
|
|
8637
9534
|
return;
|
|
8638
9535
|
}
|
|
8639
|
-
const runId2 =
|
|
9536
|
+
const runId2 = crypto19.randomUUID();
|
|
8640
9537
|
this.db.insert(runs).values({
|
|
8641
9538
|
id: runId2,
|
|
8642
9539
|
projectId,
|
|
@@ -8649,7 +9546,7 @@ var Scheduler = class {
|
|
|
8649
9546
|
lastRunAt: now,
|
|
8650
9547
|
nextRunAt,
|
|
8651
9548
|
updatedAt: now
|
|
8652
|
-
}).where(
|
|
9549
|
+
}).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
8653
9550
|
log13.info("gbp-sync.triggered", { runId: runId2, projectName: project.name });
|
|
8654
9551
|
this.callbacks.onGbpSyncRequested(runId2, projectId);
|
|
8655
9552
|
return;
|
|
@@ -8659,17 +9556,17 @@ var Scheduler = class {
|
|
|
8659
9556
|
log13.warn("ads-sync.no-callback", { scheduleId, projectId, msg: "host did not register onAdsSyncRequested" });
|
|
8660
9557
|
return;
|
|
8661
9558
|
}
|
|
8662
|
-
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(
|
|
8663
|
-
|
|
8664
|
-
|
|
9559
|
+
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(and13(
|
|
9560
|
+
eq16(runs.projectId, projectId),
|
|
9561
|
+
eq16(runs.kind, RunKinds["ads-sync"]),
|
|
8665
9562
|
inArray5(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
8666
9563
|
)).get();
|
|
8667
9564
|
if (activeAdsRun) {
|
|
8668
9565
|
log13.info("ads-sync.skipped-active", { projectName: project.name, activeRunId: activeAdsRun.id });
|
|
8669
|
-
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(
|
|
9566
|
+
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
8670
9567
|
return;
|
|
8671
9568
|
}
|
|
8672
|
-
const runId2 =
|
|
9569
|
+
const runId2 = crypto19.randomUUID();
|
|
8673
9570
|
this.db.insert(runs).values({
|
|
8674
9571
|
id: runId2,
|
|
8675
9572
|
projectId,
|
|
@@ -8682,7 +9579,7 @@ var Scheduler = class {
|
|
|
8682
9579
|
lastRunAt: now,
|
|
8683
9580
|
nextRunAt,
|
|
8684
9581
|
updatedAt: now
|
|
8685
|
-
}).where(
|
|
9582
|
+
}).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
8686
9583
|
log13.info("ads-sync.triggered", { runId: runId2, projectName: project.name });
|
|
8687
9584
|
this.callbacks.onAdsSyncRequested(runId2, projectId);
|
|
8688
9585
|
return;
|
|
@@ -8696,7 +9593,7 @@ var Scheduler = class {
|
|
|
8696
9593
|
lastRunAt: now,
|
|
8697
9594
|
nextRunAt,
|
|
8698
9595
|
updatedAt: now
|
|
8699
|
-
}).where(
|
|
9596
|
+
}).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
8700
9597
|
log13.info("data-refresh.triggered", { projectName: project.name });
|
|
8701
9598
|
this.callbacks.onDataRefreshRequested(project.name);
|
|
8702
9599
|
return;
|
|
@@ -8710,7 +9607,7 @@ var Scheduler = class {
|
|
|
8710
9607
|
lastRunAt: now,
|
|
8711
9608
|
nextRunAt,
|
|
8712
9609
|
updatedAt: now
|
|
8713
|
-
}).where(
|
|
9610
|
+
}).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
8714
9611
|
log13.info("doctor.triggered", { projectName: project.name });
|
|
8715
9612
|
this.callbacks.onDoctorRequested(project.name);
|
|
8716
9613
|
return;
|
|
@@ -8724,7 +9621,7 @@ var Scheduler = class {
|
|
|
8724
9621
|
lastRunAt: now,
|
|
8725
9622
|
nextRunAt,
|
|
8726
9623
|
updatedAt: now
|
|
8727
|
-
}).where(
|
|
9624
|
+
}).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
8728
9625
|
log13.info("backlinks-sync.triggered", { projectName: project.name });
|
|
8729
9626
|
this.callbacks.onBacklinksSyncRequested(project.name);
|
|
8730
9627
|
return;
|
|
@@ -8734,30 +9631,40 @@ var Scheduler = class {
|
|
|
8734
9631
|
log13.warn("site-audit.no-callback", { scheduleId, projectId, msg: "host did not register onSiteAuditRequested" });
|
|
8735
9632
|
return;
|
|
8736
9633
|
}
|
|
8737
|
-
const active = this.db.select({ id: runs.id }).from(runs).where(
|
|
8738
|
-
|
|
8739
|
-
|
|
9634
|
+
const active = this.db.select({ id: runs.id }).from(runs).where(and13(
|
|
9635
|
+
eq16(runs.projectId, projectId),
|
|
9636
|
+
eq16(runs.kind, RunKinds["site-audit"]),
|
|
8740
9637
|
inArray5(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
8741
9638
|
)).get();
|
|
8742
9639
|
if (active) {
|
|
8743
9640
|
log13.info("site-audit.skipped-active", { projectName: project.name, activeRunId: active.id });
|
|
8744
|
-
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(
|
|
9641
|
+
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
8745
9642
|
return;
|
|
8746
9643
|
}
|
|
8747
|
-
const runId2 =
|
|
8748
|
-
|
|
8749
|
-
|
|
8750
|
-
|
|
8751
|
-
|
|
8752
|
-
|
|
8753
|
-
|
|
8754
|
-
|
|
8755
|
-
|
|
9644
|
+
const runId2 = crypto19.randomUUID();
|
|
9645
|
+
const effectiveRequest = normalizeSiteAuditRunRequest({});
|
|
9646
|
+
this.db.transaction((tx) => {
|
|
9647
|
+
tx.insert(runs).values({
|
|
9648
|
+
id: runId2,
|
|
9649
|
+
projectId,
|
|
9650
|
+
kind: RunKinds["site-audit"],
|
|
9651
|
+
status: RunStatuses.queued,
|
|
9652
|
+
trigger: RunTriggers.scheduled,
|
|
9653
|
+
createdAt: now
|
|
9654
|
+
}).run();
|
|
9655
|
+
tx.insert(siteCrawlRunRequests).values({
|
|
9656
|
+
runId: runId2,
|
|
9657
|
+
projectId,
|
|
9658
|
+
identityKey: siteAuditRequestIdentity(effectiveRequest),
|
|
9659
|
+
effectiveOptions: effectiveRequest,
|
|
9660
|
+
createdAt: now
|
|
9661
|
+
}).run();
|
|
9662
|
+
});
|
|
8756
9663
|
this.db.update(schedules).set({
|
|
8757
9664
|
lastRunAt: now,
|
|
8758
9665
|
nextRunAt,
|
|
8759
9666
|
updatedAt: now
|
|
8760
|
-
}).where(
|
|
9667
|
+
}).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
8761
9668
|
log13.info("site-audit.triggered", { runId: runId2, projectName: project.name });
|
|
8762
9669
|
this.callbacks.onSiteAuditRequested(runId2, projectId);
|
|
8763
9670
|
return;
|
|
@@ -8790,7 +9697,7 @@ var Scheduler = class {
|
|
|
8790
9697
|
this.db.update(schedules).set({
|
|
8791
9698
|
nextRunAt,
|
|
8792
9699
|
updatedAt: now
|
|
8793
|
-
}).where(
|
|
9700
|
+
}).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
8794
9701
|
return;
|
|
8795
9702
|
}
|
|
8796
9703
|
const runId = queueResult.runId;
|
|
@@ -8798,7 +9705,7 @@ var Scheduler = class {
|
|
|
8798
9705
|
lastRunAt: now,
|
|
8799
9706
|
nextRunAt,
|
|
8800
9707
|
updatedAt: now
|
|
8801
|
-
}).where(
|
|
9708
|
+
}).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
8802
9709
|
log13.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
8803
9710
|
this.callbacks.onRunCreated(runId, projectId, providers, resolvedLocation);
|
|
8804
9711
|
} catch (err) {
|
|
@@ -8831,8 +9738,8 @@ async function refreshAllIntegrations(client, projectName) {
|
|
|
8831
9738
|
}
|
|
8832
9739
|
|
|
8833
9740
|
// src/notifier.ts
|
|
8834
|
-
import { eq as
|
|
8835
|
-
import
|
|
9741
|
+
import { eq as eq17, desc as desc7, and as and14, inArray as inArray6, or as or2 } from "drizzle-orm";
|
|
9742
|
+
import crypto20 from "crypto";
|
|
8836
9743
|
var log15 = createLogger("Notifier");
|
|
8837
9744
|
var Notifier = class {
|
|
8838
9745
|
db;
|
|
@@ -8844,18 +9751,18 @@ var Notifier = class {
|
|
|
8844
9751
|
/** Called after a run completes (success, partial, or failed). */
|
|
8845
9752
|
async onRunCompleted(runId, projectId) {
|
|
8846
9753
|
log15.info("run.completed", { runId, projectId });
|
|
8847
|
-
const notifs = this.db.select().from(notifications).where(
|
|
9754
|
+
const notifs = this.db.select().from(notifications).where(eq17(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
8848
9755
|
if (notifs.length === 0) {
|
|
8849
9756
|
log15.info("notifications.none-enabled", { projectId });
|
|
8850
9757
|
return;
|
|
8851
9758
|
}
|
|
8852
9759
|
log15.info("notifications.found", { projectId, count: notifs.length });
|
|
8853
|
-
const run = this.db.select().from(runs).where(
|
|
9760
|
+
const run = this.db.select().from(runs).where(eq17(runs.id, runId)).get();
|
|
8854
9761
|
if (!run) {
|
|
8855
9762
|
log15.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
8856
9763
|
return;
|
|
8857
9764
|
}
|
|
8858
|
-
const project = this.db.select().from(projects).where(
|
|
9765
|
+
const project = this.db.select().from(projects).where(eq17(projects.id, projectId)).get();
|
|
8859
9766
|
if (!project) {
|
|
8860
9767
|
log15.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
8861
9768
|
return;
|
|
@@ -8908,7 +9815,7 @@ var Notifier = class {
|
|
|
8908
9815
|
* inferred from the absence of a webhook.
|
|
8909
9816
|
*/
|
|
8910
9817
|
async onHealthChecked(projectId, report) {
|
|
8911
|
-
const project = this.db.select().from(projects).where(
|
|
9818
|
+
const project = this.db.select().from(projects).where(eq17(projects.id, projectId)).get();
|
|
8912
9819
|
if (!project) {
|
|
8913
9820
|
log15.error("project.not-found", { projectId, msg: "skipping health notification" });
|
|
8914
9821
|
return null;
|
|
@@ -8939,7 +9846,7 @@ var Notifier = class {
|
|
|
8939
9846
|
const status = noSignal ? "warn" : worst ? worst.status : "ok";
|
|
8940
9847
|
const code = noSignal ? "health.no-signal" : worst?.code ?? "health.ok";
|
|
8941
9848
|
const summary = noSignal ? `No health check produced a result (${report.checks.length} skipped) \u2014 health is unknown, not confirmed.` : worst?.summary ?? `All ${graded.length} health check(s) passing.`;
|
|
8942
|
-
const previous = this.db.select().from(doctorHealthState).where(
|
|
9849
|
+
const previous = this.db.select().from(doctorHealthState).where(eq17(doctorHealthState.projectId, projectId)).get();
|
|
8943
9850
|
const previousStatus = previous?.status ?? null;
|
|
8944
9851
|
let event = null;
|
|
8945
9852
|
if (previous === void 0) {
|
|
@@ -8954,13 +9861,13 @@ var Notifier = class {
|
|
|
8954
9861
|
if (previous === void 0) {
|
|
8955
9862
|
this.db.insert(doctorHealthState).values({ ...observation, notifiedAt: null }).run();
|
|
8956
9863
|
} else {
|
|
8957
|
-
this.db.update(doctorHealthState).set(observation).where(
|
|
9864
|
+
this.db.update(doctorHealthState).set(observation).where(eq17(doctorHealthState.projectId, projectId)).run();
|
|
8958
9865
|
}
|
|
8959
9866
|
if (!event) {
|
|
8960
9867
|
log15.info("health.unchanged", { projectId, status, code });
|
|
8961
9868
|
return null;
|
|
8962
9869
|
}
|
|
8963
|
-
const notifs = this.db.select().from(notifications).where(
|
|
9870
|
+
const notifs = this.db.select().from(notifications).where(eq17(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
8964
9871
|
const payload = {
|
|
8965
9872
|
source: "canonry",
|
|
8966
9873
|
event,
|
|
@@ -8984,7 +9891,7 @@ var Notifier = class {
|
|
|
8984
9891
|
delivered += 1;
|
|
8985
9892
|
}
|
|
8986
9893
|
if (delivered > 0) {
|
|
8987
|
-
this.db.update(doctorHealthState).set({ notifiedAt: now }).where(
|
|
9894
|
+
this.db.update(doctorHealthState).set({ notifiedAt: now }).where(eq17(doctorHealthState.projectId, projectId)).run();
|
|
8988
9895
|
}
|
|
8989
9896
|
log15.info("health.notified", { projectId, event, status, code, subscribers: notifs.length, delivered });
|
|
8990
9897
|
return event;
|
|
@@ -8997,11 +9904,11 @@ var Notifier = class {
|
|
|
8997
9904
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
8998
9905
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
8999
9906
|
if (insightEvents.length === 0) return;
|
|
9000
|
-
const notifs = this.db.select().from(notifications).where(
|
|
9907
|
+
const notifs = this.db.select().from(notifications).where(eq17(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
9001
9908
|
if (notifs.length === 0) return;
|
|
9002
|
-
const run = this.db.select().from(runs).where(
|
|
9909
|
+
const run = this.db.select().from(runs).where(eq17(runs.id, runId)).get();
|
|
9003
9910
|
if (!run) return;
|
|
9004
|
-
const project = this.db.select().from(projects).where(
|
|
9911
|
+
const project = this.db.select().from(projects).where(eq17(projects.id, projectId)).get();
|
|
9005
9912
|
if (!project) return;
|
|
9006
9913
|
for (const notif of notifs) {
|
|
9007
9914
|
const config = notif.config;
|
|
@@ -9031,7 +9938,7 @@ var Notifier = class {
|
|
|
9031
9938
|
}
|
|
9032
9939
|
}
|
|
9033
9940
|
computeTransitions(runId, projectId) {
|
|
9034
|
-
const thisRun = this.db.select().from(runs).where(
|
|
9941
|
+
const thisRun = this.db.select().from(runs).where(eq17(runs.id, runId)).get();
|
|
9035
9942
|
if (!thisRun) return [];
|
|
9036
9943
|
const completeness = measurementRunCompleteness(this.db, runId);
|
|
9037
9944
|
if (completeness.planned && !completeness.complete) {
|
|
@@ -9042,10 +9949,10 @@ var Notifier = class {
|
|
|
9042
9949
|
});
|
|
9043
9950
|
return [];
|
|
9044
9951
|
}
|
|
9045
|
-
const groupSiblings = this.db.select().from(runs).where(
|
|
9046
|
-
|
|
9047
|
-
|
|
9048
|
-
|
|
9952
|
+
const groupSiblings = this.db.select().from(runs).where(and14(
|
|
9953
|
+
eq17(runs.projectId, projectId),
|
|
9954
|
+
eq17(runs.kind, thisRun.kind),
|
|
9955
|
+
eq17(runs.createdAt, thisRun.createdAt)
|
|
9049
9956
|
)).all();
|
|
9050
9957
|
const stillPending = groupSiblings.some((r) => r.status === "queued" || r.status === "running");
|
|
9051
9958
|
if (stillPending) return [];
|
|
@@ -9061,19 +9968,19 @@ var Notifier = class {
|
|
|
9061
9968
|
return candidate.id > best.id ? candidate : best;
|
|
9062
9969
|
});
|
|
9063
9970
|
if (winner.id !== runId) return [];
|
|
9064
|
-
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
9971
|
+
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(eq17(projects.id, projectId)).get();
|
|
9065
9972
|
const locationCount = Math.max(
|
|
9066
9973
|
1,
|
|
9067
9974
|
(projectLocations?.locations ?? []).length
|
|
9068
9975
|
);
|
|
9069
9976
|
const RECENT_FETCH_LIMIT = Math.max(8, locationCount * 4);
|
|
9070
9977
|
const recentRuns = this.db.select().from(runs).where(
|
|
9071
|
-
|
|
9072
|
-
|
|
9073
|
-
|
|
9074
|
-
|
|
9978
|
+
and14(
|
|
9979
|
+
eq17(runs.projectId, projectId),
|
|
9980
|
+
eq17(runs.kind, thisRun.kind),
|
|
9981
|
+
or2(eq17(runs.status, "completed"), eq17(runs.status, "partial"))
|
|
9075
9982
|
)
|
|
9076
|
-
).orderBy(
|
|
9983
|
+
).orderBy(desc7(runs.createdAt), desc7(runs.id)).limit(RECENT_FETCH_LIMIT).all();
|
|
9077
9984
|
const groups = groupRunsByCreatedAt(recentRuns);
|
|
9078
9985
|
const currentGroupIdx = groups.findIndex((g) => g[0]?.createdAt === thisRun.createdAt);
|
|
9079
9986
|
if (currentGroupIdx < 0) return [];
|
|
@@ -9088,7 +9995,7 @@ var Notifier = class {
|
|
|
9088
9995
|
provider: querySnapshots.provider,
|
|
9089
9996
|
location: querySnapshots.location,
|
|
9090
9997
|
citationState: querySnapshots.citationState
|
|
9091
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
9998
|
+
}).from(querySnapshots).leftJoin(queries, eq17(querySnapshots.queryId, queries.id)).where(inArray6(querySnapshots.runId, currentRunIds)).all();
|
|
9092
9999
|
const previousSnapshots = this.db.select({
|
|
9093
10000
|
queryId: querySnapshots.queryId,
|
|
9094
10001
|
provider: querySnapshots.provider,
|
|
@@ -9158,7 +10065,7 @@ var Notifier = class {
|
|
|
9158
10065
|
}
|
|
9159
10066
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
9160
10067
|
this.db.insert(auditLog).values({
|
|
9161
|
-
id:
|
|
10068
|
+
id: crypto20.randomUUID(),
|
|
9162
10069
|
projectId,
|
|
9163
10070
|
actor: "scheduler",
|
|
9164
10071
|
action: `notification.${status}`,
|
|
@@ -9171,7 +10078,7 @@ var Notifier = class {
|
|
|
9171
10078
|
};
|
|
9172
10079
|
|
|
9173
10080
|
// src/run-coordinator.ts
|
|
9174
|
-
import { eq as
|
|
10081
|
+
import { eq as eq18 } from "drizzle-orm";
|
|
9175
10082
|
var log16 = createLogger("RunCoordinator");
|
|
9176
10083
|
var RunCoordinator = class {
|
|
9177
10084
|
constructor(db, notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
@@ -9187,7 +10094,7 @@ var RunCoordinator = class {
|
|
|
9187
10094
|
onInsightsGenerated;
|
|
9188
10095
|
onAeroEvent;
|
|
9189
10096
|
async onRunCompleted(runId, projectId) {
|
|
9190
|
-
const runRow = this.db.select().from(runs).where(
|
|
10097
|
+
const runRow = this.db.select().from(runs).where(eq18(runs.id, runId)).get();
|
|
9191
10098
|
const kind = runRow?.kind ?? RunKinds["answer-visibility"];
|
|
9192
10099
|
if (runRow?.trigger === RunTriggers.probe) {
|
|
9193
10100
|
log16.info("probe.skip-side-effects", { runId, projectId, kind });
|
|
@@ -9278,7 +10185,7 @@ var RunCoordinator = class {
|
|
|
9278
10185
|
* so the Aero queue is never starved of a follow-up.
|
|
9279
10186
|
*/
|
|
9280
10187
|
buildDiscoveryAeroContext(runId, projectId, status, error) {
|
|
9281
|
-
const session = this.db.select().from(discoverySessions).where(
|
|
10188
|
+
const session = this.db.select().from(discoverySessions).where(eq18(discoverySessions.runId, runId)).get();
|
|
9282
10189
|
const competitorMap = session ? session.competitorMap : [];
|
|
9283
10190
|
return {
|
|
9284
10191
|
kind: RunKinds["aeo-discover-probe"],
|
|
@@ -9320,8 +10227,8 @@ function analysisResultFromInsights(insights2) {
|
|
|
9320
10227
|
}
|
|
9321
10228
|
|
|
9322
10229
|
// src/agent/session-registry.ts
|
|
9323
|
-
import
|
|
9324
|
-
import { eq as
|
|
10230
|
+
import crypto24 from "crypto";
|
|
10231
|
+
import { eq as eq20 } from "drizzle-orm";
|
|
9325
10232
|
|
|
9326
10233
|
// src/agent/session.ts
|
|
9327
10234
|
import fs7 from "fs";
|
|
@@ -10064,7 +10971,7 @@ function buildAeroStateTools(ctx, opts = {}) {
|
|
|
10064
10971
|
}
|
|
10065
10972
|
|
|
10066
10973
|
// src/agent/llm-usage.ts
|
|
10067
|
-
import
|
|
10974
|
+
import crypto21 from "crypto";
|
|
10068
10975
|
var AeroLlmUsageFeatures = {
|
|
10069
10976
|
turn: "aero.turn"
|
|
10070
10977
|
};
|
|
@@ -10083,7 +10990,7 @@ function recordLlmUsageEvent(args) {
|
|
|
10083
10990
|
const usage = args.message.usage;
|
|
10084
10991
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
10085
10992
|
args.db.insert(llmUsageEvents).values({
|
|
10086
|
-
id:
|
|
10993
|
+
id: crypto21.randomUUID(),
|
|
10087
10994
|
projectId: args.projectId,
|
|
10088
10995
|
runId: args.runId,
|
|
10089
10996
|
agentSessionId: args.agentSessionId,
|
|
@@ -10159,7 +11066,7 @@ function splitAeroAnthropicSystemCachePayload(payload, model) {
|
|
|
10159
11066
|
}
|
|
10160
11067
|
|
|
10161
11068
|
// src/agent/tool-usage.ts
|
|
10162
|
-
import
|
|
11069
|
+
import crypto22 from "crypto";
|
|
10163
11070
|
var AeroToolEventStatuses = {
|
|
10164
11071
|
success: "success",
|
|
10165
11072
|
error: "error"
|
|
@@ -10222,7 +11129,7 @@ function createAeroToolUsageHooks(args) {
|
|
|
10222
11129
|
function recordAgentToolEvent(args) {
|
|
10223
11130
|
try {
|
|
10224
11131
|
args.db.insert(agentToolEvents).values({
|
|
10225
|
-
id:
|
|
11132
|
+
id: crypto22.randomUUID(),
|
|
10226
11133
|
projectId: args.projectId,
|
|
10227
11134
|
agentSessionId: args.agentSessionId,
|
|
10228
11135
|
toolCallId: args.toolCallId,
|
|
@@ -10449,8 +11356,8 @@ async function loadExternalMcpTools(servers, opts = {}) {
|
|
|
10449
11356
|
}
|
|
10450
11357
|
|
|
10451
11358
|
// src/agent/memory-store.ts
|
|
10452
|
-
import
|
|
10453
|
-
import { and as
|
|
11359
|
+
import crypto23 from "crypto";
|
|
11360
|
+
import { and as and15, desc as desc8, eq as eq19, like, sql as sql9 } from "drizzle-orm";
|
|
10454
11361
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
10455
11362
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
10456
11363
|
function rowToDto(row) {
|
|
@@ -10464,7 +11371,7 @@ function rowToDto(row) {
|
|
|
10464
11371
|
};
|
|
10465
11372
|
}
|
|
10466
11373
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
10467
|
-
const query = db.select().from(agentMemory).where(
|
|
11374
|
+
const query = db.select().from(agentMemory).where(eq19(agentMemory.projectId, projectId)).orderBy(desc8(agentMemory.updatedAt));
|
|
10468
11375
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
10469
11376
|
return rows.map(rowToDto);
|
|
10470
11377
|
}
|
|
@@ -10478,7 +11385,7 @@ function upsertMemoryEntry(db, args) {
|
|
|
10478
11385
|
throw new Error(`memory key prefix "${COMPACTION_KEY_PREFIX}" is reserved for compaction notes`);
|
|
10479
11386
|
}
|
|
10480
11387
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
10481
|
-
const id =
|
|
11388
|
+
const id = crypto23.randomUUID();
|
|
10482
11389
|
db.insert(agentMemory).values({
|
|
10483
11390
|
id,
|
|
10484
11391
|
projectId: args.projectId,
|
|
@@ -10495,12 +11402,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
10495
11402
|
updatedAt: now
|
|
10496
11403
|
}
|
|
10497
11404
|
}).run();
|
|
10498
|
-
const row = db.select().from(agentMemory).where(
|
|
11405
|
+
const row = db.select().from(agentMemory).where(and15(eq19(agentMemory.projectId, args.projectId), eq19(agentMemory.key, args.key))).get();
|
|
10499
11406
|
if (!row) throw new Error("memory upsert produced no row");
|
|
10500
11407
|
return rowToDto(row);
|
|
10501
11408
|
}
|
|
10502
11409
|
function deleteMemoryEntry(db, projectId, key) {
|
|
10503
|
-
const result = db.delete(agentMemory).where(
|
|
11410
|
+
const result = db.delete(agentMemory).where(and15(eq19(agentMemory.projectId, projectId), eq19(agentMemory.key, key))).run();
|
|
10504
11411
|
const changes = result.changes ?? 0;
|
|
10505
11412
|
return changes > 0;
|
|
10506
11413
|
}
|
|
@@ -10515,7 +11422,7 @@ function writeCompactionNote(db, args) {
|
|
|
10515
11422
|
}
|
|
10516
11423
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
10517
11424
|
const key = `${COMPACTION_KEY_PREFIX}${args.sessionId}:${now}`;
|
|
10518
|
-
const id =
|
|
11425
|
+
const id = crypto23.randomUUID();
|
|
10519
11426
|
let inserted;
|
|
10520
11427
|
db.transaction((tx) => {
|
|
10521
11428
|
tx.insert(agentMemory).values({
|
|
@@ -10529,16 +11436,16 @@ function writeCompactionNote(db, args) {
|
|
|
10529
11436
|
}).run();
|
|
10530
11437
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
10531
11438
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
10532
|
-
|
|
10533
|
-
|
|
11439
|
+
and15(
|
|
11440
|
+
eq19(agentMemory.projectId, args.projectId),
|
|
10534
11441
|
like(agentMemory.key, `${sessionPrefix}%`)
|
|
10535
11442
|
)
|
|
10536
|
-
).orderBy(
|
|
11443
|
+
).orderBy(desc8(agentMemory.updatedAt)).all();
|
|
10537
11444
|
const stale = existing.slice(COMPACTION_NOTES_PER_SESSION).map((r) => r.id);
|
|
10538
11445
|
if (stale.length > 0) {
|
|
10539
|
-
tx.delete(agentMemory).where(
|
|
11446
|
+
tx.delete(agentMemory).where(sql9`${agentMemory.id} IN (${sql9.join(stale.map((s) => sql9`${s}`), sql9`, `)})`).run();
|
|
10540
11447
|
}
|
|
10541
|
-
const row = tx.select().from(agentMemory).where(
|
|
11448
|
+
const row = tx.select().from(agentMemory).where(and15(eq19(agentMemory.projectId, args.projectId), eq19(agentMemory.key, key))).get();
|
|
10542
11449
|
if (row) inserted = rowToDto(row);
|
|
10543
11450
|
});
|
|
10544
11451
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -10765,7 +11672,7 @@ var SessionRegistry = class {
|
|
|
10765
11672
|
modelProvider: effectiveProvider,
|
|
10766
11673
|
modelId: effectiveModelId,
|
|
10767
11674
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
10768
|
-
}).where(
|
|
11675
|
+
}).where(eq20(agentSessions.projectId, projectId)).run();
|
|
10769
11676
|
}
|
|
10770
11677
|
const agent2 = createAeroSession({
|
|
10771
11678
|
projectName,
|
|
@@ -10794,7 +11701,7 @@ var SessionRegistry = class {
|
|
|
10794
11701
|
}
|
|
10795
11702
|
const { provider, modelId } = resolveSessionProviderAndModel(this.opts.config, preferences);
|
|
10796
11703
|
const systemPrompt = loadAeroSystemPrompt();
|
|
10797
|
-
const sessionId =
|
|
11704
|
+
const sessionId = crypto24.randomUUID();
|
|
10798
11705
|
const agent = createAeroSession({
|
|
10799
11706
|
projectName,
|
|
10800
11707
|
client: this.opts.client,
|
|
@@ -10999,7 +11906,7 @@ ${lines.join("\n")}
|
|
|
10999
11906
|
modelProvider: nextProvider,
|
|
11000
11907
|
modelId: nextModelId,
|
|
11001
11908
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
11002
|
-
}).where(
|
|
11909
|
+
}).where(eq20(agentSessions.projectId, projectId)).run();
|
|
11003
11910
|
}
|
|
11004
11911
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
11005
11912
|
save(projectName) {
|
|
@@ -11163,17 +12070,17 @@ ${lines.join("\n")}
|
|
|
11163
12070
|
return id;
|
|
11164
12071
|
}
|
|
11165
12072
|
tryResolveProjectId(projectName) {
|
|
11166
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
12073
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq20(projects.name, projectName)).get();
|
|
11167
12074
|
return row?.id;
|
|
11168
12075
|
}
|
|
11169
12076
|
loadRow(projectId) {
|
|
11170
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
12077
|
+
const row = this.opts.db.select().from(agentSessions).where(eq20(agentSessions.projectId, projectId)).get();
|
|
11171
12078
|
return row ?? null;
|
|
11172
12079
|
}
|
|
11173
12080
|
insertRow(params) {
|
|
11174
12081
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
11175
12082
|
this.opts.db.insert(agentSessions).values({
|
|
11176
|
-
id: params.id ??
|
|
12083
|
+
id: params.id ?? crypto24.randomUUID(),
|
|
11177
12084
|
projectId: params.projectId,
|
|
11178
12085
|
systemPrompt: params.systemPrompt,
|
|
11179
12086
|
modelProvider: params.provider ?? params.modelProvider ?? AgentProviderIds.claude,
|
|
@@ -11186,14 +12093,14 @@ ${lines.join("\n")}
|
|
|
11186
12093
|
}
|
|
11187
12094
|
updateRow(projectId, patch) {
|
|
11188
12095
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
11189
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
12096
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq20(agentSessions.projectId, projectId)).run();
|
|
11190
12097
|
}
|
|
11191
12098
|
};
|
|
11192
12099
|
|
|
11193
12100
|
// src/agent/agent-routes.ts
|
|
11194
|
-
import { eq as
|
|
12101
|
+
import { eq as eq21 } from "drizzle-orm";
|
|
11195
12102
|
function resolveProject(db, name) {
|
|
11196
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
12103
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq21(projects.name, name)).get();
|
|
11197
12104
|
if (!row) throw notFound("project", name);
|
|
11198
12105
|
return row;
|
|
11199
12106
|
}
|
|
@@ -11202,7 +12109,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
11202
12109
|
"/projects/:name/agent/transcript",
|
|
11203
12110
|
async (request) => {
|
|
11204
12111
|
const project = resolveProject(opts.db, request.params.name);
|
|
11205
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
12112
|
+
const row = opts.db.select().from(agentSessions).where(eq21(agentSessions.projectId, project.id)).get();
|
|
11206
12113
|
if (!row) {
|
|
11207
12114
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
11208
12115
|
}
|
|
@@ -11226,7 +12133,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
11226
12133
|
async (request) => {
|
|
11227
12134
|
const project = resolveProject(opts.db, request.params.name);
|
|
11228
12135
|
opts.sessionRegistry.reset(project.name);
|
|
11229
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
12136
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq21(agentSessions.projectId, project.id)).run();
|
|
11230
12137
|
return { status: "reset" };
|
|
11231
12138
|
}
|
|
11232
12139
|
);
|
|
@@ -11550,7 +12457,7 @@ Your previous reply was not valid JSON. Return ONLY a JSON object with exactly t
|
|
|
11550
12457
|
}
|
|
11551
12458
|
|
|
11552
12459
|
// src/snapshot-service.ts
|
|
11553
|
-
import { runAeoAudit } from "@
|
|
12460
|
+
import { runAeoAudit } from "@canonry/aeo-audit";
|
|
11554
12461
|
|
|
11555
12462
|
// src/site-fetch.ts
|
|
11556
12463
|
import https from "https";
|
|
@@ -12034,7 +12941,7 @@ function buildSnapshotSummary(companyName, queries2, providers, queryResults, au
|
|
|
12034
12941
|
competitorCounts.set(competitor, (competitorCounts.get(competitor) ?? 0) + 1);
|
|
12035
12942
|
}
|
|
12036
12943
|
}
|
|
12037
|
-
const topCompetitors = [...competitorCounts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 10).map(([name,
|
|
12944
|
+
const topCompetitors = [...competitorCounts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 10).map(([name, count2]) => ({ name, count: count2 }));
|
|
12038
12945
|
const defaultMeaning = totalComparisons > 0 ? `${companyName} was mentioned in ${mentionCount}/${totalComparisons} successful provider-query response${totalComparisons === 1 ? "" : "s"} across ${queries2.length} category queries.` : `No successful provider responses were returned across ${queries2.length} category queries.`;
|
|
12039
12946
|
const failureNote = failedComparisons > 0 ? [
|
|
12040
12947
|
`${failedComparisons} provider response${failedComparisons === 1 ? "" : "s"} failed and ${failedComparisons === 1 ? "was" : "were"} excluded from visibility totals.`
|
|
@@ -12169,20 +13076,20 @@ function clipText(value, length) {
|
|
|
12169
13076
|
}
|
|
12170
13077
|
|
|
12171
13078
|
// src/research-runner.ts
|
|
12172
|
-
import { and as
|
|
13079
|
+
import { and as and16, eq as eq22, inArray as inArray7, sql as sql10 } from "drizzle-orm";
|
|
12173
13080
|
var unfinishedResearchQueryStatuses = [ResearchQueryStatuses.queued, ResearchQueryStatuses.running];
|
|
12174
13081
|
function finalResearchRunStatus(completed, failed) {
|
|
12175
13082
|
if (failed === 0) return ResearchRunStatuses.completed;
|
|
12176
13083
|
return completed > 0 ? ResearchRunStatuses.partial : ResearchRunStatuses.failed;
|
|
12177
13084
|
}
|
|
12178
13085
|
async function executeResearchRun(db, registry, runId, projectId) {
|
|
12179
|
-
const run = db.select().from(researchRuns).where(
|
|
13086
|
+
const run = db.select().from(researchRuns).where(and16(eq22(researchRuns.id, runId), eq22(researchRuns.projectId, projectId))).get();
|
|
12180
13087
|
if (!run || run.status !== ResearchRunStatuses.queued) return;
|
|
12181
|
-
const project = db.select().from(projects).where(
|
|
13088
|
+
const project = db.select().from(projects).where(eq22(projects.id, projectId)).get();
|
|
12182
13089
|
if (!project) return;
|
|
12183
13090
|
const provider = registry.get(run.provider);
|
|
12184
13091
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
12185
|
-
const claim = db.update(researchRuns).set({ status: ResearchRunStatuses.running, startedAt: now }).where(
|
|
13092
|
+
const claim = db.update(researchRuns).set({ status: ResearchRunStatuses.running, startedAt: now }).where(and16(eq22(researchRuns.id, runId), eq22(researchRuns.status, ResearchRunStatuses.queued))).run();
|
|
12186
13093
|
if (claim.changes !== 1) return;
|
|
12187
13094
|
let reserved = 0;
|
|
12188
13095
|
let dispatched = 0;
|
|
@@ -12206,8 +13113,8 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
12206
13113
|
}
|
|
12207
13114
|
reserved = run.totalQueries;
|
|
12208
13115
|
reservation = { scope, period };
|
|
12209
|
-
const rows = db.select().from(researchRunQueries).where(
|
|
12210
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
13116
|
+
const rows = db.select().from(researchRunQueries).where(eq22(researchRunQueries.researchRunId, runId)).orderBy(researchRunQueries.position).all();
|
|
13117
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq22(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
12211
13118
|
const domains = effectiveDomains(project);
|
|
12212
13119
|
const brands = effectiveBrandNames(project);
|
|
12213
13120
|
const config = { ...provider.config, model: run.resolvedModel };
|
|
@@ -12215,7 +13122,7 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
12215
13122
|
await mapWithConcurrency(rows, Math.max(1, provider.config.quotaPolicy.maxConcurrency), async (row) => {
|
|
12216
13123
|
try {
|
|
12217
13124
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
12218
|
-
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.running, startedAt }).where(
|
|
13125
|
+
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.running, startedAt }).where(and16(eq22(researchRunQueries.id, row.id), eq22(researchRunQueries.status, ResearchQueryStatuses.queued))).run();
|
|
12219
13126
|
const raw = await gate.run(async () => {
|
|
12220
13127
|
dispatched++;
|
|
12221
13128
|
return provider.adapter.executeTrackedQuery({ query: row.queryText, canonicalDomains: domains, competitorDomains, ...run.location ? { location: run.location } : {} }, config);
|
|
@@ -12242,7 +13149,7 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
12242
13149
|
citationState: determineCitationState(normalized, domains),
|
|
12243
13150
|
rawResponse: raw.rawResponse,
|
|
12244
13151
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
12245
|
-
}).where(
|
|
13152
|
+
}).where(and16(eq22(researchRunQueries.id, row.id), eq22(researchRunQueries.status, ResearchQueryStatuses.running))).run();
|
|
12246
13153
|
if (completed.changes === 1) incrementResearchProgress(db, runId, "completedQueries");
|
|
12247
13154
|
} catch (error) {
|
|
12248
13155
|
try {
|
|
@@ -12266,18 +13173,18 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
12266
13173
|
}
|
|
12267
13174
|
}
|
|
12268
13175
|
function incrementResearchProgress(db, runId, column) {
|
|
12269
|
-
db.update(researchRuns).set({ [column]:
|
|
13176
|
+
db.update(researchRuns).set({ [column]: sql10`${researchRuns[column]} + 1` }).where(eq22(researchRuns.id, runId)).run();
|
|
12270
13177
|
}
|
|
12271
13178
|
function markResearchQueryFailed(db, runId, queryId, error) {
|
|
12272
13179
|
const message = error instanceof Error ? error.message : String(error);
|
|
12273
|
-
const failed = db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error: message, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
13180
|
+
const failed = db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error: message, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and16(eq22(researchRunQueries.id, queryId), inArray7(researchRunQueries.status, unfinishedResearchQueryStatuses))).run();
|
|
12274
13181
|
if (failed.changes === 1) incrementResearchProgress(db, runId, "failedQueries");
|
|
12275
13182
|
}
|
|
12276
13183
|
function markUnfinishedResearchQueriesFailed(db, runId, error) {
|
|
12277
|
-
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
13184
|
+
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and16(eq22(researchRunQueries.researchRunId, runId), inArray7(researchRunQueries.status, unfinishedResearchQueryStatuses))).run();
|
|
12278
13185
|
}
|
|
12279
13186
|
function finalizeResearchRun(db, runId, fatalError) {
|
|
12280
|
-
const rows = db.select({ status: researchRunQueries.status }).from(researchRunQueries).where(
|
|
13187
|
+
const rows = db.select({ status: researchRunQueries.status }).from(researchRunQueries).where(eq22(researchRunQueries.researchRunId, runId)).all();
|
|
12281
13188
|
const completed = rows.filter((row) => row.status === ResearchQueryStatuses.completed).length;
|
|
12282
13189
|
const failed = rows.filter((row) => row.status === ResearchQueryStatuses.failed).length;
|
|
12283
13190
|
db.update(researchRuns).set({
|
|
@@ -12286,11 +13193,11 @@ function finalizeResearchRun(db, runId, fatalError) {
|
|
|
12286
13193
|
failedQueries: failed,
|
|
12287
13194
|
error: fatalError ?? (failed === rows.length ? "Every research query failed." : null),
|
|
12288
13195
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
12289
|
-
}).where(
|
|
13196
|
+
}).where(eq22(researchRuns.id, runId)).run();
|
|
12290
13197
|
}
|
|
12291
13198
|
|
|
12292
13199
|
// src/server.ts
|
|
12293
|
-
var _require3 =
|
|
13200
|
+
var _require3 = createRequire4(import.meta.url);
|
|
12294
13201
|
var { version: PKG_VERSION2 } = _require3("../package.json");
|
|
12295
13202
|
var log20 = createLogger("Server");
|
|
12296
13203
|
var DEFAULT_QUOTA = {
|
|
@@ -12349,14 +13256,14 @@ function summarizeProviderConfig(config) {
|
|
|
12349
13256
|
};
|
|
12350
13257
|
}
|
|
12351
13258
|
function hashApiKey(key) {
|
|
12352
|
-
return
|
|
13259
|
+
return crypto25.createHash("sha256").update(key).digest("hex");
|
|
12353
13260
|
}
|
|
12354
13261
|
var DASHBOARD_SCRYPT_KEYLEN = 64;
|
|
12355
13262
|
var DASHBOARD_SCRYPT_COST = 1 << 15;
|
|
12356
13263
|
var DASHBOARD_SCRYPT_MAXMEM = 64 * 1024 * 1024;
|
|
12357
13264
|
function hashDashboardPassword(password) {
|
|
12358
|
-
const salt =
|
|
12359
|
-
const derived =
|
|
13265
|
+
const salt = crypto25.randomBytes(16);
|
|
13266
|
+
const derived = crypto25.scryptSync(password, salt, DASHBOARD_SCRYPT_KEYLEN, {
|
|
12360
13267
|
N: DASHBOARD_SCRYPT_COST,
|
|
12361
13268
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
12362
13269
|
});
|
|
@@ -12377,14 +13284,14 @@ function verifyDashboardPassword(password, storedHash) {
|
|
|
12377
13284
|
} catch {
|
|
12378
13285
|
return { ok: false, needsRehash: false };
|
|
12379
13286
|
}
|
|
12380
|
-
const derived =
|
|
13287
|
+
const derived = crypto25.scryptSync(password, salt, expected.length, {
|
|
12381
13288
|
N: DASHBOARD_SCRYPT_COST,
|
|
12382
13289
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
12383
13290
|
});
|
|
12384
13291
|
if (derived.length !== expected.length)
|
|
12385
13292
|
return { ok: false, needsRehash: false };
|
|
12386
13293
|
return {
|
|
12387
|
-
ok:
|
|
13294
|
+
ok: crypto25.timingSafeEqual(derived, expected),
|
|
12388
13295
|
needsRehash: false
|
|
12389
13296
|
};
|
|
12390
13297
|
}
|
|
@@ -12393,7 +13300,7 @@ function verifyDashboardPassword(password, storedHash) {
|
|
|
12393
13300
|
const expected = Buffer.from(storedHash, "hex");
|
|
12394
13301
|
if (candidate.length !== expected.length)
|
|
12395
13302
|
return { ok: false, needsRehash: false };
|
|
12396
|
-
const ok =
|
|
13303
|
+
const ok = crypto25.timingSafeEqual(candidate, expected);
|
|
12397
13304
|
return { ok, needsRehash: ok };
|
|
12398
13305
|
}
|
|
12399
13306
|
return { ok: false, needsRehash: false };
|
|
@@ -12601,7 +13508,7 @@ async function createServer(opts) {
|
|
|
12601
13508
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
12602
13509
|
async (ctx) => {
|
|
12603
13510
|
if (!sessionRegistry) return;
|
|
12604
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
13511
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq23(projects.id, ctx.projectId)).get();
|
|
12605
13512
|
if (!project) return;
|
|
12606
13513
|
let content;
|
|
12607
13514
|
if (ctx.kind === RunKinds["aeo-discover-probe"]) {
|
|
@@ -12651,9 +13558,16 @@ async function createServer(opts) {
|
|
|
12651
13558
|
app.log.error({ runId, err }, "Ads sync failed");
|
|
12652
13559
|
});
|
|
12653
13560
|
};
|
|
13561
|
+
const siteAuditAbortControllers = /* @__PURE__ */ new Map();
|
|
12654
13562
|
const runSiteAudit = (runId, projectId, auditOpts) => {
|
|
12655
|
-
|
|
13563
|
+
const controller = new AbortController();
|
|
13564
|
+
siteAuditAbortControllers.set(runId, controller);
|
|
13565
|
+
executeSiteAudit(opts.db, runId, projectId, { ...auditOpts ?? {}, signal: controller.signal }).then(() => runCoordinator.onRunCompleted(runId, projectId)).catch((err) => {
|
|
12656
13566
|
app.log.error({ runId, err }, "Site audit failed");
|
|
13567
|
+
}).finally(() => {
|
|
13568
|
+
if (siteAuditAbortControllers.get(runId) === controller) {
|
|
13569
|
+
siteAuditAbortControllers.delete(runId);
|
|
13570
|
+
}
|
|
12657
13571
|
});
|
|
12658
13572
|
};
|
|
12659
13573
|
const adsCredentialStore = {
|
|
@@ -12944,7 +13858,7 @@ async function createServer(opts) {
|
|
|
12944
13858
|
void (async () => {
|
|
12945
13859
|
try {
|
|
12946
13860
|
const report = await aeroClient.runDoctor({ project: projectName });
|
|
12947
|
-
const project = opts.db.select().from(projects).where(
|
|
13861
|
+
const project = opts.db.select().from(projects).where(eq23(projects.name, projectName)).get();
|
|
12948
13862
|
if (!project) {
|
|
12949
13863
|
app.log.warn({ projectName }, "doctor schedule fired for an unknown project");
|
|
12950
13864
|
return;
|
|
@@ -12972,9 +13886,9 @@ async function createServer(opts) {
|
|
|
12972
13886
|
});
|
|
12973
13887
|
if (!probed) return;
|
|
12974
13888
|
const alreadySynced = opts.db.select().from(ccReleaseSyncs).where(
|
|
12975
|
-
|
|
12976
|
-
|
|
12977
|
-
|
|
13889
|
+
and17(
|
|
13890
|
+
eq23(ccReleaseSyncs.release, probed.release),
|
|
13891
|
+
eq23(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)
|
|
12978
13892
|
)
|
|
12979
13893
|
).limit(1).get();
|
|
12980
13894
|
if (alreadySynced) {
|
|
@@ -13129,7 +14043,7 @@ async function createServer(opts) {
|
|
|
13129
14043
|
return removed;
|
|
13130
14044
|
}
|
|
13131
14045
|
};
|
|
13132
|
-
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ??
|
|
14046
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto25.randomBytes(32).toString("hex");
|
|
13133
14047
|
const googleConnectionStore = {
|
|
13134
14048
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
13135
14049
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -13204,11 +14118,11 @@ async function createServer(opts) {
|
|
|
13204
14118
|
const googlePublicUrl = resolveGooglePublicUrl(opts.config, basePath);
|
|
13205
14119
|
if (opts.config.apiKey) {
|
|
13206
14120
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
13207
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
14121
|
+
const existing = opts.db.select().from(apiKeys).where(eq23(apiKeys.keyHash, keyHash)).get();
|
|
13208
14122
|
if (!existing) {
|
|
13209
14123
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
13210
14124
|
opts.db.insert(apiKeys).values({
|
|
13211
|
-
id: `key_${
|
|
14125
|
+
id: `key_${crypto25.randomBytes(8).toString("hex")}`,
|
|
13212
14126
|
name: "default",
|
|
13213
14127
|
keyHash,
|
|
13214
14128
|
keyPrefix: prefix,
|
|
@@ -13232,7 +14146,7 @@ async function createServer(opts) {
|
|
|
13232
14146
|
};
|
|
13233
14147
|
const createSession = (apiKeyId) => {
|
|
13234
14148
|
pruneExpiredSessions();
|
|
13235
|
-
const sessionId =
|
|
14149
|
+
const sessionId = crypto25.randomBytes(32).toString("hex");
|
|
13236
14150
|
sessions.set(sessionId, {
|
|
13237
14151
|
apiKeyId,
|
|
13238
14152
|
expiresAt: Date.now() + SESSION_TTL_MS
|
|
@@ -13256,7 +14170,7 @@ async function createServer(opts) {
|
|
|
13256
14170
|
};
|
|
13257
14171
|
const getDefaultApiKey = () => {
|
|
13258
14172
|
if (!opts.config.apiKey) return void 0;
|
|
13259
|
-
return opts.db.select().from(apiKeys).where(
|
|
14173
|
+
return opts.db.select().from(apiKeys).where(eq23(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
13260
14174
|
};
|
|
13261
14175
|
const createPasswordSession = (reply) => {
|
|
13262
14176
|
const key = getDefaultApiKey();
|
|
@@ -13280,7 +14194,7 @@ async function createServer(opts) {
|
|
|
13280
14194
|
if (!header) return false;
|
|
13281
14195
|
const parts = header.split(" ");
|
|
13282
14196
|
if (parts.length !== 2 || parts[0] !== "Bearer") return false;
|
|
13283
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
14197
|
+
const key = opts.db.select().from(apiKeys).where(eq23(apiKeys.keyHash, hashApiKey(parts[1]))).get();
|
|
13284
14198
|
return Boolean(key && !key.revokedAt);
|
|
13285
14199
|
};
|
|
13286
14200
|
const namedAccountsInUse = () => anyUsersExist(opts.db);
|
|
@@ -13370,12 +14284,12 @@ async function createServer(opts) {
|
|
|
13370
14284
|
return reply.send({ authenticated: true });
|
|
13371
14285
|
}
|
|
13372
14286
|
if (apiKey) {
|
|
13373
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
14287
|
+
const key = opts.db.select().from(apiKeys).where(eq23(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
13374
14288
|
if (!key || key.revokedAt) {
|
|
13375
14289
|
const err2 = authInvalid();
|
|
13376
14290
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
13377
14291
|
}
|
|
13378
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
14292
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq23(apiKeys.id, key.id)).run();
|
|
13379
14293
|
const sessionId = createSession(key.id);
|
|
13380
14294
|
reply.header(
|
|
13381
14295
|
"set-cookie",
|
|
@@ -13592,7 +14506,7 @@ async function createServer(opts) {
|
|
|
13592
14506
|
deps: {
|
|
13593
14507
|
enqueueAutoExtract: ({ projectId, release: r }) => {
|
|
13594
14508
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
13595
|
-
const runId =
|
|
14509
|
+
const runId = crypto25.randomUUID();
|
|
13596
14510
|
opts.db.insert(runs).values({
|
|
13597
14511
|
id: runId,
|
|
13598
14512
|
projectId,
|
|
@@ -13717,7 +14631,7 @@ async function createServer(opts) {
|
|
|
13717
14631
|
...inspectOpts,
|
|
13718
14632
|
config: opts.config
|
|
13719
14633
|
}).then(() => {
|
|
13720
|
-
const finished = opts.db.select({ status: runs.status }).from(runs).where(
|
|
14634
|
+
const finished = opts.db.select({ status: runs.status }).from(runs).where(eq23(runs.id, runId)).get();
|
|
13721
14635
|
if (finished?.status === RunStatuses.completed || finished?.status === RunStatuses.partial) {
|
|
13722
14636
|
return maybeRefreshGscCoverage(
|
|
13723
14637
|
opts.db,
|
|
@@ -13759,6 +14673,12 @@ async function createServer(opts) {
|
|
|
13759
14673
|
app.log.error({ runId, err }, "Job runner failed");
|
|
13760
14674
|
});
|
|
13761
14675
|
},
|
|
14676
|
+
onRunCancelled: (runId) => {
|
|
14677
|
+
const controller = siteAuditAbortControllers.get(runId);
|
|
14678
|
+
if (controller && !controller.signal.aborted) {
|
|
14679
|
+
controller.abort(new Error("Cancelled by user"));
|
|
14680
|
+
}
|
|
14681
|
+
},
|
|
13762
14682
|
getRunnableProviderNames: () => registry.getAll().map((provider) => provider.adapter.name),
|
|
13763
14683
|
getEffectiveProviderModels: () => effectiveProviderModels(registry),
|
|
13764
14684
|
onProviderUpdate: (providerName, apiKey, model, baseUrl, incomingQuota) => {
|
|
@@ -13819,7 +14739,7 @@ async function createServer(opts) {
|
|
|
13819
14739
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
13820
14740
|
opts.db.insert(auditLog).values(
|
|
13821
14741
|
targetProjectIds.map((projectId) => ({
|
|
13822
|
-
id:
|
|
14742
|
+
id: crypto25.randomUUID(),
|
|
13823
14743
|
projectId,
|
|
13824
14744
|
actor: "api",
|
|
13825
14745
|
action: existing ? "provider.updated" : "provider.created",
|
|
@@ -13957,7 +14877,7 @@ async function createServer(opts) {
|
|
|
13957
14877
|
}
|
|
13958
14878
|
];
|
|
13959
14879
|
},
|
|
13960
|
-
onGenerateQueries: async (providerName,
|
|
14880
|
+
onGenerateQueries: async (providerName, count2, project) => {
|
|
13961
14881
|
const provider = registry.get(providerName);
|
|
13962
14882
|
if (!provider)
|
|
13963
14883
|
throw new Error(`Provider "${providerName}" is not configured`);
|
|
@@ -13969,10 +14889,10 @@ async function createServer(opts) {
|
|
|
13969
14889
|
language: project.language,
|
|
13970
14890
|
existingQueries: project.existingQueries,
|
|
13971
14891
|
siteText,
|
|
13972
|
-
count
|
|
14892
|
+
count: count2
|
|
13973
14893
|
});
|
|
13974
14894
|
const raw = await provider.adapter.generateText(prompt, provider.config);
|
|
13975
|
-
return parseQueryResponse(raw,
|
|
14895
|
+
return parseQueryResponse(raw, count2);
|
|
13976
14896
|
},
|
|
13977
14897
|
onSnapshotRequested: async (input) => {
|
|
13978
14898
|
return snapshotService.createReport(input);
|
|
@@ -14087,7 +15007,7 @@ async function createServer(opts) {
|
|
|
14087
15007
|
}
|
|
14088
15008
|
checkLatestVersionForServer();
|
|
14089
15009
|
scheduler.start();
|
|
14090
|
-
for (const run of opts.db.select({ id: researchRuns.id, projectId: researchRuns.projectId }).from(researchRuns).where(
|
|
15010
|
+
for (const run of opts.db.select({ id: researchRuns.id, projectId: researchRuns.projectId }).from(researchRuns).where(eq23(researchRuns.status, ResearchRunStatuses.queued)).all()) {
|
|
14091
15011
|
dispatchResearchRun(run.id, run.projectId);
|
|
14092
15012
|
}
|
|
14093
15013
|
app.addHook("onClose", async () => {
|
|
@@ -14131,7 +15051,7 @@ function buildQueryGenerationPrompt(ctx) {
|
|
|
14131
15051
|
);
|
|
14132
15052
|
return lines.join("\n");
|
|
14133
15053
|
}
|
|
14134
|
-
function parseQueryResponse(raw,
|
|
15054
|
+
function parseQueryResponse(raw, count2) {
|
|
14135
15055
|
const seen = /* @__PURE__ */ new Set();
|
|
14136
15056
|
const results = [];
|
|
14137
15057
|
for (const line of raw.split("\n")) {
|
|
@@ -14147,7 +15067,7 @@ function parseQueryResponse(raw, count) {
|
|
|
14147
15067
|
if (seen.has(key)) continue;
|
|
14148
15068
|
seen.add(key);
|
|
14149
15069
|
results.push(cleaned);
|
|
14150
|
-
if (results.length >=
|
|
15070
|
+
if (results.length >= count2) break;
|
|
14151
15071
|
}
|
|
14152
15072
|
return results;
|
|
14153
15073
|
}
|