@canonry/canonry 4.155.0 → 4.157.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/assets/agent-workspace/skills/aero/references/orchestration.md +1 -1
- package/assets/agent-workspace/skills/canonry/SKILL.md +9 -4
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +14 -2
- package/assets/agent-workspace/skills/canonry/references/server-side-traffic.md +454 -21
- package/assets/assets/{AuditHistoryPanel-BlbMBxOY.js → AuditHistoryPanel-D12lHs6n.js} +1 -1
- package/assets/assets/{BacklinksPage-DA9EaV9q.js → BacklinksPage-D2ScFYru.js} +1 -1
- package/assets/assets/{HistoryPage-BTrIHjMQ.js → HistoryPage-jZ_I4rwX.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-CN6o5xFg.js → MeasurementPropertyPage-BIdQjXJT.js} +1 -1
- package/assets/assets/ProjectPage-CLvLOOtj.js +15 -0
- package/assets/assets/{RunRow-ch_OlCPs.js → RunRow-Q2X0NR74.js} +1 -1
- package/assets/assets/{RunsPage-T89Y_yS8.js → RunsPage-Cw7YVsbJ.js} +1 -1
- package/assets/assets/{SettingsPage-DnmEq3F9.js → SettingsPage-bFl2EXOD.js} +1 -1
- package/assets/assets/{TrafficPage-CW1UhaJt.js → TrafficPage-DJjhk7Jw.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-r4Ji3wJm.js → TrafficSourceDetailPage-CGKwIu9A.js} +1 -1
- package/assets/assets/{extract-error-message-a8EErBoZ.js → extract-error-message-BHm98tXH.js} +1 -1
- package/assets/assets/index-BxegEwic.css +1 -0
- package/assets/assets/index-VO6nKq5b.js +83 -0
- package/assets/assets/react-sigma_core.esm.min-EzswMBkr.js +312 -0
- package/assets/assets/vendor-lucide-BtYdosYk.js +1 -0
- package/assets/assets/{vendor-markdown-CpDJch4G.js → vendor-markdown-D274_wsI.js} +1 -1
- package/assets/assets/{vendor-radix-DnzzbllM.js → vendor-radix-BxMXSwbC.js} +1 -1
- package/assets/assets/{vendor-recharts-33ACLxsD.js → vendor-recharts-C4ktw_aE.js} +1 -1
- package/assets/assets/{vendor-tanstack-BjS7iY3g.js → vendor-tanstack-pFxw0eGP.js} +1 -1
- package/assets/index.html +7 -7
- package/dist/{chunk-YSESZNVT.js → chunk-AEIKAWOW.js} +795 -239
- package/dist/{chunk-OSHRDV36.js → chunk-KMTRO56O.js} +281 -1
- package/dist/{chunk-VOL2BSM5.js → chunk-MEBL4EJO.js} +3030 -338
- package/dist/{chunk-VIKPFI6F.js → chunk-TRRVBTU5.js} +399 -1
- package/dist/cli.js +1087 -140
- package/dist/index.d.ts +32 -0
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-G4NV3X36.js → intelligence-service-YUUPSJRC.js} +2 -2
- package/dist/mcp.js +4 -4
- package/package.json +13 -10
- package/assets/assets/ProjectPage-CsDbgp_0.js +0 -12
- package/assets/assets/index-BKq6k5Of.js +0 -83
- package/assets/assets/index-DyL54s8x.css +0 -1
- package/assets/assets/vendor-lucide-ByU7vBHt.js +0 -1
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
loadConfig,
|
|
11
11
|
loadConfigRaw,
|
|
12
12
|
saveConfigPatch
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-KMTRO56O.js";
|
|
14
14
|
import {
|
|
15
15
|
CC_CACHE_DIR,
|
|
16
16
|
DUCKDB_SPEC,
|
|
@@ -128,12 +128,15 @@ import {
|
|
|
128
128
|
siteCrawlEdges,
|
|
129
129
|
siteCrawlEventReceipts,
|
|
130
130
|
siteCrawlFindings,
|
|
131
|
+
siteCrawlGraphEdges,
|
|
132
|
+
siteCrawlGraphLayouts,
|
|
133
|
+
siteCrawlGraphNodes,
|
|
131
134
|
siteCrawlPages,
|
|
132
135
|
siteCrawlRunRequests,
|
|
133
136
|
siteCrawlSnapshots,
|
|
134
137
|
toAlertView,
|
|
135
138
|
usageCounters
|
|
136
|
-
} from "./chunk-
|
|
139
|
+
} from "./chunk-MEBL4EJO.js";
|
|
137
140
|
import {
|
|
138
141
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
139
142
|
AGENT_PROVIDER_IDS,
|
|
@@ -162,6 +165,8 @@ import {
|
|
|
162
165
|
SITE_AUDIT_DEFAULT_PAGE_LIMIT,
|
|
163
166
|
SITE_AUDIT_MAX_EDGE_LIMIT,
|
|
164
167
|
SITE_AUDIT_MAX_PAGE_LIMIT,
|
|
168
|
+
SITE_CRAWL_GRAPH_MAX_EDGES,
|
|
169
|
+
SITE_CRAWL_GRAPH_MAX_NODES,
|
|
165
170
|
SKILL_MANIFEST_FILENAME,
|
|
166
171
|
SchedulableRunKinds,
|
|
167
172
|
SkillsClients,
|
|
@@ -231,7 +236,7 @@ import {
|
|
|
231
236
|
validationError,
|
|
232
237
|
winnabilityClassLabel,
|
|
233
238
|
withRetry
|
|
234
|
-
} from "./chunk-
|
|
239
|
+
} from "./chunk-TRRVBTU5.js";
|
|
235
240
|
|
|
236
241
|
// src/telemetry.ts
|
|
237
242
|
import crypto from "crypto";
|
|
@@ -561,12 +566,12 @@ function checkLatestVersionForServer(opts) {
|
|
|
561
566
|
}
|
|
562
567
|
|
|
563
568
|
// src/server.ts
|
|
564
|
-
import { createRequire as
|
|
565
|
-
import
|
|
569
|
+
import { createRequire as createRequire4 } from "module";
|
|
570
|
+
import crypto25 from "crypto";
|
|
566
571
|
import fs8 from "fs";
|
|
567
572
|
import path9 from "path";
|
|
568
573
|
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
569
|
-
import { and as
|
|
574
|
+
import { and as and17, eq as eq23 } from "drizzle-orm";
|
|
570
575
|
import Fastify from "fastify";
|
|
571
576
|
import os5 from "os";
|
|
572
577
|
|
|
@@ -3162,8 +3167,67 @@ function removeVercelTrafficConnection(config, projectName) {
|
|
|
3162
3167
|
return true;
|
|
3163
3168
|
}
|
|
3164
3169
|
|
|
3165
|
-
// src/
|
|
3170
|
+
// src/cloudflare-traffic-config.ts
|
|
3171
|
+
function unsupportedDeliveryMode(deliveryMode) {
|
|
3172
|
+
const label = typeof deliveryMode === "string" ? ` "${deliveryMode}"` : "";
|
|
3173
|
+
return new Error(`Unsupported Cloudflare traffic delivery mode${label}`);
|
|
3174
|
+
}
|
|
3166
3175
|
function ensureConnections6(config) {
|
|
3176
|
+
if (!config.cloudflareTraffic) config.cloudflareTraffic = {};
|
|
3177
|
+
if (!config.cloudflareTraffic.connections) config.cloudflareTraffic.connections = [];
|
|
3178
|
+
return config.cloudflareTraffic.connections;
|
|
3179
|
+
}
|
|
3180
|
+
function normalizeDirectPushConnection(connection) {
|
|
3181
|
+
const deliveryMode = connection.deliveryMode;
|
|
3182
|
+
if (deliveryMode !== void 0 && deliveryMode !== "direct-push") {
|
|
3183
|
+
throw unsupportedDeliveryMode(deliveryMode);
|
|
3184
|
+
}
|
|
3185
|
+
if (deliveryMode === "direct-push") return connection;
|
|
3186
|
+
return { ...connection, deliveryMode: "direct-push" };
|
|
3187
|
+
}
|
|
3188
|
+
function getCloudflareTrafficConnection(config, projectName) {
|
|
3189
|
+
const connection = (config.cloudflareTraffic?.connections ?? []).find((c) => c.projectName === projectName);
|
|
3190
|
+
return connection ? normalizeDirectPushConnection(connection) : void 0;
|
|
3191
|
+
}
|
|
3192
|
+
function getCloudflareTrafficConnectionBySourceId(config, sourceId) {
|
|
3193
|
+
const connection = (config.cloudflareTraffic?.connections ?? []).find((c) => c.sourceId === sourceId);
|
|
3194
|
+
return connection ? normalizeDirectPushConnection(connection) : void 0;
|
|
3195
|
+
}
|
|
3196
|
+
function upsertCloudflareTrafficConnection(config, connection) {
|
|
3197
|
+
const deliveryMode = connection.deliveryMode;
|
|
3198
|
+
if (deliveryMode !== "direct-push") {
|
|
3199
|
+
throw unsupportedDeliveryMode(deliveryMode);
|
|
3200
|
+
}
|
|
3201
|
+
const connections = ensureConnections6(config);
|
|
3202
|
+
const index = connections.findIndex((c) => c.projectName === connection.projectName);
|
|
3203
|
+
if (index === -1) {
|
|
3204
|
+
connections.push(connection);
|
|
3205
|
+
return connection;
|
|
3206
|
+
}
|
|
3207
|
+
connections[index] = connection;
|
|
3208
|
+
return connection;
|
|
3209
|
+
}
|
|
3210
|
+
function removeCloudflareTrafficConnection(config, projectName) {
|
|
3211
|
+
const connections = config.cloudflareTraffic?.connections;
|
|
3212
|
+
if (!connections?.length) return false;
|
|
3213
|
+
const next = connections.filter((c) => c.projectName !== projectName);
|
|
3214
|
+
if (next.length === connections.length) return false;
|
|
3215
|
+
if (!config.cloudflareTraffic) return false;
|
|
3216
|
+
config.cloudflareTraffic.connections = next;
|
|
3217
|
+
if (next.length === 0) {
|
|
3218
|
+
delete config.cloudflareTraffic;
|
|
3219
|
+
}
|
|
3220
|
+
return true;
|
|
3221
|
+
}
|
|
3222
|
+
|
|
3223
|
+
// src/cloudflare-ingest-url.ts
|
|
3224
|
+
function buildCloudflareIngestUrlTemplate(config) {
|
|
3225
|
+
const base = (config.publicUrl ?? config.apiUrl).replace(/\/+$/, "");
|
|
3226
|
+
return `${base}/api/v1/projects/{name}/traffic/cloudflare/ingest`;
|
|
3227
|
+
}
|
|
3228
|
+
|
|
3229
|
+
// src/wordpress-config.ts
|
|
3230
|
+
function ensureConnections7(config) {
|
|
3167
3231
|
if (!config.wordpress) config.wordpress = {};
|
|
3168
3232
|
if (!config.wordpress.connections) config.wordpress.connections = [];
|
|
3169
3233
|
return config.wordpress.connections;
|
|
@@ -3180,7 +3244,7 @@ function getWordpressConnection(config, projectName) {
|
|
|
3180
3244
|
return (config.wordpress?.connections ?? []).find((connection) => connection.projectName === projectName);
|
|
3181
3245
|
}
|
|
3182
3246
|
function upsertWordpressConnection(config, connection) {
|
|
3183
|
-
const connections =
|
|
3247
|
+
const connections = ensureConnections7(config);
|
|
3184
3248
|
const normalized = normalizeConnection(connection);
|
|
3185
3249
|
const index = connections.findIndex((entry) => entry.projectName === connection.projectName);
|
|
3186
3250
|
if (index === -1) {
|
|
@@ -4144,7 +4208,7 @@ var JobRunner = class {
|
|
|
4144
4208
|
}
|
|
4145
4209
|
}
|
|
4146
4210
|
}
|
|
4147
|
-
incrementUsage(scope, metric,
|
|
4211
|
+
incrementUsage(scope, metric, count2) {
|
|
4148
4212
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4149
4213
|
const period = now.slice(0, 10);
|
|
4150
4214
|
this.db.insert(usageCounters).values({
|
|
@@ -4152,11 +4216,11 @@ var JobRunner = class {
|
|
|
4152
4216
|
scope,
|
|
4153
4217
|
period,
|
|
4154
4218
|
metric,
|
|
4155
|
-
count,
|
|
4219
|
+
count: count2,
|
|
4156
4220
|
updatedAt: now
|
|
4157
4221
|
}).onConflictDoUpdate({
|
|
4158
4222
|
target: [usageCounters.scope, usageCounters.period, usageCounters.metric],
|
|
4159
|
-
set: { count: sql2`${usageCounters.count} + ${
|
|
4223
|
+
set: { count: sql2`${usageCounters.count} + ${count2}`, updatedAt: now }
|
|
4160
4224
|
}).run();
|
|
4161
4225
|
}
|
|
4162
4226
|
flushProviderUsage(providerDispatchCounts, providerReservations) {
|
|
@@ -5285,7 +5349,7 @@ async function getAdGroupInsights(apiKey, adGroupId, opts) {
|
|
|
5285
5349
|
}
|
|
5286
5350
|
|
|
5287
5351
|
// src/ads-config.ts
|
|
5288
|
-
function
|
|
5352
|
+
function ensureConnections8(config) {
|
|
5289
5353
|
if (!config.openaiAds) config.openaiAds = {};
|
|
5290
5354
|
if (!config.openaiAds.connections) config.openaiAds.connections = [];
|
|
5291
5355
|
return config.openaiAds.connections;
|
|
@@ -5294,7 +5358,7 @@ function getOpenAiAdsConnection(config, projectName) {
|
|
|
5294
5358
|
return (config.openaiAds?.connections ?? []).find((c) => c.projectName === projectName);
|
|
5295
5359
|
}
|
|
5296
5360
|
function upsertOpenAiAdsConnection(config, connection) {
|
|
5297
|
-
const connections =
|
|
5361
|
+
const connections = ensureConnections8(config);
|
|
5298
5362
|
const index = connections.findIndex((c) => c.projectName === connection.projectName);
|
|
5299
5363
|
if (index === -1) {
|
|
5300
5364
|
connections.push(connection);
|
|
@@ -6930,9 +6994,421 @@ function buildDiscoveryInsightTitle(input) {
|
|
|
6930
6994
|
}
|
|
6931
6995
|
|
|
6932
6996
|
// src/execute-site-audit.ts
|
|
6933
|
-
import
|
|
6934
|
-
import { and as
|
|
6997
|
+
import crypto17 from "crypto";
|
|
6998
|
+
import { and as and11, eq as eq14, sql as sql6 } from "drizzle-orm";
|
|
6935
6999
|
import { runSiteCrawl } from "@canonry/aeo-audit";
|
|
7000
|
+
|
|
7001
|
+
// src/site-crawl-graph-layout.ts
|
|
7002
|
+
import crypto16 from "crypto";
|
|
7003
|
+
import { createRequire as createRequire3 } from "module";
|
|
7004
|
+
import { Worker } from "worker_threads";
|
|
7005
|
+
import { and as and10, asc, count, desc as desc6, eq as eq13, isNull, ne as ne2, or, sql as sql5 } from "drizzle-orm";
|
|
7006
|
+
import { alias } from "drizzle-orm/sqlite-core";
|
|
7007
|
+
var SITE_CRAWL_GRAPH_LAYOUT_VERSION = "site-health-fa2-v1";
|
|
7008
|
+
var SITE_CRAWL_GRAPH_LAYOUT_TIMEOUT_MS = 15e3;
|
|
7009
|
+
var SITE_CRAWL_GRAPH_SEED_SCALE = 100;
|
|
7010
|
+
var graphSourcePage = alias(siteCrawlPages, "layout_graph_source_page");
|
|
7011
|
+
var graphTargetPage = alias(siteCrawlPages, "layout_graph_target_page");
|
|
7012
|
+
var FORCE_ATLAS_SETTINGS = Object.freeze({
|
|
7013
|
+
adjustSizes: false,
|
|
7014
|
+
barnesHutOptimize: true,
|
|
7015
|
+
barnesHutTheta: 0.5,
|
|
7016
|
+
edgeWeightInfluence: 0.5,
|
|
7017
|
+
gravity: 1,
|
|
7018
|
+
linLogMode: false,
|
|
7019
|
+
outboundAttractionDistribution: false,
|
|
7020
|
+
scalingRatio: 10,
|
|
7021
|
+
slowDown: 10,
|
|
7022
|
+
strongGravityMode: false
|
|
7023
|
+
});
|
|
7024
|
+
var WORKER_SOURCE = String.raw`
|
|
7025
|
+
const {parentPort, workerData} = require('node:worker_threads');
|
|
7026
|
+
try {
|
|
7027
|
+
const graphologyModule = require(workerData.graphologyPath);
|
|
7028
|
+
const forceAtlasModule = require(workerData.forceAtlasPath);
|
|
7029
|
+
const MultiDirectedGraph = graphologyModule.MultiDirectedGraph;
|
|
7030
|
+
const forceAtlas2 = forceAtlasModule.default || forceAtlasModule;
|
|
7031
|
+
const graph = new MultiDirectedGraph({allowSelfLoops: true});
|
|
7032
|
+
for (const node of workerData.input.nodes) {
|
|
7033
|
+
graph.addNode(node.nodeKey, {x: node.x, y: node.y});
|
|
7034
|
+
}
|
|
7035
|
+
for (const edge of workerData.input.edges) {
|
|
7036
|
+
graph.addDirectedEdgeWithKey(edge.edgeKey, edge.sourceNodeKey, edge.targetNodeKey, {
|
|
7037
|
+
weight: 1 + Math.log2(Math.max(1, edge.occurrences)),
|
|
7038
|
+
});
|
|
7039
|
+
}
|
|
7040
|
+
if (graph.order > 1 && graph.size > 0) {
|
|
7041
|
+
forceAtlas2.assign(graph, {
|
|
7042
|
+
iterations: workerData.input.iterations,
|
|
7043
|
+
settings: workerData.settings,
|
|
7044
|
+
});
|
|
7045
|
+
}
|
|
7046
|
+
const positions = [];
|
|
7047
|
+
graph.forEachNode((nodeKey, attributes) => {
|
|
7048
|
+
positions.push({nodeKey, x: attributes.x, y: attributes.y});
|
|
7049
|
+
});
|
|
7050
|
+
parentPort.postMessage({ok: true, positions});
|
|
7051
|
+
} catch (error) {
|
|
7052
|
+
parentPort.postMessage({
|
|
7053
|
+
ok: false,
|
|
7054
|
+
error: error && typeof error.message === 'string' ? error.message : String(error),
|
|
7055
|
+
});
|
|
7056
|
+
}
|
|
7057
|
+
`;
|
|
7058
|
+
var GraphLayoutTimeoutError = class extends Error {
|
|
7059
|
+
name = "GraphLayoutTimeoutError";
|
|
7060
|
+
};
|
|
7061
|
+
function hashUnit(value) {
|
|
7062
|
+
let hash = 2166136261;
|
|
7063
|
+
for (let i = 0; i < value.length; i++) {
|
|
7064
|
+
hash ^= value.charCodeAt(i);
|
|
7065
|
+
hash = Math.imul(hash, 16777619);
|
|
7066
|
+
}
|
|
7067
|
+
return (hash >>> 0) / 4294967296;
|
|
7068
|
+
}
|
|
7069
|
+
function pathSection(path10) {
|
|
7070
|
+
return path10.split("/").filter(Boolean)[0] ?? "__root__";
|
|
7071
|
+
}
|
|
7072
|
+
function seedSiteCrawlGraphNodes(nodes, rootNodeKey, priorPositions) {
|
|
7073
|
+
return [...nodes].sort((a, b) => a.sampleRank - b.sampleRank || a.nodeKey.localeCompare(b.nodeKey)).map((node) => {
|
|
7074
|
+
const prior = priorPositions?.get(node.nodeKey);
|
|
7075
|
+
if (prior && Number.isFinite(prior.x) && Number.isFinite(prior.y)) {
|
|
7076
|
+
return {
|
|
7077
|
+
...node,
|
|
7078
|
+
x: prior.x * SITE_CRAWL_GRAPH_SEED_SCALE,
|
|
7079
|
+
y: prior.y * SITE_CRAWL_GRAPH_SEED_SCALE
|
|
7080
|
+
};
|
|
7081
|
+
}
|
|
7082
|
+
if (node.nodeKey === rootNodeKey) return { ...node, x: 0, y: 0 };
|
|
7083
|
+
const section = pathSection(node.path);
|
|
7084
|
+
const sectionAngle = hashUnit(`section:${section}`) * Math.PI * 2;
|
|
7085
|
+
const sectionRadius = section === "__root__" ? 24 : 100;
|
|
7086
|
+
const clusterX = Math.cos(sectionAngle) * sectionRadius;
|
|
7087
|
+
const clusterY = Math.sin(sectionAngle) * sectionRadius;
|
|
7088
|
+
const localAngle = hashUnit(`node-angle:${node.nodeKey}`) * Math.PI * 2;
|
|
7089
|
+
const depth = Math.max(0, Math.min(12, node.depth ?? 4));
|
|
7090
|
+
const localRadius = 4 + depth * 2.5 + Math.sqrt(hashUnit(`node-radius:${node.nodeKey}`)) * 14;
|
|
7091
|
+
return {
|
|
7092
|
+
...node,
|
|
7093
|
+
x: clusterX + Math.cos(localAngle) * localRadius,
|
|
7094
|
+
y: clusterY + Math.sin(localAngle) * localRadius
|
|
7095
|
+
};
|
|
7096
|
+
});
|
|
7097
|
+
}
|
|
7098
|
+
function adaptiveForceAtlasIterations(nodeCount, edgeCount) {
|
|
7099
|
+
if (nodeCount <= 100 && edgeCount <= 500) return 100;
|
|
7100
|
+
if (nodeCount <= 500 && edgeCount <= 2500) return 70;
|
|
7101
|
+
if (nodeCount <= 2e3 && edgeCount <= 1e4) return 40;
|
|
7102
|
+
if (nodeCount <= 5e3 && edgeCount <= 2e4) return 24;
|
|
7103
|
+
if (nodeCount <= 1e4 && edgeCount <= 35e3) return 12;
|
|
7104
|
+
return 6;
|
|
7105
|
+
}
|
|
7106
|
+
async function computeForceAtlasPositions(input, options = {}) {
|
|
7107
|
+
if (input.nodes.length <= 1 || input.edges.length === 0) {
|
|
7108
|
+
return input.nodes.map(({ nodeKey, x, y }) => ({ nodeKey, x, y }));
|
|
7109
|
+
}
|
|
7110
|
+
const requireFromPackage = createRequire3(import.meta.url);
|
|
7111
|
+
const worker = new Worker(WORKER_SOURCE, {
|
|
7112
|
+
eval: true,
|
|
7113
|
+
workerData: {
|
|
7114
|
+
graphologyPath: requireFromPackage.resolve("graphology"),
|
|
7115
|
+
forceAtlasPath: requireFromPackage.resolve("graphology-layout-forceatlas2"),
|
|
7116
|
+
input,
|
|
7117
|
+
settings: FORCE_ATLAS_SETTINGS
|
|
7118
|
+
}
|
|
7119
|
+
});
|
|
7120
|
+
const timeoutMs = options.timeoutMs ?? SITE_CRAWL_GRAPH_LAYOUT_TIMEOUT_MS;
|
|
7121
|
+
return await new Promise((resolve, reject) => {
|
|
7122
|
+
let settled = false;
|
|
7123
|
+
const finish = (result) => {
|
|
7124
|
+
if (settled) return;
|
|
7125
|
+
settled = true;
|
|
7126
|
+
clearTimeout(timer);
|
|
7127
|
+
options.signal?.removeEventListener("abort", onAbort);
|
|
7128
|
+
void worker.terminate();
|
|
7129
|
+
if (result.error) reject(result.error instanceof Error ? result.error : new Error("Graph layout failed with a non-Error reason", { cause: result.error }));
|
|
7130
|
+
else resolve(result.positions ?? []);
|
|
7131
|
+
};
|
|
7132
|
+
const onAbort = () => finish({ error: options.signal?.reason ?? new DOMException("Layout aborted", "AbortError") });
|
|
7133
|
+
const timer = setTimeout(() => finish({ error: new GraphLayoutTimeoutError(`Graph layout exceeded ${timeoutMs}ms`) }), timeoutMs);
|
|
7134
|
+
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
7135
|
+
if (options.signal?.aborted) return onAbort();
|
|
7136
|
+
worker.once("message", (message) => {
|
|
7137
|
+
if (!message.ok) finish({ error: new Error(message.error ?? "Graph layout worker failed") });
|
|
7138
|
+
else finish({ positions: message.positions });
|
|
7139
|
+
});
|
|
7140
|
+
worker.once("error", (error) => finish({ error }));
|
|
7141
|
+
worker.once("exit", (code) => {
|
|
7142
|
+
if (code !== 0 && !settled) finish({ error: new Error(`Graph layout worker exited with code ${code}`) });
|
|
7143
|
+
});
|
|
7144
|
+
});
|
|
7145
|
+
}
|
|
7146
|
+
function normalizePositions(seededNodes, computed, rootNodeKey) {
|
|
7147
|
+
const byKey = new Map(computed.map((position) => [position.nodeKey, position]));
|
|
7148
|
+
if (byKey.size !== seededNodes.length) throw new Error("Graph layout returned an incomplete node set");
|
|
7149
|
+
const root = rootNodeKey ? byKey.get(rootNodeKey) : null;
|
|
7150
|
+
const centerX = root?.x ?? computed.reduce((sum, position) => sum + position.x, 0) / Math.max(1, computed.length);
|
|
7151
|
+
const centerY = root?.y ?? computed.reduce((sum, position) => sum + position.y, 0) / Math.max(1, computed.length);
|
|
7152
|
+
if (!Number.isFinite(centerX) || !Number.isFinite(centerY)) throw new Error("Graph layout returned a non-finite center");
|
|
7153
|
+
let scale = 0;
|
|
7154
|
+
for (const position of computed) {
|
|
7155
|
+
if (!Number.isFinite(position.x) || !Number.isFinite(position.y)) throw new Error("Graph layout returned non-finite coordinates");
|
|
7156
|
+
scale = Math.max(scale, Math.abs(position.x - centerX), Math.abs(position.y - centerY));
|
|
7157
|
+
}
|
|
7158
|
+
scale = scale > 0 ? scale : 1;
|
|
7159
|
+
return seededNodes.map((node) => {
|
|
7160
|
+
const position = byKey.get(node.nodeKey);
|
|
7161
|
+
if (!position) throw new Error(`Graph layout omitted ${node.nodeKey}`);
|
|
7162
|
+
return {
|
|
7163
|
+
nodeKey: node.nodeKey,
|
|
7164
|
+
sampleRank: node.sampleRank,
|
|
7165
|
+
x: Math.round((position.x - centerX) / scale * 1e6) / 1e6,
|
|
7166
|
+
y: Math.round((position.y - centerY) / scale * 1e6) / 1e6
|
|
7167
|
+
};
|
|
7168
|
+
});
|
|
7169
|
+
}
|
|
7170
|
+
async function layoutSiteCrawlGraphInput(input, options = {}) {
|
|
7171
|
+
options.signal?.throwIfAborted();
|
|
7172
|
+
if (input.nodes.length === 0) {
|
|
7173
|
+
return {
|
|
7174
|
+
state: "unavailable",
|
|
7175
|
+
failureCode: "empty-crawl",
|
|
7176
|
+
totalNodes: input.totalNodes,
|
|
7177
|
+
totalEdges: input.totalEdges,
|
|
7178
|
+
nodeCount: 0,
|
|
7179
|
+
edgeCount: 0,
|
|
7180
|
+
nodes: [],
|
|
7181
|
+
edges: []
|
|
7182
|
+
};
|
|
7183
|
+
}
|
|
7184
|
+
const seeded = seedSiteCrawlGraphNodes(input.nodes, input.rootNodeKey, input.priorPositions);
|
|
7185
|
+
try {
|
|
7186
|
+
const compute = options.computePositions ?? computeForceAtlasPositions;
|
|
7187
|
+
const positions = await compute({
|
|
7188
|
+
nodes: seeded,
|
|
7189
|
+
edges: input.edges,
|
|
7190
|
+
iterations: adaptiveForceAtlasIterations(input.nodes.length, input.edges.length)
|
|
7191
|
+
}, { signal: options.signal, timeoutMs: options.timeoutMs });
|
|
7192
|
+
options.signal?.throwIfAborted();
|
|
7193
|
+
return {
|
|
7194
|
+
state: "ready",
|
|
7195
|
+
layoutVersion: SITE_CRAWL_GRAPH_LAYOUT_VERSION,
|
|
7196
|
+
totalNodes: input.totalNodes,
|
|
7197
|
+
totalEdges: input.totalEdges,
|
|
7198
|
+
nodeCount: input.nodes.length,
|
|
7199
|
+
edgeCount: input.edges.length,
|
|
7200
|
+
nodes: normalizePositions(seeded, positions, input.rootNodeKey),
|
|
7201
|
+
edges: input.edges.map((edge, sampleRank) => ({ ...edge, sampleRank }))
|
|
7202
|
+
};
|
|
7203
|
+
} catch (error) {
|
|
7204
|
+
if (options.signal?.aborted) options.signal.throwIfAborted();
|
|
7205
|
+
if (error instanceof DOMException && error.name === "AbortError") throw error;
|
|
7206
|
+
return {
|
|
7207
|
+
state: "unavailable",
|
|
7208
|
+
failureCode: error instanceof GraphLayoutTimeoutError ? "layout-timeout" : "layout-error",
|
|
7209
|
+
totalNodes: input.totalNodes,
|
|
7210
|
+
totalEdges: input.totalEdges,
|
|
7211
|
+
nodeCount: 0,
|
|
7212
|
+
edgeCount: 0,
|
|
7213
|
+
nodes: [],
|
|
7214
|
+
edges: []
|
|
7215
|
+
};
|
|
7216
|
+
}
|
|
7217
|
+
}
|
|
7218
|
+
function loadPriorSiteCrawlGraphPositions(db, scope) {
|
|
7219
|
+
const prior = db.select({
|
|
7220
|
+
runId: siteCrawlSnapshots.runId,
|
|
7221
|
+
attemptId: siteCrawlSnapshots.attemptId
|
|
7222
|
+
}).from(siteCrawlSnapshots).innerJoin(siteCrawlGraphLayouts, and10(
|
|
7223
|
+
eq13(siteCrawlGraphLayouts.projectId, siteCrawlSnapshots.projectId),
|
|
7224
|
+
eq13(siteCrawlGraphLayouts.runId, siteCrawlSnapshots.runId),
|
|
7225
|
+
eq13(siteCrawlGraphLayouts.attemptId, siteCrawlSnapshots.attemptId)
|
|
7226
|
+
)).where(and10(
|
|
7227
|
+
eq13(siteCrawlSnapshots.projectId, scope.projectId),
|
|
7228
|
+
ne2(siteCrawlSnapshots.runId, scope.runId),
|
|
7229
|
+
eq13(siteCrawlSnapshots.complete, true),
|
|
7230
|
+
eq13(siteCrawlGraphLayouts.state, "ready"),
|
|
7231
|
+
eq13(siteCrawlGraphLayouts.layoutVersion, SITE_CRAWL_GRAPH_LAYOUT_VERSION)
|
|
7232
|
+
)).orderBy(desc6(siteCrawlSnapshots.createdAt), desc6(siteCrawlSnapshots.id)).limit(1).get();
|
|
7233
|
+
if (!prior?.attemptId) return /* @__PURE__ */ new Map();
|
|
7234
|
+
return new Map(db.select({
|
|
7235
|
+
nodeKey: siteCrawlGraphNodes.nodeKey,
|
|
7236
|
+
x: siteCrawlGraphNodes.x,
|
|
7237
|
+
y: siteCrawlGraphNodes.y
|
|
7238
|
+
}).from(siteCrawlGraphNodes).where(and10(
|
|
7239
|
+
eq13(siteCrawlGraphNodes.projectId, scope.projectId),
|
|
7240
|
+
eq13(siteCrawlGraphNodes.runId, prior.runId),
|
|
7241
|
+
eq13(siteCrawlGraphNodes.attemptId, prior.attemptId)
|
|
7242
|
+
)).all().map((node) => [node.nodeKey, { x: node.x, y: node.y }]));
|
|
7243
|
+
}
|
|
7244
|
+
async function prepareSiteCrawlGraphLayout(db, scope, options = {}) {
|
|
7245
|
+
let totalNodes = 0;
|
|
7246
|
+
let totalEdges = 0;
|
|
7247
|
+
try {
|
|
7248
|
+
const assertNotAborted = () => options.signal?.throwIfAborted();
|
|
7249
|
+
assertNotAborted();
|
|
7250
|
+
const pageScope = [
|
|
7251
|
+
eq13(siteCrawlPages.projectId, scope.projectId),
|
|
7252
|
+
eq13(siteCrawlPages.runId, scope.runId),
|
|
7253
|
+
eq13(siteCrawlPages.attemptId, scope.attemptId)
|
|
7254
|
+
];
|
|
7255
|
+
const edgeScope = [
|
|
7256
|
+
eq13(siteCrawlEdges.projectId, scope.projectId),
|
|
7257
|
+
eq13(siteCrawlEdges.runId, scope.runId),
|
|
7258
|
+
eq13(siteCrawlEdges.attemptId, scope.attemptId),
|
|
7259
|
+
eq13(siteCrawlEdges.internal, true),
|
|
7260
|
+
eq13(siteCrawlEdges.relation, "anchor")
|
|
7261
|
+
];
|
|
7262
|
+
totalNodes = db.select({ value: count() }).from(siteCrawlPages).where(and10(...pageScope)).get()?.value ?? 0;
|
|
7263
|
+
assertNotAborted();
|
|
7264
|
+
totalEdges = db.select({ value: count() }).from(siteCrawlEdges).innerJoin(graphSourcePage, and10(
|
|
7265
|
+
eq13(graphSourcePage.projectId, scope.projectId),
|
|
7266
|
+
eq13(graphSourcePage.runId, scope.runId),
|
|
7267
|
+
eq13(graphSourcePage.attemptId, scope.attemptId),
|
|
7268
|
+
eq13(graphSourcePage.nodeKey, siteCrawlEdges.sourceNodeKey)
|
|
7269
|
+
)).innerJoin(graphTargetPage, and10(
|
|
7270
|
+
eq13(graphTargetPage.projectId, scope.projectId),
|
|
7271
|
+
eq13(graphTargetPage.runId, scope.runId),
|
|
7272
|
+
eq13(graphTargetPage.attemptId, scope.attemptId),
|
|
7273
|
+
eq13(graphTargetPage.nodeKey, siteCrawlEdges.targetNodeKey)
|
|
7274
|
+
)).where(and10(...edgeScope)).get()?.value ?? 0;
|
|
7275
|
+
assertNotAborted();
|
|
7276
|
+
const graphSeedColumns = {
|
|
7277
|
+
nodeKey: siteCrawlPages.nodeKey,
|
|
7278
|
+
path: siteCrawlPages.path,
|
|
7279
|
+
depth: siteCrawlPages.depth
|
|
7280
|
+
};
|
|
7281
|
+
const rootRows = db.select(graphSeedColumns).from(siteCrawlPages).where(and10(...pageScope, eq13(siteCrawlPages.url, scope.rootUrl))).orderBy(asc(siteCrawlPages.nodeKey)).limit(1).all();
|
|
7282
|
+
assertNotAborted();
|
|
7283
|
+
const retainedRows = [...rootRows];
|
|
7284
|
+
if (retainedRows.length < SITE_CRAWL_GRAPH_MAX_NODES) {
|
|
7285
|
+
retainedRows.push(...db.select(graphSeedColumns).from(siteCrawlPages).where(and10(...pageScope, eq13(siteCrawlPages.depth, 0), ne2(siteCrawlPages.url, scope.rootUrl))).orderBy(asc(siteCrawlPages.nodeKey)).limit(SITE_CRAWL_GRAPH_MAX_NODES - retainedRows.length).all());
|
|
7286
|
+
}
|
|
7287
|
+
assertNotAborted();
|
|
7288
|
+
if (retainedRows.length < SITE_CRAWL_GRAPH_MAX_NODES) {
|
|
7289
|
+
retainedRows.push(...db.select(graphSeedColumns).from(siteCrawlPages).where(and10(
|
|
7290
|
+
...pageScope,
|
|
7291
|
+
or(isNull(siteCrawlPages.depth), ne2(siteCrawlPages.depth, 0)),
|
|
7292
|
+
ne2(siteCrawlPages.url, scope.rootUrl)
|
|
7293
|
+
)).orderBy(desc6(siteCrawlPages.linkScoreNormalized), asc(siteCrawlPages.nodeKey)).limit(SITE_CRAWL_GRAPH_MAX_NODES - retainedRows.length).all());
|
|
7294
|
+
}
|
|
7295
|
+
assertNotAborted();
|
|
7296
|
+
const nodes = retainedRows.map((row, sampleRank) => ({
|
|
7297
|
+
nodeKey: row.nodeKey,
|
|
7298
|
+
path: row.path,
|
|
7299
|
+
depth: row.depth,
|
|
7300
|
+
sampleRank
|
|
7301
|
+
}));
|
|
7302
|
+
const rootNodeKey = rootRows[0]?.nodeKey ?? null;
|
|
7303
|
+
const priorPositions = loadPriorSiteCrawlGraphPositions(db, scope);
|
|
7304
|
+
assertNotAborted();
|
|
7305
|
+
const selectedKeys = JSON.stringify(nodes.map((node) => node.nodeKey));
|
|
7306
|
+
const edgeRows = nodes.length === 0 ? [] : db.all(sql5`
|
|
7307
|
+
WITH selected(node_key) AS (SELECT value FROM json_each(${selectedKeys}))
|
|
7308
|
+
SELECT
|
|
7309
|
+
edge.edge_key AS edgeKey,
|
|
7310
|
+
edge.source_node_key AS sourceNodeKey,
|
|
7311
|
+
edge.target_node_key AS targetNodeKey,
|
|
7312
|
+
edge.followable AS followable,
|
|
7313
|
+
edge.occurrences AS occurrences
|
|
7314
|
+
FROM site_crawl_edges AS edge
|
|
7315
|
+
INNER JOIN selected AS source ON source.node_key = edge.source_node_key
|
|
7316
|
+
INNER JOIN selected AS target ON target.node_key = edge.target_node_key
|
|
7317
|
+
WHERE edge.project_id = ${scope.projectId}
|
|
7318
|
+
AND edge.run_id = ${scope.runId}
|
|
7319
|
+
AND edge.attempt_id = ${scope.attemptId}
|
|
7320
|
+
AND edge.internal = 1
|
|
7321
|
+
AND edge.relation = 'anchor'
|
|
7322
|
+
ORDER BY edge.occurrences DESC, edge.edge_key ASC
|
|
7323
|
+
LIMIT ${SITE_CRAWL_GRAPH_MAX_EDGES}
|
|
7324
|
+
`);
|
|
7325
|
+
assertNotAborted();
|
|
7326
|
+
return await layoutSiteCrawlGraphInput({
|
|
7327
|
+
totalNodes,
|
|
7328
|
+
totalEdges,
|
|
7329
|
+
nodes,
|
|
7330
|
+
edges: edgeRows,
|
|
7331
|
+
rootNodeKey,
|
|
7332
|
+
priorPositions
|
|
7333
|
+
}, options);
|
|
7334
|
+
} catch (error) {
|
|
7335
|
+
if (options.signal?.aborted) options.signal.throwIfAborted();
|
|
7336
|
+
if (error instanceof DOMException && error.name === "AbortError") throw error;
|
|
7337
|
+
return {
|
|
7338
|
+
state: "unavailable",
|
|
7339
|
+
failureCode: "layout-error",
|
|
7340
|
+
totalNodes,
|
|
7341
|
+
totalEdges,
|
|
7342
|
+
nodeCount: 0,
|
|
7343
|
+
edgeCount: 0,
|
|
7344
|
+
nodes: [],
|
|
7345
|
+
edges: []
|
|
7346
|
+
};
|
|
7347
|
+
}
|
|
7348
|
+
}
|
|
7349
|
+
function persistSiteCrawlGraphLayout(db, scope, layout, now = (/* @__PURE__ */ new Date()).toISOString()) {
|
|
7350
|
+
db.transaction((tx) => {
|
|
7351
|
+
tx.delete(siteCrawlGraphLayouts).where(and10(
|
|
7352
|
+
eq13(siteCrawlGraphLayouts.projectId, scope.projectId),
|
|
7353
|
+
eq13(siteCrawlGraphLayouts.runId, scope.runId),
|
|
7354
|
+
eq13(siteCrawlGraphLayouts.attemptId, scope.attemptId)
|
|
7355
|
+
)).run();
|
|
7356
|
+
tx.insert(siteCrawlGraphLayouts).values({
|
|
7357
|
+
id: crypto16.randomUUID(),
|
|
7358
|
+
projectId: scope.projectId,
|
|
7359
|
+
runId: scope.runId,
|
|
7360
|
+
attemptId: scope.attemptId,
|
|
7361
|
+
state: layout.state,
|
|
7362
|
+
layoutVersion: layout.state === "ready" ? layout.layoutVersion : null,
|
|
7363
|
+
failureCode: layout.state === "unavailable" ? layout.failureCode : null,
|
|
7364
|
+
totalNodes: layout.totalNodes,
|
|
7365
|
+
totalEdges: layout.totalEdges,
|
|
7366
|
+
nodeCount: layout.nodeCount,
|
|
7367
|
+
edgeCount: layout.edgeCount,
|
|
7368
|
+
createdAt: now,
|
|
7369
|
+
updatedAt: now
|
|
7370
|
+
}).run();
|
|
7371
|
+
if (layout.state !== "ready") return;
|
|
7372
|
+
const batchSize = 250;
|
|
7373
|
+
for (let offset = 0; offset < layout.nodes.length; offset += batchSize) {
|
|
7374
|
+
tx.insert(siteCrawlGraphNodes).values(layout.nodes.slice(offset, offset + batchSize).map((node) => ({
|
|
7375
|
+
id: crypto16.randomUUID(),
|
|
7376
|
+
projectId: scope.projectId,
|
|
7377
|
+
runId: scope.runId,
|
|
7378
|
+
attemptId: scope.attemptId,
|
|
7379
|
+
nodeKey: node.nodeKey,
|
|
7380
|
+
sampleRank: node.sampleRank,
|
|
7381
|
+
x: node.x,
|
|
7382
|
+
y: node.y,
|
|
7383
|
+
createdAt: now
|
|
7384
|
+
}))).run();
|
|
7385
|
+
}
|
|
7386
|
+
for (let offset = 0; offset < layout.edges.length; offset += batchSize) {
|
|
7387
|
+
tx.insert(siteCrawlGraphEdges).values(layout.edges.slice(offset, offset + batchSize).map((edge) => ({
|
|
7388
|
+
id: crypto16.randomUUID(),
|
|
7389
|
+
projectId: scope.projectId,
|
|
7390
|
+
runId: scope.runId,
|
|
7391
|
+
attemptId: scope.attemptId,
|
|
7392
|
+
edgeKey: edge.edgeKey,
|
|
7393
|
+
sampleRank: edge.sampleRank,
|
|
7394
|
+
sourceNodeKey: edge.sourceNodeKey,
|
|
7395
|
+
targetNodeKey: edge.targetNodeKey,
|
|
7396
|
+
followable: edge.followable,
|
|
7397
|
+
occurrences: edge.occurrences,
|
|
7398
|
+
createdAt: now
|
|
7399
|
+
}))).run();
|
|
7400
|
+
}
|
|
7401
|
+
});
|
|
7402
|
+
}
|
|
7403
|
+
function deleteSiteCrawlGraphLayout(db, scope) {
|
|
7404
|
+
db.delete(siteCrawlGraphLayouts).where(and10(
|
|
7405
|
+
eq13(siteCrawlGraphLayouts.projectId, scope.projectId),
|
|
7406
|
+
eq13(siteCrawlGraphLayouts.runId, scope.runId),
|
|
7407
|
+
eq13(siteCrawlGraphLayouts.attemptId, scope.attemptId)
|
|
7408
|
+
)).run();
|
|
7409
|
+
}
|
|
7410
|
+
|
|
7411
|
+
// src/execute-site-audit.ts
|
|
6936
7412
|
var log12 = createLogger("SiteAudit");
|
|
6937
7413
|
var SiteAuditCancelledError = class extends Error {
|
|
6938
7414
|
name = "SiteAuditCancelledError";
|
|
@@ -6956,6 +7432,22 @@ function clampSiteAuditEdgeLimit(limit) {
|
|
|
6956
7432
|
function toPageFactor(factor) {
|
|
6957
7433
|
return { id: factor.id, name: factor.name, weight: factor.weight, score: factor.score };
|
|
6958
7434
|
}
|
|
7435
|
+
function toCrawlAuditFactor(factor) {
|
|
7436
|
+
return {
|
|
7437
|
+
...toPageFactor(factor),
|
|
7438
|
+
status: factorStatusFromScore(factor.score),
|
|
7439
|
+
applicable: factor.applicable ?? null,
|
|
7440
|
+
findings: factor.findings,
|
|
7441
|
+
recommendations: factor.recommendations
|
|
7442
|
+
};
|
|
7443
|
+
}
|
|
7444
|
+
function crawlAuditFields(audit) {
|
|
7445
|
+
return {
|
|
7446
|
+
schemaVersion: "1.0",
|
|
7447
|
+
factors: audit.factors.map(toCrawlAuditFactor),
|
|
7448
|
+
criticalDefects: audit.criticalDefects
|
|
7449
|
+
};
|
|
7450
|
+
}
|
|
6959
7451
|
function computeFactorAverages(pages) {
|
|
6960
7452
|
const byId = /* @__PURE__ */ new Map();
|
|
6961
7453
|
for (const page of pages) {
|
|
@@ -7063,10 +7555,10 @@ function deadLinkCheckedCount(edges, pages) {
|
|
|
7063
7555
|
}
|
|
7064
7556
|
async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
7065
7557
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
7066
|
-
const claim = db.update(runs).set({ status: "running", startedAt }).where(
|
|
7558
|
+
const claim = db.update(runs).set({ status: "running", startedAt }).where(and11(eq14(runs.id, runId), eq14(runs.projectId, projectId), eq14(runs.status, "queued"))).run();
|
|
7067
7559
|
if (claim.changes === 0) return;
|
|
7068
|
-
const attemptNumber = (db.select({ value:
|
|
7069
|
-
const attemptId =
|
|
7560
|
+
const attemptNumber = (db.select({ value: sql6`coalesce(max(${siteCrawlAttempts.attemptNumber}), 0)` }).from(siteCrawlAttempts).where(eq14(siteCrawlAttempts.runId, runId)).get()?.value ?? 0) + 1;
|
|
7561
|
+
const attemptId = crypto17.randomUUID();
|
|
7070
7562
|
db.insert(siteCrawlAttempts).values({
|
|
7071
7563
|
id: attemptId,
|
|
7072
7564
|
projectId,
|
|
@@ -7100,30 +7592,30 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7100
7592
|
const nodeKey = nodeKeyByUrl.get(url);
|
|
7101
7593
|
if (!nodeKey) continue;
|
|
7102
7594
|
const scope = [
|
|
7103
|
-
|
|
7104
|
-
|
|
7105
|
-
|
|
7595
|
+
eq14(siteCrawlEdges.projectId, projectId),
|
|
7596
|
+
eq14(siteCrawlEdges.runId, runId),
|
|
7597
|
+
eq14(siteCrawlEdges.attemptId, attemptId)
|
|
7106
7598
|
];
|
|
7107
|
-
tx.update(siteCrawlEdges).set({ sourceNodeKey: nodeKey, updatedAt: now }).where(
|
|
7599
|
+
tx.update(siteCrawlEdges).set({ sourceNodeKey: nodeKey, updatedAt: now }).where(and11(
|
|
7108
7600
|
...scope,
|
|
7109
|
-
|
|
7601
|
+
eq14(siteCrawlEdges.sourceUrl, url)
|
|
7110
7602
|
)).run();
|
|
7111
|
-
tx.update(siteCrawlEdges).set({ targetNodeKey: nodeKey, updatedAt: now }).where(
|
|
7603
|
+
tx.update(siteCrawlEdges).set({ targetNodeKey: nodeKey, updatedAt: now }).where(and11(
|
|
7112
7604
|
...scope,
|
|
7113
|
-
|
|
7605
|
+
eq14(siteCrawlEdges.targetUrl, url)
|
|
7114
7606
|
)).run();
|
|
7115
|
-
tx.update(siteCrawlPages).set({ canonicalNodeKey: nodeKey, updatedAt: now }).where(
|
|
7116
|
-
|
|
7117
|
-
|
|
7118
|
-
|
|
7119
|
-
|
|
7607
|
+
tx.update(siteCrawlPages).set({ canonicalNodeKey: nodeKey, updatedAt: now }).where(and11(
|
|
7608
|
+
eq14(siteCrawlPages.projectId, projectId),
|
|
7609
|
+
eq14(siteCrawlPages.runId, runId),
|
|
7610
|
+
eq14(siteCrawlPages.attemptId, attemptId),
|
|
7611
|
+
eq14(siteCrawlPages.canonicalUrl, url)
|
|
7120
7612
|
)).run();
|
|
7121
7613
|
}
|
|
7122
7614
|
};
|
|
7123
7615
|
const persistPage = (tx, page, now) => {
|
|
7124
7616
|
const audit = page.audit;
|
|
7125
7617
|
tx.insert(siteCrawlPages).values({
|
|
7126
|
-
id:
|
|
7618
|
+
id: crypto17.randomUUID(),
|
|
7127
7619
|
projectId,
|
|
7128
7620
|
runId,
|
|
7129
7621
|
attemptId,
|
|
@@ -7151,7 +7643,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7151
7643
|
indexabilityReasons: page.indexability.reasons,
|
|
7152
7644
|
auditState: pageAuditState(page),
|
|
7153
7645
|
auditScore: audit?.overallScore ?? null,
|
|
7154
|
-
auditFields: audit ?
|
|
7646
|
+
auditFields: audit ? crawlAuditFields(audit) : {},
|
|
7155
7647
|
inventoryEligible: isInventoryEligible(page),
|
|
7156
7648
|
depth: page.depth,
|
|
7157
7649
|
inboundUniqueEdges: page.metrics.inbound.uniqueEdges,
|
|
@@ -7172,7 +7664,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7172
7664
|
discoveryProvenance: [{ discoveredFrom: page.provenance.discoveredFrom, sitemapSources: page.provenance.sitemapSources, root: page.provenance.root }],
|
|
7173
7665
|
sitemapMetadata: { sources: page.provenance.sitemapSources },
|
|
7174
7666
|
fetchState: page.state,
|
|
7175
|
-
fetchedAt: page.state === "discovered" || page.state === "robots-blocked" ? null :
|
|
7667
|
+
fetchedAt: page.state === "discovered" || page.state === "robots-blocked" ? null : sql6`coalesce(${siteCrawlPages.fetchedAt}, ${now})`,
|
|
7176
7668
|
httpStatus: page.statusCode,
|
|
7177
7669
|
contentType: page.contentType,
|
|
7178
7670
|
finalUrl: page.finalUrl,
|
|
@@ -7184,7 +7676,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7184
7676
|
indexabilityReasons: page.indexability.reasons,
|
|
7185
7677
|
auditState: pageAuditState(page),
|
|
7186
7678
|
auditScore: audit?.overallScore ?? null,
|
|
7187
|
-
auditFields: audit ?
|
|
7679
|
+
auditFields: audit ? crawlAuditFields(audit) : {},
|
|
7188
7680
|
inventoryEligible: isInventoryEligible(page),
|
|
7189
7681
|
depth: page.depth,
|
|
7190
7682
|
inboundUniqueEdges: page.metrics.inbound.uniqueEdges,
|
|
@@ -7206,7 +7698,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7206
7698
|
const persistEdge = (tx, edge, now) => {
|
|
7207
7699
|
observedEdges.set(edge.key, edge);
|
|
7208
7700
|
tx.insert(siteCrawlEdges).values({
|
|
7209
|
-
id:
|
|
7701
|
+
id: crypto17.randomUUID(),
|
|
7210
7702
|
projectId,
|
|
7211
7703
|
runId,
|
|
7212
7704
|
attemptId,
|
|
@@ -7252,20 +7744,20 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7252
7744
|
linkScoreRaw: metrics.linkScoreRaw,
|
|
7253
7745
|
linkScoreNormalized: metrics.linkScore,
|
|
7254
7746
|
updatedAt: now
|
|
7255
|
-
}).where(
|
|
7256
|
-
|
|
7257
|
-
|
|
7258
|
-
|
|
7259
|
-
|
|
7747
|
+
}).where(and11(
|
|
7748
|
+
eq14(siteCrawlPages.projectId, projectId),
|
|
7749
|
+
eq14(siteCrawlPages.runId, runId),
|
|
7750
|
+
eq14(siteCrawlPages.attemptId, attemptId),
|
|
7751
|
+
eq14(siteCrawlPages.nodeKey, key)
|
|
7260
7752
|
)).run();
|
|
7261
7753
|
};
|
|
7262
7754
|
const persistEvent = async (event) => {
|
|
7263
7755
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7264
7756
|
db.transaction((tx) => {
|
|
7265
|
-
const receipt = tx.select({ checksum: siteCrawlEventReceipts.checksum }).from(siteCrawlEventReceipts).where(
|
|
7266
|
-
|
|
7267
|
-
|
|
7268
|
-
|
|
7757
|
+
const receipt = tx.select({ checksum: siteCrawlEventReceipts.checksum }).from(siteCrawlEventReceipts).where(and11(
|
|
7758
|
+
eq14(siteCrawlEventReceipts.attemptId, attemptId),
|
|
7759
|
+
eq14(siteCrawlEventReceipts.sequence, event.sequence),
|
|
7760
|
+
eq14(siteCrawlEventReceipts.batchId, event.batchId)
|
|
7269
7761
|
)).get();
|
|
7270
7762
|
if (receipt) {
|
|
7271
7763
|
if (receipt.checksum !== event.checksum) {
|
|
@@ -7274,7 +7766,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7274
7766
|
return;
|
|
7275
7767
|
}
|
|
7276
7768
|
tx.insert(siteCrawlEventReceipts).values({
|
|
7277
|
-
id:
|
|
7769
|
+
id: crypto17.randomUUID(),
|
|
7278
7770
|
projectId,
|
|
7279
7771
|
runId,
|
|
7280
7772
|
attemptId,
|
|
@@ -7294,22 +7786,22 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7294
7786
|
lastEventChecksum: event.checksum,
|
|
7295
7787
|
pagesDiscovered: progress.pagesDiscovered,
|
|
7296
7788
|
pagesFetched: progress.pagesFetched,
|
|
7297
|
-
pagesEligible: event.type === "summary" ? observedPages.size === 0 ? 0 : [...observedPages.values()].filter(isInventoryEligible).length :
|
|
7298
|
-
pagesErrored: event.type === "summary" ? observedErrorCount(observedPages.values()) :
|
|
7789
|
+
pagesEligible: event.type === "summary" ? observedPages.size === 0 ? 0 : [...observedPages.values()].filter(isInventoryEligible).length : sql6`${siteCrawlAttempts.pagesEligible}`,
|
|
7790
|
+
pagesErrored: event.type === "summary" ? observedErrorCount(observedPages.values()) : sql6`${siteCrawlAttempts.pagesErrored}`,
|
|
7299
7791
|
edgesDiscovered: progress.edgesObserved,
|
|
7300
7792
|
updatedAt: now
|
|
7301
|
-
}).where(
|
|
7793
|
+
}).where(eq14(siteCrawlAttempts.id, attemptId)).run();
|
|
7302
7794
|
} else {
|
|
7303
7795
|
tx.update(siteCrawlAttempts).set({
|
|
7304
7796
|
lastEventSequence: event.sequence,
|
|
7305
7797
|
lastEventChecksum: event.checksum,
|
|
7306
7798
|
updatedAt: now
|
|
7307
|
-
}).where(
|
|
7799
|
+
}).where(eq14(siteCrawlAttempts.id, attemptId)).run();
|
|
7308
7800
|
}
|
|
7309
7801
|
});
|
|
7310
7802
|
};
|
|
7311
7803
|
try {
|
|
7312
|
-
const project = db.select().from(projects).where(
|
|
7804
|
+
const project = db.select().from(projects).where(eq14(projects.id, projectId)).get();
|
|
7313
7805
|
if (!project) throw new Error(`Project not found: ${projectId}`);
|
|
7314
7806
|
const homepageUrl = toHomepageUrl(project.canonicalDomain);
|
|
7315
7807
|
const maxPages = clampSiteAuditLimit(opts.maxPages ?? opts.limit);
|
|
@@ -7332,6 +7824,12 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7332
7824
|
if (crawlSummary.complete && !hasAuditedPages) {
|
|
7333
7825
|
throw emptyCompleteCrawlError(crawlSummary.rootUrl, crawlSummary.finalRootUrl, crawlSummary.pagesObserved);
|
|
7334
7826
|
}
|
|
7827
|
+
const graphLayout = await prepareSiteCrawlGraphLayout(db, {
|
|
7828
|
+
projectId,
|
|
7829
|
+
runId,
|
|
7830
|
+
attemptId,
|
|
7831
|
+
rootUrl: crawlSummary.finalRootUrl ?? crawlSummary.rootUrl
|
|
7832
|
+
}, { signal: opts.signal, timeoutMs: opts.graphLayoutTimeoutMs });
|
|
7335
7833
|
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
7336
7834
|
const factors = computeFactorAverages([...observedPages.values()]);
|
|
7337
7835
|
const errorCount = observedErrorCount(observedPages.values());
|
|
@@ -7339,12 +7837,39 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7339
7837
|
const deadLinksChecked = opts.checkDeadLinks ? deadLinkCheckedCount(observedEdges.values(), observedPages.values()) : 0;
|
|
7340
7838
|
const deadLinksFound = report.deadLinks.findings.length;
|
|
7341
7839
|
const legacyIssues = factors.map((factor) => toLegacyIssue(factor, crawlSummary.auditRollup.auditedPages)).filter((issue) => issue !== null);
|
|
7840
|
+
try {
|
|
7841
|
+
persistSiteCrawlGraphLayout(db, { projectId, runId, attemptId }, graphLayout, finishedAt);
|
|
7842
|
+
} catch (error) {
|
|
7843
|
+
log12.warn("graph-layout.persist-failed", {
|
|
7844
|
+
runId,
|
|
7845
|
+
projectId,
|
|
7846
|
+
error: error instanceof Error ? error.message : String(error)
|
|
7847
|
+
});
|
|
7848
|
+
try {
|
|
7849
|
+
persistSiteCrawlGraphLayout(db, { projectId, runId, attemptId }, {
|
|
7850
|
+
state: "unavailable",
|
|
7851
|
+
failureCode: "layout-error",
|
|
7852
|
+
totalNodes: graphLayout.totalNodes,
|
|
7853
|
+
totalEdges: graphLayout.totalEdges,
|
|
7854
|
+
nodeCount: 0,
|
|
7855
|
+
edgeCount: 0,
|
|
7856
|
+
nodes: [],
|
|
7857
|
+
edges: []
|
|
7858
|
+
}, finishedAt);
|
|
7859
|
+
} catch (markerError) {
|
|
7860
|
+
log12.warn("graph-layout.marker-persist-failed", {
|
|
7861
|
+
runId,
|
|
7862
|
+
projectId,
|
|
7863
|
+
error: markerError instanceof Error ? markerError.message : String(markerError)
|
|
7864
|
+
});
|
|
7865
|
+
}
|
|
7866
|
+
}
|
|
7342
7867
|
const published = db.transaction((tx) => {
|
|
7343
|
-
const claim2 = tx.update(runs).set({ status: terminalStatus, finishedAt }).where(
|
|
7868
|
+
const claim2 = tx.update(runs).set({ status: terminalStatus, finishedAt }).where(and11(eq14(runs.id, runId), eq14(runs.status, "running"))).run();
|
|
7344
7869
|
if (claim2.changes === 0) return false;
|
|
7345
7870
|
if (hasAuditedPages) {
|
|
7346
7871
|
tx.insert(siteAuditSnapshots).values({
|
|
7347
|
-
id:
|
|
7872
|
+
id: crypto17.randomUUID(),
|
|
7348
7873
|
projectId,
|
|
7349
7874
|
runId,
|
|
7350
7875
|
sitemapUrl: opts.sitemapUrl ?? `${homepageUrl}/sitemap.xml`,
|
|
@@ -7362,7 +7887,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7362
7887
|
for (const page of observedPages.values()) {
|
|
7363
7888
|
if (!isLegacyScorecardPage(page)) continue;
|
|
7364
7889
|
tx.insert(siteAuditPages).values({
|
|
7365
|
-
id:
|
|
7890
|
+
id: crypto17.randomUUID(),
|
|
7366
7891
|
projectId,
|
|
7367
7892
|
runId,
|
|
7368
7893
|
url: page.finalUrl ?? page.requestedUrl,
|
|
@@ -7375,11 +7900,12 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7375
7900
|
}
|
|
7376
7901
|
}
|
|
7377
7902
|
tx.insert(siteCrawlSnapshots).values({
|
|
7378
|
-
id:
|
|
7903
|
+
id: crypto17.randomUUID(),
|
|
7379
7904
|
projectId,
|
|
7380
7905
|
runId,
|
|
7381
7906
|
attemptId,
|
|
7382
|
-
|
|
7907
|
+
requestedRootUrl: crawlSummary.rootUrl,
|
|
7908
|
+
rootUrl: crawlSummary.finalRootUrl ?? crawlSummary.rootUrl,
|
|
7383
7909
|
crawlSchemaVersion: crawlSummary.crawlSchemaVersion,
|
|
7384
7910
|
engineVersion: crawlSummary.engineVersion,
|
|
7385
7911
|
normalizationVersion: crawlSummary.urlNormalizationVersion,
|
|
@@ -7415,7 +7941,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7415
7941
|
if (opts.checkDeadLinks) {
|
|
7416
7942
|
for (const finding of report.deadLinks.findings) {
|
|
7417
7943
|
tx.insert(siteCrawlFindings).values({
|
|
7418
|
-
id:
|
|
7944
|
+
id: crypto17.randomUUID(),
|
|
7419
7945
|
projectId,
|
|
7420
7946
|
runId,
|
|
7421
7947
|
attemptId,
|
|
@@ -7441,11 +7967,20 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7441
7967
|
edgesDiscovered: crawlSummary.edgesObserved,
|
|
7442
7968
|
finishedAt,
|
|
7443
7969
|
updatedAt: finishedAt
|
|
7444
|
-
}).where(
|
|
7970
|
+
}).where(eq14(siteCrawlAttempts.id, attemptId)).run();
|
|
7445
7971
|
return true;
|
|
7446
7972
|
});
|
|
7447
7973
|
if (!published) {
|
|
7448
|
-
|
|
7974
|
+
try {
|
|
7975
|
+
deleteSiteCrawlGraphLayout(db, { projectId, runId, attemptId });
|
|
7976
|
+
} catch (cleanupError) {
|
|
7977
|
+
log12.warn("graph-layout.cleanup-failed", {
|
|
7978
|
+
runId,
|
|
7979
|
+
projectId,
|
|
7980
|
+
error: cleanupError instanceof Error ? cleanupError.message : String(cleanupError)
|
|
7981
|
+
});
|
|
7982
|
+
}
|
|
7983
|
+
const current = db.select({ status: runs.status }).from(runs).where(eq14(runs.id, runId)).get();
|
|
7449
7984
|
if (current?.status === "cancelled") throw new SiteAuditCancelledError(`Site audit cancelled before publication: ${runId}`);
|
|
7450
7985
|
throw new Error(`Site audit run was no longer running before publication: ${runId}`);
|
|
7451
7986
|
}
|
|
@@ -7460,12 +7995,12 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7460
7995
|
error: message,
|
|
7461
7996
|
finishedAt,
|
|
7462
7997
|
updatedAt: finishedAt
|
|
7463
|
-
}).where(
|
|
7998
|
+
}).where(eq14(siteCrawlAttempts.id, attemptId)).run();
|
|
7464
7999
|
tx.update(runs).set({
|
|
7465
8000
|
status: cancelled ? "cancelled" : "failed",
|
|
7466
8001
|
error: message,
|
|
7467
8002
|
finishedAt
|
|
7468
|
-
}).where(
|
|
8003
|
+
}).where(and11(eq14(runs.id, runId), eq14(runs.status, "running"))).run();
|
|
7469
8004
|
});
|
|
7470
8005
|
log12.error("failed", { runId, projectId, cancelled, error: message });
|
|
7471
8006
|
throw error;
|
|
@@ -7473,7 +8008,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7473
8008
|
}
|
|
7474
8009
|
|
|
7475
8010
|
// src/commands/backfill.ts
|
|
7476
|
-
import { and as
|
|
8011
|
+
import { and as and12, eq as eq15, inArray as inArray4, isNull as isNull2, sql as sql7 } from "drizzle-orm";
|
|
7477
8012
|
var SNAPSHOT_BATCH_SIZE = 500;
|
|
7478
8013
|
async function backfillAnswerVisibilityCommand(opts) {
|
|
7479
8014
|
const config = loadConfig();
|
|
@@ -7481,7 +8016,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
7481
8016
|
migrate(db);
|
|
7482
8017
|
const projectFilter = opts?.project?.trim();
|
|
7483
8018
|
const isDryRun = opts?.dryRun === true;
|
|
7484
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
8019
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq15(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
7485
8020
|
let examined = 0;
|
|
7486
8021
|
let updated = 0;
|
|
7487
8022
|
let wouldUpdate = 0;
|
|
@@ -7489,10 +8024,10 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
7489
8024
|
let reparsed = 0;
|
|
7490
8025
|
let providerErrors = 0;
|
|
7491
8026
|
if (scopedProjects.length > 0) {
|
|
7492
|
-
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
7493
|
-
|
|
8027
|
+
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(and12(
|
|
8028
|
+
eq15(runs.kind, RunKinds["answer-visibility"]),
|
|
7494
8029
|
inArray4(runs.projectId, scopedProjects.map((project) => project.id))
|
|
7495
|
-
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
8030
|
+
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(eq15(runs.kind, RunKinds["answer-visibility"])).all();
|
|
7496
8031
|
const runIdsByProject = /* @__PURE__ */ new Map();
|
|
7497
8032
|
for (const run of runRows) {
|
|
7498
8033
|
const existing = runIdsByProject.get(run.projectId);
|
|
@@ -7500,7 +8035,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
7500
8035
|
else runIdsByProject.set(run.projectId, [run.id]);
|
|
7501
8036
|
}
|
|
7502
8037
|
for (const project of scopedProjects) {
|
|
7503
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
8038
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq15(competitors.projectId, project.id)).all().map((row) => row.domain);
|
|
7504
8039
|
const runIds = runIdsByProject.get(project.id) ?? [];
|
|
7505
8040
|
if (runIds.length === 0) continue;
|
|
7506
8041
|
const projectDomains = effectiveDomains({
|
|
@@ -7593,7 +8128,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
7593
8128
|
} else {
|
|
7594
8129
|
db.transaction((tx) => {
|
|
7595
8130
|
for (const update of pendingUpdates) {
|
|
7596
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
8131
|
+
tx.update(querySnapshots).set(update.patch).where(eq15(querySnapshots.id, update.id)).run();
|
|
7597
8132
|
}
|
|
7598
8133
|
});
|
|
7599
8134
|
updated += pendingUpdates.length;
|
|
@@ -7642,13 +8177,13 @@ No DB writes performed. Re-run without --dry-run to apply.`);
|
|
|
7642
8177
|
function backfillNormalizedPaths(db, opts) {
|
|
7643
8178
|
const baseConditions = [];
|
|
7644
8179
|
if (opts?.projectId) {
|
|
7645
|
-
baseConditions.push(
|
|
8180
|
+
baseConditions.push(eq15(gaTrafficSnapshots.projectId, opts.projectId));
|
|
7646
8181
|
}
|
|
7647
8182
|
const rows = db.select({
|
|
7648
8183
|
id: gaTrafficSnapshots.id,
|
|
7649
8184
|
landingPage: gaTrafficSnapshots.landingPage,
|
|
7650
8185
|
landingPageNormalized: gaTrafficSnapshots.landingPageNormalized
|
|
7651
|
-
}).from(gaTrafficSnapshots).where(baseConditions.length > 0 ?
|
|
8186
|
+
}).from(gaTrafficSnapshots).where(baseConditions.length > 0 ? and12(...baseConditions) : void 0).all();
|
|
7652
8187
|
let updated = 0;
|
|
7653
8188
|
let unchanged = 0;
|
|
7654
8189
|
if (rows.length > 0) {
|
|
@@ -7663,7 +8198,7 @@ function backfillNormalizedPaths(db, opts) {
|
|
|
7663
8198
|
unchanged++;
|
|
7664
8199
|
continue;
|
|
7665
8200
|
}
|
|
7666
|
-
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(
|
|
8201
|
+
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(eq15(gaTrafficSnapshots.id, row.id)).run();
|
|
7667
8202
|
updated++;
|
|
7668
8203
|
}
|
|
7669
8204
|
});
|
|
@@ -7677,7 +8212,7 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
7677
8212
|
const projectFilter = opts?.project?.trim();
|
|
7678
8213
|
let projectId;
|
|
7679
8214
|
if (projectFilter) {
|
|
7680
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
8215
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq15(projects.name, projectFilter)).get();
|
|
7681
8216
|
if (!project) {
|
|
7682
8217
|
const result2 = {
|
|
7683
8218
|
project: projectFilter,
|
|
@@ -7714,13 +8249,13 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
7714
8249
|
function backfillAiReferralPaths(db, opts) {
|
|
7715
8250
|
const baseConditions = [];
|
|
7716
8251
|
if (opts?.projectId) {
|
|
7717
|
-
baseConditions.push(
|
|
8252
|
+
baseConditions.push(eq15(gaAiReferrals.projectId, opts.projectId));
|
|
7718
8253
|
}
|
|
7719
8254
|
const rows = db.select({
|
|
7720
8255
|
id: gaAiReferrals.id,
|
|
7721
8256
|
landingPage: gaAiReferrals.landingPage,
|
|
7722
8257
|
landingPageNormalized: gaAiReferrals.landingPageNormalized
|
|
7723
|
-
}).from(gaAiReferrals).where(baseConditions.length > 0 ?
|
|
8258
|
+
}).from(gaAiReferrals).where(baseConditions.length > 0 ? and12(...baseConditions) : void 0).all();
|
|
7724
8259
|
let updated = 0;
|
|
7725
8260
|
let unchanged = 0;
|
|
7726
8261
|
if (rows.length > 0) {
|
|
@@ -7735,7 +8270,7 @@ function backfillAiReferralPaths(db, opts) {
|
|
|
7735
8270
|
unchanged++;
|
|
7736
8271
|
continue;
|
|
7737
8272
|
}
|
|
7738
|
-
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(
|
|
8273
|
+
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(eq15(gaAiReferrals.id, row.id)).run();
|
|
7739
8274
|
updated++;
|
|
7740
8275
|
}
|
|
7741
8276
|
});
|
|
@@ -7749,7 +8284,7 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
7749
8284
|
const projectFilter = opts?.project?.trim();
|
|
7750
8285
|
let projectId;
|
|
7751
8286
|
if (projectFilter) {
|
|
7752
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
8287
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq15(projects.name, projectFilter)).get();
|
|
7753
8288
|
if (!project) {
|
|
7754
8289
|
const result2 = {
|
|
7755
8290
|
project: projectFilter,
|
|
@@ -7785,10 +8320,10 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
7785
8320
|
}
|
|
7786
8321
|
function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
7787
8322
|
const isDryRun = opts?.dryRun === true;
|
|
7788
|
-
const project = db.select().from(projects).where(
|
|
8323
|
+
const project = db.select().from(projects).where(eq15(projects.id, projectId)).get();
|
|
7789
8324
|
if (!project) return { examined: 0, updated: 0, mentioned: 0 };
|
|
7790
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
7791
|
-
const runRows = db.select({ id: runs.id }).from(runs).where(
|
|
8325
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq15(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
8326
|
+
const runRows = db.select({ id: runs.id }).from(runs).where(and12(eq15(runs.kind, RunKinds["answer-visibility"]), eq15(runs.projectId, projectId))).all();
|
|
7792
8327
|
const runIds = runRows.map((r) => r.id);
|
|
7793
8328
|
let examined = 0;
|
|
7794
8329
|
let updated = 0;
|
|
@@ -7862,7 +8397,7 @@ function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
|
7862
8397
|
} else {
|
|
7863
8398
|
db.transaction((tx) => {
|
|
7864
8399
|
for (const update of pendingUpdates) {
|
|
7865
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
8400
|
+
tx.update(querySnapshots).set(update.patch).where(eq15(querySnapshots.id, update.id)).run();
|
|
7866
8401
|
}
|
|
7867
8402
|
});
|
|
7868
8403
|
updated += pendingUpdates.length;
|
|
@@ -7877,7 +8412,7 @@ async function backfillAnswerMentionsCommand(opts) {
|
|
|
7877
8412
|
migrate(db);
|
|
7878
8413
|
const projectFilter = opts?.project?.trim();
|
|
7879
8414
|
const isDryRun = opts?.dryRun === true;
|
|
7880
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
8415
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq15(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
7881
8416
|
let examined = 0;
|
|
7882
8417
|
let updated = 0;
|
|
7883
8418
|
let wouldUpdate = 0;
|
|
@@ -7937,7 +8472,7 @@ function readStoredGroundingSources(rawResponse) {
|
|
|
7937
8472
|
return result;
|
|
7938
8473
|
}
|
|
7939
8474
|
async function backfillInsightsCommand(project, opts) {
|
|
7940
|
-
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-
|
|
8475
|
+
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-YUUPSJRC.js");
|
|
7941
8476
|
const config = loadConfig();
|
|
7942
8477
|
const db = createClient(config.database);
|
|
7943
8478
|
migrate(db);
|
|
@@ -8096,7 +8631,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
8096
8631
|
const config = loadConfig();
|
|
8097
8632
|
const db = createClient(config.database);
|
|
8098
8633
|
migrate(db);
|
|
8099
|
-
const project = db.select().from(projects).where(
|
|
8634
|
+
const project = db.select().from(projects).where(eq15(projects.name, opts.project)).get();
|
|
8100
8635
|
if (!project) {
|
|
8101
8636
|
throw new Error(`Project "${opts.project}" not found`);
|
|
8102
8637
|
}
|
|
@@ -8107,8 +8642,8 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
8107
8642
|
process.stderr.write(`Recovering orphan snapshot attribution for "${project.name}"${mode}...
|
|
8108
8643
|
`);
|
|
8109
8644
|
}
|
|
8110
|
-
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(
|
|
8111
|
-
|
|
8645
|
+
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(and12(
|
|
8646
|
+
eq15(auditLog.projectId, project.id),
|
|
8112
8647
|
inArray4(auditLog.action, ["keywords.appended", "keywords.deleted", "queries.appended", "queries.deleted", "queries.replaced"])
|
|
8113
8648
|
)).orderBy(auditLog.createdAt).all();
|
|
8114
8649
|
const history = replayQueryAuditLog(events);
|
|
@@ -8116,10 +8651,10 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
8116
8651
|
runId: runs.id,
|
|
8117
8652
|
createdAt: runs.createdAt,
|
|
8118
8653
|
location: runs.location
|
|
8119
|
-
}).from(runs).innerJoin(querySnapshots,
|
|
8120
|
-
|
|
8121
|
-
|
|
8122
|
-
|
|
8654
|
+
}).from(runs).innerJoin(querySnapshots, eq15(querySnapshots.runId, runs.id)).where(and12(
|
|
8655
|
+
eq15(runs.projectId, project.id),
|
|
8656
|
+
isNull2(querySnapshots.queryId),
|
|
8657
|
+
isNull2(querySnapshots.queryText)
|
|
8123
8658
|
)).groupBy(runs.id).orderBy(runs.createdAt).all();
|
|
8124
8659
|
const result = {
|
|
8125
8660
|
project: project.name,
|
|
@@ -8139,10 +8674,10 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
8139
8674
|
provider: querySnapshots.provider,
|
|
8140
8675
|
createdAt: querySnapshots.createdAt,
|
|
8141
8676
|
answerText: querySnapshots.answerText
|
|
8142
|
-
}).from(querySnapshots).where(
|
|
8143
|
-
|
|
8144
|
-
|
|
8145
|
-
|
|
8677
|
+
}).from(querySnapshots).where(and12(
|
|
8678
|
+
eq15(querySnapshots.runId, run.runId),
|
|
8679
|
+
isNull2(querySnapshots.queryId),
|
|
8680
|
+
isNull2(querySnapshots.queryText)
|
|
8146
8681
|
)).orderBy(querySnapshots.provider, querySnapshots.createdAt).all();
|
|
8147
8682
|
const byProvider = /* @__PURE__ */ new Map();
|
|
8148
8683
|
for (const s of orphanSnaps) {
|
|
@@ -8206,7 +8741,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
8206
8741
|
if (!isDryRun && updates.length > 0) {
|
|
8207
8742
|
db.transaction((tx) => {
|
|
8208
8743
|
for (const u of updates) {
|
|
8209
|
-
tx.update(querySnapshots).set({ queryText: u.queryText }).where(
|
|
8744
|
+
tx.update(querySnapshots).set({ queryText: u.queryText }).where(eq15(querySnapshots.id, u.id)).run();
|
|
8210
8745
|
}
|
|
8211
8746
|
});
|
|
8212
8747
|
}
|
|
@@ -8280,7 +8815,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
8280
8815
|
const projectFilter = opts?.project?.trim();
|
|
8281
8816
|
const isDryRun = opts?.dryRun === true;
|
|
8282
8817
|
const isJson = isMachineFormat(opts?.format);
|
|
8283
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
8818
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq15(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
8284
8819
|
if (scopedProjects.length === 0) {
|
|
8285
8820
|
if (projectFilter && !isJson) {
|
|
8286
8821
|
process.stderr.write(`No project named "${projectFilter}".
|
|
@@ -8305,8 +8840,8 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
8305
8840
|
dryRun: isDryRun,
|
|
8306
8841
|
byBot: {}
|
|
8307
8842
|
};
|
|
8308
|
-
const unknownCountRow = db.select({ n:
|
|
8309
|
-
|
|
8843
|
+
const unknownCountRow = db.select({ n: sql7`count(*)` }).from(rawEventSamples).where(and12(
|
|
8844
|
+
eq15(rawEventSamples.eventType, "unknown"),
|
|
8310
8845
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
8311
8846
|
)).get();
|
|
8312
8847
|
result.unknownBefore = Number(unknownCountRow?.n ?? 0);
|
|
@@ -8318,8 +8853,8 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
8318
8853
|
userAgent: rawEventSamples.userAgent,
|
|
8319
8854
|
pathNormalized: rawEventSamples.pathNormalized,
|
|
8320
8855
|
status: rawEventSamples.status
|
|
8321
|
-
}).from(rawEventSamples).where(
|
|
8322
|
-
|
|
8856
|
+
}).from(rawEventSamples).where(and12(
|
|
8857
|
+
eq15(rawEventSamples.eventType, "unknown"),
|
|
8323
8858
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
8324
8859
|
)).all();
|
|
8325
8860
|
result.examined = unknownSamples.length;
|
|
@@ -8358,7 +8893,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
8358
8893
|
result.reclassified++;
|
|
8359
8894
|
result.byBot[classified.botId] = (result.byBot[classified.botId] ?? 0) + 1;
|
|
8360
8895
|
if (isDryRun) continue;
|
|
8361
|
-
db.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(
|
|
8896
|
+
db.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(eq15(rawEventSamples.id, snap.id)).run();
|
|
8362
8897
|
const tsHour = new Date(snap.ts);
|
|
8363
8898
|
tsHour.setUTCMinutes(0, 0, 0);
|
|
8364
8899
|
if (userFetch) {
|
|
@@ -8386,7 +8921,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
8386
8921
|
aiUserFetchEventsHourly.status
|
|
8387
8922
|
],
|
|
8388
8923
|
set: {
|
|
8389
|
-
hits:
|
|
8924
|
+
hits: sql7`${aiUserFetchEventsHourly.hits} + 1`,
|
|
8390
8925
|
updatedAt: now
|
|
8391
8926
|
}
|
|
8392
8927
|
}).run();
|
|
@@ -8415,15 +8950,15 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
8415
8950
|
crawlerEventsHourly.status
|
|
8416
8951
|
],
|
|
8417
8952
|
set: {
|
|
8418
|
-
hits:
|
|
8953
|
+
hits: sql7`${crawlerEventsHourly.hits} + 1`,
|
|
8419
8954
|
updatedAt: now
|
|
8420
8955
|
}
|
|
8421
8956
|
}).run();
|
|
8422
8957
|
}
|
|
8423
8958
|
}
|
|
8424
8959
|
if (!isDryRun) {
|
|
8425
|
-
const afterRow = db.select({ n:
|
|
8426
|
-
|
|
8960
|
+
const afterRow = db.select({ n: sql7`count(*)` }).from(rawEventSamples).where(and12(
|
|
8961
|
+
eq15(rawEventSamples.eventType, "unknown"),
|
|
8427
8962
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
8428
8963
|
)).get();
|
|
8429
8964
|
result.unknownAfter = Number(afterRow?.n ?? 0);
|
|
@@ -8447,8 +8982,8 @@ Traffic classification ${result.dryRun ? "preview" : "recovery"} complete.`);
|
|
|
8447
8982
|
if (result.reclassified > 0) {
|
|
8448
8983
|
console.log(` By bot:`);
|
|
8449
8984
|
const sorted = Object.entries(result.byBot).sort(([, a], [, b]) => b - a);
|
|
8450
|
-
for (const [bot,
|
|
8451
|
-
console.log(` ${
|
|
8985
|
+
for (const [bot, count2] of sorted) {
|
|
8986
|
+
console.log(` ${count2.toString().padStart(5)} ${bot}`);
|
|
8452
8987
|
}
|
|
8453
8988
|
}
|
|
8454
8989
|
if (result.dryRun) {
|
|
@@ -8458,7 +8993,7 @@ No DB writes performed. Re-run without --dry-run to apply.`);
|
|
|
8458
8993
|
}
|
|
8459
8994
|
|
|
8460
8995
|
// src/commands/skills.ts
|
|
8461
|
-
import
|
|
8996
|
+
import crypto18 from "crypto";
|
|
8462
8997
|
import fs4 from "fs";
|
|
8463
8998
|
import os4 from "os";
|
|
8464
8999
|
import path5 from "path";
|
|
@@ -8513,7 +9048,7 @@ function walkRelative(dir, prefix = "") {
|
|
|
8513
9048
|
return out.sort();
|
|
8514
9049
|
}
|
|
8515
9050
|
function sha256File(filePath) {
|
|
8516
|
-
return
|
|
9051
|
+
return crypto18.createHash("sha256").update(fs4.readFileSync(filePath)).digest("hex");
|
|
8517
9052
|
}
|
|
8518
9053
|
function readSkillManifest(skillDir) {
|
|
8519
9054
|
try {
|
|
@@ -8855,9 +9390,9 @@ var ProviderRegistry = class {
|
|
|
8855
9390
|
};
|
|
8856
9391
|
|
|
8857
9392
|
// src/scheduler.ts
|
|
8858
|
-
import
|
|
9393
|
+
import crypto19 from "crypto";
|
|
8859
9394
|
import cron from "node-cron";
|
|
8860
|
-
import { and as
|
|
9395
|
+
import { and as and13, eq as eq16, inArray as inArray5, notExists, sql as sql8 } from "drizzle-orm";
|
|
8861
9396
|
var log13 = createLogger("Scheduler");
|
|
8862
9397
|
var DEFAULT_HEALTH_CRON = "0 */6 * * *";
|
|
8863
9398
|
function taskKey(projectId, kind) {
|
|
@@ -8889,16 +9424,16 @@ var Scheduler = class {
|
|
|
8889
9424
|
*/
|
|
8890
9425
|
ensureHealthSchedules() {
|
|
8891
9426
|
const projectsWithoutHealth = this.db.select({ id: projects.id }).from(projects).where(notExists(
|
|
8892
|
-
this.db.select({ one:
|
|
8893
|
-
|
|
8894
|
-
|
|
9427
|
+
this.db.select({ one: sql8`1` }).from(schedules).where(and13(
|
|
9428
|
+
eq16(schedules.projectId, projects.id),
|
|
9429
|
+
eq16(schedules.kind, SchedulableRunKinds.doctor)
|
|
8895
9430
|
))
|
|
8896
9431
|
)).all();
|
|
8897
9432
|
if (projectsWithoutHealth.length === 0) return;
|
|
8898
9433
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8899
9434
|
for (const project of projectsWithoutHealth) {
|
|
8900
9435
|
this.db.insert(schedules).values({
|
|
8901
|
-
id:
|
|
9436
|
+
id: crypto19.randomUUID(),
|
|
8902
9437
|
projectId: project.id,
|
|
8903
9438
|
kind: SchedulableRunKinds.doctor,
|
|
8904
9439
|
cronExpr: DEFAULT_HEALTH_CRON,
|
|
@@ -8942,7 +9477,7 @@ var Scheduler = class {
|
|
|
8942
9477
|
start() {
|
|
8943
9478
|
this.ensureHealthSchedules();
|
|
8944
9479
|
this.ensureQueryBaskets();
|
|
8945
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
9480
|
+
const allSchedules = this.db.select().from(schedules).where(eq16(schedules.enabled, true)).all();
|
|
8946
9481
|
for (const schedule of allSchedules) {
|
|
8947
9482
|
const missedRunAt = schedule.nextRunAt;
|
|
8948
9483
|
this.registerCronTask(schedule);
|
|
@@ -8972,7 +9507,7 @@ var Scheduler = class {
|
|
|
8972
9507
|
this.stopTask(key, existing, "Stopped");
|
|
8973
9508
|
this.tasks.delete(key);
|
|
8974
9509
|
}
|
|
8975
|
-
const schedule = this.db.select().from(schedules).where(
|
|
9510
|
+
const schedule = this.db.select().from(schedules).where(and13(eq16(schedules.projectId, projectId), eq16(schedules.kind, kind))).get();
|
|
8976
9511
|
if (schedule && schedule.enabled) {
|
|
8977
9512
|
this.registerCronTask(schedule);
|
|
8978
9513
|
}
|
|
@@ -9013,21 +9548,21 @@ var Scheduler = class {
|
|
|
9013
9548
|
this.db.update(schedules).set({
|
|
9014
9549
|
nextRunAt: nextRunFromCron(cronExpr, timezone),
|
|
9015
9550
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
9016
|
-
}).where(
|
|
9551
|
+
}).where(eq16(schedules.id, scheduleId)).run();
|
|
9017
9552
|
const label = schedule.preset ?? cronExpr;
|
|
9018
9553
|
log13.info("cron.registered", { projectId, kind, schedule: label, timezone });
|
|
9019
9554
|
}
|
|
9020
9555
|
triggerRun(scheduleId, projectId, kind) {
|
|
9021
9556
|
try {
|
|
9022
9557
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9023
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
9558
|
+
const currentSchedule = this.db.select().from(schedules).where(eq16(schedules.id, scheduleId)).get();
|
|
9024
9559
|
if (!currentSchedule || !currentSchedule.enabled) {
|
|
9025
9560
|
log13.warn("schedule.stale", { scheduleId, projectId, kind, msg: "schedule no longer exists or is disabled" });
|
|
9026
9561
|
this.remove(projectId, kind);
|
|
9027
9562
|
return;
|
|
9028
9563
|
}
|
|
9029
9564
|
const nextRunAt = nextRunFromCron(currentSchedule.cronExpr, currentSchedule.timezone);
|
|
9030
|
-
const project = this.db.select().from(projects).where(
|
|
9565
|
+
const project = this.db.select().from(projects).where(eq16(projects.id, projectId)).get();
|
|
9031
9566
|
if (!project) {
|
|
9032
9567
|
log13.error("project.not-found", { projectId, kind, msg: "skipping scheduled run" });
|
|
9033
9568
|
this.remove(projectId, kind);
|
|
@@ -9047,7 +9582,7 @@ var Scheduler = class {
|
|
|
9047
9582
|
lastRunAt: now,
|
|
9048
9583
|
nextRunAt,
|
|
9049
9584
|
updatedAt: now
|
|
9050
|
-
}).where(
|
|
9585
|
+
}).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
9051
9586
|
log13.info("traffic-sync.triggered", { projectName: project.name, sourceId });
|
|
9052
9587
|
this.callbacks.onTrafficSyncRequested(project.name, sourceId);
|
|
9053
9588
|
return;
|
|
@@ -9057,7 +9592,7 @@ var Scheduler = class {
|
|
|
9057
9592
|
log13.warn("gbp-sync.no-callback", { scheduleId, projectId, msg: "host did not register onGbpSyncRequested" });
|
|
9058
9593
|
return;
|
|
9059
9594
|
}
|
|
9060
|
-
const runId2 =
|
|
9595
|
+
const runId2 = crypto19.randomUUID();
|
|
9061
9596
|
this.db.insert(runs).values({
|
|
9062
9597
|
id: runId2,
|
|
9063
9598
|
projectId,
|
|
@@ -9070,7 +9605,7 @@ var Scheduler = class {
|
|
|
9070
9605
|
lastRunAt: now,
|
|
9071
9606
|
nextRunAt,
|
|
9072
9607
|
updatedAt: now
|
|
9073
|
-
}).where(
|
|
9608
|
+
}).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
9074
9609
|
log13.info("gbp-sync.triggered", { runId: runId2, projectName: project.name });
|
|
9075
9610
|
this.callbacks.onGbpSyncRequested(runId2, projectId);
|
|
9076
9611
|
return;
|
|
@@ -9080,17 +9615,17 @@ var Scheduler = class {
|
|
|
9080
9615
|
log13.warn("ads-sync.no-callback", { scheduleId, projectId, msg: "host did not register onAdsSyncRequested" });
|
|
9081
9616
|
return;
|
|
9082
9617
|
}
|
|
9083
|
-
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(
|
|
9084
|
-
|
|
9085
|
-
|
|
9618
|
+
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(and13(
|
|
9619
|
+
eq16(runs.projectId, projectId),
|
|
9620
|
+
eq16(runs.kind, RunKinds["ads-sync"]),
|
|
9086
9621
|
inArray5(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
9087
9622
|
)).get();
|
|
9088
9623
|
if (activeAdsRun) {
|
|
9089
9624
|
log13.info("ads-sync.skipped-active", { projectName: project.name, activeRunId: activeAdsRun.id });
|
|
9090
|
-
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(
|
|
9625
|
+
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
9091
9626
|
return;
|
|
9092
9627
|
}
|
|
9093
|
-
const runId2 =
|
|
9628
|
+
const runId2 = crypto19.randomUUID();
|
|
9094
9629
|
this.db.insert(runs).values({
|
|
9095
9630
|
id: runId2,
|
|
9096
9631
|
projectId,
|
|
@@ -9103,7 +9638,7 @@ var Scheduler = class {
|
|
|
9103
9638
|
lastRunAt: now,
|
|
9104
9639
|
nextRunAt,
|
|
9105
9640
|
updatedAt: now
|
|
9106
|
-
}).where(
|
|
9641
|
+
}).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
9107
9642
|
log13.info("ads-sync.triggered", { runId: runId2, projectName: project.name });
|
|
9108
9643
|
this.callbacks.onAdsSyncRequested(runId2, projectId);
|
|
9109
9644
|
return;
|
|
@@ -9117,7 +9652,7 @@ var Scheduler = class {
|
|
|
9117
9652
|
lastRunAt: now,
|
|
9118
9653
|
nextRunAt,
|
|
9119
9654
|
updatedAt: now
|
|
9120
|
-
}).where(
|
|
9655
|
+
}).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
9121
9656
|
log13.info("data-refresh.triggered", { projectName: project.name });
|
|
9122
9657
|
this.callbacks.onDataRefreshRequested(project.name);
|
|
9123
9658
|
return;
|
|
@@ -9131,7 +9666,7 @@ var Scheduler = class {
|
|
|
9131
9666
|
lastRunAt: now,
|
|
9132
9667
|
nextRunAt,
|
|
9133
9668
|
updatedAt: now
|
|
9134
|
-
}).where(
|
|
9669
|
+
}).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
9135
9670
|
log13.info("doctor.triggered", { projectName: project.name });
|
|
9136
9671
|
this.callbacks.onDoctorRequested(project.name);
|
|
9137
9672
|
return;
|
|
@@ -9145,7 +9680,7 @@ var Scheduler = class {
|
|
|
9145
9680
|
lastRunAt: now,
|
|
9146
9681
|
nextRunAt,
|
|
9147
9682
|
updatedAt: now
|
|
9148
|
-
}).where(
|
|
9683
|
+
}).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
9149
9684
|
log13.info("backlinks-sync.triggered", { projectName: project.name });
|
|
9150
9685
|
this.callbacks.onBacklinksSyncRequested(project.name);
|
|
9151
9686
|
return;
|
|
@@ -9155,17 +9690,17 @@ var Scheduler = class {
|
|
|
9155
9690
|
log13.warn("site-audit.no-callback", { scheduleId, projectId, msg: "host did not register onSiteAuditRequested" });
|
|
9156
9691
|
return;
|
|
9157
9692
|
}
|
|
9158
|
-
const active = this.db.select({ id: runs.id }).from(runs).where(
|
|
9159
|
-
|
|
9160
|
-
|
|
9693
|
+
const active = this.db.select({ id: runs.id }).from(runs).where(and13(
|
|
9694
|
+
eq16(runs.projectId, projectId),
|
|
9695
|
+
eq16(runs.kind, RunKinds["site-audit"]),
|
|
9161
9696
|
inArray5(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
9162
9697
|
)).get();
|
|
9163
9698
|
if (active) {
|
|
9164
9699
|
log13.info("site-audit.skipped-active", { projectName: project.name, activeRunId: active.id });
|
|
9165
|
-
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(
|
|
9700
|
+
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
9166
9701
|
return;
|
|
9167
9702
|
}
|
|
9168
|
-
const runId2 =
|
|
9703
|
+
const runId2 = crypto19.randomUUID();
|
|
9169
9704
|
const effectiveRequest = normalizeSiteAuditRunRequest({});
|
|
9170
9705
|
this.db.transaction((tx) => {
|
|
9171
9706
|
tx.insert(runs).values({
|
|
@@ -9188,7 +9723,7 @@ var Scheduler = class {
|
|
|
9188
9723
|
lastRunAt: now,
|
|
9189
9724
|
nextRunAt,
|
|
9190
9725
|
updatedAt: now
|
|
9191
|
-
}).where(
|
|
9726
|
+
}).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
9192
9727
|
log13.info("site-audit.triggered", { runId: runId2, projectName: project.name });
|
|
9193
9728
|
this.callbacks.onSiteAuditRequested(runId2, projectId);
|
|
9194
9729
|
return;
|
|
@@ -9221,7 +9756,7 @@ var Scheduler = class {
|
|
|
9221
9756
|
this.db.update(schedules).set({
|
|
9222
9757
|
nextRunAt,
|
|
9223
9758
|
updatedAt: now
|
|
9224
|
-
}).where(
|
|
9759
|
+
}).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
9225
9760
|
return;
|
|
9226
9761
|
}
|
|
9227
9762
|
const runId = queueResult.runId;
|
|
@@ -9229,7 +9764,7 @@ var Scheduler = class {
|
|
|
9229
9764
|
lastRunAt: now,
|
|
9230
9765
|
nextRunAt,
|
|
9231
9766
|
updatedAt: now
|
|
9232
|
-
}).where(
|
|
9767
|
+
}).where(eq16(schedules.id, currentSchedule.id)).run();
|
|
9233
9768
|
log13.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
9234
9769
|
this.callbacks.onRunCreated(runId, projectId, providers, resolvedLocation);
|
|
9235
9770
|
} catch (err) {
|
|
@@ -9262,8 +9797,8 @@ async function refreshAllIntegrations(client, projectName) {
|
|
|
9262
9797
|
}
|
|
9263
9798
|
|
|
9264
9799
|
// src/notifier.ts
|
|
9265
|
-
import { eq as
|
|
9266
|
-
import
|
|
9800
|
+
import { eq as eq17, desc as desc7, and as and14, inArray as inArray6, or as or2 } from "drizzle-orm";
|
|
9801
|
+
import crypto20 from "crypto";
|
|
9267
9802
|
var log15 = createLogger("Notifier");
|
|
9268
9803
|
var Notifier = class {
|
|
9269
9804
|
db;
|
|
@@ -9275,18 +9810,18 @@ var Notifier = class {
|
|
|
9275
9810
|
/** Called after a run completes (success, partial, or failed). */
|
|
9276
9811
|
async onRunCompleted(runId, projectId) {
|
|
9277
9812
|
log15.info("run.completed", { runId, projectId });
|
|
9278
|
-
const notifs = this.db.select().from(notifications).where(
|
|
9813
|
+
const notifs = this.db.select().from(notifications).where(eq17(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
9279
9814
|
if (notifs.length === 0) {
|
|
9280
9815
|
log15.info("notifications.none-enabled", { projectId });
|
|
9281
9816
|
return;
|
|
9282
9817
|
}
|
|
9283
9818
|
log15.info("notifications.found", { projectId, count: notifs.length });
|
|
9284
|
-
const run = this.db.select().from(runs).where(
|
|
9819
|
+
const run = this.db.select().from(runs).where(eq17(runs.id, runId)).get();
|
|
9285
9820
|
if (!run) {
|
|
9286
9821
|
log15.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
9287
9822
|
return;
|
|
9288
9823
|
}
|
|
9289
|
-
const project = this.db.select().from(projects).where(
|
|
9824
|
+
const project = this.db.select().from(projects).where(eq17(projects.id, projectId)).get();
|
|
9290
9825
|
if (!project) {
|
|
9291
9826
|
log15.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
9292
9827
|
return;
|
|
@@ -9339,7 +9874,7 @@ var Notifier = class {
|
|
|
9339
9874
|
* inferred from the absence of a webhook.
|
|
9340
9875
|
*/
|
|
9341
9876
|
async onHealthChecked(projectId, report) {
|
|
9342
|
-
const project = this.db.select().from(projects).where(
|
|
9877
|
+
const project = this.db.select().from(projects).where(eq17(projects.id, projectId)).get();
|
|
9343
9878
|
if (!project) {
|
|
9344
9879
|
log15.error("project.not-found", { projectId, msg: "skipping health notification" });
|
|
9345
9880
|
return null;
|
|
@@ -9370,7 +9905,7 @@ var Notifier = class {
|
|
|
9370
9905
|
const status = noSignal ? "warn" : worst ? worst.status : "ok";
|
|
9371
9906
|
const code = noSignal ? "health.no-signal" : worst?.code ?? "health.ok";
|
|
9372
9907
|
const summary = noSignal ? `No health check produced a result (${report.checks.length} skipped) \u2014 health is unknown, not confirmed.` : worst?.summary ?? `All ${graded.length} health check(s) passing.`;
|
|
9373
|
-
const previous = this.db.select().from(doctorHealthState).where(
|
|
9908
|
+
const previous = this.db.select().from(doctorHealthState).where(eq17(doctorHealthState.projectId, projectId)).get();
|
|
9374
9909
|
const previousStatus = previous?.status ?? null;
|
|
9375
9910
|
let event = null;
|
|
9376
9911
|
if (previous === void 0) {
|
|
@@ -9385,13 +9920,13 @@ var Notifier = class {
|
|
|
9385
9920
|
if (previous === void 0) {
|
|
9386
9921
|
this.db.insert(doctorHealthState).values({ ...observation, notifiedAt: null }).run();
|
|
9387
9922
|
} else {
|
|
9388
|
-
this.db.update(doctorHealthState).set(observation).where(
|
|
9923
|
+
this.db.update(doctorHealthState).set(observation).where(eq17(doctorHealthState.projectId, projectId)).run();
|
|
9389
9924
|
}
|
|
9390
9925
|
if (!event) {
|
|
9391
9926
|
log15.info("health.unchanged", { projectId, status, code });
|
|
9392
9927
|
return null;
|
|
9393
9928
|
}
|
|
9394
|
-
const notifs = this.db.select().from(notifications).where(
|
|
9929
|
+
const notifs = this.db.select().from(notifications).where(eq17(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
9395
9930
|
const payload = {
|
|
9396
9931
|
source: "canonry",
|
|
9397
9932
|
event,
|
|
@@ -9415,7 +9950,7 @@ var Notifier = class {
|
|
|
9415
9950
|
delivered += 1;
|
|
9416
9951
|
}
|
|
9417
9952
|
if (delivered > 0) {
|
|
9418
|
-
this.db.update(doctorHealthState).set({ notifiedAt: now }).where(
|
|
9953
|
+
this.db.update(doctorHealthState).set({ notifiedAt: now }).where(eq17(doctorHealthState.projectId, projectId)).run();
|
|
9419
9954
|
}
|
|
9420
9955
|
log15.info("health.notified", { projectId, event, status, code, subscribers: notifs.length, delivered });
|
|
9421
9956
|
return event;
|
|
@@ -9428,11 +9963,11 @@ var Notifier = class {
|
|
|
9428
9963
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
9429
9964
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
9430
9965
|
if (insightEvents.length === 0) return;
|
|
9431
|
-
const notifs = this.db.select().from(notifications).where(
|
|
9966
|
+
const notifs = this.db.select().from(notifications).where(eq17(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
9432
9967
|
if (notifs.length === 0) return;
|
|
9433
|
-
const run = this.db.select().from(runs).where(
|
|
9968
|
+
const run = this.db.select().from(runs).where(eq17(runs.id, runId)).get();
|
|
9434
9969
|
if (!run) return;
|
|
9435
|
-
const project = this.db.select().from(projects).where(
|
|
9970
|
+
const project = this.db.select().from(projects).where(eq17(projects.id, projectId)).get();
|
|
9436
9971
|
if (!project) return;
|
|
9437
9972
|
for (const notif of notifs) {
|
|
9438
9973
|
const config = notif.config;
|
|
@@ -9462,7 +9997,7 @@ var Notifier = class {
|
|
|
9462
9997
|
}
|
|
9463
9998
|
}
|
|
9464
9999
|
computeTransitions(runId, projectId) {
|
|
9465
|
-
const thisRun = this.db.select().from(runs).where(
|
|
10000
|
+
const thisRun = this.db.select().from(runs).where(eq17(runs.id, runId)).get();
|
|
9466
10001
|
if (!thisRun) return [];
|
|
9467
10002
|
const completeness = measurementRunCompleteness(this.db, runId);
|
|
9468
10003
|
if (completeness.planned && !completeness.complete) {
|
|
@@ -9473,10 +10008,10 @@ var Notifier = class {
|
|
|
9473
10008
|
});
|
|
9474
10009
|
return [];
|
|
9475
10010
|
}
|
|
9476
|
-
const groupSiblings = this.db.select().from(runs).where(
|
|
9477
|
-
|
|
9478
|
-
|
|
9479
|
-
|
|
10011
|
+
const groupSiblings = this.db.select().from(runs).where(and14(
|
|
10012
|
+
eq17(runs.projectId, projectId),
|
|
10013
|
+
eq17(runs.kind, thisRun.kind),
|
|
10014
|
+
eq17(runs.createdAt, thisRun.createdAt)
|
|
9480
10015
|
)).all();
|
|
9481
10016
|
const stillPending = groupSiblings.some((r) => r.status === "queued" || r.status === "running");
|
|
9482
10017
|
if (stillPending) return [];
|
|
@@ -9492,19 +10027,19 @@ var Notifier = class {
|
|
|
9492
10027
|
return candidate.id > best.id ? candidate : best;
|
|
9493
10028
|
});
|
|
9494
10029
|
if (winner.id !== runId) return [];
|
|
9495
|
-
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
10030
|
+
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(eq17(projects.id, projectId)).get();
|
|
9496
10031
|
const locationCount = Math.max(
|
|
9497
10032
|
1,
|
|
9498
10033
|
(projectLocations?.locations ?? []).length
|
|
9499
10034
|
);
|
|
9500
10035
|
const RECENT_FETCH_LIMIT = Math.max(8, locationCount * 4);
|
|
9501
10036
|
const recentRuns = this.db.select().from(runs).where(
|
|
9502
|
-
|
|
9503
|
-
|
|
9504
|
-
|
|
9505
|
-
|
|
10037
|
+
and14(
|
|
10038
|
+
eq17(runs.projectId, projectId),
|
|
10039
|
+
eq17(runs.kind, thisRun.kind),
|
|
10040
|
+
or2(eq17(runs.status, "completed"), eq17(runs.status, "partial"))
|
|
9506
10041
|
)
|
|
9507
|
-
).orderBy(
|
|
10042
|
+
).orderBy(desc7(runs.createdAt), desc7(runs.id)).limit(RECENT_FETCH_LIMIT).all();
|
|
9508
10043
|
const groups = groupRunsByCreatedAt(recentRuns);
|
|
9509
10044
|
const currentGroupIdx = groups.findIndex((g) => g[0]?.createdAt === thisRun.createdAt);
|
|
9510
10045
|
if (currentGroupIdx < 0) return [];
|
|
@@ -9519,7 +10054,7 @@ var Notifier = class {
|
|
|
9519
10054
|
provider: querySnapshots.provider,
|
|
9520
10055
|
location: querySnapshots.location,
|
|
9521
10056
|
citationState: querySnapshots.citationState
|
|
9522
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
10057
|
+
}).from(querySnapshots).leftJoin(queries, eq17(querySnapshots.queryId, queries.id)).where(inArray6(querySnapshots.runId, currentRunIds)).all();
|
|
9523
10058
|
const previousSnapshots = this.db.select({
|
|
9524
10059
|
queryId: querySnapshots.queryId,
|
|
9525
10060
|
provider: querySnapshots.provider,
|
|
@@ -9589,7 +10124,7 @@ var Notifier = class {
|
|
|
9589
10124
|
}
|
|
9590
10125
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
9591
10126
|
this.db.insert(auditLog).values({
|
|
9592
|
-
id:
|
|
10127
|
+
id: crypto20.randomUUID(),
|
|
9593
10128
|
projectId,
|
|
9594
10129
|
actor: "scheduler",
|
|
9595
10130
|
action: `notification.${status}`,
|
|
@@ -9602,7 +10137,7 @@ var Notifier = class {
|
|
|
9602
10137
|
};
|
|
9603
10138
|
|
|
9604
10139
|
// src/run-coordinator.ts
|
|
9605
|
-
import { eq as
|
|
10140
|
+
import { eq as eq18 } from "drizzle-orm";
|
|
9606
10141
|
var log16 = createLogger("RunCoordinator");
|
|
9607
10142
|
var RunCoordinator = class {
|
|
9608
10143
|
constructor(db, notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
@@ -9618,7 +10153,7 @@ var RunCoordinator = class {
|
|
|
9618
10153
|
onInsightsGenerated;
|
|
9619
10154
|
onAeroEvent;
|
|
9620
10155
|
async onRunCompleted(runId, projectId) {
|
|
9621
|
-
const runRow = this.db.select().from(runs).where(
|
|
10156
|
+
const runRow = this.db.select().from(runs).where(eq18(runs.id, runId)).get();
|
|
9622
10157
|
const kind = runRow?.kind ?? RunKinds["answer-visibility"];
|
|
9623
10158
|
if (runRow?.trigger === RunTriggers.probe) {
|
|
9624
10159
|
log16.info("probe.skip-side-effects", { runId, projectId, kind });
|
|
@@ -9709,7 +10244,7 @@ var RunCoordinator = class {
|
|
|
9709
10244
|
* so the Aero queue is never starved of a follow-up.
|
|
9710
10245
|
*/
|
|
9711
10246
|
buildDiscoveryAeroContext(runId, projectId, status, error) {
|
|
9712
|
-
const session = this.db.select().from(discoverySessions).where(
|
|
10247
|
+
const session = this.db.select().from(discoverySessions).where(eq18(discoverySessions.runId, runId)).get();
|
|
9713
10248
|
const competitorMap = session ? session.competitorMap : [];
|
|
9714
10249
|
return {
|
|
9715
10250
|
kind: RunKinds["aeo-discover-probe"],
|
|
@@ -9751,8 +10286,8 @@ function analysisResultFromInsights(insights2) {
|
|
|
9751
10286
|
}
|
|
9752
10287
|
|
|
9753
10288
|
// src/agent/session-registry.ts
|
|
9754
|
-
import
|
|
9755
|
-
import { eq as
|
|
10289
|
+
import crypto24 from "crypto";
|
|
10290
|
+
import { eq as eq20 } from "drizzle-orm";
|
|
9756
10291
|
|
|
9757
10292
|
// src/agent/session.ts
|
|
9758
10293
|
import fs7 from "fs";
|
|
@@ -10495,7 +11030,7 @@ function buildAeroStateTools(ctx, opts = {}) {
|
|
|
10495
11030
|
}
|
|
10496
11031
|
|
|
10497
11032
|
// src/agent/llm-usage.ts
|
|
10498
|
-
import
|
|
11033
|
+
import crypto21 from "crypto";
|
|
10499
11034
|
var AeroLlmUsageFeatures = {
|
|
10500
11035
|
turn: "aero.turn"
|
|
10501
11036
|
};
|
|
@@ -10514,7 +11049,7 @@ function recordLlmUsageEvent(args) {
|
|
|
10514
11049
|
const usage = args.message.usage;
|
|
10515
11050
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
10516
11051
|
args.db.insert(llmUsageEvents).values({
|
|
10517
|
-
id:
|
|
11052
|
+
id: crypto21.randomUUID(),
|
|
10518
11053
|
projectId: args.projectId,
|
|
10519
11054
|
runId: args.runId,
|
|
10520
11055
|
agentSessionId: args.agentSessionId,
|
|
@@ -10590,7 +11125,7 @@ function splitAeroAnthropicSystemCachePayload(payload, model) {
|
|
|
10590
11125
|
}
|
|
10591
11126
|
|
|
10592
11127
|
// src/agent/tool-usage.ts
|
|
10593
|
-
import
|
|
11128
|
+
import crypto22 from "crypto";
|
|
10594
11129
|
var AeroToolEventStatuses = {
|
|
10595
11130
|
success: "success",
|
|
10596
11131
|
error: "error"
|
|
@@ -10653,7 +11188,7 @@ function createAeroToolUsageHooks(args) {
|
|
|
10653
11188
|
function recordAgentToolEvent(args) {
|
|
10654
11189
|
try {
|
|
10655
11190
|
args.db.insert(agentToolEvents).values({
|
|
10656
|
-
id:
|
|
11191
|
+
id: crypto22.randomUUID(),
|
|
10657
11192
|
projectId: args.projectId,
|
|
10658
11193
|
agentSessionId: args.agentSessionId,
|
|
10659
11194
|
toolCallId: args.toolCallId,
|
|
@@ -10880,8 +11415,8 @@ async function loadExternalMcpTools(servers, opts = {}) {
|
|
|
10880
11415
|
}
|
|
10881
11416
|
|
|
10882
11417
|
// src/agent/memory-store.ts
|
|
10883
|
-
import
|
|
10884
|
-
import { and as
|
|
11418
|
+
import crypto23 from "crypto";
|
|
11419
|
+
import { and as and15, desc as desc8, eq as eq19, like, sql as sql9 } from "drizzle-orm";
|
|
10885
11420
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
10886
11421
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
10887
11422
|
function rowToDto(row) {
|
|
@@ -10895,7 +11430,7 @@ function rowToDto(row) {
|
|
|
10895
11430
|
};
|
|
10896
11431
|
}
|
|
10897
11432
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
10898
|
-
const query = db.select().from(agentMemory).where(
|
|
11433
|
+
const query = db.select().from(agentMemory).where(eq19(agentMemory.projectId, projectId)).orderBy(desc8(agentMemory.updatedAt));
|
|
10899
11434
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
10900
11435
|
return rows.map(rowToDto);
|
|
10901
11436
|
}
|
|
@@ -10909,7 +11444,7 @@ function upsertMemoryEntry(db, args) {
|
|
|
10909
11444
|
throw new Error(`memory key prefix "${COMPACTION_KEY_PREFIX}" is reserved for compaction notes`);
|
|
10910
11445
|
}
|
|
10911
11446
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
10912
|
-
const id =
|
|
11447
|
+
const id = crypto23.randomUUID();
|
|
10913
11448
|
db.insert(agentMemory).values({
|
|
10914
11449
|
id,
|
|
10915
11450
|
projectId: args.projectId,
|
|
@@ -10926,12 +11461,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
10926
11461
|
updatedAt: now
|
|
10927
11462
|
}
|
|
10928
11463
|
}).run();
|
|
10929
|
-
const row = db.select().from(agentMemory).where(
|
|
11464
|
+
const row = db.select().from(agentMemory).where(and15(eq19(agentMemory.projectId, args.projectId), eq19(agentMemory.key, args.key))).get();
|
|
10930
11465
|
if (!row) throw new Error("memory upsert produced no row");
|
|
10931
11466
|
return rowToDto(row);
|
|
10932
11467
|
}
|
|
10933
11468
|
function deleteMemoryEntry(db, projectId, key) {
|
|
10934
|
-
const result = db.delete(agentMemory).where(
|
|
11469
|
+
const result = db.delete(agentMemory).where(and15(eq19(agentMemory.projectId, projectId), eq19(agentMemory.key, key))).run();
|
|
10935
11470
|
const changes = result.changes ?? 0;
|
|
10936
11471
|
return changes > 0;
|
|
10937
11472
|
}
|
|
@@ -10946,7 +11481,7 @@ function writeCompactionNote(db, args) {
|
|
|
10946
11481
|
}
|
|
10947
11482
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
10948
11483
|
const key = `${COMPACTION_KEY_PREFIX}${args.sessionId}:${now}`;
|
|
10949
|
-
const id =
|
|
11484
|
+
const id = crypto23.randomUUID();
|
|
10950
11485
|
let inserted;
|
|
10951
11486
|
db.transaction((tx) => {
|
|
10952
11487
|
tx.insert(agentMemory).values({
|
|
@@ -10960,16 +11495,16 @@ function writeCompactionNote(db, args) {
|
|
|
10960
11495
|
}).run();
|
|
10961
11496
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
10962
11497
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
10963
|
-
|
|
10964
|
-
|
|
11498
|
+
and15(
|
|
11499
|
+
eq19(agentMemory.projectId, args.projectId),
|
|
10965
11500
|
like(agentMemory.key, `${sessionPrefix}%`)
|
|
10966
11501
|
)
|
|
10967
|
-
).orderBy(
|
|
11502
|
+
).orderBy(desc8(agentMemory.updatedAt)).all();
|
|
10968
11503
|
const stale = existing.slice(COMPACTION_NOTES_PER_SESSION).map((r) => r.id);
|
|
10969
11504
|
if (stale.length > 0) {
|
|
10970
|
-
tx.delete(agentMemory).where(
|
|
11505
|
+
tx.delete(agentMemory).where(sql9`${agentMemory.id} IN (${sql9.join(stale.map((s) => sql9`${s}`), sql9`, `)})`).run();
|
|
10971
11506
|
}
|
|
10972
|
-
const row = tx.select().from(agentMemory).where(
|
|
11507
|
+
const row = tx.select().from(agentMemory).where(and15(eq19(agentMemory.projectId, args.projectId), eq19(agentMemory.key, key))).get();
|
|
10973
11508
|
if (row) inserted = rowToDto(row);
|
|
10974
11509
|
});
|
|
10975
11510
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -11196,7 +11731,7 @@ var SessionRegistry = class {
|
|
|
11196
11731
|
modelProvider: effectiveProvider,
|
|
11197
11732
|
modelId: effectiveModelId,
|
|
11198
11733
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
11199
|
-
}).where(
|
|
11734
|
+
}).where(eq20(agentSessions.projectId, projectId)).run();
|
|
11200
11735
|
}
|
|
11201
11736
|
const agent2 = createAeroSession({
|
|
11202
11737
|
projectName,
|
|
@@ -11225,7 +11760,7 @@ var SessionRegistry = class {
|
|
|
11225
11760
|
}
|
|
11226
11761
|
const { provider, modelId } = resolveSessionProviderAndModel(this.opts.config, preferences);
|
|
11227
11762
|
const systemPrompt = loadAeroSystemPrompt();
|
|
11228
|
-
const sessionId =
|
|
11763
|
+
const sessionId = crypto24.randomUUID();
|
|
11229
11764
|
const agent = createAeroSession({
|
|
11230
11765
|
projectName,
|
|
11231
11766
|
client: this.opts.client,
|
|
@@ -11430,7 +11965,7 @@ ${lines.join("\n")}
|
|
|
11430
11965
|
modelProvider: nextProvider,
|
|
11431
11966
|
modelId: nextModelId,
|
|
11432
11967
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
11433
|
-
}).where(
|
|
11968
|
+
}).where(eq20(agentSessions.projectId, projectId)).run();
|
|
11434
11969
|
}
|
|
11435
11970
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
11436
11971
|
save(projectName) {
|
|
@@ -11594,17 +12129,17 @@ ${lines.join("\n")}
|
|
|
11594
12129
|
return id;
|
|
11595
12130
|
}
|
|
11596
12131
|
tryResolveProjectId(projectName) {
|
|
11597
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
12132
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq20(projects.name, projectName)).get();
|
|
11598
12133
|
return row?.id;
|
|
11599
12134
|
}
|
|
11600
12135
|
loadRow(projectId) {
|
|
11601
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
12136
|
+
const row = this.opts.db.select().from(agentSessions).where(eq20(agentSessions.projectId, projectId)).get();
|
|
11602
12137
|
return row ?? null;
|
|
11603
12138
|
}
|
|
11604
12139
|
insertRow(params) {
|
|
11605
12140
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
11606
12141
|
this.opts.db.insert(agentSessions).values({
|
|
11607
|
-
id: params.id ??
|
|
12142
|
+
id: params.id ?? crypto24.randomUUID(),
|
|
11608
12143
|
projectId: params.projectId,
|
|
11609
12144
|
systemPrompt: params.systemPrompt,
|
|
11610
12145
|
modelProvider: params.provider ?? params.modelProvider ?? AgentProviderIds.claude,
|
|
@@ -11617,14 +12152,14 @@ ${lines.join("\n")}
|
|
|
11617
12152
|
}
|
|
11618
12153
|
updateRow(projectId, patch) {
|
|
11619
12154
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
11620
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
12155
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq20(agentSessions.projectId, projectId)).run();
|
|
11621
12156
|
}
|
|
11622
12157
|
};
|
|
11623
12158
|
|
|
11624
12159
|
// src/agent/agent-routes.ts
|
|
11625
|
-
import { eq as
|
|
12160
|
+
import { eq as eq21 } from "drizzle-orm";
|
|
11626
12161
|
function resolveProject(db, name) {
|
|
11627
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
12162
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq21(projects.name, name)).get();
|
|
11628
12163
|
if (!row) throw notFound("project", name);
|
|
11629
12164
|
return row;
|
|
11630
12165
|
}
|
|
@@ -11633,7 +12168,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
11633
12168
|
"/projects/:name/agent/transcript",
|
|
11634
12169
|
async (request) => {
|
|
11635
12170
|
const project = resolveProject(opts.db, request.params.name);
|
|
11636
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
12171
|
+
const row = opts.db.select().from(agentSessions).where(eq21(agentSessions.projectId, project.id)).get();
|
|
11637
12172
|
if (!row) {
|
|
11638
12173
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
11639
12174
|
}
|
|
@@ -11657,7 +12192,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
11657
12192
|
async (request) => {
|
|
11658
12193
|
const project = resolveProject(opts.db, request.params.name);
|
|
11659
12194
|
opts.sessionRegistry.reset(project.name);
|
|
11660
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
12195
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq21(agentSessions.projectId, project.id)).run();
|
|
11661
12196
|
return { status: "reset" };
|
|
11662
12197
|
}
|
|
11663
12198
|
);
|
|
@@ -12465,7 +13000,7 @@ function buildSnapshotSummary(companyName, queries2, providers, queryResults, au
|
|
|
12465
13000
|
competitorCounts.set(competitor, (competitorCounts.get(competitor) ?? 0) + 1);
|
|
12466
13001
|
}
|
|
12467
13002
|
}
|
|
12468
|
-
const topCompetitors = [...competitorCounts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 10).map(([name,
|
|
13003
|
+
const topCompetitors = [...competitorCounts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 10).map(([name, count2]) => ({ name, count: count2 }));
|
|
12469
13004
|
const defaultMeaning = totalComparisons > 0 ? `${companyName} was mentioned in ${mentionCount}/${totalComparisons} successful provider-query response${totalComparisons === 1 ? "" : "s"} across ${queries2.length} category queries.` : `No successful provider responses were returned across ${queries2.length} category queries.`;
|
|
12470
13005
|
const failureNote = failedComparisons > 0 ? [
|
|
12471
13006
|
`${failedComparisons} provider response${failedComparisons === 1 ? "" : "s"} failed and ${failedComparisons === 1 ? "was" : "were"} excluded from visibility totals.`
|
|
@@ -12600,20 +13135,20 @@ function clipText(value, length) {
|
|
|
12600
13135
|
}
|
|
12601
13136
|
|
|
12602
13137
|
// src/research-runner.ts
|
|
12603
|
-
import { and as
|
|
13138
|
+
import { and as and16, eq as eq22, inArray as inArray7, sql as sql10 } from "drizzle-orm";
|
|
12604
13139
|
var unfinishedResearchQueryStatuses = [ResearchQueryStatuses.queued, ResearchQueryStatuses.running];
|
|
12605
13140
|
function finalResearchRunStatus(completed, failed) {
|
|
12606
13141
|
if (failed === 0) return ResearchRunStatuses.completed;
|
|
12607
13142
|
return completed > 0 ? ResearchRunStatuses.partial : ResearchRunStatuses.failed;
|
|
12608
13143
|
}
|
|
12609
13144
|
async function executeResearchRun(db, registry, runId, projectId) {
|
|
12610
|
-
const run = db.select().from(researchRuns).where(
|
|
13145
|
+
const run = db.select().from(researchRuns).where(and16(eq22(researchRuns.id, runId), eq22(researchRuns.projectId, projectId))).get();
|
|
12611
13146
|
if (!run || run.status !== ResearchRunStatuses.queued) return;
|
|
12612
|
-
const project = db.select().from(projects).where(
|
|
13147
|
+
const project = db.select().from(projects).where(eq22(projects.id, projectId)).get();
|
|
12613
13148
|
if (!project) return;
|
|
12614
13149
|
const provider = registry.get(run.provider);
|
|
12615
13150
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
12616
|
-
const claim = db.update(researchRuns).set({ status: ResearchRunStatuses.running, startedAt: now }).where(
|
|
13151
|
+
const claim = db.update(researchRuns).set({ status: ResearchRunStatuses.running, startedAt: now }).where(and16(eq22(researchRuns.id, runId), eq22(researchRuns.status, ResearchRunStatuses.queued))).run();
|
|
12617
13152
|
if (claim.changes !== 1) return;
|
|
12618
13153
|
let reserved = 0;
|
|
12619
13154
|
let dispatched = 0;
|
|
@@ -12637,8 +13172,8 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
12637
13172
|
}
|
|
12638
13173
|
reserved = run.totalQueries;
|
|
12639
13174
|
reservation = { scope, period };
|
|
12640
|
-
const rows = db.select().from(researchRunQueries).where(
|
|
12641
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
13175
|
+
const rows = db.select().from(researchRunQueries).where(eq22(researchRunQueries.researchRunId, runId)).orderBy(researchRunQueries.position).all();
|
|
13176
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq22(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
12642
13177
|
const domains = effectiveDomains(project);
|
|
12643
13178
|
const brands = effectiveBrandNames(project);
|
|
12644
13179
|
const config = { ...provider.config, model: run.resolvedModel };
|
|
@@ -12646,7 +13181,7 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
12646
13181
|
await mapWithConcurrency(rows, Math.max(1, provider.config.quotaPolicy.maxConcurrency), async (row) => {
|
|
12647
13182
|
try {
|
|
12648
13183
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
12649
|
-
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.running, startedAt }).where(
|
|
13184
|
+
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.running, startedAt }).where(and16(eq22(researchRunQueries.id, row.id), eq22(researchRunQueries.status, ResearchQueryStatuses.queued))).run();
|
|
12650
13185
|
const raw = await gate.run(async () => {
|
|
12651
13186
|
dispatched++;
|
|
12652
13187
|
return provider.adapter.executeTrackedQuery({ query: row.queryText, canonicalDomains: domains, competitorDomains, ...run.location ? { location: run.location } : {} }, config);
|
|
@@ -12673,7 +13208,7 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
12673
13208
|
citationState: determineCitationState(normalized, domains),
|
|
12674
13209
|
rawResponse: raw.rawResponse,
|
|
12675
13210
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
12676
|
-
}).where(
|
|
13211
|
+
}).where(and16(eq22(researchRunQueries.id, row.id), eq22(researchRunQueries.status, ResearchQueryStatuses.running))).run();
|
|
12677
13212
|
if (completed.changes === 1) incrementResearchProgress(db, runId, "completedQueries");
|
|
12678
13213
|
} catch (error) {
|
|
12679
13214
|
try {
|
|
@@ -12697,18 +13232,18 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
12697
13232
|
}
|
|
12698
13233
|
}
|
|
12699
13234
|
function incrementResearchProgress(db, runId, column) {
|
|
12700
|
-
db.update(researchRuns).set({ [column]:
|
|
13235
|
+
db.update(researchRuns).set({ [column]: sql10`${researchRuns[column]} + 1` }).where(eq22(researchRuns.id, runId)).run();
|
|
12701
13236
|
}
|
|
12702
13237
|
function markResearchQueryFailed(db, runId, queryId, error) {
|
|
12703
13238
|
const message = error instanceof Error ? error.message : String(error);
|
|
12704
|
-
const failed = db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error: message, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
13239
|
+
const failed = db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error: message, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and16(eq22(researchRunQueries.id, queryId), inArray7(researchRunQueries.status, unfinishedResearchQueryStatuses))).run();
|
|
12705
13240
|
if (failed.changes === 1) incrementResearchProgress(db, runId, "failedQueries");
|
|
12706
13241
|
}
|
|
12707
13242
|
function markUnfinishedResearchQueriesFailed(db, runId, error) {
|
|
12708
|
-
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
13243
|
+
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and16(eq22(researchRunQueries.researchRunId, runId), inArray7(researchRunQueries.status, unfinishedResearchQueryStatuses))).run();
|
|
12709
13244
|
}
|
|
12710
13245
|
function finalizeResearchRun(db, runId, fatalError) {
|
|
12711
|
-
const rows = db.select({ status: researchRunQueries.status }).from(researchRunQueries).where(
|
|
13246
|
+
const rows = db.select({ status: researchRunQueries.status }).from(researchRunQueries).where(eq22(researchRunQueries.researchRunId, runId)).all();
|
|
12712
13247
|
const completed = rows.filter((row) => row.status === ResearchQueryStatuses.completed).length;
|
|
12713
13248
|
const failed = rows.filter((row) => row.status === ResearchQueryStatuses.failed).length;
|
|
12714
13249
|
db.update(researchRuns).set({
|
|
@@ -12717,11 +13252,11 @@ function finalizeResearchRun(db, runId, fatalError) {
|
|
|
12717
13252
|
failedQueries: failed,
|
|
12718
13253
|
error: fatalError ?? (failed === rows.length ? "Every research query failed." : null),
|
|
12719
13254
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
12720
|
-
}).where(
|
|
13255
|
+
}).where(eq22(researchRuns.id, runId)).run();
|
|
12721
13256
|
}
|
|
12722
13257
|
|
|
12723
13258
|
// src/server.ts
|
|
12724
|
-
var _require3 =
|
|
13259
|
+
var _require3 = createRequire4(import.meta.url);
|
|
12725
13260
|
var { version: PKG_VERSION2 } = _require3("../package.json");
|
|
12726
13261
|
var log20 = createLogger("Server");
|
|
12727
13262
|
var DEFAULT_QUOTA = {
|
|
@@ -12780,14 +13315,14 @@ function summarizeProviderConfig(config) {
|
|
|
12780
13315
|
};
|
|
12781
13316
|
}
|
|
12782
13317
|
function hashApiKey(key) {
|
|
12783
|
-
return
|
|
13318
|
+
return crypto25.createHash("sha256").update(key).digest("hex");
|
|
12784
13319
|
}
|
|
12785
13320
|
var DASHBOARD_SCRYPT_KEYLEN = 64;
|
|
12786
13321
|
var DASHBOARD_SCRYPT_COST = 1 << 15;
|
|
12787
13322
|
var DASHBOARD_SCRYPT_MAXMEM = 64 * 1024 * 1024;
|
|
12788
13323
|
function hashDashboardPassword(password) {
|
|
12789
|
-
const salt =
|
|
12790
|
-
const derived =
|
|
13324
|
+
const salt = crypto25.randomBytes(16);
|
|
13325
|
+
const derived = crypto25.scryptSync(password, salt, DASHBOARD_SCRYPT_KEYLEN, {
|
|
12791
13326
|
N: DASHBOARD_SCRYPT_COST,
|
|
12792
13327
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
12793
13328
|
});
|
|
@@ -12808,14 +13343,14 @@ function verifyDashboardPassword(password, storedHash) {
|
|
|
12808
13343
|
} catch {
|
|
12809
13344
|
return { ok: false, needsRehash: false };
|
|
12810
13345
|
}
|
|
12811
|
-
const derived =
|
|
13346
|
+
const derived = crypto25.scryptSync(password, salt, expected.length, {
|
|
12812
13347
|
N: DASHBOARD_SCRYPT_COST,
|
|
12813
13348
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
12814
13349
|
});
|
|
12815
13350
|
if (derived.length !== expected.length)
|
|
12816
13351
|
return { ok: false, needsRehash: false };
|
|
12817
13352
|
return {
|
|
12818
|
-
ok:
|
|
13353
|
+
ok: crypto25.timingSafeEqual(derived, expected),
|
|
12819
13354
|
needsRehash: false
|
|
12820
13355
|
};
|
|
12821
13356
|
}
|
|
@@ -12824,7 +13359,7 @@ function verifyDashboardPassword(password, storedHash) {
|
|
|
12824
13359
|
const expected = Buffer.from(storedHash, "hex");
|
|
12825
13360
|
if (candidate.length !== expected.length)
|
|
12826
13361
|
return { ok: false, needsRehash: false };
|
|
12827
|
-
const ok =
|
|
13362
|
+
const ok = crypto25.timingSafeEqual(candidate, expected);
|
|
12828
13363
|
return { ok, needsRehash: ok };
|
|
12829
13364
|
}
|
|
12830
13365
|
return { ok: false, needsRehash: false };
|
|
@@ -13032,7 +13567,7 @@ async function createServer(opts) {
|
|
|
13032
13567
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
13033
13568
|
async (ctx) => {
|
|
13034
13569
|
if (!sessionRegistry) return;
|
|
13035
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
13570
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq23(projects.id, ctx.projectId)).get();
|
|
13036
13571
|
if (!project) return;
|
|
13037
13572
|
let content;
|
|
13038
13573
|
if (ctx.kind === RunKinds["aeo-discover-probe"]) {
|
|
@@ -13382,7 +13917,7 @@ async function createServer(opts) {
|
|
|
13382
13917
|
void (async () => {
|
|
13383
13918
|
try {
|
|
13384
13919
|
const report = await aeroClient.runDoctor({ project: projectName });
|
|
13385
|
-
const project = opts.db.select().from(projects).where(
|
|
13920
|
+
const project = opts.db.select().from(projects).where(eq23(projects.name, projectName)).get();
|
|
13386
13921
|
if (!project) {
|
|
13387
13922
|
app.log.warn({ projectName }, "doctor schedule fired for an unknown project");
|
|
13388
13923
|
return;
|
|
@@ -13410,9 +13945,9 @@ async function createServer(opts) {
|
|
|
13410
13945
|
});
|
|
13411
13946
|
if (!probed) return;
|
|
13412
13947
|
const alreadySynced = opts.db.select().from(ccReleaseSyncs).where(
|
|
13413
|
-
|
|
13414
|
-
|
|
13415
|
-
|
|
13948
|
+
and17(
|
|
13949
|
+
eq23(ccReleaseSyncs.release, probed.release),
|
|
13950
|
+
eq23(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)
|
|
13416
13951
|
)
|
|
13417
13952
|
).limit(1).get();
|
|
13418
13953
|
if (alreadySynced) {
|
|
@@ -13567,7 +14102,25 @@ async function createServer(opts) {
|
|
|
13567
14102
|
return removed;
|
|
13568
14103
|
}
|
|
13569
14104
|
};
|
|
13570
|
-
const
|
|
14105
|
+
const cloudflareTrafficCredentialStore = {
|
|
14106
|
+
getConnection: (projectName) => {
|
|
14107
|
+
return getCloudflareTrafficConnection(opts.config, projectName);
|
|
14108
|
+
},
|
|
14109
|
+
getConnectionBySourceId: (sourceId) => {
|
|
14110
|
+
return getCloudflareTrafficConnectionBySourceId(opts.config, sourceId);
|
|
14111
|
+
},
|
|
14112
|
+
upsertConnection: (record) => {
|
|
14113
|
+
const updated = upsertCloudflareTrafficConnection(opts.config, record);
|
|
14114
|
+
saveConfigPatch(opts.config);
|
|
14115
|
+
return updated;
|
|
14116
|
+
},
|
|
14117
|
+
deleteConnection: (projectName) => {
|
|
14118
|
+
const removed = removeCloudflareTrafficConnection(opts.config, projectName);
|
|
14119
|
+
if (removed) saveConfigPatch(opts.config);
|
|
14120
|
+
return removed;
|
|
14121
|
+
}
|
|
14122
|
+
};
|
|
14123
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto25.randomBytes(32).toString("hex");
|
|
13571
14124
|
const googleConnectionStore = {
|
|
13572
14125
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
13573
14126
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -13642,11 +14195,11 @@ async function createServer(opts) {
|
|
|
13642
14195
|
const googlePublicUrl = resolveGooglePublicUrl(opts.config, basePath);
|
|
13643
14196
|
if (opts.config.apiKey) {
|
|
13644
14197
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
13645
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
14198
|
+
const existing = opts.db.select().from(apiKeys).where(eq23(apiKeys.keyHash, keyHash)).get();
|
|
13646
14199
|
if (!existing) {
|
|
13647
14200
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
13648
14201
|
opts.db.insert(apiKeys).values({
|
|
13649
|
-
id: `key_${
|
|
14202
|
+
id: `key_${crypto25.randomBytes(8).toString("hex")}`,
|
|
13650
14203
|
name: "default",
|
|
13651
14204
|
keyHash,
|
|
13652
14205
|
keyPrefix: prefix,
|
|
@@ -13670,7 +14223,7 @@ async function createServer(opts) {
|
|
|
13670
14223
|
};
|
|
13671
14224
|
const createSession = (apiKeyId) => {
|
|
13672
14225
|
pruneExpiredSessions();
|
|
13673
|
-
const sessionId =
|
|
14226
|
+
const sessionId = crypto25.randomBytes(32).toString("hex");
|
|
13674
14227
|
sessions.set(sessionId, {
|
|
13675
14228
|
apiKeyId,
|
|
13676
14229
|
expiresAt: Date.now() + SESSION_TTL_MS
|
|
@@ -13694,7 +14247,7 @@ async function createServer(opts) {
|
|
|
13694
14247
|
};
|
|
13695
14248
|
const getDefaultApiKey = () => {
|
|
13696
14249
|
if (!opts.config.apiKey) return void 0;
|
|
13697
|
-
return opts.db.select().from(apiKeys).where(
|
|
14250
|
+
return opts.db.select().from(apiKeys).where(eq23(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
13698
14251
|
};
|
|
13699
14252
|
const createPasswordSession = (reply) => {
|
|
13700
14253
|
const key = getDefaultApiKey();
|
|
@@ -13718,7 +14271,7 @@ async function createServer(opts) {
|
|
|
13718
14271
|
if (!header) return false;
|
|
13719
14272
|
const parts = header.split(" ");
|
|
13720
14273
|
if (parts.length !== 2 || parts[0] !== "Bearer") return false;
|
|
13721
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
14274
|
+
const key = opts.db.select().from(apiKeys).where(eq23(apiKeys.keyHash, hashApiKey(parts[1]))).get();
|
|
13722
14275
|
return Boolean(key && !key.revokedAt);
|
|
13723
14276
|
};
|
|
13724
14277
|
const namedAccountsInUse = () => anyUsersExist(opts.db);
|
|
@@ -13808,12 +14361,12 @@ async function createServer(opts) {
|
|
|
13808
14361
|
return reply.send({ authenticated: true });
|
|
13809
14362
|
}
|
|
13810
14363
|
if (apiKey) {
|
|
13811
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
14364
|
+
const key = opts.db.select().from(apiKeys).where(eq23(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
13812
14365
|
if (!key || key.revokedAt) {
|
|
13813
14366
|
const err2 = authInvalid();
|
|
13814
14367
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
13815
14368
|
}
|
|
13816
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
14369
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq23(apiKeys.id, key.id)).run();
|
|
13817
14370
|
const sessionId = createSession(key.id);
|
|
13818
14371
|
reply.header(
|
|
13819
14372
|
"set-cookie",
|
|
@@ -14030,7 +14583,7 @@ async function createServer(opts) {
|
|
|
14030
14583
|
deps: {
|
|
14031
14584
|
enqueueAutoExtract: ({ projectId, release: r }) => {
|
|
14032
14585
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
14033
|
-
const runId =
|
|
14586
|
+
const runId = crypto25.randomUUID();
|
|
14034
14587
|
opts.db.insert(runs).values({
|
|
14035
14588
|
id: runId,
|
|
14036
14589
|
projectId,
|
|
@@ -14155,7 +14708,7 @@ async function createServer(opts) {
|
|
|
14155
14708
|
...inspectOpts,
|
|
14156
14709
|
config: opts.config
|
|
14157
14710
|
}).then(() => {
|
|
14158
|
-
const finished = opts.db.select({ status: runs.status }).from(runs).where(
|
|
14711
|
+
const finished = opts.db.select({ status: runs.status }).from(runs).where(eq23(runs.id, runId)).get();
|
|
14159
14712
|
if (finished?.status === RunStatuses.completed || finished?.status === RunStatuses.partial) {
|
|
14160
14713
|
return maybeRefreshGscCoverage(
|
|
14161
14714
|
opts.db,
|
|
@@ -14176,6 +14729,8 @@ async function createServer(opts) {
|
|
|
14176
14729
|
cloudRunCredentialStore,
|
|
14177
14730
|
wordpressTrafficCredentialStore,
|
|
14178
14731
|
vercelTrafficCredentialStore,
|
|
14732
|
+
cloudflareTrafficCredentialStore,
|
|
14733
|
+
cloudflareTrafficIngestUrl: buildCloudflareIngestUrlTemplate(opts.config),
|
|
14179
14734
|
onTrafficSynced: (event) => {
|
|
14180
14735
|
trackEvent(
|
|
14181
14736
|
"traffic.synced",
|
|
@@ -14263,7 +14818,7 @@ async function createServer(opts) {
|
|
|
14263
14818
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
14264
14819
|
opts.db.insert(auditLog).values(
|
|
14265
14820
|
targetProjectIds.map((projectId) => ({
|
|
14266
|
-
id:
|
|
14821
|
+
id: crypto25.randomUUID(),
|
|
14267
14822
|
projectId,
|
|
14268
14823
|
actor: "api",
|
|
14269
14824
|
action: existing ? "provider.updated" : "provider.created",
|
|
@@ -14401,7 +14956,7 @@ async function createServer(opts) {
|
|
|
14401
14956
|
}
|
|
14402
14957
|
];
|
|
14403
14958
|
},
|
|
14404
|
-
onGenerateQueries: async (providerName,
|
|
14959
|
+
onGenerateQueries: async (providerName, count2, project) => {
|
|
14405
14960
|
const provider = registry.get(providerName);
|
|
14406
14961
|
if (!provider)
|
|
14407
14962
|
throw new Error(`Provider "${providerName}" is not configured`);
|
|
@@ -14413,10 +14968,10 @@ async function createServer(opts) {
|
|
|
14413
14968
|
language: project.language,
|
|
14414
14969
|
existingQueries: project.existingQueries,
|
|
14415
14970
|
siteText,
|
|
14416
|
-
count
|
|
14971
|
+
count: count2
|
|
14417
14972
|
});
|
|
14418
14973
|
const raw = await provider.adapter.generateText(prompt, provider.config);
|
|
14419
|
-
return parseQueryResponse(raw,
|
|
14974
|
+
return parseQueryResponse(raw, count2);
|
|
14420
14975
|
},
|
|
14421
14976
|
onSnapshotRequested: async (input) => {
|
|
14422
14977
|
return snapshotService.createReport(input);
|
|
@@ -14531,7 +15086,7 @@ async function createServer(opts) {
|
|
|
14531
15086
|
}
|
|
14532
15087
|
checkLatestVersionForServer();
|
|
14533
15088
|
scheduler.start();
|
|
14534
|
-
for (const run of opts.db.select({ id: researchRuns.id, projectId: researchRuns.projectId }).from(researchRuns).where(
|
|
15089
|
+
for (const run of opts.db.select({ id: researchRuns.id, projectId: researchRuns.projectId }).from(researchRuns).where(eq23(researchRuns.status, ResearchRunStatuses.queued)).all()) {
|
|
14535
15090
|
dispatchResearchRun(run.id, run.projectId);
|
|
14536
15091
|
}
|
|
14537
15092
|
app.addHook("onClose", async () => {
|
|
@@ -14575,7 +15130,7 @@ function buildQueryGenerationPrompt(ctx) {
|
|
|
14575
15130
|
);
|
|
14576
15131
|
return lines.join("\n");
|
|
14577
15132
|
}
|
|
14578
|
-
function parseQueryResponse(raw,
|
|
15133
|
+
function parseQueryResponse(raw, count2) {
|
|
14579
15134
|
const seen = /* @__PURE__ */ new Set();
|
|
14580
15135
|
const results = [];
|
|
14581
15136
|
for (const line of raw.split("\n")) {
|
|
@@ -14591,7 +15146,7 @@ function parseQueryResponse(raw, count) {
|
|
|
14591
15146
|
if (seen.has(key)) continue;
|
|
14592
15147
|
seen.add(key);
|
|
14593
15148
|
results.push(cleaned);
|
|
14594
|
-
if (results.length >=
|
|
15149
|
+
if (results.length >= count2) break;
|
|
14595
15150
|
}
|
|
14596
15151
|
return results;
|
|
14597
15152
|
}
|
|
@@ -14614,6 +15169,7 @@ export {
|
|
|
14614
15169
|
backfillInsightsCommand,
|
|
14615
15170
|
backfillSnapshotAttributionCommand,
|
|
14616
15171
|
backfillTrafficClassificationCommand,
|
|
15172
|
+
getCloudflareTrafficConnection,
|
|
14617
15173
|
setGoogleAuthConfig,
|
|
14618
15174
|
installSkills,
|
|
14619
15175
|
listSkills,
|