@ainyc/canonry 5.1.3 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agent-workspace/skills/aero/SKILL.md +43 -9
- package/assets/agent-workspace/skills/aero/references/agent-operations.md +231 -0
- package/assets/agent-workspace/skills/aero/references/orchestration.md +11 -5
- package/assets/agent-workspace/skills/aero/references/portfolio-analysis.md +93 -0
- package/assets/agent-workspace/skills/aero/references/reporting.md +10 -1
- package/assets/agent-workspace/skills/aero/references/site-health.md +104 -0
- package/assets/agent-workspace/skills/aero/soul.md +5 -0
- package/assets/assets/{AuditHistoryPanel-Dc_pymFg.js → AuditHistoryPanel-D0H5TtxC.js} +1 -1
- package/assets/assets/{BacklinksPage-BkSo48YX.js → BacklinksPage-DRihwG5O.js} +1 -1
- package/assets/assets/{HistoryPage-BbswsdU7.js → HistoryPage-BQ2nEXly.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-BbgtalkM.js → MeasurementPropertyPage-Bp6EURzJ.js} +1 -1
- package/assets/assets/ProjectPage-Cd03Phwx.js +8 -0
- package/assets/assets/{RunRow-2cNW8FHz.js → RunRow-Bo_Adiy0.js} +1 -1
- package/assets/assets/{RunsPage-Cz3lJ6VJ.js → RunsPage-CDkbzjuo.js} +1 -1
- package/assets/assets/{SettingsPage-BQwB0r9k.js → SettingsPage-CF4i_EzR.js} +1 -1
- package/assets/assets/SiteHealthSection-BrzBV7dt.js +4 -0
- package/assets/assets/{TrafficPage-DTic3zhO.js → TrafficPage-BHSz1N4P.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-BKfabnCw.js → TrafficSourceDetailPage-DzHLwhpy.js} +1 -1
- package/assets/assets/{extract-error-message-BzB8j3fC.js → extract-error-message-WHM24Sai.js} +1 -1
- package/assets/assets/{index-DQGEzxrN.js → index-5W4ZZnwg.js} +22 -22
- package/assets/assets/index-DCLmKL1s.css +1 -0
- package/assets/assets/{react-sigma_core.esm.min-rYv0aKKx.js → react-sigma_core.esm.min-BB2Lcq4W.js} +1 -1
- package/assets/assets/{v2-overview-adapter-CWWPfsm-.js → v2-overview-adapter-ebrNjcUG.js} +1 -1
- package/assets/assets/vendor-lucide-D_WBzb76.js +1 -0
- package/assets/index.html +3 -3
- package/dist/{chunk-MMSPU72Z.js → chunk-2U5IW3G5.js} +4 -0
- package/dist/chunk-5VONCLVQ.js +652 -0
- package/dist/chunk-AOJU5LKX.js +8 -0
- package/dist/{chunk-KJM4DRZN.js → chunk-C7PITYL3.js} +5 -3
- package/dist/{chunk-5ZOLPHTO.js → chunk-CW2KPN3L.js} +331 -943
- package/dist/{chunk-AEE5DGTE.js → chunk-IOP36L3C.js} +4 -7
- package/dist/chunk-RB2F4D3Z.js +67 -0
- package/dist/{chunk-EBUX3C5H.js → chunk-YQNKULAD.js} +30 -10
- package/dist/cli.js +100 -5
- package/dist/demo-server-BIPT53EY.js +2125 -0
- package/dist/index.js +6 -4
- package/dist/intelligence-service-SJO5E7WN.js +9 -0
- package/dist/mcp.js +4 -3
- package/package.json +11 -11
- package/assets/assets/ProjectPage-CAcRQp-T.js +0 -8
- package/assets/assets/SiteHealthSection-C0aJ_UhN.js +0 -4
- package/assets/assets/index-CjEprBri.css +0 -1
- package/assets/assets/vendor-lucide-CW2qe0uH.js +0 -1
- package/dist/intelligence-service-25NQCETC.js +0 -7
|
@@ -4,7 +4,6 @@ import {
|
|
|
4
4
|
CANONRY_MCP_TOOLKIT_NAMES,
|
|
5
5
|
CanonryMcpToolNames,
|
|
6
6
|
CliError,
|
|
7
|
-
PACKAGE_VERSION,
|
|
8
7
|
canonryMcpTools,
|
|
9
8
|
configExists,
|
|
10
9
|
createCanonryMcpServer,
|
|
@@ -17,7 +16,15 @@ import {
|
|
|
17
16
|
resolveOperatorApiKeyIds,
|
|
18
17
|
runWithUsageTags,
|
|
19
18
|
saveConfigPatch
|
|
20
|
-
} from "./chunk-
|
|
19
|
+
} from "./chunk-IOP36L3C.js";
|
|
20
|
+
import {
|
|
21
|
+
deleteSiteCrawlGraphLayout,
|
|
22
|
+
persistSiteCrawlGraphLayout,
|
|
23
|
+
prepareSiteCrawlGraphLayout
|
|
24
|
+
} from "./chunk-5VONCLVQ.js";
|
|
25
|
+
import {
|
|
26
|
+
PACKAGE_VERSION
|
|
27
|
+
} from "./chunk-AOJU5LKX.js";
|
|
21
28
|
import {
|
|
22
29
|
CC_CACHE_DIR,
|
|
23
30
|
DUCKDB_SPEC,
|
|
@@ -160,15 +167,12 @@ import {
|
|
|
160
167
|
siteCrawlEdges,
|
|
161
168
|
siteCrawlEventReceipts,
|
|
162
169
|
siteCrawlFindings,
|
|
163
|
-
siteCrawlGraphEdges,
|
|
164
|
-
siteCrawlGraphLayouts,
|
|
165
|
-
siteCrawlGraphNodes,
|
|
166
170
|
siteCrawlPages,
|
|
167
171
|
siteCrawlRunRequests,
|
|
168
172
|
siteCrawlSnapshots,
|
|
169
173
|
toAlertView,
|
|
170
174
|
usageCounters
|
|
171
|
-
} from "./chunk-
|
|
175
|
+
} from "./chunk-YQNKULAD.js";
|
|
172
176
|
import {
|
|
173
177
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
174
178
|
AGENT_NONE,
|
|
@@ -208,12 +212,9 @@ import {
|
|
|
208
212
|
SITE_AUDIT_DEFAULT_PAGE_LIMIT,
|
|
209
213
|
SITE_AUDIT_MAX_EDGE_LIMIT,
|
|
210
214
|
SITE_AUDIT_MAX_PAGE_LIMIT,
|
|
211
|
-
SITE_CRAWL_GRAPH_MAX_EDGES,
|
|
212
|
-
SITE_CRAWL_GRAPH_MAX_NODES,
|
|
213
215
|
SchedulableRunKinds,
|
|
214
216
|
SiteHealthLinkClassificationSources,
|
|
215
217
|
SnapshotProviderModes,
|
|
216
|
-
TEMPLATE_LINK_RATIO_THRESHOLD,
|
|
217
218
|
TrafficEventConfidences,
|
|
218
219
|
TrafficEventKinds,
|
|
219
220
|
TrafficEvidenceKinds,
|
|
@@ -324,7 +325,7 @@ import {
|
|
|
324
325
|
validationError,
|
|
325
326
|
winnabilityClassLabel,
|
|
326
327
|
withRetry
|
|
327
|
-
} from "./chunk-
|
|
328
|
+
} from "./chunk-2U5IW3G5.js";
|
|
328
329
|
|
|
329
330
|
// src/runtime-context.ts
|
|
330
331
|
function cliRuntimeContext(env = process.env, stdio = process) {
|
|
@@ -3531,12 +3532,12 @@ function createGoogleMarketingRuntime(options) {
|
|
|
3531
3532
|
}
|
|
3532
3533
|
|
|
3533
3534
|
// src/server.ts
|
|
3534
|
-
import { createRequire as
|
|
3535
|
-
import
|
|
3535
|
+
import { createRequire as createRequire3 } from "module";
|
|
3536
|
+
import crypto26 from "crypto";
|
|
3536
3537
|
import fs8 from "fs";
|
|
3537
3538
|
import path8 from "path";
|
|
3538
3539
|
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
3539
|
-
import { and as
|
|
3540
|
+
import { and as and19, eq as eq25 } from "drizzle-orm";
|
|
3540
3541
|
import Fastify from "fastify";
|
|
3541
3542
|
import os4 from "os";
|
|
3542
3543
|
|
|
@@ -6456,7 +6457,7 @@ function nonBlank(value) {
|
|
|
6456
6457
|
return trimmed ? trimmed : void 0;
|
|
6457
6458
|
}
|
|
6458
6459
|
function resolveBuildCommit(env = process.env) {
|
|
6459
|
-
const embedded = true ? "
|
|
6460
|
+
const embedded = true ? "40be8574db3459209f5f8887911f27d256df9ab3" : void 0;
|
|
6460
6461
|
return nonBlank(embedded) ?? nonBlank(env.CANONRY_COMMIT);
|
|
6461
6462
|
}
|
|
6462
6463
|
function resolveInstanceIdentity(env = process.env) {
|
|
@@ -7456,7 +7457,7 @@ var JobRunner = class {
|
|
|
7456
7457
|
}
|
|
7457
7458
|
}
|
|
7458
7459
|
}
|
|
7459
|
-
incrementUsage(scope, metric,
|
|
7460
|
+
incrementUsage(scope, metric, count) {
|
|
7460
7461
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7461
7462
|
const period = now.slice(0, 10);
|
|
7462
7463
|
this.db.insert(usageCounters).values({
|
|
@@ -7464,11 +7465,11 @@ var JobRunner = class {
|
|
|
7464
7465
|
scope,
|
|
7465
7466
|
period,
|
|
7466
7467
|
metric,
|
|
7467
|
-
count
|
|
7468
|
+
count,
|
|
7468
7469
|
updatedAt: now
|
|
7469
7470
|
}).onConflictDoUpdate({
|
|
7470
7471
|
target: [usageCounters.scope, usageCounters.period, usageCounters.metric],
|
|
7471
|
-
set: { count: sql2`${usageCounters.count} + ${
|
|
7472
|
+
set: { count: sql2`${usageCounters.count} + ${count}`, updatedAt: now }
|
|
7472
7473
|
}).run();
|
|
7473
7474
|
}
|
|
7474
7475
|
flushProviderUsage(providerDispatchCounts, providerReservations) {
|
|
@@ -10261,8 +10262,8 @@ function buildDiscoveryInsightTitle(input) {
|
|
|
10261
10262
|
}
|
|
10262
10263
|
|
|
10263
10264
|
// src/execute-site-audit.ts
|
|
10264
|
-
import
|
|
10265
|
-
import { and as
|
|
10265
|
+
import crypto18 from "crypto";
|
|
10266
|
+
import { and as and12, eq as eq14, ne as ne2, sql as sql5 } from "drizzle-orm";
|
|
10266
10267
|
import { runSiteCrawl } from "@canonry/aeo-audit";
|
|
10267
10268
|
|
|
10268
10269
|
// src/site-audit-root.ts
|
|
@@ -10449,640 +10450,8 @@ async function requestPinnedSiteAuditRoot(target, options) {
|
|
|
10449
10450
|
});
|
|
10450
10451
|
}
|
|
10451
10452
|
|
|
10452
|
-
// src/site-crawl-graph-layout.ts
|
|
10453
|
-
import crypto18 from "crypto";
|
|
10454
|
-
import { createRequire as createRequire3 } from "module";
|
|
10455
|
-
import { Worker } from "worker_threads";
|
|
10456
|
-
import { and as and11, asc, count, desc as desc6, eq as eq13, isNull, ne as ne2, or, sql as sql5 } from "drizzle-orm";
|
|
10457
|
-
import { alias } from "drizzle-orm/sqlite-core";
|
|
10458
|
-
var SITE_CRAWL_GRAPH_LAYOUT_TIMEOUT_MS = 15e3;
|
|
10459
|
-
var SITE_CRAWL_GRAPH_SEED_SCALE = 100;
|
|
10460
|
-
var graphSourcePage = alias(siteCrawlPages, "layout_graph_source_page");
|
|
10461
|
-
var graphTargetPage = alias(siteCrawlPages, "layout_graph_target_page");
|
|
10462
|
-
var FORCE_ATLAS_SETTINGS = Object.freeze({
|
|
10463
|
-
adjustSizes: false,
|
|
10464
|
-
barnesHutOptimize: true,
|
|
10465
|
-
barnesHutTheta: 0.5,
|
|
10466
|
-
edgeWeightInfluence: 0.5,
|
|
10467
|
-
gravity: 0.2,
|
|
10468
|
-
linLogMode: true,
|
|
10469
|
-
outboundAttractionDistribution: true,
|
|
10470
|
-
scalingRatio: 2,
|
|
10471
|
-
slowDown: 10,
|
|
10472
|
-
strongGravityMode: false
|
|
10473
|
-
});
|
|
10474
|
-
var RENDER_MAX_NODE_SIZE = 14;
|
|
10475
|
-
var RENDER_HALF_EXTENT_PX = 420;
|
|
10476
|
-
var NODE_SIZE_EXTENT_FRACTION = RENDER_MAX_NODE_SIZE / RENDER_HALF_EXTENT_PX;
|
|
10477
|
-
function siteCrawlGraphNodeSize(extent, nodeCount) {
|
|
10478
|
-
if (!Number.isFinite(extent) || extent <= 0 || nodeCount <= 0) return 0;
|
|
10479
|
-
const byExtent = extent * NODE_SIZE_EXTENT_FRACTION;
|
|
10480
|
-
const byArea = Math.sqrt(NOVERLAP_SETTINGS.maxPackingFraction * (2 * extent) ** 2 / nodeCount) / 2;
|
|
10481
|
-
return Math.min(byExtent, byArea);
|
|
10482
|
-
}
|
|
10483
|
-
function siteCrawlGraphExtent(positions) {
|
|
10484
|
-
if (positions.length === 0) return 0;
|
|
10485
|
-
const centerX = positions.reduce((sum, p) => sum + p.x, 0) / positions.length;
|
|
10486
|
-
const centerY = positions.reduce((sum, p) => sum + p.y, 0) / positions.length;
|
|
10487
|
-
let extent = 0;
|
|
10488
|
-
for (const position of positions) {
|
|
10489
|
-
extent = Math.max(extent, Math.abs(position.x - centerX), Math.abs(position.y - centerY));
|
|
10490
|
-
}
|
|
10491
|
-
return extent;
|
|
10492
|
-
}
|
|
10493
|
-
var NOVERLAP_SETTINGS = Object.freeze({
|
|
10494
|
-
/** Half a node of clear space between two touching nodes. */
|
|
10495
|
-
marginFactor: 0.5,
|
|
10496
|
-
expansion: 1.1,
|
|
10497
|
-
ratio: 1,
|
|
10498
|
-
speed: 3,
|
|
10499
|
-
/**
|
|
10500
|
-
* Node discs may claim at most this share of the graph's bounding box. At
|
|
10501
|
-
* the 20k cap the extent-derived size asks for spacing the graph physically
|
|
10502
|
-
* does not have, and noverlap either runs for minutes or overflows its
|
|
10503
|
-
* collision set. This bound keeps the pass solvable at every graph size,
|
|
10504
|
-
* and it only binds on graphs far too dense to read node-by-node anyway.
|
|
10505
|
-
*/
|
|
10506
|
-
maxPackingFraction: 0.15,
|
|
10507
|
-
/** Cells hold a handful of nodes each, which is what keeps the pass linear. */
|
|
10508
|
-
gridSizeMin: 10,
|
|
10509
|
-
gridSizeMax: 200,
|
|
10510
|
-
iterationLadder: [
|
|
10511
|
-
{ maxNodes: 1e3, iterations: 120 },
|
|
10512
|
-
{ maxNodes: 5e3, iterations: 60 }
|
|
10513
|
-
],
|
|
10514
|
-
iterationsFloor: 20
|
|
10515
|
-
});
|
|
10516
|
-
function noverlapGridSize(nodeCount) {
|
|
10517
|
-
return Math.max(
|
|
10518
|
-
NOVERLAP_SETTINGS.gridSizeMin,
|
|
10519
|
-
Math.min(NOVERLAP_SETTINGS.gridSizeMax, Math.ceil(Math.sqrt(Math.max(1, nodeCount))))
|
|
10520
|
-
);
|
|
10521
|
-
}
|
|
10522
|
-
function noverlapMaxIterations(nodeCount) {
|
|
10523
|
-
for (const step of NOVERLAP_SETTINGS.iterationLadder) {
|
|
10524
|
-
if (nodeCount <= step.maxNodes) return step.iterations;
|
|
10525
|
-
}
|
|
10526
|
-
return NOVERLAP_SETTINGS.iterationsFloor;
|
|
10527
|
-
}
|
|
10528
|
-
var COMPONENT_PACKING_SPACING_FACTOR = 3;
|
|
10529
|
-
function siteCrawlGraphLayoutSettingsFingerprint(settings) {
|
|
10530
|
-
return crypto18.createHash("sha256").update(JSON.stringify(settings)).digest("hex").slice(0, 8);
|
|
10531
|
-
}
|
|
10532
|
-
var SITE_CRAWL_GRAPH_LAYOUT_ALGORITHM = "site-health-fa2-v5";
|
|
10533
|
-
var SITE_CRAWL_GRAPH_LAYOUT_VERSION = `${SITE_CRAWL_GRAPH_LAYOUT_ALGORITHM}-${siteCrawlGraphLayoutSettingsFingerprint({
|
|
10534
|
-
forceAtlas: FORCE_ATLAS_SETTINGS,
|
|
10535
|
-
noverlap: NOVERLAP_SETTINGS,
|
|
10536
|
-
nodeSizeExtentFraction: NODE_SIZE_EXTENT_FRACTION,
|
|
10537
|
-
componentPackingSpacingFactor: COMPONENT_PACKING_SPACING_FACTOR,
|
|
10538
|
-
templateLinkRatioThreshold: TEMPLATE_LINK_RATIO_THRESHOLD
|
|
10539
|
-
})}`;
|
|
10540
|
-
var WORKER_SOURCE = String.raw`
|
|
10541
|
-
const {parentPort, workerData} = require('node:worker_threads');
|
|
10542
|
-
try {
|
|
10543
|
-
const graphologyModule = require(workerData.graphologyPath);
|
|
10544
|
-
const forceAtlasModule = require(workerData.forceAtlasPath);
|
|
10545
|
-
const noverlapModule = require(workerData.noverlapPath);
|
|
10546
|
-
const MultiDirectedGraph = graphologyModule.MultiDirectedGraph;
|
|
10547
|
-
const forceAtlas2 = forceAtlasModule.default || forceAtlasModule;
|
|
10548
|
-
const noverlap = noverlapModule.default || noverlapModule;
|
|
10549
|
-
const graph = new MultiDirectedGraph({allowSelfLoops: true});
|
|
10550
|
-
for (const node of workerData.input.nodes) {
|
|
10551
|
-
graph.addNode(node.nodeKey, {x: node.x, y: node.y});
|
|
10552
|
-
}
|
|
10553
|
-
for (const edge of workerData.input.edges) {
|
|
10554
|
-
graph.addDirectedEdgeWithKey(edge.edgeKey, edge.sourceNodeKey, edge.targetNodeKey, {
|
|
10555
|
-
weight: 1 + Math.log2(Math.max(1, edge.occurrences)),
|
|
10556
|
-
});
|
|
10557
|
-
}
|
|
10558
|
-
if (graph.order > 1 && graph.size > 0) {
|
|
10559
|
-
forceAtlas2.assign(graph, {
|
|
10560
|
-
iterations: workerData.input.iterations,
|
|
10561
|
-
settings: workerData.settings,
|
|
10562
|
-
});
|
|
10563
|
-
}
|
|
10564
|
-
let nodeSize = 0;
|
|
10565
|
-
// Anti-overlap runs on the spatialization ForceAtlas2 just produced. Node
|
|
10566
|
-
// size is uniform and derived from the graph's own extent, so the pass is
|
|
10567
|
-
// scale invariant and stays deterministic for a fixed input.
|
|
10568
|
-
if (graph.order > 1) {
|
|
10569
|
-
let sumX = 0;
|
|
10570
|
-
let sumY = 0;
|
|
10571
|
-
graph.forEachNode(function (nodeKey, attributes) {
|
|
10572
|
-
sumX += attributes.x;
|
|
10573
|
-
sumY += attributes.y;
|
|
10574
|
-
});
|
|
10575
|
-
const centerX = sumX / graph.order;
|
|
10576
|
-
const centerY = sumY / graph.order;
|
|
10577
|
-
let extent = 0;
|
|
10578
|
-
graph.forEachNode(function (nodeKey, attributes) {
|
|
10579
|
-
extent = Math.max(extent, Math.abs(attributes.x - centerX), Math.abs(attributes.y - centerY));
|
|
10580
|
-
});
|
|
10581
|
-
if (Number.isFinite(extent) && extent > 0) {
|
|
10582
|
-
// Whichever bound is tighter: one render-sized node, or the largest disc
|
|
10583
|
-
// that still leaves the graph mostly empty space at this node count.
|
|
10584
|
-
// Mirrors siteCrawlGraphNodeSize; a test asserts the two agree.
|
|
10585
|
-
const byExtent = extent * workerData.nodeSizeExtentFraction;
|
|
10586
|
-
const byArea = Math.sqrt(
|
|
10587
|
-
(workerData.noverlap.maxPackingFraction * Math.pow(2 * extent, 2)) / graph.order,
|
|
10588
|
-
) / 2;
|
|
10589
|
-
nodeSize = Math.min(byExtent, byArea);
|
|
10590
|
-
graph.forEachNode(function (nodeKey) {
|
|
10591
|
-
graph.setNodeAttribute(nodeKey, 'size', nodeSize);
|
|
10592
|
-
});
|
|
10593
|
-
noverlap.assign(graph, {
|
|
10594
|
-
maxIterations: workerData.noverlap.maxIterations,
|
|
10595
|
-
settings: {
|
|
10596
|
-
gridSize: workerData.noverlap.gridSize,
|
|
10597
|
-
margin: nodeSize * workerData.noverlap.marginFactor,
|
|
10598
|
-
expansion: workerData.noverlap.expansion,
|
|
10599
|
-
ratio: workerData.noverlap.ratio,
|
|
10600
|
-
speed: workerData.noverlap.speed,
|
|
10601
|
-
},
|
|
10602
|
-
});
|
|
10603
|
-
}
|
|
10604
|
-
}
|
|
10605
|
-
const positions = [];
|
|
10606
|
-
graph.forEachNode((nodeKey, attributes) => {
|
|
10607
|
-
positions.push({nodeKey, x: attributes.x, y: attributes.y});
|
|
10608
|
-
});
|
|
10609
|
-
parentPort.postMessage({ok: true, positions, nodeSize});
|
|
10610
|
-
} catch (error) {
|
|
10611
|
-
parentPort.postMessage({
|
|
10612
|
-
ok: false,
|
|
10613
|
-
error: error && typeof error.message === 'string' ? error.message : String(error),
|
|
10614
|
-
});
|
|
10615
|
-
}
|
|
10616
|
-
`;
|
|
10617
|
-
var GraphLayoutTimeoutError = class extends Error {
|
|
10618
|
-
name = "GraphLayoutTimeoutError";
|
|
10619
|
-
};
|
|
10620
|
-
function hashUnit(value) {
|
|
10621
|
-
let hash = 2166136261;
|
|
10622
|
-
for (let i = 0; i < value.length; i++) {
|
|
10623
|
-
hash ^= value.charCodeAt(i);
|
|
10624
|
-
hash = Math.imul(hash, 16777619);
|
|
10625
|
-
}
|
|
10626
|
-
return (hash >>> 0) / 4294967296;
|
|
10627
|
-
}
|
|
10628
|
-
function pathSection(path9) {
|
|
10629
|
-
return path9.split("/").filter(Boolean)[0] ?? "__root__";
|
|
10630
|
-
}
|
|
10631
|
-
function seedSiteCrawlGraphNodes(nodes, rootNodeKey, priorPositions) {
|
|
10632
|
-
return [...nodes].sort((a, b) => a.sampleRank - b.sampleRank || a.nodeKey.localeCompare(b.nodeKey)).map((node) => {
|
|
10633
|
-
const prior = priorPositions?.get(node.nodeKey);
|
|
10634
|
-
if (prior && Number.isFinite(prior.x) && Number.isFinite(prior.y)) {
|
|
10635
|
-
return {
|
|
10636
|
-
...node,
|
|
10637
|
-
x: prior.x * SITE_CRAWL_GRAPH_SEED_SCALE,
|
|
10638
|
-
y: prior.y * SITE_CRAWL_GRAPH_SEED_SCALE
|
|
10639
|
-
};
|
|
10640
|
-
}
|
|
10641
|
-
if (node.nodeKey === rootNodeKey) return { ...node, x: 0, y: 0 };
|
|
10642
|
-
const section = pathSection(node.path);
|
|
10643
|
-
const sectionAngle = hashUnit(`section:${section}`) * Math.PI * 2;
|
|
10644
|
-
const sectionRadius = section === "__root__" ? 24 : 100;
|
|
10645
|
-
const clusterX = Math.cos(sectionAngle) * sectionRadius;
|
|
10646
|
-
const clusterY = Math.sin(sectionAngle) * sectionRadius;
|
|
10647
|
-
const localAngle = hashUnit(`node-angle:${node.nodeKey}`) * Math.PI * 2;
|
|
10648
|
-
const depth = Math.max(0, Math.min(12, node.depth ?? 4));
|
|
10649
|
-
const localRadius = 4 + depth * 2.5 + Math.sqrt(hashUnit(`node-radius:${node.nodeKey}`)) * 14;
|
|
10650
|
-
return {
|
|
10651
|
-
...node,
|
|
10652
|
-
x: clusterX + Math.cos(localAngle) * localRadius,
|
|
10653
|
-
y: clusterY + Math.sin(localAngle) * localRadius
|
|
10654
|
-
};
|
|
10655
|
-
});
|
|
10656
|
-
}
|
|
10657
|
-
function adaptiveForceAtlasIterations(nodeCount, edgeCount) {
|
|
10658
|
-
if (nodeCount <= 100 && edgeCount <= 500) return 100;
|
|
10659
|
-
if (nodeCount <= 500 && edgeCount <= 2500) return 70;
|
|
10660
|
-
if (nodeCount <= 2e3 && edgeCount <= 1e4) return 40;
|
|
10661
|
-
if (nodeCount <= 5e3 && edgeCount <= 2e4) return 24;
|
|
10662
|
-
if (nodeCount <= 1e4 && edgeCount <= 35e3) return 12;
|
|
10663
|
-
return 6;
|
|
10664
|
-
}
|
|
10665
|
-
async function computeForceAtlasPositions(input, options = {}) {
|
|
10666
|
-
if (input.nodes.length <= 1 || input.edges.length === 0) {
|
|
10667
|
-
const positions = input.nodes.map(({ nodeKey, x, y }) => ({ nodeKey, x, y }));
|
|
10668
|
-
return { positions, nodeSize: siteCrawlGraphNodeSize(siteCrawlGraphExtent(positions), positions.length) };
|
|
10669
|
-
}
|
|
10670
|
-
const requireFromPackage = createRequire3(import.meta.url);
|
|
10671
|
-
const worker = new Worker(WORKER_SOURCE, {
|
|
10672
|
-
eval: true,
|
|
10673
|
-
workerData: {
|
|
10674
|
-
graphologyPath: requireFromPackage.resolve("graphology"),
|
|
10675
|
-
forceAtlasPath: requireFromPackage.resolve("graphology-layout-forceatlas2"),
|
|
10676
|
-
noverlapPath: requireFromPackage.resolve("graphology-layout-noverlap"),
|
|
10677
|
-
input,
|
|
10678
|
-
settings: FORCE_ATLAS_SETTINGS,
|
|
10679
|
-
noverlap: {
|
|
10680
|
-
...NOVERLAP_SETTINGS,
|
|
10681
|
-
gridSize: noverlapGridSize(input.nodes.length),
|
|
10682
|
-
maxIterations: noverlapMaxIterations(input.nodes.length)
|
|
10683
|
-
},
|
|
10684
|
-
nodeSizeExtentFraction: NODE_SIZE_EXTENT_FRACTION
|
|
10685
|
-
}
|
|
10686
|
-
});
|
|
10687
|
-
const timeoutMs = options.timeoutMs ?? SITE_CRAWL_GRAPH_LAYOUT_TIMEOUT_MS;
|
|
10688
|
-
return await new Promise((resolve, reject) => {
|
|
10689
|
-
let settled = false;
|
|
10690
|
-
const finish = (result) => {
|
|
10691
|
-
if (settled) return;
|
|
10692
|
-
settled = true;
|
|
10693
|
-
clearTimeout(timer);
|
|
10694
|
-
options.signal?.removeEventListener("abort", onAbort);
|
|
10695
|
-
void worker.terminate();
|
|
10696
|
-
if (result.error) reject(result.error instanceof Error ? result.error : new Error("Graph layout failed with a non-Error reason", { cause: result.error }));
|
|
10697
|
-
else resolve({ positions: result.positions ?? [], nodeSize: result.nodeSize ?? 0 });
|
|
10698
|
-
};
|
|
10699
|
-
const onAbort = () => finish({ error: options.signal?.reason ?? new DOMException("Layout aborted", "AbortError") });
|
|
10700
|
-
const timer = setTimeout(() => finish({ error: new GraphLayoutTimeoutError(`Graph layout exceeded ${timeoutMs}ms`) }), timeoutMs);
|
|
10701
|
-
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
10702
|
-
if (options.signal?.aborted) return onAbort();
|
|
10703
|
-
worker.once("message", (message) => {
|
|
10704
|
-
if (!message.ok) finish({ error: new Error(message.error ?? "Graph layout worker failed") });
|
|
10705
|
-
else finish({ positions: message.positions, nodeSize: message.nodeSize });
|
|
10706
|
-
});
|
|
10707
|
-
worker.once("error", (error) => finish({ error }));
|
|
10708
|
-
worker.once("exit", (code) => {
|
|
10709
|
-
if (code !== 0 && !settled) finish({ error: new Error(`Graph layout worker exited with code ${code}`) });
|
|
10710
|
-
});
|
|
10711
|
-
});
|
|
10712
|
-
}
|
|
10713
|
-
function siteCrawlGraphComponents(nodeKeys, edges) {
|
|
10714
|
-
const parent = new Map(nodeKeys.map((key) => [key, key]));
|
|
10715
|
-
const find = (key) => {
|
|
10716
|
-
let root = key;
|
|
10717
|
-
while (parent.get(root) !== root) root = parent.get(root);
|
|
10718
|
-
let cursor = key;
|
|
10719
|
-
while (parent.get(cursor) !== root) {
|
|
10720
|
-
const next = parent.get(cursor);
|
|
10721
|
-
parent.set(cursor, root);
|
|
10722
|
-
cursor = next;
|
|
10723
|
-
}
|
|
10724
|
-
return root;
|
|
10725
|
-
};
|
|
10726
|
-
for (const edge of edges) {
|
|
10727
|
-
if (!parent.has(edge.sourceNodeKey) || !parent.has(edge.targetNodeKey)) continue;
|
|
10728
|
-
const sourceRoot = find(edge.sourceNodeKey);
|
|
10729
|
-
const targetRoot = find(edge.targetNodeKey);
|
|
10730
|
-
if (sourceRoot === targetRoot) continue;
|
|
10731
|
-
if (sourceRoot < targetRoot) parent.set(targetRoot, sourceRoot);
|
|
10732
|
-
else parent.set(sourceRoot, targetRoot);
|
|
10733
|
-
}
|
|
10734
|
-
const members = /* @__PURE__ */ new Map();
|
|
10735
|
-
for (const key of [...nodeKeys].sort()) {
|
|
10736
|
-
const root = find(key);
|
|
10737
|
-
const group = members.get(root);
|
|
10738
|
-
if (group) group.push(key);
|
|
10739
|
-
else members.set(root, [key]);
|
|
10740
|
-
}
|
|
10741
|
-
return [...members.values()].sort((left, right) => right.length - left.length || (left[0] < right[0] ? -1 : left[0] > right[0] ? 1 : 0));
|
|
10742
|
-
}
|
|
10743
|
-
function describeComponent(nodeKeys, byKey) {
|
|
10744
|
-
let sumX = 0;
|
|
10745
|
-
let sumY = 0;
|
|
10746
|
-
for (const nodeKey of nodeKeys) {
|
|
10747
|
-
const position = byKey.get(nodeKey);
|
|
10748
|
-
sumX += position.x;
|
|
10749
|
-
sumY += position.y;
|
|
10750
|
-
}
|
|
10751
|
-
const centerX = sumX / nodeKeys.length;
|
|
10752
|
-
const centerY = sumY / nodeKeys.length;
|
|
10753
|
-
let radius = 0;
|
|
10754
|
-
for (const nodeKey of nodeKeys) {
|
|
10755
|
-
const position = byKey.get(nodeKey);
|
|
10756
|
-
radius = Math.max(radius, Math.hypot(position.x - centerX, position.y - centerY));
|
|
10757
|
-
}
|
|
10758
|
-
return { nodeKeys, centerX, centerY, radius };
|
|
10759
|
-
}
|
|
10760
|
-
function packSiteCrawlGraphComponents(positions, edges, nodeSpacing) {
|
|
10761
|
-
const components = siteCrawlGraphComponents(positions.map((position) => position.nodeKey), edges);
|
|
10762
|
-
if (components.length <= 1) return [...positions];
|
|
10763
|
-
const byKey = new Map(positions.map((position) => [position.nodeKey, position]));
|
|
10764
|
-
if (positions.some((position) => !Number.isFinite(position.x) || !Number.isFinite(position.y))) {
|
|
10765
|
-
return [...positions];
|
|
10766
|
-
}
|
|
10767
|
-
const described = components.map((nodeKeys) => describeComponent(nodeKeys, byKey));
|
|
10768
|
-
const [anchor, ...rest] = described;
|
|
10769
|
-
const spacing = Math.max(nodeSpacing, Number.EPSILON) * COMPONENT_PACKING_SPACING_FACTOR;
|
|
10770
|
-
const ordered = [...rest].sort((left, right) => right.radius - left.radius || (left.nodeKeys[0] < right.nodeKeys[0] ? -1 : left.nodeKeys[0] > right.nodeKeys[0] ? 1 : 0));
|
|
10771
|
-
const moved = /* @__PURE__ */ new Map();
|
|
10772
|
-
let ringRadius = anchor.radius + spacing;
|
|
10773
|
-
let ringPitch = 0;
|
|
10774
|
-
let placedOnRing = 0;
|
|
10775
|
-
let ringCapacity = 0;
|
|
10776
|
-
for (const component of ordered) {
|
|
10777
|
-
if (placedOnRing >= ringCapacity) {
|
|
10778
|
-
ringPitch = 2 * component.radius + spacing;
|
|
10779
|
-
ringRadius += ringPitch / 2 + (placedOnRing === 0 ? 0 : ringPitch / 2);
|
|
10780
|
-
ringCapacity = Math.max(1, Math.floor(2 * Math.PI * ringRadius / ringPitch));
|
|
10781
|
-
placedOnRing = 0;
|
|
10782
|
-
}
|
|
10783
|
-
const angle = placedOnRing / ringCapacity * Math.PI * 2;
|
|
10784
|
-
placedOnRing += 1;
|
|
10785
|
-
const targetX = anchor.centerX + Math.cos(angle) * ringRadius;
|
|
10786
|
-
const targetY = anchor.centerY + Math.sin(angle) * ringRadius;
|
|
10787
|
-
const shiftX = targetX - component.centerX;
|
|
10788
|
-
const shiftY = targetY - component.centerY;
|
|
10789
|
-
for (const nodeKey of component.nodeKeys) {
|
|
10790
|
-
const position = byKey.get(nodeKey);
|
|
10791
|
-
moved.set(nodeKey, { nodeKey, x: position.x + shiftX, y: position.y + shiftY });
|
|
10792
|
-
}
|
|
10793
|
-
}
|
|
10794
|
-
return positions.map((position) => moved.get(position.nodeKey) ?? position);
|
|
10795
|
-
}
|
|
10796
|
-
function normalizePositions(seededNodes, computed, rootNodeKey) {
|
|
10797
|
-
const byKey = new Map(computed.map((position) => [position.nodeKey, position]));
|
|
10798
|
-
if (byKey.size !== seededNodes.length) throw new Error("Graph layout returned an incomplete node set");
|
|
10799
|
-
const root = rootNodeKey ? byKey.get(rootNodeKey) : null;
|
|
10800
|
-
const centerX = root?.x ?? computed.reduce((sum, position) => sum + position.x, 0) / Math.max(1, computed.length);
|
|
10801
|
-
const centerY = root?.y ?? computed.reduce((sum, position) => sum + position.y, 0) / Math.max(1, computed.length);
|
|
10802
|
-
if (!Number.isFinite(centerX) || !Number.isFinite(centerY)) throw new Error("Graph layout returned a non-finite center");
|
|
10803
|
-
let scale = 0;
|
|
10804
|
-
for (const position of computed) {
|
|
10805
|
-
if (!Number.isFinite(position.x) || !Number.isFinite(position.y)) throw new Error("Graph layout returned non-finite coordinates");
|
|
10806
|
-
scale = Math.max(scale, Math.abs(position.x - centerX), Math.abs(position.y - centerY));
|
|
10807
|
-
}
|
|
10808
|
-
scale = scale > 0 ? scale : 1;
|
|
10809
|
-
return seededNodes.map((node) => {
|
|
10810
|
-
const position = byKey.get(node.nodeKey);
|
|
10811
|
-
if (!position) throw new Error(`Graph layout omitted ${node.nodeKey}`);
|
|
10812
|
-
return {
|
|
10813
|
-
nodeKey: node.nodeKey,
|
|
10814
|
-
sampleRank: node.sampleRank,
|
|
10815
|
-
x: Math.round((position.x - centerX) / scale * 1e6) / 1e6,
|
|
10816
|
-
y: Math.round((position.y - centerY) / scale * 1e6) / 1e6
|
|
10817
|
-
};
|
|
10818
|
-
});
|
|
10819
|
-
}
|
|
10820
|
-
async function layoutSiteCrawlGraphInput(input, options = {}) {
|
|
10821
|
-
options.signal?.throwIfAborted();
|
|
10822
|
-
if (input.nodes.length === 0) {
|
|
10823
|
-
return {
|
|
10824
|
-
state: "unavailable",
|
|
10825
|
-
failureCode: "empty-crawl",
|
|
10826
|
-
totalNodes: input.totalNodes,
|
|
10827
|
-
totalEdges: input.totalEdges,
|
|
10828
|
-
totalTemplateEdges: input.totalTemplateEdges,
|
|
10829
|
-
nodeCount: 0,
|
|
10830
|
-
edgeCount: 0,
|
|
10831
|
-
nodes: [],
|
|
10832
|
-
edges: []
|
|
10833
|
-
};
|
|
10834
|
-
}
|
|
10835
|
-
const layoutEdges = input.edges.filter((edge) => !edge.isTemplate);
|
|
10836
|
-
const seeded = seedSiteCrawlGraphNodes(input.nodes, input.rootNodeKey, input.priorPositions);
|
|
10837
|
-
const timeoutMs = options.timeoutMs ?? SITE_CRAWL_GRAPH_LAYOUT_TIMEOUT_MS;
|
|
10838
|
-
const deadline = Date.now() + timeoutMs;
|
|
10839
|
-
const assertWithinBudget = () => {
|
|
10840
|
-
options.signal?.throwIfAborted();
|
|
10841
|
-
if (Date.now() > deadline) throw new GraphLayoutTimeoutError(`Graph layout exceeded ${timeoutMs}ms`);
|
|
10842
|
-
};
|
|
10843
|
-
try {
|
|
10844
|
-
const compute = options.computePositions ?? computeForceAtlasPositions;
|
|
10845
|
-
const { positions, nodeSize } = await compute({
|
|
10846
|
-
nodes: seeded,
|
|
10847
|
-
edges: layoutEdges,
|
|
10848
|
-
iterations: adaptiveForceAtlasIterations(input.nodes.length, layoutEdges.length)
|
|
10849
|
-
}, { signal: options.signal, timeoutMs: Math.max(1, deadline - Date.now()) });
|
|
10850
|
-
assertWithinBudget();
|
|
10851
|
-
const framed = packSiteCrawlGraphComponents(positions, layoutEdges, nodeSize);
|
|
10852
|
-
assertWithinBudget();
|
|
10853
|
-
return {
|
|
10854
|
-
state: "ready",
|
|
10855
|
-
layoutVersion: SITE_CRAWL_GRAPH_LAYOUT_VERSION,
|
|
10856
|
-
totalNodes: input.totalNodes,
|
|
10857
|
-
totalEdges: input.totalEdges,
|
|
10858
|
-
totalTemplateEdges: input.totalTemplateEdges,
|
|
10859
|
-
nodeCount: input.nodes.length,
|
|
10860
|
-
edgeCount: input.edges.length,
|
|
10861
|
-
templateLinksExcluded: true,
|
|
10862
|
-
nodes: normalizePositions(seeded, framed, input.rootNodeKey),
|
|
10863
|
-
edges: input.edges.map((edge, sampleRank) => ({ ...edge, sampleRank }))
|
|
10864
|
-
};
|
|
10865
|
-
} catch (error) {
|
|
10866
|
-
if (options.signal?.aborted) options.signal.throwIfAborted();
|
|
10867
|
-
if (error instanceof DOMException && error.name === "AbortError") throw error;
|
|
10868
|
-
return {
|
|
10869
|
-
state: "unavailable",
|
|
10870
|
-
failureCode: error instanceof GraphLayoutTimeoutError ? "layout-timeout" : "layout-error",
|
|
10871
|
-
totalNodes: input.totalNodes,
|
|
10872
|
-
totalEdges: input.totalEdges,
|
|
10873
|
-
totalTemplateEdges: input.totalTemplateEdges,
|
|
10874
|
-
nodeCount: 0,
|
|
10875
|
-
edgeCount: 0,
|
|
10876
|
-
nodes: [],
|
|
10877
|
-
edges: []
|
|
10878
|
-
};
|
|
10879
|
-
}
|
|
10880
|
-
}
|
|
10881
|
-
function loadPriorSiteCrawlGraphPositions(db, scope) {
|
|
10882
|
-
const prior = db.select({
|
|
10883
|
-
runId: siteCrawlSnapshots.runId,
|
|
10884
|
-
attemptId: siteCrawlSnapshots.attemptId
|
|
10885
|
-
}).from(siteCrawlSnapshots).innerJoin(siteCrawlGraphLayouts, and11(
|
|
10886
|
-
eq13(siteCrawlGraphLayouts.projectId, siteCrawlSnapshots.projectId),
|
|
10887
|
-
eq13(siteCrawlGraphLayouts.runId, siteCrawlSnapshots.runId),
|
|
10888
|
-
eq13(siteCrawlGraphLayouts.attemptId, siteCrawlSnapshots.attemptId)
|
|
10889
|
-
)).where(and11(
|
|
10890
|
-
eq13(siteCrawlSnapshots.projectId, scope.projectId),
|
|
10891
|
-
ne2(siteCrawlSnapshots.runId, scope.runId),
|
|
10892
|
-
eq13(siteCrawlSnapshots.complete, true),
|
|
10893
|
-
eq13(siteCrawlGraphLayouts.state, "ready"),
|
|
10894
|
-
eq13(siteCrawlGraphLayouts.layoutVersion, SITE_CRAWL_GRAPH_LAYOUT_VERSION)
|
|
10895
|
-
)).orderBy(desc6(siteCrawlSnapshots.createdAt), desc6(siteCrawlSnapshots.id)).limit(1).get();
|
|
10896
|
-
if (!prior?.attemptId) return /* @__PURE__ */ new Map();
|
|
10897
|
-
return new Map(db.select({
|
|
10898
|
-
nodeKey: siteCrawlGraphNodes.nodeKey,
|
|
10899
|
-
x: siteCrawlGraphNodes.x,
|
|
10900
|
-
y: siteCrawlGraphNodes.y
|
|
10901
|
-
}).from(siteCrawlGraphNodes).where(and11(
|
|
10902
|
-
eq13(siteCrawlGraphNodes.projectId, scope.projectId),
|
|
10903
|
-
eq13(siteCrawlGraphNodes.runId, prior.runId),
|
|
10904
|
-
eq13(siteCrawlGraphNodes.attemptId, prior.attemptId)
|
|
10905
|
-
)).all().map((node) => [node.nodeKey, { x: node.x, y: node.y }]));
|
|
10906
|
-
}
|
|
10907
|
-
async function prepareSiteCrawlGraphLayout(db, scope, options = {}) {
|
|
10908
|
-
let totalNodes = 0;
|
|
10909
|
-
let totalEdges = 0;
|
|
10910
|
-
let totalTemplateEdges = 0;
|
|
10911
|
-
try {
|
|
10912
|
-
const assertNotAborted = () => options.signal?.throwIfAborted();
|
|
10913
|
-
assertNotAborted();
|
|
10914
|
-
const pageScope = [
|
|
10915
|
-
eq13(siteCrawlPages.projectId, scope.projectId),
|
|
10916
|
-
eq13(siteCrawlPages.runId, scope.runId),
|
|
10917
|
-
eq13(siteCrawlPages.attemptId, scope.attemptId)
|
|
10918
|
-
];
|
|
10919
|
-
const edgeScope = [
|
|
10920
|
-
eq13(siteCrawlEdges.projectId, scope.projectId),
|
|
10921
|
-
eq13(siteCrawlEdges.runId, scope.runId),
|
|
10922
|
-
eq13(siteCrawlEdges.attemptId, scope.attemptId),
|
|
10923
|
-
eq13(siteCrawlEdges.internal, true),
|
|
10924
|
-
eq13(siteCrawlEdges.relation, "anchor")
|
|
10925
|
-
];
|
|
10926
|
-
totalNodes = db.select({ value: count() }).from(siteCrawlPages).where(and11(...pageScope)).get()?.value ?? 0;
|
|
10927
|
-
assertNotAborted();
|
|
10928
|
-
const countGraphCompatibleEdges = (...extra) => db.select({ value: count() }).from(siteCrawlEdges).innerJoin(graphSourcePage, and11(
|
|
10929
|
-
eq13(graphSourcePage.projectId, scope.projectId),
|
|
10930
|
-
eq13(graphSourcePage.runId, scope.runId),
|
|
10931
|
-
eq13(graphSourcePage.attemptId, scope.attemptId),
|
|
10932
|
-
eq13(graphSourcePage.nodeKey, siteCrawlEdges.sourceNodeKey)
|
|
10933
|
-
)).innerJoin(graphTargetPage, and11(
|
|
10934
|
-
eq13(graphTargetPage.projectId, scope.projectId),
|
|
10935
|
-
eq13(graphTargetPage.runId, scope.runId),
|
|
10936
|
-
eq13(graphTargetPage.attemptId, scope.attemptId),
|
|
10937
|
-
eq13(graphTargetPage.nodeKey, siteCrawlEdges.targetNodeKey)
|
|
10938
|
-
)).where(and11(...edgeScope, ...extra)).get()?.value ?? 0;
|
|
10939
|
-
totalEdges = countGraphCompatibleEdges();
|
|
10940
|
-
assertNotAborted();
|
|
10941
|
-
totalTemplateEdges = countGraphCompatibleEdges(eq13(siteCrawlEdges.isTemplate, true));
|
|
10942
|
-
assertNotAborted();
|
|
10943
|
-
const graphSeedColumns = {
|
|
10944
|
-
nodeKey: siteCrawlPages.nodeKey,
|
|
10945
|
-
path: siteCrawlPages.path,
|
|
10946
|
-
depth: siteCrawlPages.depth
|
|
10947
|
-
};
|
|
10948
|
-
const rootRows = db.select(graphSeedColumns).from(siteCrawlPages).where(and11(...pageScope, eq13(siteCrawlPages.url, scope.rootUrl))).orderBy(asc(siteCrawlPages.nodeKey)).limit(1).all();
|
|
10949
|
-
assertNotAborted();
|
|
10950
|
-
const retainedRows = [...rootRows];
|
|
10951
|
-
if (retainedRows.length < SITE_CRAWL_GRAPH_MAX_NODES) {
|
|
10952
|
-
retainedRows.push(...db.select(graphSeedColumns).from(siteCrawlPages).where(and11(...pageScope, eq13(siteCrawlPages.depth, 0), ne2(siteCrawlPages.url, scope.rootUrl))).orderBy(asc(siteCrawlPages.nodeKey)).limit(SITE_CRAWL_GRAPH_MAX_NODES - retainedRows.length).all());
|
|
10953
|
-
}
|
|
10954
|
-
assertNotAborted();
|
|
10955
|
-
if (retainedRows.length < SITE_CRAWL_GRAPH_MAX_NODES) {
|
|
10956
|
-
retainedRows.push(...db.select(graphSeedColumns).from(siteCrawlPages).where(and11(
|
|
10957
|
-
...pageScope,
|
|
10958
|
-
or(isNull(siteCrawlPages.depth), ne2(siteCrawlPages.depth, 0)),
|
|
10959
|
-
ne2(siteCrawlPages.url, scope.rootUrl)
|
|
10960
|
-
)).orderBy(desc6(siteCrawlPages.linkScoreNormalized), asc(siteCrawlPages.nodeKey)).limit(SITE_CRAWL_GRAPH_MAX_NODES - retainedRows.length).all());
|
|
10961
|
-
}
|
|
10962
|
-
assertNotAborted();
|
|
10963
|
-
const nodes = retainedRows.map((row, sampleRank) => ({
|
|
10964
|
-
nodeKey: row.nodeKey,
|
|
10965
|
-
path: row.path,
|
|
10966
|
-
depth: row.depth,
|
|
10967
|
-
sampleRank
|
|
10968
|
-
}));
|
|
10969
|
-
const rootNodeKey = rootRows[0]?.nodeKey ?? null;
|
|
10970
|
-
const priorPositions = loadPriorSiteCrawlGraphPositions(db, scope);
|
|
10971
|
-
assertNotAborted();
|
|
10972
|
-
const selectedKeys = JSON.stringify(nodes.map((node) => node.nodeKey));
|
|
10973
|
-
const edgeRows = nodes.length === 0 ? [] : db.all(sql5`
|
|
10974
|
-
WITH selected(node_key) AS (SELECT value FROM json_each(${selectedKeys}))
|
|
10975
|
-
SELECT
|
|
10976
|
-
edge.edge_key AS edgeKey,
|
|
10977
|
-
edge.source_node_key AS sourceNodeKey,
|
|
10978
|
-
edge.target_node_key AS targetNodeKey,
|
|
10979
|
-
edge.followable AS followable,
|
|
10980
|
-
edge.occurrences AS occurrences,
|
|
10981
|
-
edge.is_template AS isTemplate
|
|
10982
|
-
FROM site_crawl_edges AS edge
|
|
10983
|
-
INNER JOIN selected AS source ON source.node_key = edge.source_node_key
|
|
10984
|
-
INNER JOIN selected AS target ON target.node_key = edge.target_node_key
|
|
10985
|
-
WHERE edge.project_id = ${scope.projectId}
|
|
10986
|
-
AND edge.run_id = ${scope.runId}
|
|
10987
|
-
AND edge.attempt_id = ${scope.attemptId}
|
|
10988
|
-
AND edge.internal = 1
|
|
10989
|
-
AND edge.relation = 'anchor'
|
|
10990
|
-
ORDER BY edge.occurrences DESC, edge.edge_key ASC
|
|
10991
|
-
LIMIT ${SITE_CRAWL_GRAPH_MAX_EDGES}
|
|
10992
|
-
`);
|
|
10993
|
-
assertNotAborted();
|
|
10994
|
-
return await layoutSiteCrawlGraphInput({
|
|
10995
|
-
totalNodes,
|
|
10996
|
-
totalEdges,
|
|
10997
|
-
totalTemplateEdges,
|
|
10998
|
-
nodes,
|
|
10999
|
-
edges: edgeRows.map((row) => ({ ...row, isTemplate: Boolean(row.isTemplate) })),
|
|
11000
|
-
rootNodeKey,
|
|
11001
|
-
priorPositions
|
|
11002
|
-
}, options);
|
|
11003
|
-
} catch (error) {
|
|
11004
|
-
if (options.signal?.aborted) options.signal.throwIfAborted();
|
|
11005
|
-
if (error instanceof DOMException && error.name === "AbortError") throw error;
|
|
11006
|
-
return {
|
|
11007
|
-
state: "unavailable",
|
|
11008
|
-
failureCode: "layout-error",
|
|
11009
|
-
totalNodes,
|
|
11010
|
-
totalEdges,
|
|
11011
|
-
totalTemplateEdges,
|
|
11012
|
-
nodeCount: 0,
|
|
11013
|
-
edgeCount: 0,
|
|
11014
|
-
nodes: [],
|
|
11015
|
-
edges: []
|
|
11016
|
-
};
|
|
11017
|
-
}
|
|
11018
|
-
}
|
|
11019
|
-
function persistSiteCrawlGraphLayout(db, scope, layout, now = (/* @__PURE__ */ new Date()).toISOString()) {
|
|
11020
|
-
db.transaction((tx) => {
|
|
11021
|
-
tx.delete(siteCrawlGraphLayouts).where(and11(
|
|
11022
|
-
eq13(siteCrawlGraphLayouts.projectId, scope.projectId),
|
|
11023
|
-
eq13(siteCrawlGraphLayouts.runId, scope.runId),
|
|
11024
|
-
eq13(siteCrawlGraphLayouts.attemptId, scope.attemptId)
|
|
11025
|
-
)).run();
|
|
11026
|
-
tx.insert(siteCrawlGraphLayouts).values({
|
|
11027
|
-
id: crypto18.randomUUID(),
|
|
11028
|
-
projectId: scope.projectId,
|
|
11029
|
-
runId: scope.runId,
|
|
11030
|
-
attemptId: scope.attemptId,
|
|
11031
|
-
state: layout.state,
|
|
11032
|
-
layoutVersion: layout.state === "ready" ? layout.layoutVersion : null,
|
|
11033
|
-
failureCode: layout.state === "unavailable" ? layout.failureCode : null,
|
|
11034
|
-
totalNodes: layout.totalNodes,
|
|
11035
|
-
totalEdges: layout.totalEdges,
|
|
11036
|
-
totalTemplateEdges: layout.totalTemplateEdges,
|
|
11037
|
-
nodeCount: layout.nodeCount,
|
|
11038
|
-
edgeCount: layout.edgeCount,
|
|
11039
|
-
templateLinksExcluded: layout.state === "ready",
|
|
11040
|
-
createdAt: now,
|
|
11041
|
-
updatedAt: now
|
|
11042
|
-
}).run();
|
|
11043
|
-
if (layout.state !== "ready") return;
|
|
11044
|
-
const batchSize = 250;
|
|
11045
|
-
for (let offset = 0; offset < layout.nodes.length; offset += batchSize) {
|
|
11046
|
-
tx.insert(siteCrawlGraphNodes).values(layout.nodes.slice(offset, offset + batchSize).map((node) => ({
|
|
11047
|
-
id: crypto18.randomUUID(),
|
|
11048
|
-
projectId: scope.projectId,
|
|
11049
|
-
runId: scope.runId,
|
|
11050
|
-
attemptId: scope.attemptId,
|
|
11051
|
-
nodeKey: node.nodeKey,
|
|
11052
|
-
sampleRank: node.sampleRank,
|
|
11053
|
-
x: node.x,
|
|
11054
|
-
y: node.y,
|
|
11055
|
-
createdAt: now
|
|
11056
|
-
}))).run();
|
|
11057
|
-
}
|
|
11058
|
-
for (let offset = 0; offset < layout.edges.length; offset += batchSize) {
|
|
11059
|
-
tx.insert(siteCrawlGraphEdges).values(layout.edges.slice(offset, offset + batchSize).map((edge) => ({
|
|
11060
|
-
id: crypto18.randomUUID(),
|
|
11061
|
-
projectId: scope.projectId,
|
|
11062
|
-
runId: scope.runId,
|
|
11063
|
-
attemptId: scope.attemptId,
|
|
11064
|
-
edgeKey: edge.edgeKey,
|
|
11065
|
-
sampleRank: edge.sampleRank,
|
|
11066
|
-
sourceNodeKey: edge.sourceNodeKey,
|
|
11067
|
-
targetNodeKey: edge.targetNodeKey,
|
|
11068
|
-
followable: edge.followable,
|
|
11069
|
-
occurrences: edge.occurrences,
|
|
11070
|
-
isTemplate: edge.isTemplate,
|
|
11071
|
-
createdAt: now
|
|
11072
|
-
}))).run();
|
|
11073
|
-
}
|
|
11074
|
-
});
|
|
11075
|
-
}
|
|
11076
|
-
function deleteSiteCrawlGraphLayout(db, scope) {
|
|
11077
|
-
db.delete(siteCrawlGraphLayouts).where(and11(
|
|
11078
|
-
eq13(siteCrawlGraphLayouts.projectId, scope.projectId),
|
|
11079
|
-
eq13(siteCrawlGraphLayouts.runId, scope.runId),
|
|
11080
|
-
eq13(siteCrawlGraphLayouts.attemptId, scope.attemptId)
|
|
11081
|
-
)).run();
|
|
11082
|
-
}
|
|
11083
|
-
|
|
11084
10453
|
// src/site-crawl-template-links.ts
|
|
11085
|
-
import { and as
|
|
10454
|
+
import { and as and11, asc, eq as eq13, gt, inArray as inArray5 } from "drizzle-orm";
|
|
11086
10455
|
var TEMPLATE_LINK_READ_BATCH = 2e3;
|
|
11087
10456
|
var TEMPLATE_LINK_WRITE_BATCH = 500;
|
|
11088
10457
|
function placementOccurrencesOf(row) {
|
|
@@ -11104,12 +10473,12 @@ function* streamAttemptEdges(db, scope) {
|
|
|
11104
10473
|
placementNavigationOccurrences: siteCrawlEdges.placementNavigationOccurrences,
|
|
11105
10474
|
placementContentOccurrences: siteCrawlEdges.placementContentOccurrences,
|
|
11106
10475
|
placementUnknownOccurrences: siteCrawlEdges.placementUnknownOccurrences
|
|
11107
|
-
}).from(siteCrawlEdges).where(
|
|
11108
|
-
|
|
11109
|
-
|
|
11110
|
-
|
|
10476
|
+
}).from(siteCrawlEdges).where(and11(
|
|
10477
|
+
eq13(siteCrawlEdges.projectId, scope.projectId),
|
|
10478
|
+
eq13(siteCrawlEdges.runId, scope.runId),
|
|
10479
|
+
eq13(siteCrawlEdges.attemptId, scope.attemptId),
|
|
11111
10480
|
gt(siteCrawlEdges.edgeKey, after)
|
|
11112
|
-
)).orderBy(
|
|
10481
|
+
)).orderBy(asc(siteCrawlEdges.edgeKey)).limit(TEMPLATE_LINK_READ_BATCH).all();
|
|
11113
10482
|
if (rows.length === 0) return;
|
|
11114
10483
|
yield rows.map((row) => ({
|
|
11115
10484
|
edgeKey: row.edgeKey,
|
|
@@ -11132,10 +10501,10 @@ function writeGroupKey(isTemplate, templateRatio) {
|
|
|
11132
10501
|
return `${isTemplate ? "1" : "0"}${templateRatio ?? ""}`;
|
|
11133
10502
|
}
|
|
11134
10503
|
function classifySiteCrawlTemplateLinks(db, scope, pagesFetched, placementRulesetVersion) {
|
|
11135
|
-
const scopeFilter =
|
|
11136
|
-
|
|
11137
|
-
|
|
11138
|
-
|
|
10504
|
+
const scopeFilter = and11(
|
|
10505
|
+
eq13(siteCrawlEdges.projectId, scope.projectId),
|
|
10506
|
+
eq13(siteCrawlEdges.runId, scope.runId),
|
|
10507
|
+
eq13(siteCrawlEdges.attemptId, scope.attemptId)
|
|
11139
10508
|
);
|
|
11140
10509
|
const placementAvailable = templateLinkPlacementAvailable(placementRulesetVersion);
|
|
11141
10510
|
const ubiquityAvailable = templateLinkUbiquityAvailable(pagesFetched);
|
|
@@ -11170,7 +10539,7 @@ function classifySiteCrawlTemplateLinks(db, scope, pagesFetched, placementRulese
|
|
|
11170
10539
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
11171
10540
|
for (const group of edgeKeysByValue.values()) {
|
|
11172
10541
|
for (const chunk of chunked(group.edgeKeys)) {
|
|
11173
|
-
db.update(siteCrawlEdges).set({ isTemplate: group.isTemplate, templateRatio: group.templateRatio, updatedAt: now }).where(
|
|
10542
|
+
db.update(siteCrawlEdges).set({ isTemplate: group.isTemplate, templateRatio: group.templateRatio, updatedAt: now }).where(and11(scopeFilter, inArray5(siteCrawlEdges.edgeKey, chunk))).run();
|
|
11174
10543
|
}
|
|
11175
10544
|
}
|
|
11176
10545
|
return {
|
|
@@ -11349,7 +10718,7 @@ function trackSiteAuditOutcome(db, input) {
|
|
|
11349
10718
|
);
|
|
11350
10719
|
const pagesAudited = input.crawl?.pagesAudited ?? 0;
|
|
11351
10720
|
if (pagesAudited === 0 || input.trigger === RunTriggers.probe) return;
|
|
11352
|
-
const prior = db.select({ id: siteAuditSnapshots.id }).from(siteAuditSnapshots).where(
|
|
10721
|
+
const prior = db.select({ id: siteAuditSnapshots.id }).from(siteAuditSnapshots).where(and12(eq14(siteAuditSnapshots.projectId, input.projectId), ne2(siteAuditSnapshots.runId, input.runId))).limit(1).get();
|
|
11353
10722
|
if (prior) return;
|
|
11354
10723
|
trackEvent("activation.completed", {
|
|
11355
10724
|
flowVersion: ONBOARDING_FLOW_VERSION,
|
|
@@ -11363,13 +10732,13 @@ function trackSiteAuditOutcome(db, input) {
|
|
|
11363
10732
|
}
|
|
11364
10733
|
async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
11365
10734
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
11366
|
-
const claim = db.update(runs).set({ status: "running", startedAt }).where(
|
|
10735
|
+
const claim = db.update(runs).set({ status: "running", startedAt }).where(and12(eq14(runs.id, runId), eq14(runs.projectId, projectId), eq14(runs.status, "queued"))).run();
|
|
11367
10736
|
if (claim.changes === 0) return;
|
|
11368
10737
|
const startTime = Date.now();
|
|
11369
|
-
const trigger = db.select({ trigger: runs.trigger }).from(runs).where(
|
|
10738
|
+
const trigger = db.select({ trigger: runs.trigger }).from(runs).where(eq14(runs.id, runId)).get()?.trigger ?? null;
|
|
11370
10739
|
let canonicalDomain = null;
|
|
11371
|
-
const attemptNumber = (db.select({ value:
|
|
11372
|
-
const attemptId =
|
|
10740
|
+
const attemptNumber = (db.select({ value: sql5`coalesce(max(${siteCrawlAttempts.attemptNumber}), 0)` }).from(siteCrawlAttempts).where(eq14(siteCrawlAttempts.runId, runId)).get()?.value ?? 0) + 1;
|
|
10741
|
+
const attemptId = crypto18.randomUUID();
|
|
11373
10742
|
db.insert(siteCrawlAttempts).values({
|
|
11374
10743
|
id: attemptId,
|
|
11375
10744
|
projectId,
|
|
@@ -11403,22 +10772,22 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11403
10772
|
const nodeKey = nodeKeyByUrl.get(url);
|
|
11404
10773
|
if (!nodeKey) continue;
|
|
11405
10774
|
const scope = [
|
|
11406
|
-
|
|
11407
|
-
|
|
11408
|
-
|
|
10775
|
+
eq14(siteCrawlEdges.projectId, projectId),
|
|
10776
|
+
eq14(siteCrawlEdges.runId, runId),
|
|
10777
|
+
eq14(siteCrawlEdges.attemptId, attemptId)
|
|
11409
10778
|
];
|
|
11410
|
-
tx.update(siteCrawlEdges).set({ sourceNodeKey: nodeKey, updatedAt: now }).where(
|
|
10779
|
+
tx.update(siteCrawlEdges).set({ sourceNodeKey: nodeKey, updatedAt: now }).where(and12(
|
|
11411
10780
|
...scope,
|
|
11412
|
-
|
|
10781
|
+
eq14(siteCrawlEdges.sourceUrl, url)
|
|
11413
10782
|
)).run();
|
|
11414
|
-
tx.update(siteCrawlEdges).set({ targetNodeKey: nodeKey, updatedAt: now }).where(
|
|
10783
|
+
tx.update(siteCrawlEdges).set({ targetNodeKey: nodeKey, updatedAt: now }).where(and12(
|
|
11415
10784
|
...scope,
|
|
11416
|
-
|
|
10785
|
+
eq14(siteCrawlEdges.targetUrl, url)
|
|
11417
10786
|
)).run();
|
|
11418
10787
|
const pageScope = [
|
|
11419
|
-
|
|
11420
|
-
|
|
11421
|
-
|
|
10788
|
+
eq14(siteCrawlPages.projectId, projectId),
|
|
10789
|
+
eq14(siteCrawlPages.runId, runId),
|
|
10790
|
+
eq14(siteCrawlPages.attemptId, attemptId)
|
|
11422
10791
|
];
|
|
11423
10792
|
const canonicalTargets = tx.select({
|
|
11424
10793
|
id: siteCrawlPages.id,
|
|
@@ -11426,13 +10795,13 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11426
10795
|
fetchState: siteCrawlPages.fetchState,
|
|
11427
10796
|
indexabilityState: siteCrawlPages.indexabilityState,
|
|
11428
10797
|
indexabilityReasons: siteCrawlPages.indexabilityReasons
|
|
11429
|
-
}).from(siteCrawlPages).where(
|
|
10798
|
+
}).from(siteCrawlPages).where(and12(...pageScope, eq14(siteCrawlPages.canonicalUrl, url))).all();
|
|
11430
10799
|
for (const target of canonicalTargets) {
|
|
11431
10800
|
tx.update(siteCrawlPages).set({
|
|
11432
10801
|
canonicalNodeKey: nodeKey,
|
|
11433
10802
|
healthState: deriveSiteHealthState({ ...target, canonicalNodeKey: nodeKey }),
|
|
11434
10803
|
updatedAt: now
|
|
11435
|
-
}).where(
|
|
10804
|
+
}).where(eq14(siteCrawlPages.id, target.id)).run();
|
|
11436
10805
|
}
|
|
11437
10806
|
}
|
|
11438
10807
|
};
|
|
@@ -11447,7 +10816,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11447
10816
|
nodeKey: page.key
|
|
11448
10817
|
});
|
|
11449
10818
|
tx.insert(siteCrawlPages).values({
|
|
11450
|
-
id:
|
|
10819
|
+
id: crypto18.randomUUID(),
|
|
11451
10820
|
projectId,
|
|
11452
10821
|
runId,
|
|
11453
10822
|
attemptId,
|
|
@@ -11497,7 +10866,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11497
10866
|
discoveryProvenance: [{ discoveredFrom: page.provenance.discoveredFrom, sitemapSources: page.provenance.sitemapSources, root: page.provenance.root }],
|
|
11498
10867
|
sitemapMetadata: { sources: page.provenance.sitemapSources },
|
|
11499
10868
|
fetchState: page.state,
|
|
11500
|
-
fetchedAt: page.state === "discovered" || page.state === "robots-blocked" ? null :
|
|
10869
|
+
fetchedAt: page.state === "discovered" || page.state === "robots-blocked" ? null : sql5`coalesce(${siteCrawlPages.fetchedAt}, ${now})`,
|
|
11501
10870
|
httpStatus: page.statusCode,
|
|
11502
10871
|
contentType: page.contentType,
|
|
11503
10872
|
finalUrl: page.finalUrl,
|
|
@@ -11540,7 +10909,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11540
10909
|
if (isSelfLink(edge.from, edge.to)) return;
|
|
11541
10910
|
observedEdges.set(edge.key, edge);
|
|
11542
10911
|
tx.insert(siteCrawlEdges).values({
|
|
11543
|
-
id:
|
|
10912
|
+
id: crypto18.randomUUID(),
|
|
11544
10913
|
projectId,
|
|
11545
10914
|
runId,
|
|
11546
10915
|
attemptId,
|
|
@@ -11588,20 +10957,20 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11588
10957
|
linkScoreRaw: metrics.linkScoreRaw,
|
|
11589
10958
|
linkScoreNormalized: metrics.linkScore,
|
|
11590
10959
|
updatedAt: now
|
|
11591
|
-
}).where(
|
|
11592
|
-
|
|
11593
|
-
|
|
11594
|
-
|
|
11595
|
-
|
|
10960
|
+
}).where(and12(
|
|
10961
|
+
eq14(siteCrawlPages.projectId, projectId),
|
|
10962
|
+
eq14(siteCrawlPages.runId, runId),
|
|
10963
|
+
eq14(siteCrawlPages.attemptId, attemptId),
|
|
10964
|
+
eq14(siteCrawlPages.nodeKey, key)
|
|
11596
10965
|
)).run();
|
|
11597
10966
|
};
|
|
11598
10967
|
const persistEvent = async (event) => {
|
|
11599
10968
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
11600
10969
|
db.transaction((tx) => {
|
|
11601
|
-
const receipt = tx.select({ checksum: siteCrawlEventReceipts.checksum }).from(siteCrawlEventReceipts).where(
|
|
11602
|
-
|
|
11603
|
-
|
|
11604
|
-
|
|
10970
|
+
const receipt = tx.select({ checksum: siteCrawlEventReceipts.checksum }).from(siteCrawlEventReceipts).where(and12(
|
|
10971
|
+
eq14(siteCrawlEventReceipts.attemptId, attemptId),
|
|
10972
|
+
eq14(siteCrawlEventReceipts.sequence, event.sequence),
|
|
10973
|
+
eq14(siteCrawlEventReceipts.batchId, event.batchId)
|
|
11605
10974
|
)).get();
|
|
11606
10975
|
if (receipt) {
|
|
11607
10976
|
if (receipt.checksum !== event.checksum) {
|
|
@@ -11610,7 +10979,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11610
10979
|
return;
|
|
11611
10980
|
}
|
|
11612
10981
|
tx.insert(siteCrawlEventReceipts).values({
|
|
11613
|
-
id:
|
|
10982
|
+
id: crypto18.randomUUID(),
|
|
11614
10983
|
projectId,
|
|
11615
10984
|
runId,
|
|
11616
10985
|
attemptId,
|
|
@@ -11630,22 +10999,22 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11630
10999
|
lastEventChecksum: event.checksum,
|
|
11631
11000
|
pagesDiscovered: progress.pagesDiscovered,
|
|
11632
11001
|
pagesFetched: progress.pagesFetched,
|
|
11633
|
-
pagesEligible: event.type === "summary" ? observedPages.size === 0 ? 0 : [...observedPages.values()].filter(isInventoryEligible).length :
|
|
11634
|
-
pagesErrored: event.type === "summary" ? observedErrorCount(observedPages.values()) :
|
|
11002
|
+
pagesEligible: event.type === "summary" ? observedPages.size === 0 ? 0 : [...observedPages.values()].filter(isInventoryEligible).length : sql5`${siteCrawlAttempts.pagesEligible}`,
|
|
11003
|
+
pagesErrored: event.type === "summary" ? observedErrorCount(observedPages.values()) : sql5`${siteCrawlAttempts.pagesErrored}`,
|
|
11635
11004
|
edgesDiscovered: progress.edgesObserved,
|
|
11636
11005
|
updatedAt: now
|
|
11637
|
-
}).where(
|
|
11006
|
+
}).where(eq14(siteCrawlAttempts.id, attemptId)).run();
|
|
11638
11007
|
} else {
|
|
11639
11008
|
tx.update(siteCrawlAttempts).set({
|
|
11640
11009
|
lastEventSequence: event.sequence,
|
|
11641
11010
|
lastEventChecksum: event.checksum,
|
|
11642
11011
|
updatedAt: now
|
|
11643
|
-
}).where(
|
|
11012
|
+
}).where(eq14(siteCrawlAttempts.id, attemptId)).run();
|
|
11644
11013
|
}
|
|
11645
11014
|
});
|
|
11646
11015
|
};
|
|
11647
11016
|
try {
|
|
11648
|
-
const project = db.select().from(projects).where(
|
|
11017
|
+
const project = db.select().from(projects).where(eq14(projects.id, projectId)).get();
|
|
11649
11018
|
if (!project) throw new Error(`Project not found: ${projectId}`);
|
|
11650
11019
|
canonicalDomain = project.canonicalDomain;
|
|
11651
11020
|
const homepageUrl = toHomepageUrl(project.canonicalDomain);
|
|
@@ -11742,11 +11111,11 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11742
11111
|
}
|
|
11743
11112
|
}
|
|
11744
11113
|
const published = db.transaction((tx) => {
|
|
11745
|
-
const claim2 = tx.update(runs).set({ status: terminalStatus, finishedAt }).where(
|
|
11114
|
+
const claim2 = tx.update(runs).set({ status: terminalStatus, finishedAt }).where(and12(eq14(runs.id, runId), eq14(runs.status, "running"))).run();
|
|
11746
11115
|
if (claim2.changes === 0) return false;
|
|
11747
11116
|
if (hasAuditedPages) {
|
|
11748
11117
|
tx.insert(siteAuditSnapshots).values({
|
|
11749
|
-
id:
|
|
11118
|
+
id: crypto18.randomUUID(),
|
|
11750
11119
|
projectId,
|
|
11751
11120
|
runId,
|
|
11752
11121
|
sitemapUrl: opts.sitemapUrl ?? new URL("/sitemap.xml", crawlRootUrl).href,
|
|
@@ -11764,7 +11133,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11764
11133
|
for (const page of observedPages.values()) {
|
|
11765
11134
|
if (!isLegacyScorecardPage(page)) continue;
|
|
11766
11135
|
tx.insert(siteAuditPages).values({
|
|
11767
|
-
id:
|
|
11136
|
+
id: crypto18.randomUUID(),
|
|
11768
11137
|
projectId,
|
|
11769
11138
|
runId,
|
|
11770
11139
|
url: page.finalUrl ?? page.requestedUrl,
|
|
@@ -11777,7 +11146,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11777
11146
|
}
|
|
11778
11147
|
}
|
|
11779
11148
|
tx.insert(siteCrawlSnapshots).values({
|
|
11780
|
-
id:
|
|
11149
|
+
id: crypto18.randomUUID(),
|
|
11781
11150
|
projectId,
|
|
11782
11151
|
runId,
|
|
11783
11152
|
attemptId,
|
|
@@ -11823,7 +11192,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11823
11192
|
if (opts.checkDeadLinks) {
|
|
11824
11193
|
for (const finding of deadLinkFindings) {
|
|
11825
11194
|
tx.insert(siteCrawlFindings).values({
|
|
11826
|
-
id:
|
|
11195
|
+
id: crypto18.randomUUID(),
|
|
11827
11196
|
projectId,
|
|
11828
11197
|
runId,
|
|
11829
11198
|
attemptId,
|
|
@@ -11849,7 +11218,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11849
11218
|
edgesDiscovered: crawlSummary.edgesObserved,
|
|
11850
11219
|
finishedAt,
|
|
11851
11220
|
updatedAt: finishedAt
|
|
11852
|
-
}).where(
|
|
11221
|
+
}).where(eq14(siteCrawlAttempts.id, attemptId)).run();
|
|
11853
11222
|
return true;
|
|
11854
11223
|
});
|
|
11855
11224
|
if (!published) {
|
|
@@ -11862,7 +11231,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11862
11231
|
error: describeError(cleanupError)
|
|
11863
11232
|
});
|
|
11864
11233
|
}
|
|
11865
|
-
const current = db.select({ status: runs.status }).from(runs).where(
|
|
11234
|
+
const current = db.select({ status: runs.status }).from(runs).where(eq14(runs.id, runId)).get();
|
|
11866
11235
|
if (current?.status === "cancelled") throw new SiteAuditCancelledError(`Site audit cancelled before publication: ${runId}`);
|
|
11867
11236
|
throw new Error(`Site audit run was no longer running before publication: ${runId}`);
|
|
11868
11237
|
}
|
|
@@ -11897,12 +11266,12 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11897
11266
|
error: message,
|
|
11898
11267
|
finishedAt,
|
|
11899
11268
|
updatedAt: finishedAt
|
|
11900
|
-
}).where(
|
|
11269
|
+
}).where(eq14(siteCrawlAttempts.id, attemptId)).run();
|
|
11901
11270
|
tx.update(runs).set({
|
|
11902
11271
|
status: cancelled ? "cancelled" : "failed",
|
|
11903
11272
|
error: message,
|
|
11904
11273
|
finishedAt
|
|
11905
|
-
}).where(
|
|
11274
|
+
}).where(and12(eq14(runs.id, runId), eq14(runs.status, "running"))).run();
|
|
11906
11275
|
});
|
|
11907
11276
|
log13.error("failed", { runId, projectId, cancelled, error: message });
|
|
11908
11277
|
trackSiteAuditOutcome(db, {
|
|
@@ -11918,7 +11287,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11918
11287
|
}
|
|
11919
11288
|
|
|
11920
11289
|
// src/commands/backfill.ts
|
|
11921
|
-
import { and as
|
|
11290
|
+
import { and as and13, eq as eq15, inArray as inArray6, isNull, sql as sql6 } from "drizzle-orm";
|
|
11922
11291
|
var SNAPSHOT_BATCH_SIZE = 500;
|
|
11923
11292
|
async function backfillAnswerVisibilityCommand(opts) {
|
|
11924
11293
|
const config = loadConfig();
|
|
@@ -11926,7 +11295,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
11926
11295
|
migrate(db);
|
|
11927
11296
|
const projectFilter = opts?.project?.trim();
|
|
11928
11297
|
const isDryRun = opts?.dryRun === true;
|
|
11929
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
11298
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq15(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
11930
11299
|
let examined = 0;
|
|
11931
11300
|
let updated = 0;
|
|
11932
11301
|
let wouldUpdate = 0;
|
|
@@ -11934,10 +11303,10 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
11934
11303
|
let reparsed = 0;
|
|
11935
11304
|
let providerErrors = 0;
|
|
11936
11305
|
if (scopedProjects.length > 0) {
|
|
11937
|
-
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
11938
|
-
|
|
11306
|
+
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(and13(
|
|
11307
|
+
eq15(runs.kind, RunKinds["answer-visibility"]),
|
|
11939
11308
|
inArray6(runs.projectId, scopedProjects.map((project) => project.id))
|
|
11940
|
-
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
11309
|
+
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(eq15(runs.kind, RunKinds["answer-visibility"])).all();
|
|
11941
11310
|
const runIdsByProject = /* @__PURE__ */ new Map();
|
|
11942
11311
|
for (const run of runRows) {
|
|
11943
11312
|
const existing = runIdsByProject.get(run.projectId);
|
|
@@ -11945,7 +11314,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
11945
11314
|
else runIdsByProject.set(run.projectId, [run.id]);
|
|
11946
11315
|
}
|
|
11947
11316
|
for (const project of scopedProjects) {
|
|
11948
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
11317
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq15(competitors.projectId, project.id)).all().map((row) => row.domain);
|
|
11949
11318
|
const runIds = runIdsByProject.get(project.id) ?? [];
|
|
11950
11319
|
if (runIds.length === 0) continue;
|
|
11951
11320
|
const projectDomains = effectiveDomains({
|
|
@@ -12038,7 +11407,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
12038
11407
|
} else {
|
|
12039
11408
|
db.transaction((tx) => {
|
|
12040
11409
|
for (const update of pendingUpdates) {
|
|
12041
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
11410
|
+
tx.update(querySnapshots).set(update.patch).where(eq15(querySnapshots.id, update.id)).run();
|
|
12042
11411
|
}
|
|
12043
11412
|
});
|
|
12044
11413
|
updated += pendingUpdates.length;
|
|
@@ -12087,13 +11456,13 @@ No DB writes performed. Re-run without --dry-run to apply.`);
|
|
|
12087
11456
|
function backfillNormalizedPaths(db, opts) {
|
|
12088
11457
|
const baseConditions = [];
|
|
12089
11458
|
if (opts?.projectId) {
|
|
12090
|
-
baseConditions.push(
|
|
11459
|
+
baseConditions.push(eq15(gaTrafficSnapshots.projectId, opts.projectId));
|
|
12091
11460
|
}
|
|
12092
11461
|
const rows = db.select({
|
|
12093
11462
|
id: gaTrafficSnapshots.id,
|
|
12094
11463
|
landingPage: gaTrafficSnapshots.landingPage,
|
|
12095
11464
|
landingPageNormalized: gaTrafficSnapshots.landingPageNormalized
|
|
12096
|
-
}).from(gaTrafficSnapshots).where(baseConditions.length > 0 ?
|
|
11465
|
+
}).from(gaTrafficSnapshots).where(baseConditions.length > 0 ? and13(...baseConditions) : void 0).all();
|
|
12097
11466
|
let updated = 0;
|
|
12098
11467
|
let unchanged = 0;
|
|
12099
11468
|
if (rows.length > 0) {
|
|
@@ -12108,7 +11477,7 @@ function backfillNormalizedPaths(db, opts) {
|
|
|
12108
11477
|
unchanged++;
|
|
12109
11478
|
continue;
|
|
12110
11479
|
}
|
|
12111
|
-
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(
|
|
11480
|
+
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(eq15(gaTrafficSnapshots.id, row.id)).run();
|
|
12112
11481
|
updated++;
|
|
12113
11482
|
}
|
|
12114
11483
|
});
|
|
@@ -12122,7 +11491,7 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
12122
11491
|
const projectFilter = opts?.project?.trim();
|
|
12123
11492
|
let projectId;
|
|
12124
11493
|
if (projectFilter) {
|
|
12125
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
11494
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq15(projects.name, projectFilter)).get();
|
|
12126
11495
|
if (!project) {
|
|
12127
11496
|
const result2 = {
|
|
12128
11497
|
project: projectFilter,
|
|
@@ -12159,13 +11528,13 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
12159
11528
|
function backfillAiReferralPaths(db, opts) {
|
|
12160
11529
|
const baseConditions = [];
|
|
12161
11530
|
if (opts?.projectId) {
|
|
12162
|
-
baseConditions.push(
|
|
11531
|
+
baseConditions.push(eq15(gaAiReferrals.projectId, opts.projectId));
|
|
12163
11532
|
}
|
|
12164
11533
|
const rows = db.select({
|
|
12165
11534
|
id: gaAiReferrals.id,
|
|
12166
11535
|
landingPage: gaAiReferrals.landingPage,
|
|
12167
11536
|
landingPageNormalized: gaAiReferrals.landingPageNormalized
|
|
12168
|
-
}).from(gaAiReferrals).where(baseConditions.length > 0 ?
|
|
11537
|
+
}).from(gaAiReferrals).where(baseConditions.length > 0 ? and13(...baseConditions) : void 0).all();
|
|
12169
11538
|
let updated = 0;
|
|
12170
11539
|
let unchanged = 0;
|
|
12171
11540
|
if (rows.length > 0) {
|
|
@@ -12180,7 +11549,7 @@ function backfillAiReferralPaths(db, opts) {
|
|
|
12180
11549
|
unchanged++;
|
|
12181
11550
|
continue;
|
|
12182
11551
|
}
|
|
12183
|
-
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(
|
|
11552
|
+
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(eq15(gaAiReferrals.id, row.id)).run();
|
|
12184
11553
|
updated++;
|
|
12185
11554
|
}
|
|
12186
11555
|
});
|
|
@@ -12194,7 +11563,7 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
12194
11563
|
const projectFilter = opts?.project?.trim();
|
|
12195
11564
|
let projectId;
|
|
12196
11565
|
if (projectFilter) {
|
|
12197
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
11566
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq15(projects.name, projectFilter)).get();
|
|
12198
11567
|
if (!project) {
|
|
12199
11568
|
const result2 = {
|
|
12200
11569
|
project: projectFilter,
|
|
@@ -12230,10 +11599,10 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
12230
11599
|
}
|
|
12231
11600
|
function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
12232
11601
|
const isDryRun = opts?.dryRun === true;
|
|
12233
|
-
const project = db.select().from(projects).where(
|
|
11602
|
+
const project = db.select().from(projects).where(eq15(projects.id, projectId)).get();
|
|
12234
11603
|
if (!project) return { examined: 0, updated: 0, mentioned: 0 };
|
|
12235
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
12236
|
-
const runRows = db.select({ id: runs.id }).from(runs).where(
|
|
11604
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq15(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
11605
|
+
const runRows = db.select({ id: runs.id }).from(runs).where(and13(eq15(runs.kind, RunKinds["answer-visibility"]), eq15(runs.projectId, projectId))).all();
|
|
12237
11606
|
const runIds = runRows.map((r) => r.id);
|
|
12238
11607
|
let examined = 0;
|
|
12239
11608
|
let updated = 0;
|
|
@@ -12307,7 +11676,7 @@ function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
|
12307
11676
|
} else {
|
|
12308
11677
|
db.transaction((tx) => {
|
|
12309
11678
|
for (const update of pendingUpdates) {
|
|
12310
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
11679
|
+
tx.update(querySnapshots).set(update.patch).where(eq15(querySnapshots.id, update.id)).run();
|
|
12311
11680
|
}
|
|
12312
11681
|
});
|
|
12313
11682
|
updated += pendingUpdates.length;
|
|
@@ -12322,7 +11691,7 @@ async function backfillAnswerMentionsCommand(opts) {
|
|
|
12322
11691
|
migrate(db);
|
|
12323
11692
|
const projectFilter = opts?.project?.trim();
|
|
12324
11693
|
const isDryRun = opts?.dryRun === true;
|
|
12325
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
11694
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq15(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
12326
11695
|
let examined = 0;
|
|
12327
11696
|
let updated = 0;
|
|
12328
11697
|
let wouldUpdate = 0;
|
|
@@ -12382,7 +11751,7 @@ function readStoredGroundingSources(rawResponse) {
|
|
|
12382
11751
|
return result;
|
|
12383
11752
|
}
|
|
12384
11753
|
async function backfillInsightsCommand(project, opts) {
|
|
12385
|
-
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-
|
|
11754
|
+
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-SJO5E7WN.js");
|
|
12386
11755
|
const config = loadConfig();
|
|
12387
11756
|
const db = createClient(config.database);
|
|
12388
11757
|
migrate(db);
|
|
@@ -12541,7 +11910,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
12541
11910
|
const config = loadConfig();
|
|
12542
11911
|
const db = createClient(config.database);
|
|
12543
11912
|
migrate(db);
|
|
12544
|
-
const project = db.select().from(projects).where(
|
|
11913
|
+
const project = db.select().from(projects).where(eq15(projects.name, opts.project)).get();
|
|
12545
11914
|
if (!project) {
|
|
12546
11915
|
throw new Error(`Project "${opts.project}" not found`);
|
|
12547
11916
|
}
|
|
@@ -12552,8 +11921,8 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
12552
11921
|
process.stderr.write(`Recovering orphan snapshot attribution for "${project.name}"${mode}...
|
|
12553
11922
|
`);
|
|
12554
11923
|
}
|
|
12555
|
-
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(
|
|
12556
|
-
|
|
11924
|
+
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(and13(
|
|
11925
|
+
eq15(auditLog.projectId, project.id),
|
|
12557
11926
|
inArray6(auditLog.action, ["keywords.appended", "keywords.deleted", "queries.appended", "queries.deleted", "queries.replaced"])
|
|
12558
11927
|
)).orderBy(auditLog.createdAt).all();
|
|
12559
11928
|
const history = replayQueryAuditLog(events);
|
|
@@ -12561,10 +11930,10 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
12561
11930
|
runId: runs.id,
|
|
12562
11931
|
createdAt: runs.createdAt,
|
|
12563
11932
|
location: runs.location
|
|
12564
|
-
}).from(runs).innerJoin(querySnapshots,
|
|
12565
|
-
|
|
12566
|
-
|
|
12567
|
-
|
|
11933
|
+
}).from(runs).innerJoin(querySnapshots, eq15(querySnapshots.runId, runs.id)).where(and13(
|
|
11934
|
+
eq15(runs.projectId, project.id),
|
|
11935
|
+
isNull(querySnapshots.queryId),
|
|
11936
|
+
isNull(querySnapshots.queryText)
|
|
12568
11937
|
)).groupBy(runs.id).orderBy(runs.createdAt).all();
|
|
12569
11938
|
const result = {
|
|
12570
11939
|
project: project.name,
|
|
@@ -12584,10 +11953,10 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
12584
11953
|
provider: querySnapshots.provider,
|
|
12585
11954
|
createdAt: querySnapshots.createdAt,
|
|
12586
11955
|
answerText: querySnapshots.answerText
|
|
12587
|
-
}).from(querySnapshots).where(
|
|
12588
|
-
|
|
12589
|
-
|
|
12590
|
-
|
|
11956
|
+
}).from(querySnapshots).where(and13(
|
|
11957
|
+
eq15(querySnapshots.runId, run.runId),
|
|
11958
|
+
isNull(querySnapshots.queryId),
|
|
11959
|
+
isNull(querySnapshots.queryText)
|
|
12591
11960
|
)).orderBy(querySnapshots.provider, querySnapshots.createdAt).all();
|
|
12592
11961
|
const byProvider = /* @__PURE__ */ new Map();
|
|
12593
11962
|
for (const s of orphanSnaps) {
|
|
@@ -12651,7 +12020,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
12651
12020
|
if (!isDryRun && updates.length > 0) {
|
|
12652
12021
|
db.transaction((tx) => {
|
|
12653
12022
|
for (const u of updates) {
|
|
12654
|
-
tx.update(querySnapshots).set({ queryText: u.queryText }).where(
|
|
12023
|
+
tx.update(querySnapshots).set({ queryText: u.queryText }).where(eq15(querySnapshots.id, u.id)).run();
|
|
12655
12024
|
}
|
|
12656
12025
|
});
|
|
12657
12026
|
}
|
|
@@ -12725,7 +12094,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
12725
12094
|
const projectFilter = opts?.project?.trim();
|
|
12726
12095
|
const isDryRun = opts?.dryRun === true;
|
|
12727
12096
|
const isJson = isMachineFormat(opts?.format);
|
|
12728
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
12097
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq15(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
12729
12098
|
if (scopedProjects.length === 0) {
|
|
12730
12099
|
if (projectFilter && !isJson) {
|
|
12731
12100
|
process.stderr.write(`No project named "${projectFilter}".
|
|
@@ -12750,8 +12119,8 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
12750
12119
|
dryRun: isDryRun,
|
|
12751
12120
|
byBot: {}
|
|
12752
12121
|
};
|
|
12753
|
-
const unknownCountRow = db.select({ n:
|
|
12754
|
-
|
|
12122
|
+
const unknownCountRow = db.select({ n: sql6`count(*)` }).from(rawEventSamples).where(and13(
|
|
12123
|
+
eq15(rawEventSamples.eventType, "unknown"),
|
|
12755
12124
|
inArray6(rawEventSamples.projectId, projectIds)
|
|
12756
12125
|
)).get();
|
|
12757
12126
|
result.unknownBefore = Number(unknownCountRow?.n ?? 0);
|
|
@@ -12763,8 +12132,8 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
12763
12132
|
userAgent: rawEventSamples.userAgent,
|
|
12764
12133
|
pathNormalized: rawEventSamples.pathNormalized,
|
|
12765
12134
|
status: rawEventSamples.status
|
|
12766
|
-
}).from(rawEventSamples).where(
|
|
12767
|
-
|
|
12135
|
+
}).from(rawEventSamples).where(and13(
|
|
12136
|
+
eq15(rawEventSamples.eventType, "unknown"),
|
|
12768
12137
|
inArray6(rawEventSamples.projectId, projectIds)
|
|
12769
12138
|
)).all();
|
|
12770
12139
|
result.examined = unknownSamples.length;
|
|
@@ -12808,7 +12177,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
12808
12177
|
const tsHourIso = tsHour.toISOString();
|
|
12809
12178
|
const status = snap.status ?? 200;
|
|
12810
12179
|
db.transaction((tx) => {
|
|
12811
|
-
tx.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(
|
|
12180
|
+
tx.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(eq15(rawEventSamples.id, snap.id)).run();
|
|
12812
12181
|
if (userFetch) {
|
|
12813
12182
|
tx.insert(aiUserFetchEventsHourly).values({
|
|
12814
12183
|
projectId: snap.projectId,
|
|
@@ -12834,7 +12203,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
12834
12203
|
aiUserFetchEventsHourly.status
|
|
12835
12204
|
],
|
|
12836
12205
|
set: {
|
|
12837
|
-
hits:
|
|
12206
|
+
hits: sql6`${aiUserFetchEventsHourly.hits} + 1`,
|
|
12838
12207
|
updatedAt: now
|
|
12839
12208
|
}
|
|
12840
12209
|
}).run();
|
|
@@ -12863,7 +12232,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
12863
12232
|
crawlerEventsHourly.status
|
|
12864
12233
|
],
|
|
12865
12234
|
set: {
|
|
12866
|
-
hits:
|
|
12235
|
+
hits: sql6`${crawlerEventsHourly.hits} + 1`,
|
|
12867
12236
|
updatedAt: now
|
|
12868
12237
|
}
|
|
12869
12238
|
}).run();
|
|
@@ -12871,8 +12240,8 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
12871
12240
|
});
|
|
12872
12241
|
}
|
|
12873
12242
|
if (!isDryRun) {
|
|
12874
|
-
const afterRow = db.select({ n:
|
|
12875
|
-
|
|
12243
|
+
const afterRow = db.select({ n: sql6`count(*)` }).from(rawEventSamples).where(and13(
|
|
12244
|
+
eq15(rawEventSamples.eventType, "unknown"),
|
|
12876
12245
|
inArray6(rawEventSamples.projectId, projectIds)
|
|
12877
12246
|
)).get();
|
|
12878
12247
|
result.unknownAfter = Number(afterRow?.n ?? 0);
|
|
@@ -12896,8 +12265,8 @@ Traffic classification ${result.dryRun ? "preview" : "recovery"} complete.`);
|
|
|
12896
12265
|
if (result.reclassified > 0) {
|
|
12897
12266
|
console.log(` By bot:`);
|
|
12898
12267
|
const sorted = Object.entries(result.byBot).sort(([, a], [, b]) => b - a);
|
|
12899
|
-
for (const [bot,
|
|
12900
|
-
console.log(` ${
|
|
12268
|
+
for (const [bot, count] of sorted) {
|
|
12269
|
+
console.log(` ${count.toString().padStart(5)} ${bot}`);
|
|
12901
12270
|
}
|
|
12902
12271
|
}
|
|
12903
12272
|
if (result.dryRun) {
|
|
@@ -13018,14 +12387,14 @@ function createProviderModelCatalog(registry) {
|
|
|
13018
12387
|
}
|
|
13019
12388
|
|
|
13020
12389
|
// src/scheduler.ts
|
|
13021
|
-
import
|
|
12390
|
+
import crypto19 from "crypto";
|
|
13022
12391
|
import cron from "node-cron";
|
|
13023
|
-
import { and as
|
|
12392
|
+
import { and as and14, eq as eq16, inArray as inArray7, notExists, sql as sql7 } from "drizzle-orm";
|
|
13024
12393
|
var log14 = createLogger("Scheduler");
|
|
13025
12394
|
var DEFAULT_HEALTH_CRON = "0 */6 * * *";
|
|
13026
12395
|
function ensureDefaultHealthSchedule(db, projectId, now = (/* @__PURE__ */ new Date()).toISOString()) {
|
|
13027
12396
|
const result = db.insert(schedules).values({
|
|
13028
|
-
id:
|
|
12397
|
+
id: crypto19.randomUUID(),
|
|
13029
12398
|
projectId,
|
|
13030
12399
|
kind: SchedulableRunKinds.doctor,
|
|
13031
12400
|
cronExpr: DEFAULT_HEALTH_CRON,
|
|
@@ -13071,9 +12440,9 @@ var Scheduler = class {
|
|
|
13071
12440
|
*/
|
|
13072
12441
|
ensureHealthSchedules() {
|
|
13073
12442
|
const projectsWithoutHealth = this.db.select({ id: projects.id }).from(projects).where(notExists(
|
|
13074
|
-
this.db.select({ one:
|
|
13075
|
-
|
|
13076
|
-
|
|
12443
|
+
this.db.select({ one: sql7`1` }).from(schedules).where(and14(
|
|
12444
|
+
eq16(schedules.projectId, projects.id),
|
|
12445
|
+
eq16(schedules.kind, SchedulableRunKinds.doctor)
|
|
13077
12446
|
))
|
|
13078
12447
|
)).all();
|
|
13079
12448
|
if (projectsWithoutHealth.length === 0) return;
|
|
@@ -13116,7 +12485,7 @@ var Scheduler = class {
|
|
|
13116
12485
|
start() {
|
|
13117
12486
|
this.ensureHealthSchedules();
|
|
13118
12487
|
this.ensureQueryBaskets();
|
|
13119
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
12488
|
+
const allSchedules = this.db.select().from(schedules).where(eq16(schedules.enabled, true)).all();
|
|
13120
12489
|
for (const schedule of allSchedules) {
|
|
13121
12490
|
const missedRunAt = schedule.nextRunAt;
|
|
13122
12491
|
const recurrence = scheduleRecurrence(schedule);
|
|
@@ -13150,7 +12519,7 @@ var Scheduler = class {
|
|
|
13150
12519
|
this.stopTask(key, existing, "Stopped");
|
|
13151
12520
|
this.tasks.delete(key);
|
|
13152
12521
|
}
|
|
13153
|
-
const schedule = this.db.select().from(schedules).where(
|
|
12522
|
+
const schedule = this.db.select().from(schedules).where(and14(eq16(schedules.projectId, projectId), eq16(schedules.kind, kind))).get();
|
|
13154
12523
|
if (schedule && schedule.enabled) {
|
|
13155
12524
|
this.registerCronTask(schedule);
|
|
13156
12525
|
}
|
|
@@ -13178,12 +12547,12 @@ var Scheduler = class {
|
|
|
13178
12547
|
/** Runtime timing writes must not restore a version already issued to a settings caller. */
|
|
13179
12548
|
updateScheduleTiming(scheduleId, timing) {
|
|
13180
12549
|
this.db.transaction((tx) => {
|
|
13181
|
-
const current = tx.select({ updatedAt: schedules.updatedAt }).from(schedules).where(
|
|
12550
|
+
const current = tx.select({ updatedAt: schedules.updatedAt }).from(schedules).where(eq16(schedules.id, scheduleId)).get();
|
|
13182
12551
|
if (!current) return;
|
|
13183
12552
|
tx.update(schedules).set({
|
|
13184
12553
|
...timing,
|
|
13185
12554
|
updatedAt: nextScheduleUpdatedAt(current.updatedAt)
|
|
13186
|
-
}).where(
|
|
12555
|
+
}).where(eq16(schedules.id, scheduleId)).run();
|
|
13187
12556
|
});
|
|
13188
12557
|
}
|
|
13189
12558
|
registerCronTask(schedule, options = {}) {
|
|
@@ -13244,7 +12613,7 @@ var Scheduler = class {
|
|
|
13244
12613
|
};
|
|
13245
12614
|
const tick = () => {
|
|
13246
12615
|
if (stopped) return;
|
|
13247
|
-
const current = this.db.select().from(schedules).where(
|
|
12616
|
+
const current = this.db.select().from(schedules).where(eq16(schedules.id, scheduleId)).get();
|
|
13248
12617
|
if (!current || !current.enabled || !scheduleRecurrence(current)) {
|
|
13249
12618
|
this.remove(projectId, kind);
|
|
13250
12619
|
return;
|
|
@@ -13262,7 +12631,7 @@ var Scheduler = class {
|
|
|
13262
12631
|
this.triggerRun(scheduleId, projectId, kind);
|
|
13263
12632
|
}
|
|
13264
12633
|
}
|
|
13265
|
-
arm(this.db.select({ nextRunAt: schedules.nextRunAt }).from(schedules).where(
|
|
12634
|
+
arm(this.db.select({ nextRunAt: schedules.nextRunAt }).from(schedules).where(eq16(schedules.id, scheduleId)).get()?.nextRunAt);
|
|
13266
12635
|
};
|
|
13267
12636
|
this.tasks.set(key, task);
|
|
13268
12637
|
let registeredNextRunAt = schedule.nextRunAt;
|
|
@@ -13277,7 +12646,7 @@ var Scheduler = class {
|
|
|
13277
12646
|
/** Atomically advance a due callback-only calendar occurrence before dispatch. */
|
|
13278
12647
|
claimCalendarOccurrence(schedule, dueAt, now) {
|
|
13279
12648
|
return this.db.transaction((tx) => {
|
|
13280
|
-
const current = tx.select().from(schedules).where(
|
|
12649
|
+
const current = tx.select().from(schedules).where(eq16(schedules.id, schedule.id)).get();
|
|
13281
12650
|
if (!current || !current.enabled || current.updatedAt !== schedule.updatedAt || current.nextRunAt !== dueAt) return false;
|
|
13282
12651
|
const recurrence = scheduleRecurrence(current);
|
|
13283
12652
|
if (!recurrence) return false;
|
|
@@ -13286,11 +12655,11 @@ var Scheduler = class {
|
|
|
13286
12655
|
const result = tx.update(schedules).set({
|
|
13287
12656
|
nextRunAt,
|
|
13288
12657
|
updatedAt: nextScheduleUpdatedAt(current.updatedAt)
|
|
13289
|
-
}).where(
|
|
13290
|
-
|
|
13291
|
-
|
|
13292
|
-
|
|
13293
|
-
|
|
12658
|
+
}).where(and14(
|
|
12659
|
+
eq16(schedules.id, current.id),
|
|
12660
|
+
eq16(schedules.enabled, true),
|
|
12661
|
+
eq16(schedules.nextRunAt, dueAt),
|
|
12662
|
+
eq16(schedules.updatedAt, schedule.updatedAt)
|
|
13294
12663
|
)).run();
|
|
13295
12664
|
return result.changes === 1;
|
|
13296
12665
|
});
|
|
@@ -13298,7 +12667,7 @@ var Scheduler = class {
|
|
|
13298
12667
|
triggerRun(scheduleId, projectId, kind, claimedOccurrence) {
|
|
13299
12668
|
try {
|
|
13300
12669
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
13301
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
12670
|
+
const currentSchedule = this.db.select().from(schedules).where(eq16(schedules.id, scheduleId)).get();
|
|
13302
12671
|
if (!currentSchedule || !currentSchedule.enabled) {
|
|
13303
12672
|
log14.warn("schedule.stale", { scheduleId, projectId, kind, msg: "schedule no longer exists or is disabled" });
|
|
13304
12673
|
this.remove(projectId, kind);
|
|
@@ -13314,7 +12683,7 @@ var Scheduler = class {
|
|
|
13314
12683
|
timezone: currentSchedule.timezone,
|
|
13315
12684
|
recurrence
|
|
13316
12685
|
});
|
|
13317
|
-
const project = this.db.select().from(projects).where(
|
|
12686
|
+
const project = this.db.select().from(projects).where(eq16(projects.id, projectId)).get();
|
|
13318
12687
|
if (!project) {
|
|
13319
12688
|
log14.error("project.not-found", { projectId, kind, msg: "skipping scheduled run" });
|
|
13320
12689
|
this.remove(projectId, kind);
|
|
@@ -13343,7 +12712,7 @@ var Scheduler = class {
|
|
|
13343
12712
|
log14.warn("gbp-sync.no-callback", { scheduleId, projectId, msg: "host did not register onGbpSyncRequested" });
|
|
13344
12713
|
return;
|
|
13345
12714
|
}
|
|
13346
|
-
const runId2 =
|
|
12715
|
+
const runId2 = crypto19.randomUUID();
|
|
13347
12716
|
this.db.insert(runs).values({
|
|
13348
12717
|
id: runId2,
|
|
13349
12718
|
projectId,
|
|
@@ -13365,9 +12734,9 @@ var Scheduler = class {
|
|
|
13365
12734
|
log14.warn("ads-sync.no-callback", { scheduleId, projectId, msg: "host did not register onAdsSyncRequested" });
|
|
13366
12735
|
return;
|
|
13367
12736
|
}
|
|
13368
|
-
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(
|
|
13369
|
-
|
|
13370
|
-
|
|
12737
|
+
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(and14(
|
|
12738
|
+
eq16(runs.projectId, projectId),
|
|
12739
|
+
eq16(runs.kind, RunKinds["ads-sync"]),
|
|
13371
12740
|
inArray7(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
13372
12741
|
)).get();
|
|
13373
12742
|
if (activeAdsRun) {
|
|
@@ -13375,7 +12744,7 @@ var Scheduler = class {
|
|
|
13375
12744
|
this.updateScheduleTiming(currentSchedule.id, { nextRunAt });
|
|
13376
12745
|
return;
|
|
13377
12746
|
}
|
|
13378
|
-
const runId2 =
|
|
12747
|
+
const runId2 = crypto19.randomUUID();
|
|
13379
12748
|
this.db.insert(runs).values({
|
|
13380
12749
|
id: runId2,
|
|
13381
12750
|
projectId,
|
|
@@ -13436,9 +12805,9 @@ var Scheduler = class {
|
|
|
13436
12805
|
log14.warn("site-audit.no-callback", { scheduleId, projectId, msg: "host did not register onSiteAuditRequested" });
|
|
13437
12806
|
return;
|
|
13438
12807
|
}
|
|
13439
|
-
const active = this.db.select({ id: runs.id }).from(runs).where(
|
|
13440
|
-
|
|
13441
|
-
|
|
12808
|
+
const active = this.db.select({ id: runs.id }).from(runs).where(and14(
|
|
12809
|
+
eq16(runs.projectId, projectId),
|
|
12810
|
+
eq16(runs.kind, RunKinds["site-audit"]),
|
|
13442
12811
|
inArray7(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
13443
12812
|
)).get();
|
|
13444
12813
|
if (active) {
|
|
@@ -13446,7 +12815,7 @@ var Scheduler = class {
|
|
|
13446
12815
|
this.updateScheduleTiming(currentSchedule.id, { nextRunAt });
|
|
13447
12816
|
return;
|
|
13448
12817
|
}
|
|
13449
|
-
const runId2 =
|
|
12818
|
+
const runId2 = crypto19.randomUUID();
|
|
13450
12819
|
const effectiveRequest = normalizeSiteAuditRunRequest({});
|
|
13451
12820
|
this.db.transaction((tx) => {
|
|
13452
12821
|
tx.insert(runs).values({
|
|
@@ -13560,8 +12929,8 @@ async function refreshAllIntegrations(client, projectName) {
|
|
|
13560
12929
|
}
|
|
13561
12930
|
|
|
13562
12931
|
// src/notifier.ts
|
|
13563
|
-
import { eq as
|
|
13564
|
-
import
|
|
12932
|
+
import { eq as eq17, desc as desc6, and as and15, inArray as inArray8, or } from "drizzle-orm";
|
|
12933
|
+
import crypto20 from "crypto";
|
|
13565
12934
|
var log16 = createLogger("Notifier");
|
|
13566
12935
|
var MATERIAL_MAGNITUDE_DELTA = 20;
|
|
13567
12936
|
var SEVERITY_RANK = { low: 0, medium: 1, high: 2, critical: 3 };
|
|
@@ -13588,18 +12957,18 @@ var Notifier = class {
|
|
|
13588
12957
|
/** Called after a run completes (success, partial, or failed). */
|
|
13589
12958
|
async onRunCompleted(runId, projectId) {
|
|
13590
12959
|
log16.info("run.completed", { runId, projectId });
|
|
13591
|
-
const notifs = this.db.select().from(notifications).where(
|
|
12960
|
+
const notifs = this.db.select().from(notifications).where(eq17(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
13592
12961
|
if (notifs.length === 0) {
|
|
13593
12962
|
log16.info("notifications.none-enabled", { projectId });
|
|
13594
12963
|
return;
|
|
13595
12964
|
}
|
|
13596
12965
|
log16.info("notifications.found", { projectId, count: notifs.length });
|
|
13597
|
-
const run = this.db.select().from(runs).where(
|
|
12966
|
+
const run = this.db.select().from(runs).where(eq17(runs.id, runId)).get();
|
|
13598
12967
|
if (!run) {
|
|
13599
12968
|
log16.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
13600
12969
|
return;
|
|
13601
12970
|
}
|
|
13602
|
-
const project = this.db.select().from(projects).where(
|
|
12971
|
+
const project = this.db.select().from(projects).where(eq17(projects.id, projectId)).get();
|
|
13603
12972
|
if (!project) {
|
|
13604
12973
|
log16.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
13605
12974
|
return;
|
|
@@ -13652,7 +13021,7 @@ var Notifier = class {
|
|
|
13652
13021
|
* inferred from the absence of a webhook.
|
|
13653
13022
|
*/
|
|
13654
13023
|
async onHealthChecked(projectId, report) {
|
|
13655
|
-
const project = this.db.select().from(projects).where(
|
|
13024
|
+
const project = this.db.select().from(projects).where(eq17(projects.id, projectId)).get();
|
|
13656
13025
|
if (!project) {
|
|
13657
13026
|
log16.error("project.not-found", { projectId, msg: "skipping health notification" });
|
|
13658
13027
|
return null;
|
|
@@ -13683,7 +13052,7 @@ var Notifier = class {
|
|
|
13683
13052
|
const status = noSignal ? "warn" : worst ? worst.status : "ok";
|
|
13684
13053
|
const code = noSignal ? "health.no-signal" : worst?.code ?? "health.ok";
|
|
13685
13054
|
const summary = noSignal ? `No health check produced a result (${report.checks.length} skipped) \u2014 health is unknown, not confirmed.` : worst?.summary ?? `All ${graded.length} health check(s) passing.`;
|
|
13686
|
-
const previous = this.db.select().from(doctorHealthState).where(
|
|
13055
|
+
const previous = this.db.select().from(doctorHealthState).where(eq17(doctorHealthState.projectId, projectId)).get();
|
|
13687
13056
|
const previousStatus = previous?.status ?? null;
|
|
13688
13057
|
let event = null;
|
|
13689
13058
|
if (previous === void 0) {
|
|
@@ -13698,13 +13067,13 @@ var Notifier = class {
|
|
|
13698
13067
|
if (previous === void 0) {
|
|
13699
13068
|
this.db.insert(doctorHealthState).values({ ...observation, notifiedAt: null }).run();
|
|
13700
13069
|
} else {
|
|
13701
|
-
this.db.update(doctorHealthState).set(observation).where(
|
|
13070
|
+
this.db.update(doctorHealthState).set(observation).where(eq17(doctorHealthState.projectId, projectId)).run();
|
|
13702
13071
|
}
|
|
13703
13072
|
if (!event) {
|
|
13704
13073
|
log16.info("health.unchanged", { projectId, status, code });
|
|
13705
13074
|
return null;
|
|
13706
13075
|
}
|
|
13707
|
-
const notifs = this.db.select().from(notifications).where(
|
|
13076
|
+
const notifs = this.db.select().from(notifications).where(eq17(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
13708
13077
|
const payload = {
|
|
13709
13078
|
source: "canonry",
|
|
13710
13079
|
event,
|
|
@@ -13728,7 +13097,7 @@ var Notifier = class {
|
|
|
13728
13097
|
delivered += 1;
|
|
13729
13098
|
}
|
|
13730
13099
|
if (delivered > 0) {
|
|
13731
|
-
this.db.update(doctorHealthState).set({ notifiedAt: now }).where(
|
|
13100
|
+
this.db.update(doctorHealthState).set({ notifiedAt: now }).where(eq17(doctorHealthState.projectId, projectId)).run();
|
|
13732
13101
|
}
|
|
13733
13102
|
log16.info("health.notified", { projectId, event, status, code, subscribers: notifs.length, delivered });
|
|
13734
13103
|
return event;
|
|
@@ -13751,7 +13120,7 @@ var Notifier = class {
|
|
|
13751
13120
|
*/
|
|
13752
13121
|
isNewInsightNews(projectId, insight) {
|
|
13753
13122
|
const key = insightNotifyKey(projectId, insight);
|
|
13754
|
-
const previous = this.db.select().from(insightNotifyState).where(
|
|
13123
|
+
const previous = this.db.select().from(insightNotifyState).where(eq17(insightNotifyState.key, key)).get();
|
|
13755
13124
|
if (!previous) return true;
|
|
13756
13125
|
if (severityRank(insight.severity) > severityRank(previous.severity)) return true;
|
|
13757
13126
|
const magnitude = insightMagnitude(insight);
|
|
@@ -13773,9 +13142,9 @@ var Notifier = class {
|
|
|
13773
13142
|
magnitude: insightMagnitude(insight),
|
|
13774
13143
|
notifiedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
13775
13144
|
};
|
|
13776
|
-
const existing = this.db.select().from(insightNotifyState).where(
|
|
13145
|
+
const existing = this.db.select().from(insightNotifyState).where(eq17(insightNotifyState.key, key)).get();
|
|
13777
13146
|
if (existing) {
|
|
13778
|
-
this.db.update(insightNotifyState).set(row).where(
|
|
13147
|
+
this.db.update(insightNotifyState).set(row).where(eq17(insightNotifyState.key, key)).run();
|
|
13779
13148
|
} else {
|
|
13780
13149
|
this.db.insert(insightNotifyState).values(row).run();
|
|
13781
13150
|
}
|
|
@@ -13789,11 +13158,11 @@ var Notifier = class {
|
|
|
13789
13158
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
13790
13159
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
13791
13160
|
if (insightEvents.length === 0) return;
|
|
13792
|
-
const notifs = this.db.select().from(notifications).where(
|
|
13161
|
+
const notifs = this.db.select().from(notifications).where(eq17(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
13793
13162
|
if (notifs.length === 0) return;
|
|
13794
|
-
const run = this.db.select().from(runs).where(
|
|
13163
|
+
const run = this.db.select().from(runs).where(eq17(runs.id, runId)).get();
|
|
13795
13164
|
if (!run) return;
|
|
13796
|
-
const project = this.db.select().from(projects).where(
|
|
13165
|
+
const project = this.db.select().from(projects).where(eq17(projects.id, projectId)).get();
|
|
13797
13166
|
if (!project) return;
|
|
13798
13167
|
for (const notif of notifs) {
|
|
13799
13168
|
const config = notif.config;
|
|
@@ -13824,7 +13193,7 @@ var Notifier = class {
|
|
|
13824
13193
|
}
|
|
13825
13194
|
}
|
|
13826
13195
|
computeTransitions(runId, projectId) {
|
|
13827
|
-
const thisRun = this.db.select().from(runs).where(
|
|
13196
|
+
const thisRun = this.db.select().from(runs).where(eq17(runs.id, runId)).get();
|
|
13828
13197
|
if (!thisRun) return [];
|
|
13829
13198
|
const completeness = measurementRunCompleteness(this.db, runId);
|
|
13830
13199
|
if (completeness.planned && !completeness.complete) {
|
|
@@ -13835,10 +13204,10 @@ var Notifier = class {
|
|
|
13835
13204
|
});
|
|
13836
13205
|
return [];
|
|
13837
13206
|
}
|
|
13838
|
-
const groupSiblings = this.db.select().from(runs).where(
|
|
13839
|
-
|
|
13840
|
-
|
|
13841
|
-
|
|
13207
|
+
const groupSiblings = this.db.select().from(runs).where(and15(
|
|
13208
|
+
eq17(runs.projectId, projectId),
|
|
13209
|
+
eq17(runs.kind, thisRun.kind),
|
|
13210
|
+
eq17(runs.createdAt, thisRun.createdAt)
|
|
13842
13211
|
)).all();
|
|
13843
13212
|
const stillPending = groupSiblings.some((r) => r.status === "queued" || r.status === "running");
|
|
13844
13213
|
if (stillPending) return [];
|
|
@@ -13854,19 +13223,19 @@ var Notifier = class {
|
|
|
13854
13223
|
return candidate.id > best.id ? candidate : best;
|
|
13855
13224
|
});
|
|
13856
13225
|
if (winner.id !== runId) return [];
|
|
13857
|
-
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
13226
|
+
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(eq17(projects.id, projectId)).get();
|
|
13858
13227
|
const locationCount = Math.max(
|
|
13859
13228
|
1,
|
|
13860
13229
|
(projectLocations?.locations ?? []).length
|
|
13861
13230
|
);
|
|
13862
13231
|
const RECENT_FETCH_LIMIT = Math.max(8, locationCount * 4);
|
|
13863
13232
|
const recentRuns = this.db.select().from(runs).where(
|
|
13864
|
-
|
|
13865
|
-
|
|
13866
|
-
|
|
13867
|
-
|
|
13233
|
+
and15(
|
|
13234
|
+
eq17(runs.projectId, projectId),
|
|
13235
|
+
eq17(runs.kind, thisRun.kind),
|
|
13236
|
+
or(eq17(runs.status, "completed"), eq17(runs.status, "partial"))
|
|
13868
13237
|
)
|
|
13869
|
-
).orderBy(
|
|
13238
|
+
).orderBy(desc6(runs.createdAt), desc6(runs.id)).limit(RECENT_FETCH_LIMIT).all();
|
|
13870
13239
|
const groups = groupRunsByCreatedAt(recentRuns);
|
|
13871
13240
|
const currentGroupIdx = groups.findIndex((g) => g[0]?.createdAt === thisRun.createdAt);
|
|
13872
13241
|
if (currentGroupIdx < 0) return [];
|
|
@@ -13881,7 +13250,7 @@ var Notifier = class {
|
|
|
13881
13250
|
provider: querySnapshots.provider,
|
|
13882
13251
|
location: querySnapshots.location,
|
|
13883
13252
|
citationState: querySnapshots.citationState
|
|
13884
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
13253
|
+
}).from(querySnapshots).leftJoin(queries, eq17(querySnapshots.queryId, queries.id)).where(inArray8(querySnapshots.runId, currentRunIds)).all();
|
|
13885
13254
|
const previousSnapshots = this.db.select({
|
|
13886
13255
|
queryId: querySnapshots.queryId,
|
|
13887
13256
|
provider: querySnapshots.provider,
|
|
@@ -13951,7 +13320,7 @@ var Notifier = class {
|
|
|
13951
13320
|
}
|
|
13952
13321
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
13953
13322
|
this.db.insert(auditLog).values({
|
|
13954
|
-
id:
|
|
13323
|
+
id: crypto20.randomUUID(),
|
|
13955
13324
|
projectId,
|
|
13956
13325
|
actor: "scheduler",
|
|
13957
13326
|
action: `notification.${status}`,
|
|
@@ -13964,7 +13333,7 @@ var Notifier = class {
|
|
|
13964
13333
|
};
|
|
13965
13334
|
|
|
13966
13335
|
// src/run-coordinator.ts
|
|
13967
|
-
import { eq as
|
|
13336
|
+
import { eq as eq18 } from "drizzle-orm";
|
|
13968
13337
|
var log17 = createLogger("RunCoordinator");
|
|
13969
13338
|
var RunCoordinator = class {
|
|
13970
13339
|
constructor(db, notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
@@ -13980,7 +13349,7 @@ var RunCoordinator = class {
|
|
|
13980
13349
|
onInsightsGenerated;
|
|
13981
13350
|
onAeroEvent;
|
|
13982
13351
|
async onRunCompleted(runId, projectId) {
|
|
13983
|
-
const runRow = this.db.select().from(runs).where(
|
|
13352
|
+
const runRow = this.db.select().from(runs).where(eq18(runs.id, runId)).get();
|
|
13984
13353
|
const kind = runRow?.kind ?? RunKinds["answer-visibility"];
|
|
13985
13354
|
if (runRow?.trigger === RunTriggers.probe) {
|
|
13986
13355
|
log17.info("probe.skip-side-effects", { runId, projectId, kind });
|
|
@@ -14071,7 +13440,7 @@ var RunCoordinator = class {
|
|
|
14071
13440
|
* so the Aero queue is never starved of a follow-up.
|
|
14072
13441
|
*/
|
|
14073
13442
|
buildDiscoveryAeroContext(runId, projectId, status, error) {
|
|
14074
|
-
const session = this.db.select().from(discoverySessions).where(
|
|
13443
|
+
const session = this.db.select().from(discoverySessions).where(eq18(discoverySessions.runId, runId)).get();
|
|
14075
13444
|
const competitorMap = session ? session.competitorMap : [];
|
|
14076
13445
|
return {
|
|
14077
13446
|
kind: RunKinds["aeo-discover-probe"],
|
|
@@ -14113,8 +13482,8 @@ function analysisResultFromInsights(insights2) {
|
|
|
14113
13482
|
}
|
|
14114
13483
|
|
|
14115
13484
|
// src/agent/session-registry.ts
|
|
14116
|
-
import
|
|
14117
|
-
import { eq as
|
|
13485
|
+
import crypto24 from "crypto";
|
|
13486
|
+
import { eq as eq20 } from "drizzle-orm";
|
|
14118
13487
|
|
|
14119
13488
|
// src/agent/session.ts
|
|
14120
13489
|
import fs7 from "fs";
|
|
@@ -14546,9 +13915,9 @@ function trimNestedArrays(full) {
|
|
|
14546
13915
|
const priorCount = typeof omitted[key] === "number" ? omitted[key] : 0;
|
|
14547
13916
|
owner.__truncated = true;
|
|
14548
13917
|
owner.__omittedRowsByField = omitted;
|
|
14549
|
-
const keep = (
|
|
14550
|
-
owner[key] = rows.slice(0,
|
|
14551
|
-
omitted[key] = priorCount + rows.length -
|
|
13918
|
+
const keep = (count) => {
|
|
13919
|
+
owner[key] = rows.slice(0, count);
|
|
13920
|
+
omitted[key] = priorCount + rows.length - count;
|
|
14552
13921
|
return serializeResult(copy);
|
|
14553
13922
|
};
|
|
14554
13923
|
out = keep(0);
|
|
@@ -14923,7 +14292,7 @@ function buildAeroStateTools(ctx, opts = {}) {
|
|
|
14923
14292
|
}
|
|
14924
14293
|
|
|
14925
14294
|
// src/agent/llm-usage.ts
|
|
14926
|
-
import
|
|
14295
|
+
import crypto21 from "crypto";
|
|
14927
14296
|
var AeroLlmUsageFeatures = {
|
|
14928
14297
|
turn: "aero.turn"
|
|
14929
14298
|
};
|
|
@@ -14942,7 +14311,7 @@ function recordLlmUsageEvent(args) {
|
|
|
14942
14311
|
const usage = args.message.usage;
|
|
14943
14312
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
14944
14313
|
args.db.insert(llmUsageEvents).values({
|
|
14945
|
-
id:
|
|
14314
|
+
id: crypto21.randomUUID(),
|
|
14946
14315
|
projectId: args.projectId,
|
|
14947
14316
|
runId: args.runId,
|
|
14948
14317
|
agentSessionId: args.agentSessionId,
|
|
@@ -15018,7 +14387,7 @@ function splitAeroAnthropicSystemCachePayload(payload, model) {
|
|
|
15018
14387
|
}
|
|
15019
14388
|
|
|
15020
14389
|
// src/agent/tool-usage.ts
|
|
15021
|
-
import
|
|
14390
|
+
import crypto22 from "crypto";
|
|
15022
14391
|
var AeroToolEventStatuses = {
|
|
15023
14392
|
success: "success",
|
|
15024
14393
|
error: "error"
|
|
@@ -15081,7 +14450,7 @@ function createAeroToolUsageHooks(args) {
|
|
|
15081
14450
|
function recordAgentToolEvent(args) {
|
|
15082
14451
|
try {
|
|
15083
14452
|
args.db.insert(agentToolEvents).values({
|
|
15084
|
-
id:
|
|
14453
|
+
id: crypto22.randomUUID(),
|
|
15085
14454
|
projectId: args.projectId,
|
|
15086
14455
|
agentSessionId: args.agentSessionId,
|
|
15087
14456
|
toolCallId: args.toolCallId,
|
|
@@ -15308,8 +14677,8 @@ async function loadExternalMcpTools(servers, opts = {}) {
|
|
|
15308
14677
|
}
|
|
15309
14678
|
|
|
15310
14679
|
// src/agent/memory-store.ts
|
|
15311
|
-
import
|
|
15312
|
-
import { and as
|
|
14680
|
+
import crypto23 from "crypto";
|
|
14681
|
+
import { and as and16, desc as desc7, eq as eq19, like, sql as sql8 } from "drizzle-orm";
|
|
15313
14682
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
15314
14683
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
15315
14684
|
function rowToDto(row) {
|
|
@@ -15323,7 +14692,7 @@ function rowToDto(row) {
|
|
|
15323
14692
|
};
|
|
15324
14693
|
}
|
|
15325
14694
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
15326
|
-
const query = db.select().from(agentMemory).where(
|
|
14695
|
+
const query = db.select().from(agentMemory).where(eq19(agentMemory.projectId, projectId)).orderBy(desc7(agentMemory.updatedAt));
|
|
15327
14696
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
15328
14697
|
return rows.map(rowToDto);
|
|
15329
14698
|
}
|
|
@@ -15337,7 +14706,7 @@ function upsertMemoryEntry(db, args) {
|
|
|
15337
14706
|
throw new Error(`memory key prefix "${COMPACTION_KEY_PREFIX}" is reserved for compaction notes`);
|
|
15338
14707
|
}
|
|
15339
14708
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
15340
|
-
const id =
|
|
14709
|
+
const id = crypto23.randomUUID();
|
|
15341
14710
|
db.insert(agentMemory).values({
|
|
15342
14711
|
id,
|
|
15343
14712
|
projectId: args.projectId,
|
|
@@ -15354,12 +14723,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
15354
14723
|
updatedAt: now
|
|
15355
14724
|
}
|
|
15356
14725
|
}).run();
|
|
15357
|
-
const row = db.select().from(agentMemory).where(
|
|
14726
|
+
const row = db.select().from(agentMemory).where(and16(eq19(agentMemory.projectId, args.projectId), eq19(agentMemory.key, args.key))).get();
|
|
15358
14727
|
if (!row) throw new Error("memory upsert produced no row");
|
|
15359
14728
|
return rowToDto(row);
|
|
15360
14729
|
}
|
|
15361
14730
|
function deleteMemoryEntry(db, projectId, key) {
|
|
15362
|
-
const result = db.delete(agentMemory).where(
|
|
14731
|
+
const result = db.delete(agentMemory).where(and16(eq19(agentMemory.projectId, projectId), eq19(agentMemory.key, key))).run();
|
|
15363
14732
|
const changes = result.changes ?? 0;
|
|
15364
14733
|
return changes > 0;
|
|
15365
14734
|
}
|
|
@@ -15374,7 +14743,7 @@ function writeCompactionNote(db, args) {
|
|
|
15374
14743
|
}
|
|
15375
14744
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
15376
14745
|
const key = `${COMPACTION_KEY_PREFIX}${args.sessionId}:${now}`;
|
|
15377
|
-
const id =
|
|
14746
|
+
const id = crypto23.randomUUID();
|
|
15378
14747
|
let inserted;
|
|
15379
14748
|
db.transaction((tx) => {
|
|
15380
14749
|
tx.insert(agentMemory).values({
|
|
@@ -15388,16 +14757,16 @@ function writeCompactionNote(db, args) {
|
|
|
15388
14757
|
}).run();
|
|
15389
14758
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
15390
14759
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
15391
|
-
|
|
15392
|
-
|
|
14760
|
+
and16(
|
|
14761
|
+
eq19(agentMemory.projectId, args.projectId),
|
|
15393
14762
|
like(agentMemory.key, `${sessionPrefix}%`)
|
|
15394
14763
|
)
|
|
15395
|
-
).orderBy(
|
|
14764
|
+
).orderBy(desc7(agentMemory.updatedAt)).all();
|
|
15396
14765
|
const stale = existing.slice(COMPACTION_NOTES_PER_SESSION).map((r) => r.id);
|
|
15397
14766
|
if (stale.length > 0) {
|
|
15398
|
-
tx.delete(agentMemory).where(
|
|
14767
|
+
tx.delete(agentMemory).where(sql8`${agentMemory.id} IN (${sql8.join(stale.map((s) => sql8`${s}`), sql8`, `)})`).run();
|
|
15399
14768
|
}
|
|
15400
|
-
const row = tx.select().from(agentMemory).where(
|
|
14769
|
+
const row = tx.select().from(agentMemory).where(and16(eq19(agentMemory.projectId, args.projectId), eq19(agentMemory.key, key))).get();
|
|
15401
14770
|
if (row) inserted = rowToDto(row);
|
|
15402
14771
|
});
|
|
15403
14772
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -15614,9 +14983,13 @@ var SessionRegistry = class {
|
|
|
15614
14983
|
if (cached) return cached;
|
|
15615
14984
|
const projectId = this.resolveProjectId(projectName);
|
|
15616
14985
|
const row = this.loadRow(projectId);
|
|
14986
|
+
const systemPrompt = loadAeroSystemPrompt();
|
|
15617
14987
|
if (row) {
|
|
15618
14988
|
const persistedMessages = parseJsonColumn(row.messages, []);
|
|
15619
14989
|
const queued = parseJsonColumn(row.followUpQueue, []);
|
|
14990
|
+
if (row.systemPrompt !== systemPrompt) {
|
|
14991
|
+
this.persistPromptSnapshot(projectId, systemPrompt);
|
|
14992
|
+
}
|
|
15620
14993
|
const effectiveProvider = preferences?.provider ?? row.modelProvider;
|
|
15621
14994
|
const effectiveModelId = preferences?.modelId ?? row.modelId;
|
|
15622
14995
|
if (preferences?.provider || preferences?.modelId) {
|
|
@@ -15624,7 +14997,7 @@ var SessionRegistry = class {
|
|
|
15624
14997
|
modelProvider: effectiveProvider,
|
|
15625
14998
|
modelId: effectiveModelId,
|
|
15626
14999
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
15627
|
-
}).where(
|
|
15000
|
+
}).where(eq20(agentSessions.projectId, projectId)).run();
|
|
15628
15001
|
}
|
|
15629
15002
|
const agent2 = createAeroSession({
|
|
15630
15003
|
projectName,
|
|
@@ -15632,7 +15005,7 @@ var SessionRegistry = class {
|
|
|
15632
15005
|
config: this.opts.config,
|
|
15633
15006
|
provider: effectiveProvider,
|
|
15634
15007
|
modelId: effectiveModelId,
|
|
15635
|
-
systemPromptOverride: this.buildHydratedSystemPrompt(projectId,
|
|
15008
|
+
systemPromptOverride: this.buildHydratedSystemPrompt(projectId, systemPrompt),
|
|
15636
15009
|
initialMessages: persistedMessages,
|
|
15637
15010
|
toolScope: preferences?.toolScope,
|
|
15638
15011
|
toolProfile: preferences?.toolProfile,
|
|
@@ -15652,8 +15025,7 @@ var SessionRegistry = class {
|
|
|
15652
15025
|
return agent2;
|
|
15653
15026
|
}
|
|
15654
15027
|
const { provider, modelId } = resolveSessionProviderAndModel(this.opts.config, preferences);
|
|
15655
|
-
const
|
|
15656
|
-
const sessionId = crypto25.randomUUID();
|
|
15028
|
+
const sessionId = crypto24.randomUUID();
|
|
15657
15029
|
const agent = createAeroSession({
|
|
15658
15030
|
projectName,
|
|
15659
15031
|
client: this.opts.client,
|
|
@@ -15675,7 +15047,7 @@ var SessionRegistry = class {
|
|
|
15675
15047
|
this.insertRow({
|
|
15676
15048
|
id: sessionId,
|
|
15677
15049
|
projectId,
|
|
15678
|
-
// Persist the raw (unhydrated) prompt
|
|
15050
|
+
// Persist the raw (unhydrated) installed prompt snapshot —
|
|
15679
15051
|
// the `<memory>` block is rebuilt from the notes table on every load.
|
|
15680
15052
|
systemPrompt,
|
|
15681
15053
|
modelProvider: provider,
|
|
@@ -15765,6 +15137,13 @@ ${lines.join("\n")}
|
|
|
15765
15137
|
if (agent.state.isStreaming) {
|
|
15766
15138
|
throw agentBusy(projectName);
|
|
15767
15139
|
}
|
|
15140
|
+
const projectId = this.resolveProjectId(projectName);
|
|
15141
|
+
const row = this.loadRow(projectId);
|
|
15142
|
+
const systemPrompt = loadAeroSystemPrompt();
|
|
15143
|
+
if (row && row.systemPrompt !== systemPrompt) {
|
|
15144
|
+
this.persistPromptSnapshot(projectId, systemPrompt);
|
|
15145
|
+
agent.state.systemPrompt = this.buildHydratedSystemPrompt(projectId, systemPrompt);
|
|
15146
|
+
}
|
|
15768
15147
|
this.alignToolSurface(projectName, agent, {
|
|
15769
15148
|
scope: preferences?.toolScope ?? AeroToolScopes.all,
|
|
15770
15149
|
profile: preferences?.toolProfile ?? AeroToolProfiles.default
|
|
@@ -15858,7 +15237,7 @@ ${lines.join("\n")}
|
|
|
15858
15237
|
modelProvider: nextProvider,
|
|
15859
15238
|
modelId: nextModelId,
|
|
15860
15239
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
15861
|
-
}).where(
|
|
15240
|
+
}).where(eq20(agentSessions.projectId, projectId)).run();
|
|
15862
15241
|
}
|
|
15863
15242
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
15864
15243
|
save(projectName) {
|
|
@@ -16022,17 +15401,17 @@ ${lines.join("\n")}
|
|
|
16022
15401
|
return id;
|
|
16023
15402
|
}
|
|
16024
15403
|
tryResolveProjectId(projectName) {
|
|
16025
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
15404
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq20(projects.name, projectName)).get();
|
|
16026
15405
|
return row?.id;
|
|
16027
15406
|
}
|
|
16028
15407
|
loadRow(projectId) {
|
|
16029
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
15408
|
+
const row = this.opts.db.select().from(agentSessions).where(eq20(agentSessions.projectId, projectId)).get();
|
|
16030
15409
|
return row ?? null;
|
|
16031
15410
|
}
|
|
16032
15411
|
insertRow(params) {
|
|
16033
15412
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
16034
15413
|
this.opts.db.insert(agentSessions).values({
|
|
16035
|
-
id: params.id ??
|
|
15414
|
+
id: params.id ?? crypto24.randomUUID(),
|
|
16036
15415
|
projectId: params.projectId,
|
|
16037
15416
|
systemPrompt: params.systemPrompt,
|
|
16038
15417
|
modelProvider: params.provider ?? params.modelProvider ?? AgentProviderIds.claude,
|
|
@@ -16045,20 +15424,29 @@ ${lines.join("\n")}
|
|
|
16045
15424
|
}
|
|
16046
15425
|
updateRow(projectId, patch) {
|
|
16047
15426
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
16048
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
15427
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq20(agentSessions.projectId, projectId)).run();
|
|
15428
|
+
}
|
|
15429
|
+
/**
|
|
15430
|
+
* Store a refreshed prompt snapshot without touching `updatedAt`, which the
|
|
15431
|
+
* transcript API reports as the conversation's last activity. Adopting a new
|
|
15432
|
+
* bundled skill is not activity, so it must not make every session look
|
|
15433
|
+
* recently used right after an upgrade.
|
|
15434
|
+
*/
|
|
15435
|
+
persistPromptSnapshot(projectId, systemPrompt) {
|
|
15436
|
+
this.opts.db.update(agentSessions).set({ systemPrompt }).where(eq20(agentSessions.projectId, projectId)).run();
|
|
16049
15437
|
}
|
|
16050
15438
|
};
|
|
16051
15439
|
|
|
16052
15440
|
// src/mcp-http.ts
|
|
16053
|
-
import
|
|
16054
|
-
import { eq as
|
|
15441
|
+
import crypto25 from "crypto";
|
|
15442
|
+
import { eq as eq21 } from "drizzle-orm";
|
|
16055
15443
|
import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
|
|
16056
15444
|
import { isInitializeRequest } from "@modelcontextprotocol/sdk/types.js";
|
|
16057
15445
|
var SESSION_IDLE_MS = 10 * 60 * 1e3;
|
|
16058
15446
|
var SWEEP_INTERVAL_MS = 60 * 1e3;
|
|
16059
15447
|
function mintSessionKey(db, scopes, userId) {
|
|
16060
|
-
const raw = `cnry_${
|
|
16061
|
-
const id =
|
|
15448
|
+
const raw = `cnry_${crypto25.randomBytes(24).toString("hex")}`;
|
|
15449
|
+
const id = crypto25.randomUUID();
|
|
16062
15450
|
db.insert(apiKeys).values({
|
|
16063
15451
|
id,
|
|
16064
15452
|
name: `mcp-session:${userId}`,
|
|
@@ -16072,7 +15460,7 @@ function mintSessionKey(db, scopes, userId) {
|
|
|
16072
15460
|
}
|
|
16073
15461
|
function revokeSessionKey(db, id) {
|
|
16074
15462
|
try {
|
|
16075
|
-
db.update(apiKeys).set({ revokedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
15463
|
+
db.update(apiKeys).set({ revokedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq21(apiKeys.id, id)).run();
|
|
16076
15464
|
} catch {
|
|
16077
15465
|
}
|
|
16078
15466
|
}
|
|
@@ -16142,7 +15530,7 @@ function registerMcpHttpRoutes(scope, opts) {
|
|
|
16142
15530
|
clientName: `canonry-mcp/${PACKAGE_VERSION}`,
|
|
16143
15531
|
surface: "mcp-http",
|
|
16144
15532
|
// Correlation only: this is never accepted as caller identity or authority.
|
|
16145
|
-
actorSession:
|
|
15533
|
+
actorSession: crypto25.randomUUID()
|
|
16146
15534
|
});
|
|
16147
15535
|
let server;
|
|
16148
15536
|
try {
|
|
@@ -16159,7 +15547,7 @@ function registerMcpHttpRoutes(scope, opts) {
|
|
|
16159
15547
|
throw error;
|
|
16160
15548
|
}
|
|
16161
15549
|
const transport = new StreamableHTTPServerTransport({
|
|
16162
|
-
sessionIdGenerator: () =>
|
|
15550
|
+
sessionIdGenerator: () => crypto25.randomUUID(),
|
|
16163
15551
|
onsessioninitialized: (id) => {
|
|
16164
15552
|
sessions.set(id, { transport, close, authorizationId, segmentId: segment.id, lastSeenAt: now() });
|
|
16165
15553
|
}
|
|
@@ -16255,9 +15643,9 @@ function registerMcpHttpRoutes(scope, opts) {
|
|
|
16255
15643
|
}
|
|
16256
15644
|
|
|
16257
15645
|
// src/agent/agent-routes.ts
|
|
16258
|
-
import { eq as
|
|
15646
|
+
import { eq as eq22 } from "drizzle-orm";
|
|
16259
15647
|
function resolveProject(db, name) {
|
|
16260
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
15648
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq22(projects.name, name)).get();
|
|
16261
15649
|
if (!row) throw notFound("project", name);
|
|
16262
15650
|
return row;
|
|
16263
15651
|
}
|
|
@@ -16266,7 +15654,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
16266
15654
|
"/projects/:name/agent/transcript",
|
|
16267
15655
|
async (request) => {
|
|
16268
15656
|
const project = resolveProject(opts.db, request.params.name);
|
|
16269
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
15657
|
+
const row = opts.db.select().from(agentSessions).where(eq22(agentSessions.projectId, project.id)).get();
|
|
16270
15658
|
if (!row) {
|
|
16271
15659
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
16272
15660
|
}
|
|
@@ -16290,7 +15678,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
16290
15678
|
async (request) => {
|
|
16291
15679
|
const project = resolveProject(opts.db, request.params.name);
|
|
16292
15680
|
opts.sessionRegistry.reset(project.name);
|
|
16293
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
15681
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq22(agentSessions.projectId, project.id)).run();
|
|
16294
15682
|
return { status: "reset" };
|
|
16295
15683
|
}
|
|
16296
15684
|
);
|
|
@@ -17113,7 +16501,7 @@ function buildSnapshotSummary(companyName, queries2, providers, queryResults, au
|
|
|
17113
16501
|
competitorCounts.set(competitor, (competitorCounts.get(competitor) ?? 0) + 1);
|
|
17114
16502
|
}
|
|
17115
16503
|
}
|
|
17116
|
-
const topCompetitors = [...competitorCounts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 10).map(([name,
|
|
16504
|
+
const topCompetitors = [...competitorCounts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 10).map(([name, count]) => ({ name, count }));
|
|
17117
16505
|
const defaultMeaning = totalComparisons > 0 ? `${companyName} was mentioned in ${mentionCount}/${totalComparisons} successful provider-query response${totalComparisons === 1 ? "" : "s"} across ${queries2.length} category queries.` : `No successful provider responses were returned across ${queries2.length} category queries.`;
|
|
17118
16506
|
const failureNote = failedComparisons > 0 ? [
|
|
17119
16507
|
`${failedComparisons} provider response${failedComparisons === 1 ? "" : "s"} failed and ${failedComparisons === 1 ? "was" : "were"} excluded from visibility totals.`
|
|
@@ -17248,20 +16636,20 @@ function clipText(value, length) {
|
|
|
17248
16636
|
}
|
|
17249
16637
|
|
|
17250
16638
|
// src/research-runner.ts
|
|
17251
|
-
import { and as
|
|
16639
|
+
import { and as and17, eq as eq23, inArray as inArray9, sql as sql9 } from "drizzle-orm";
|
|
17252
16640
|
var unfinishedResearchQueryStatuses = [ResearchQueryStatuses.queued, ResearchQueryStatuses.running];
|
|
17253
16641
|
function finalResearchRunStatus(completed, failed) {
|
|
17254
16642
|
if (failed === 0) return ResearchRunStatuses.completed;
|
|
17255
16643
|
return completed > 0 ? ResearchRunStatuses.partial : ResearchRunStatuses.failed;
|
|
17256
16644
|
}
|
|
17257
16645
|
async function executeResearchRun(db, registry, runId, projectId) {
|
|
17258
|
-
const run = db.select().from(researchRuns).where(
|
|
16646
|
+
const run = db.select().from(researchRuns).where(and17(eq23(researchRuns.id, runId), eq23(researchRuns.projectId, projectId))).get();
|
|
17259
16647
|
if (!run || run.status !== ResearchRunStatuses.queued) return;
|
|
17260
|
-
const project = db.select().from(projects).where(
|
|
16648
|
+
const project = db.select().from(projects).where(eq23(projects.id, projectId)).get();
|
|
17261
16649
|
if (!project) return;
|
|
17262
16650
|
const provider = registry.get(run.provider);
|
|
17263
16651
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
17264
|
-
const claim = db.update(researchRuns).set({ status: ResearchRunStatuses.running, startedAt: now }).where(
|
|
16652
|
+
const claim = db.update(researchRuns).set({ status: ResearchRunStatuses.running, startedAt: now }).where(and17(eq23(researchRuns.id, runId), eq23(researchRuns.status, ResearchRunStatuses.queued))).run();
|
|
17265
16653
|
if (claim.changes !== 1) return;
|
|
17266
16654
|
let reserved = 0;
|
|
17267
16655
|
let dispatched = 0;
|
|
@@ -17285,8 +16673,8 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
17285
16673
|
}
|
|
17286
16674
|
reserved = run.totalQueries;
|
|
17287
16675
|
reservation = { scope, period };
|
|
17288
|
-
const rows = db.select().from(researchRunQueries).where(
|
|
17289
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
16676
|
+
const rows = db.select().from(researchRunQueries).where(eq23(researchRunQueries.researchRunId, runId)).orderBy(researchRunQueries.position).all();
|
|
16677
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq23(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
17290
16678
|
const domains = effectiveDomains(project);
|
|
17291
16679
|
const brands = effectiveBrandNames(project);
|
|
17292
16680
|
const config = { ...provider.config, model: run.resolvedModel };
|
|
@@ -17294,7 +16682,7 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
17294
16682
|
await mapWithConcurrency(rows, Math.max(1, provider.config.quotaPolicy.maxConcurrency), async (row) => {
|
|
17295
16683
|
try {
|
|
17296
16684
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
17297
|
-
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.running, startedAt }).where(
|
|
16685
|
+
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.running, startedAt }).where(and17(eq23(researchRunQueries.id, row.id), eq23(researchRunQueries.status, ResearchQueryStatuses.queued))).run();
|
|
17298
16686
|
const raw = await gate.run(async () => {
|
|
17299
16687
|
dispatched++;
|
|
17300
16688
|
return provider.adapter.executeTrackedQuery({ query: row.queryText, canonicalDomains: domains, competitorDomains, ...run.location ? { location: run.location } : {} }, config);
|
|
@@ -17321,7 +16709,7 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
17321
16709
|
citationState: determineCitationState(normalized, domains),
|
|
17322
16710
|
rawResponse: raw.rawResponse,
|
|
17323
16711
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
17324
|
-
}).where(
|
|
16712
|
+
}).where(and17(eq23(researchRunQueries.id, row.id), eq23(researchRunQueries.status, ResearchQueryStatuses.running))).run();
|
|
17325
16713
|
if (completed.changes === 1) incrementResearchProgress(db, runId, "completedQueries");
|
|
17326
16714
|
} catch (error) {
|
|
17327
16715
|
try {
|
|
@@ -17345,18 +16733,18 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
17345
16733
|
}
|
|
17346
16734
|
}
|
|
17347
16735
|
function incrementResearchProgress(db, runId, column) {
|
|
17348
|
-
db.update(researchRuns).set({ [column]:
|
|
16736
|
+
db.update(researchRuns).set({ [column]: sql9`${researchRuns[column]} + 1` }).where(eq23(researchRuns.id, runId)).run();
|
|
17349
16737
|
}
|
|
17350
16738
|
function markResearchQueryFailed(db, runId, queryId, error) {
|
|
17351
16739
|
const message = describeError(error);
|
|
17352
|
-
const failed = db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error: message, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
16740
|
+
const failed = db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error: message, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and17(eq23(researchRunQueries.id, queryId), inArray9(researchRunQueries.status, unfinishedResearchQueryStatuses))).run();
|
|
17353
16741
|
if (failed.changes === 1) incrementResearchProgress(db, runId, "failedQueries");
|
|
17354
16742
|
}
|
|
17355
16743
|
function markUnfinishedResearchQueriesFailed(db, runId, error) {
|
|
17356
|
-
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
16744
|
+
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and17(eq23(researchRunQueries.researchRunId, runId), inArray9(researchRunQueries.status, unfinishedResearchQueryStatuses))).run();
|
|
17357
16745
|
}
|
|
17358
16746
|
function finalizeResearchRun(db, runId, fatalError) {
|
|
17359
|
-
const rows = db.select({ status: researchRunQueries.status }).from(researchRunQueries).where(
|
|
16747
|
+
const rows = db.select({ status: researchRunQueries.status }).from(researchRunQueries).where(eq23(researchRunQueries.researchRunId, runId)).all();
|
|
17360
16748
|
const completed = rows.filter((row) => row.status === ResearchQueryStatuses.completed).length;
|
|
17361
16749
|
const failed = rows.filter((row) => row.status === ResearchQueryStatuses.failed).length;
|
|
17362
16750
|
db.update(researchRuns).set({
|
|
@@ -17365,11 +16753,11 @@ function finalizeResearchRun(db, runId, fatalError) {
|
|
|
17365
16753
|
failedQueries: failed,
|
|
17366
16754
|
error: fatalError ?? (failed === rows.length ? "Every research query failed." : null),
|
|
17367
16755
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
17368
|
-
}).where(
|
|
16756
|
+
}).where(eq23(researchRuns.id, runId)).run();
|
|
17369
16757
|
}
|
|
17370
16758
|
|
|
17371
16759
|
// src/google-marketing-sync.ts
|
|
17372
|
-
import { and as
|
|
16760
|
+
import { and as and18, eq as eq24, isNull as isNull2 } from "drizzle-orm";
|
|
17373
16761
|
function safeErrorMessage(error) {
|
|
17374
16762
|
const raw = describeError(error);
|
|
17375
16763
|
return raw.replace(/(["']?authorization["']?\s*[:=]\s*["']?bearer\s+)[^\s,"'}]+/gi, "$1[redacted]").replace(/(["']?(?:developer|access|refresh|client)[_-]?(?:token|secret)["']?\s*[:=]\s*["']?)[^\s,"'}]+/gi, "$1[redacted]").replace(/[\r\n\t]+/g, " ").slice(0, 1e3);
|
|
@@ -17392,10 +16780,10 @@ function assertSnapshotContext(metadata, expected) {
|
|
|
17392
16780
|
}
|
|
17393
16781
|
}
|
|
17394
16782
|
function requireSyncRun(db, runId, projectId, kind) {
|
|
17395
|
-
const run = db.select().from(runs).where(
|
|
17396
|
-
|
|
17397
|
-
|
|
17398
|
-
|
|
16783
|
+
const run = db.select().from(runs).where(and18(
|
|
16784
|
+
eq24(runs.id, runId),
|
|
16785
|
+
eq24(runs.projectId, projectId),
|
|
16786
|
+
eq24(runs.kind, kind)
|
|
17399
16787
|
)).get();
|
|
17400
16788
|
if (!run) throw new Error(`Run ${runId} is not a ${kind} run for this project.`);
|
|
17401
16789
|
if (run.status !== RunStatuses.queued) {
|
|
@@ -17408,19 +16796,19 @@ function markFailed(db, runId, projectId, kind, expectedStatus, error) {
|
|
|
17408
16796
|
status: RunStatuses.failed,
|
|
17409
16797
|
error: serializeRunError({ message: safeErrorMessage(error) }),
|
|
17410
16798
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
17411
|
-
}).where(
|
|
17412
|
-
|
|
17413
|
-
|
|
17414
|
-
|
|
17415
|
-
|
|
16799
|
+
}).where(and18(
|
|
16800
|
+
eq24(runs.id, runId),
|
|
16801
|
+
eq24(runs.projectId, projectId),
|
|
16802
|
+
eq24(runs.kind, kind),
|
|
16803
|
+
eq24(runs.status, expectedStatus)
|
|
17416
16804
|
)).run();
|
|
17417
16805
|
}
|
|
17418
16806
|
function claimSyncRun(db, runId, projectId, kind, startedAt) {
|
|
17419
|
-
const claim = db.update(runs).set({ status: RunStatuses.running, startedAt, error: null }).where(
|
|
17420
|
-
|
|
17421
|
-
|
|
17422
|
-
|
|
17423
|
-
|
|
16807
|
+
const claim = db.update(runs).set({ status: RunStatuses.running, startedAt, error: null }).where(and18(
|
|
16808
|
+
eq24(runs.id, runId),
|
|
16809
|
+
eq24(runs.projectId, projectId),
|
|
16810
|
+
eq24(runs.kind, kind),
|
|
16811
|
+
eq24(runs.status, RunStatuses.queued)
|
|
17424
16812
|
)).run();
|
|
17425
16813
|
return claim.changes === 1;
|
|
17426
16814
|
}
|
|
@@ -17428,9 +16816,9 @@ async function executeGoogleAdsMarketingSync(db, runtime, runId, projectId) {
|
|
|
17428
16816
|
let claimed = false;
|
|
17429
16817
|
try {
|
|
17430
16818
|
requireSyncRun(db, runId, projectId, RunKinds["google-ads-sync"]);
|
|
17431
|
-
const project = db.select().from(projects).where(
|
|
16819
|
+
const project = db.select().from(projects).where(eq24(projects.id, projectId)).get();
|
|
17432
16820
|
if (!project) throw new Error(`Project ${projectId} was not found.`);
|
|
17433
|
-
const connection = db.select().from(googleAdsConnections).where(
|
|
16821
|
+
const connection = db.select().from(googleAdsConnections).where(eq24(googleAdsConnections.projectId, projectId)).get();
|
|
17434
16822
|
if (!connection?.selectedCustomerId) {
|
|
17435
16823
|
throw new Error("Google Ads needs a selected customer before it can sync.");
|
|
17436
16824
|
}
|
|
@@ -17481,11 +16869,11 @@ async function executeGoogleAdsMarketingSync(db, runtime, runId, projectId) {
|
|
|
17481
16869
|
status: RunStatuses.completed,
|
|
17482
16870
|
finishedAt,
|
|
17483
16871
|
error: null
|
|
17484
|
-
}).where(
|
|
17485
|
-
|
|
17486
|
-
|
|
17487
|
-
|
|
17488
|
-
|
|
16872
|
+
}).where(and18(
|
|
16873
|
+
eq24(runs.id, runId),
|
|
16874
|
+
eq24(runs.projectId, projectId),
|
|
16875
|
+
eq24(runs.kind, RunKinds["google-ads-sync"]),
|
|
16876
|
+
eq24(runs.status, RunStatuses.running)
|
|
17489
16877
|
)).run();
|
|
17490
16878
|
if (terminalClaim.changes !== 1) {
|
|
17491
16879
|
throw new Error(`Run ${runId} was no longer running when Google Ads evidence was ready.`);
|
|
@@ -17502,12 +16890,12 @@ async function executeGoogleAdsMarketingSync(db, runtime, runId, projectId) {
|
|
|
17502
16890
|
lastMetricsSnapshotAt: result.metrics?.metadata.capturedAt ?? connection.lastMetricsSnapshotAt,
|
|
17503
16891
|
lastMetricsSnapshotId: result.metrics?.metadata.id ?? connection.lastMetricsSnapshotId,
|
|
17504
16892
|
updatedAt: finishedAt
|
|
17505
|
-
}).where(
|
|
17506
|
-
|
|
17507
|
-
|
|
17508
|
-
|
|
17509
|
-
connection.selectedLoginCustomerId === null ?
|
|
17510
|
-
|
|
16893
|
+
}).where(and18(
|
|
16894
|
+
eq24(googleAdsConnections.id, connection.id),
|
|
16895
|
+
eq24(googleAdsConnections.projectId, projectId),
|
|
16896
|
+
eq24(googleAdsConnections.selectedCustomerId, connection.selectedCustomerId),
|
|
16897
|
+
connection.selectedLoginCustomerId === null ? isNull2(googleAdsConnections.selectedLoginCustomerId) : eq24(googleAdsConnections.selectedLoginCustomerId, connection.selectedLoginCustomerId),
|
|
16898
|
+
eq24(googleAdsConnections.selectionGeneration, connection.selectionGeneration)
|
|
17511
16899
|
)).run();
|
|
17512
16900
|
if (connectionClaim.changes !== 1) {
|
|
17513
16901
|
throw new Error("Google Ads selection changed while its evidence was being collected.");
|
|
@@ -17534,9 +16922,9 @@ async function executeGtmMarketingSync(db, runtime, runId, projectId) {
|
|
|
17534
16922
|
let claimed = false;
|
|
17535
16923
|
try {
|
|
17536
16924
|
requireSyncRun(db, runId, projectId, RunKinds["gtm-sync"]);
|
|
17537
|
-
const project = db.select().from(projects).where(
|
|
16925
|
+
const project = db.select().from(projects).where(eq24(projects.id, projectId)).get();
|
|
17538
16926
|
if (!project) throw new Error(`Project ${projectId} was not found.`);
|
|
17539
|
-
const connection = db.select().from(gtmConnections).where(
|
|
16927
|
+
const connection = db.select().from(gtmConnections).where(eq24(gtmConnections.projectId, projectId)).get();
|
|
17540
16928
|
if (!connection?.selectedAccountId || !connection.selectedContainerId) {
|
|
17541
16929
|
throw new Error("GTM needs a selected account and container before it can sync.");
|
|
17542
16930
|
}
|
|
@@ -17571,11 +16959,11 @@ async function executeGtmMarketingSync(db, runtime, runId, projectId) {
|
|
|
17571
16959
|
status: RunStatuses.completed,
|
|
17572
16960
|
finishedAt,
|
|
17573
16961
|
error: null
|
|
17574
|
-
}).where(
|
|
17575
|
-
|
|
17576
|
-
|
|
17577
|
-
|
|
17578
|
-
|
|
16962
|
+
}).where(and18(
|
|
16963
|
+
eq24(runs.id, runId),
|
|
16964
|
+
eq24(runs.projectId, projectId),
|
|
16965
|
+
eq24(runs.kind, RunKinds["gtm-sync"]),
|
|
16966
|
+
eq24(runs.status, RunStatuses.running)
|
|
17579
16967
|
)).run();
|
|
17580
16968
|
if (terminalClaim.changes !== 1) {
|
|
17581
16969
|
throw new Error(`Run ${runId} was no longer running when GTM evidence was ready.`);
|
|
@@ -17589,13 +16977,13 @@ async function executeGtmMarketingSync(db, runtime, runId, projectId) {
|
|
|
17589
16977
|
lastSnapshotAt: snapshot.metadata.capturedAt,
|
|
17590
16978
|
lastSnapshotId: snapshot.metadata.id,
|
|
17591
16979
|
updatedAt: finishedAt
|
|
17592
|
-
}).where(
|
|
17593
|
-
|
|
17594
|
-
|
|
17595
|
-
|
|
17596
|
-
|
|
17597
|
-
connection.selectedWorkspaceId === null ?
|
|
17598
|
-
|
|
16980
|
+
}).where(and18(
|
|
16981
|
+
eq24(gtmConnections.id, connection.id),
|
|
16982
|
+
eq24(gtmConnections.projectId, projectId),
|
|
16983
|
+
eq24(gtmConnections.selectedAccountId, connection.selectedAccountId),
|
|
16984
|
+
eq24(gtmConnections.selectedContainerId, connection.selectedContainerId),
|
|
16985
|
+
connection.selectedWorkspaceId === null ? isNull2(gtmConnections.selectedWorkspaceId) : eq24(gtmConnections.selectedWorkspaceId, connection.selectedWorkspaceId),
|
|
16986
|
+
eq24(gtmConnections.selectionGeneration, connection.selectionGeneration)
|
|
17599
16987
|
)).run();
|
|
17600
16988
|
if (connectionClaim.changes !== 1) {
|
|
17601
16989
|
throw new Error("GTM selection changed while its evidence was being collected.");
|
|
@@ -17616,7 +17004,7 @@ async function executeGtmMarketingSync(db, runtime, runId, projectId) {
|
|
|
17616
17004
|
}
|
|
17617
17005
|
|
|
17618
17006
|
// src/server.ts
|
|
17619
|
-
var _require3 =
|
|
17007
|
+
var _require3 = createRequire3(import.meta.url);
|
|
17620
17008
|
var { version: PKG_VERSION2 } = _require3("../package.json");
|
|
17621
17009
|
var log21 = createLogger("Server");
|
|
17622
17010
|
var runtimeStartupByServer = /* @__PURE__ */ new WeakMap();
|
|
@@ -17716,14 +17104,14 @@ function summarizeProviderConfig(config) {
|
|
|
17716
17104
|
};
|
|
17717
17105
|
}
|
|
17718
17106
|
function hashApiKey2(key) {
|
|
17719
|
-
return
|
|
17107
|
+
return crypto26.createHash("sha256").update(key).digest("hex");
|
|
17720
17108
|
}
|
|
17721
17109
|
var DASHBOARD_SCRYPT_KEYLEN = 64;
|
|
17722
17110
|
var DASHBOARD_SCRYPT_COST = 1 << 15;
|
|
17723
17111
|
var DASHBOARD_SCRYPT_MAXMEM = 64 * 1024 * 1024;
|
|
17724
17112
|
function hashDashboardPassword(password) {
|
|
17725
|
-
const salt =
|
|
17726
|
-
const derived =
|
|
17113
|
+
const salt = crypto26.randomBytes(16);
|
|
17114
|
+
const derived = crypto26.scryptSync(password, salt, DASHBOARD_SCRYPT_KEYLEN, {
|
|
17727
17115
|
N: DASHBOARD_SCRYPT_COST,
|
|
17728
17116
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
17729
17117
|
});
|
|
@@ -17744,14 +17132,14 @@ function verifyDashboardPassword(password, storedHash) {
|
|
|
17744
17132
|
} catch {
|
|
17745
17133
|
return { ok: false, needsRehash: false };
|
|
17746
17134
|
}
|
|
17747
|
-
const derived =
|
|
17135
|
+
const derived = crypto26.scryptSync(password, salt, expected.length, {
|
|
17748
17136
|
N: DASHBOARD_SCRYPT_COST,
|
|
17749
17137
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
17750
17138
|
});
|
|
17751
17139
|
if (derived.length !== expected.length)
|
|
17752
17140
|
return { ok: false, needsRehash: false };
|
|
17753
17141
|
return {
|
|
17754
|
-
ok:
|
|
17142
|
+
ok: crypto26.timingSafeEqual(derived, expected),
|
|
17755
17143
|
needsRehash: false
|
|
17756
17144
|
};
|
|
17757
17145
|
}
|
|
@@ -17760,7 +17148,7 @@ function verifyDashboardPassword(password, storedHash) {
|
|
|
17760
17148
|
const expected = Buffer.from(storedHash, "hex");
|
|
17761
17149
|
if (candidate.length !== expected.length)
|
|
17762
17150
|
return { ok: false, needsRehash: false };
|
|
17763
|
-
const ok =
|
|
17151
|
+
const ok = crypto26.timingSafeEqual(candidate, expected);
|
|
17764
17152
|
return { ok, needsRehash: ok };
|
|
17765
17153
|
}
|
|
17766
17154
|
return { ok: false, needsRehash: false };
|
|
@@ -17896,7 +17284,7 @@ function cloneGtmConfig(config) {
|
|
|
17896
17284
|
function createGoogleMarketingConfigCredentialStore(input) {
|
|
17897
17285
|
const persistConfigPatch = input.saveConfigPatch ?? saveConfigPatch;
|
|
17898
17286
|
const env = input.env ?? process.env;
|
|
17899
|
-
const randomUUID2 = input.randomUUID ??
|
|
17287
|
+
const randomUUID2 = input.randomUUID ?? crypto26.randomUUID;
|
|
17900
17288
|
const snapshot = () => ({
|
|
17901
17289
|
googleAds: cloneGoogleAdsConfig(input.config.googleAds),
|
|
17902
17290
|
gtm: cloneGtmConfig(input.config.gtm)
|
|
@@ -17999,7 +17387,7 @@ async function createServer(opts) {
|
|
|
17999
17387
|
const trustProxy = resolveTrustProxy(process.env.CANONRY_TRUST_PROXY);
|
|
18000
17388
|
const app = Fastify({
|
|
18001
17389
|
loggerInstance: logger,
|
|
18002
|
-
genReqId: () =>
|
|
17390
|
+
genReqId: () => crypto26.randomUUID(),
|
|
18003
17391
|
trustProxy
|
|
18004
17392
|
});
|
|
18005
17393
|
const registry = new ProviderRegistry();
|
|
@@ -18088,7 +17476,7 @@ async function createServer(opts) {
|
|
|
18088
17476
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
18089
17477
|
async (ctx) => {
|
|
18090
17478
|
if (!sessionRegistry) return;
|
|
18091
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
17479
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq25(projects.id, ctx.projectId)).get();
|
|
18092
17480
|
if (!project) return;
|
|
18093
17481
|
let content;
|
|
18094
17482
|
if (ctx.kind === RunKinds["aeo-discover-probe"]) {
|
|
@@ -18534,7 +17922,7 @@ async function createServer(opts) {
|
|
|
18534
17922
|
void (async () => {
|
|
18535
17923
|
try {
|
|
18536
17924
|
const report = await schedulerClient.runDoctor({ project: projectName });
|
|
18537
|
-
const project = opts.db.select().from(projects).where(
|
|
17925
|
+
const project = opts.db.select().from(projects).where(eq25(projects.name, projectName)).get();
|
|
18538
17926
|
if (!project) {
|
|
18539
17927
|
app.log.warn({ projectName }, "doctor schedule fired for an unknown project");
|
|
18540
17928
|
return;
|
|
@@ -18562,9 +17950,9 @@ async function createServer(opts) {
|
|
|
18562
17950
|
});
|
|
18563
17951
|
if (!probed) return;
|
|
18564
17952
|
const alreadySynced = opts.db.select().from(ccReleaseSyncs).where(
|
|
18565
|
-
|
|
18566
|
-
|
|
18567
|
-
|
|
17953
|
+
and19(
|
|
17954
|
+
eq25(ccReleaseSyncs.release, probed.release),
|
|
17955
|
+
eq25(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)
|
|
18568
17956
|
)
|
|
18569
17957
|
).limit(1).get();
|
|
18570
17958
|
if (alreadySynced) {
|
|
@@ -18745,7 +18133,7 @@ async function createServer(opts) {
|
|
|
18745
18133
|
return removed;
|
|
18746
18134
|
}
|
|
18747
18135
|
};
|
|
18748
|
-
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ??
|
|
18136
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto26.randomBytes(32).toString("hex");
|
|
18749
18137
|
const googleConnectionStore = {
|
|
18750
18138
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
18751
18139
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -18835,11 +18223,11 @@ async function createServer(opts) {
|
|
|
18835
18223
|
})();
|
|
18836
18224
|
if (opts.config.apiKey) {
|
|
18837
18225
|
const keyHash = hashApiKey2(opts.config.apiKey);
|
|
18838
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
18226
|
+
const existing = opts.db.select().from(apiKeys).where(eq25(apiKeys.keyHash, keyHash)).get();
|
|
18839
18227
|
if (!existing) {
|
|
18840
18228
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
18841
18229
|
opts.db.insert(apiKeys).values({
|
|
18842
|
-
id: `key_${
|
|
18230
|
+
id: `key_${crypto26.randomBytes(8).toString("hex")}`,
|
|
18843
18231
|
name: "default",
|
|
18844
18232
|
keyHash,
|
|
18845
18233
|
keyPrefix: prefix,
|
|
@@ -18863,7 +18251,7 @@ async function createServer(opts) {
|
|
|
18863
18251
|
};
|
|
18864
18252
|
const createSession = (apiKeyId) => {
|
|
18865
18253
|
pruneExpiredSessions();
|
|
18866
|
-
const sessionId =
|
|
18254
|
+
const sessionId = crypto26.randomBytes(32).toString("hex");
|
|
18867
18255
|
sessions.set(sessionId, {
|
|
18868
18256
|
apiKeyId,
|
|
18869
18257
|
expiresAt: Date.now() + SESSION_TTL_MS
|
|
@@ -18887,7 +18275,7 @@ async function createServer(opts) {
|
|
|
18887
18275
|
};
|
|
18888
18276
|
const getDefaultApiKey = () => {
|
|
18889
18277
|
if (!opts.config.apiKey) return void 0;
|
|
18890
|
-
return opts.db.select().from(apiKeys).where(
|
|
18278
|
+
return opts.db.select().from(apiKeys).where(eq25(apiKeys.keyHash, hashApiKey2(opts.config.apiKey))).get();
|
|
18891
18279
|
};
|
|
18892
18280
|
const createPasswordSession = (reply) => {
|
|
18893
18281
|
const key = getDefaultApiKey();
|
|
@@ -18911,7 +18299,7 @@ async function createServer(opts) {
|
|
|
18911
18299
|
if (!header) return false;
|
|
18912
18300
|
const parts = header.split(" ");
|
|
18913
18301
|
if (parts.length !== 2 || parts[0] !== "Bearer") return false;
|
|
18914
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
18302
|
+
const key = opts.db.select().from(apiKeys).where(eq25(apiKeys.keyHash, hashApiKey2(parts[1]))).get();
|
|
18915
18303
|
return Boolean(key && !key.revokedAt);
|
|
18916
18304
|
};
|
|
18917
18305
|
const namedAccountsInUse = () => anyUsersExist(opts.db);
|
|
@@ -19001,12 +18389,12 @@ async function createServer(opts) {
|
|
|
19001
18389
|
return reply.send({ authenticated: true });
|
|
19002
18390
|
}
|
|
19003
18391
|
if (apiKey) {
|
|
19004
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
18392
|
+
const key = opts.db.select().from(apiKeys).where(eq25(apiKeys.keyHash, hashApiKey2(apiKey))).get();
|
|
19005
18393
|
if (!key || key.revokedAt) {
|
|
19006
18394
|
const err2 = authInvalid();
|
|
19007
18395
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
19008
18396
|
}
|
|
19009
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
18397
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq25(apiKeys.id, key.id)).run();
|
|
19010
18398
|
const sessionId = createSession(key.id);
|
|
19011
18399
|
reply.header(
|
|
19012
18400
|
"set-cookie",
|
|
@@ -19168,8 +18556,8 @@ async function createServer(opts) {
|
|
|
19168
18556
|
getAgentProviderSummary: () => buildAgentProvidersResponse(opts.config).providers,
|
|
19169
18557
|
getGoogleMarketingDoctorInput: (ctx) => {
|
|
19170
18558
|
if (!ctx.project) return null;
|
|
19171
|
-
const googleAds = opts.db.select().from(googleAdsConnections).where(
|
|
19172
|
-
const gtm = opts.db.select().from(gtmConnections).where(
|
|
18559
|
+
const googleAds = opts.db.select().from(googleAdsConnections).where(eq25(googleAdsConnections.projectId, ctx.project.id)).get();
|
|
18560
|
+
const gtm = opts.db.select().from(gtmConnections).where(eq25(gtmConnections.projectId, ctx.project.id)).get();
|
|
19173
18561
|
const googleAdsCredential = getGoogleAdsConnection(opts.config, ctx.project.id);
|
|
19174
18562
|
const gtmCredential = getGtmConnection(opts.config, ctx.project.id);
|
|
19175
18563
|
const googleAdsConnected = Boolean(googleAdsCredential?.accessToken);
|
|
@@ -19293,7 +18681,7 @@ async function createServer(opts) {
|
|
|
19293
18681
|
deps: {
|
|
19294
18682
|
enqueueAutoExtract: ({ projectId, release: r }) => {
|
|
19295
18683
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
19296
|
-
const runId =
|
|
18684
|
+
const runId = crypto26.randomUUID();
|
|
19297
18685
|
opts.db.insert(runs).values({
|
|
19298
18686
|
id: runId,
|
|
19299
18687
|
projectId,
|
|
@@ -19420,7 +18808,7 @@ async function createServer(opts) {
|
|
|
19420
18808
|
...inspectOpts,
|
|
19421
18809
|
config: opts.config
|
|
19422
18810
|
}).then(() => {
|
|
19423
|
-
const finished = opts.db.select({ status: runs.status }).from(runs).where(
|
|
18811
|
+
const finished = opts.db.select({ status: runs.status }).from(runs).where(eq25(runs.id, runId)).get();
|
|
19424
18812
|
if (finished?.status === RunStatuses.completed || finished?.status === RunStatuses.partial) {
|
|
19425
18813
|
return maybeRefreshGscCoverage(
|
|
19426
18814
|
opts.db,
|
|
@@ -19530,7 +18918,7 @@ async function createServer(opts) {
|
|
|
19530
18918
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
19531
18919
|
opts.db.insert(auditLog).values(
|
|
19532
18920
|
targetProjectIds.map((projectId) => ({
|
|
19533
|
-
id:
|
|
18921
|
+
id: crypto26.randomUUID(),
|
|
19534
18922
|
projectId,
|
|
19535
18923
|
actor: auditContext?.actor ?? "api",
|
|
19536
18924
|
userAgent: auditContext?.userAgent ?? null,
|
|
@@ -19678,7 +19066,7 @@ async function createServer(opts) {
|
|
|
19678
19066
|
}
|
|
19679
19067
|
];
|
|
19680
19068
|
},
|
|
19681
|
-
onGenerateQueries: async (providerName,
|
|
19069
|
+
onGenerateQueries: async (providerName, count, project) => {
|
|
19682
19070
|
const provider = registry.get(providerName);
|
|
19683
19071
|
if (!provider)
|
|
19684
19072
|
throw new Error(`Provider "${providerName}" is not configured`);
|
|
@@ -19690,10 +19078,10 @@ async function createServer(opts) {
|
|
|
19690
19078
|
language: project.language,
|
|
19691
19079
|
existingQueries: project.existingQueries,
|
|
19692
19080
|
siteText,
|
|
19693
|
-
count
|
|
19081
|
+
count
|
|
19694
19082
|
});
|
|
19695
19083
|
const raw = await provider.adapter.generateText(prompt, provider.config);
|
|
19696
|
-
return parseQueryResponse(raw,
|
|
19084
|
+
return parseQueryResponse(raw, count);
|
|
19697
19085
|
},
|
|
19698
19086
|
onSnapshotRequested: async (input) => {
|
|
19699
19087
|
return snapshotService.createReport(input);
|
|
@@ -19873,7 +19261,7 @@ async function createServer(opts) {
|
|
|
19873
19261
|
if (runtimeStartupSettled) return;
|
|
19874
19262
|
try {
|
|
19875
19263
|
scheduler.start();
|
|
19876
|
-
for (const run of opts.db.select({ id: researchRuns.id, projectId: researchRuns.projectId }).from(researchRuns).where(
|
|
19264
|
+
for (const run of opts.db.select({ id: researchRuns.id, projectId: researchRuns.projectId }).from(researchRuns).where(eq25(researchRuns.status, ResearchRunStatuses.queued)).all()) {
|
|
19877
19265
|
dispatchResearchRun(run.id, run.projectId);
|
|
19878
19266
|
}
|
|
19879
19267
|
runtimeStartupSettled = true;
|
|
@@ -19926,7 +19314,7 @@ function buildQueryGenerationPrompt(ctx) {
|
|
|
19926
19314
|
);
|
|
19927
19315
|
return lines.join("\n");
|
|
19928
19316
|
}
|
|
19929
|
-
function parseQueryResponse(raw,
|
|
19317
|
+
function parseQueryResponse(raw, count) {
|
|
19930
19318
|
const seen = /* @__PURE__ */ new Set();
|
|
19931
19319
|
const results = [];
|
|
19932
19320
|
for (const line of raw.split("\n")) {
|
|
@@ -19942,7 +19330,7 @@ function parseQueryResponse(raw, count2) {
|
|
|
19942
19330
|
if (seen.has(key)) continue;
|
|
19943
19331
|
seen.add(key);
|
|
19944
19332
|
results.push(cleaned);
|
|
19945
|
-
if (results.length >=
|
|
19333
|
+
if (results.length >= count) break;
|
|
19946
19334
|
}
|
|
19947
19335
|
return results;
|
|
19948
19336
|
}
|