@ainyc/canonry 5.1.4 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/assets/{AuditHistoryPanel-Dc_pymFg.js → AuditHistoryPanel-D0H5TtxC.js} +1 -1
- package/assets/assets/{BacklinksPage-BkSo48YX.js → BacklinksPage-DRihwG5O.js} +1 -1
- package/assets/assets/{HistoryPage-BbswsdU7.js → HistoryPage-BQ2nEXly.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-BbgtalkM.js → MeasurementPropertyPage-Bp6EURzJ.js} +1 -1
- package/assets/assets/ProjectPage-Cd03Phwx.js +8 -0
- package/assets/assets/{RunRow-2cNW8FHz.js → RunRow-Bo_Adiy0.js} +1 -1
- package/assets/assets/{RunsPage-Cz3lJ6VJ.js → RunsPage-CDkbzjuo.js} +1 -1
- package/assets/assets/{SettingsPage-BQwB0r9k.js → SettingsPage-CF4i_EzR.js} +1 -1
- package/assets/assets/SiteHealthSection-BrzBV7dt.js +4 -0
- package/assets/assets/{TrafficPage-DTic3zhO.js → TrafficPage-BHSz1N4P.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-BKfabnCw.js → TrafficSourceDetailPage-DzHLwhpy.js} +1 -1
- package/assets/assets/{extract-error-message-BzB8j3fC.js → extract-error-message-WHM24Sai.js} +1 -1
- package/assets/assets/{index-DQGEzxrN.js → index-5W4ZZnwg.js} +22 -22
- package/assets/assets/index-DCLmKL1s.css +1 -0
- package/assets/assets/{react-sigma_core.esm.min-rYv0aKKx.js → react-sigma_core.esm.min-BB2Lcq4W.js} +1 -1
- package/assets/assets/{v2-overview-adapter-CWWPfsm-.js → v2-overview-adapter-ebrNjcUG.js} +1 -1
- package/assets/assets/vendor-lucide-D_WBzb76.js +1 -0
- package/assets/index.html +3 -3
- package/dist/{chunk-MMSPU72Z.js → chunk-2U5IW3G5.js} +4 -0
- package/dist/chunk-5VONCLVQ.js +652 -0
- package/dist/chunk-AOJU5LKX.js +8 -0
- package/dist/{chunk-KJM4DRZN.js → chunk-C7PITYL3.js} +5 -3
- package/dist/{chunk-3KXI4C3S.js → chunk-CW2KPN3L.js} +310 -941
- package/dist/{chunk-AEE5DGTE.js → chunk-IOP36L3C.js} +4 -7
- package/dist/chunk-RB2F4D3Z.js +67 -0
- package/dist/{chunk-EBUX3C5H.js → chunk-YQNKULAD.js} +30 -10
- package/dist/cli.js +100 -5
- package/dist/demo-server-BIPT53EY.js +2125 -0
- package/dist/index.js +6 -4
- package/dist/intelligence-service-SJO5E7WN.js +9 -0
- package/dist/mcp.js +4 -3
- package/package.json +13 -13
- package/assets/assets/ProjectPage-CAcRQp-T.js +0 -8
- package/assets/assets/SiteHealthSection-C0aJ_UhN.js +0 -4
- package/assets/assets/index-CjEprBri.css +0 -1
- package/assets/assets/vendor-lucide-CW2qe0uH.js +0 -1
- package/dist/intelligence-service-25NQCETC.js +0 -7
|
@@ -4,7 +4,6 @@ import {
|
|
|
4
4
|
CANONRY_MCP_TOOLKIT_NAMES,
|
|
5
5
|
CanonryMcpToolNames,
|
|
6
6
|
CliError,
|
|
7
|
-
PACKAGE_VERSION,
|
|
8
7
|
canonryMcpTools,
|
|
9
8
|
configExists,
|
|
10
9
|
createCanonryMcpServer,
|
|
@@ -17,7 +16,15 @@ import {
|
|
|
17
16
|
resolveOperatorApiKeyIds,
|
|
18
17
|
runWithUsageTags,
|
|
19
18
|
saveConfigPatch
|
|
20
|
-
} from "./chunk-
|
|
19
|
+
} from "./chunk-IOP36L3C.js";
|
|
20
|
+
import {
|
|
21
|
+
deleteSiteCrawlGraphLayout,
|
|
22
|
+
persistSiteCrawlGraphLayout,
|
|
23
|
+
prepareSiteCrawlGraphLayout
|
|
24
|
+
} from "./chunk-5VONCLVQ.js";
|
|
25
|
+
import {
|
|
26
|
+
PACKAGE_VERSION
|
|
27
|
+
} from "./chunk-AOJU5LKX.js";
|
|
21
28
|
import {
|
|
22
29
|
CC_CACHE_DIR,
|
|
23
30
|
DUCKDB_SPEC,
|
|
@@ -160,15 +167,12 @@ import {
|
|
|
160
167
|
siteCrawlEdges,
|
|
161
168
|
siteCrawlEventReceipts,
|
|
162
169
|
siteCrawlFindings,
|
|
163
|
-
siteCrawlGraphEdges,
|
|
164
|
-
siteCrawlGraphLayouts,
|
|
165
|
-
siteCrawlGraphNodes,
|
|
166
170
|
siteCrawlPages,
|
|
167
171
|
siteCrawlRunRequests,
|
|
168
172
|
siteCrawlSnapshots,
|
|
169
173
|
toAlertView,
|
|
170
174
|
usageCounters
|
|
171
|
-
} from "./chunk-
|
|
175
|
+
} from "./chunk-YQNKULAD.js";
|
|
172
176
|
import {
|
|
173
177
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
174
178
|
AGENT_NONE,
|
|
@@ -208,12 +212,9 @@ import {
|
|
|
208
212
|
SITE_AUDIT_DEFAULT_PAGE_LIMIT,
|
|
209
213
|
SITE_AUDIT_MAX_EDGE_LIMIT,
|
|
210
214
|
SITE_AUDIT_MAX_PAGE_LIMIT,
|
|
211
|
-
SITE_CRAWL_GRAPH_MAX_EDGES,
|
|
212
|
-
SITE_CRAWL_GRAPH_MAX_NODES,
|
|
213
215
|
SchedulableRunKinds,
|
|
214
216
|
SiteHealthLinkClassificationSources,
|
|
215
217
|
SnapshotProviderModes,
|
|
216
|
-
TEMPLATE_LINK_RATIO_THRESHOLD,
|
|
217
218
|
TrafficEventConfidences,
|
|
218
219
|
TrafficEventKinds,
|
|
219
220
|
TrafficEvidenceKinds,
|
|
@@ -324,7 +325,7 @@ import {
|
|
|
324
325
|
validationError,
|
|
325
326
|
winnabilityClassLabel,
|
|
326
327
|
withRetry
|
|
327
|
-
} from "./chunk-
|
|
328
|
+
} from "./chunk-2U5IW3G5.js";
|
|
328
329
|
|
|
329
330
|
// src/runtime-context.ts
|
|
330
331
|
function cliRuntimeContext(env = process.env, stdio = process) {
|
|
@@ -3531,12 +3532,12 @@ function createGoogleMarketingRuntime(options) {
|
|
|
3531
3532
|
}
|
|
3532
3533
|
|
|
3533
3534
|
// src/server.ts
|
|
3534
|
-
import { createRequire as
|
|
3535
|
-
import
|
|
3535
|
+
import { createRequire as createRequire3 } from "module";
|
|
3536
|
+
import crypto26 from "crypto";
|
|
3536
3537
|
import fs8 from "fs";
|
|
3537
3538
|
import path8 from "path";
|
|
3538
3539
|
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
3539
|
-
import { and as
|
|
3540
|
+
import { and as and19, eq as eq25 } from "drizzle-orm";
|
|
3540
3541
|
import Fastify from "fastify";
|
|
3541
3542
|
import os4 from "os";
|
|
3542
3543
|
|
|
@@ -6456,7 +6457,7 @@ function nonBlank(value) {
|
|
|
6456
6457
|
return trimmed ? trimmed : void 0;
|
|
6457
6458
|
}
|
|
6458
6459
|
function resolveBuildCommit(env = process.env) {
|
|
6459
|
-
const embedded = true ? "
|
|
6460
|
+
const embedded = true ? "40be8574db3459209f5f8887911f27d256df9ab3" : void 0;
|
|
6460
6461
|
return nonBlank(embedded) ?? nonBlank(env.CANONRY_COMMIT);
|
|
6461
6462
|
}
|
|
6462
6463
|
function resolveInstanceIdentity(env = process.env) {
|
|
@@ -7456,7 +7457,7 @@ var JobRunner = class {
|
|
|
7456
7457
|
}
|
|
7457
7458
|
}
|
|
7458
7459
|
}
|
|
7459
|
-
incrementUsage(scope, metric,
|
|
7460
|
+
incrementUsage(scope, metric, count) {
|
|
7460
7461
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7461
7462
|
const period = now.slice(0, 10);
|
|
7462
7463
|
this.db.insert(usageCounters).values({
|
|
@@ -7464,11 +7465,11 @@ var JobRunner = class {
|
|
|
7464
7465
|
scope,
|
|
7465
7466
|
period,
|
|
7466
7467
|
metric,
|
|
7467
|
-
count
|
|
7468
|
+
count,
|
|
7468
7469
|
updatedAt: now
|
|
7469
7470
|
}).onConflictDoUpdate({
|
|
7470
7471
|
target: [usageCounters.scope, usageCounters.period, usageCounters.metric],
|
|
7471
|
-
set: { count: sql2`${usageCounters.count} + ${
|
|
7472
|
+
set: { count: sql2`${usageCounters.count} + ${count}`, updatedAt: now }
|
|
7472
7473
|
}).run();
|
|
7473
7474
|
}
|
|
7474
7475
|
flushProviderUsage(providerDispatchCounts, providerReservations) {
|
|
@@ -10261,8 +10262,8 @@ function buildDiscoveryInsightTitle(input) {
|
|
|
10261
10262
|
}
|
|
10262
10263
|
|
|
10263
10264
|
// src/execute-site-audit.ts
|
|
10264
|
-
import
|
|
10265
|
-
import { and as
|
|
10265
|
+
import crypto18 from "crypto";
|
|
10266
|
+
import { and as and12, eq as eq14, ne as ne2, sql as sql5 } from "drizzle-orm";
|
|
10266
10267
|
import { runSiteCrawl } from "@canonry/aeo-audit";
|
|
10267
10268
|
|
|
10268
10269
|
// src/site-audit-root.ts
|
|
@@ -10449,640 +10450,8 @@ async function requestPinnedSiteAuditRoot(target, options) {
|
|
|
10449
10450
|
});
|
|
10450
10451
|
}
|
|
10451
10452
|
|
|
10452
|
-
// src/site-crawl-graph-layout.ts
|
|
10453
|
-
import crypto18 from "crypto";
|
|
10454
|
-
import { createRequire as createRequire3 } from "module";
|
|
10455
|
-
import { Worker } from "worker_threads";
|
|
10456
|
-
import { and as and11, asc, count, desc as desc6, eq as eq13, isNull, ne as ne2, or, sql as sql5 } from "drizzle-orm";
|
|
10457
|
-
import { alias } from "drizzle-orm/sqlite-core";
|
|
10458
|
-
var SITE_CRAWL_GRAPH_LAYOUT_TIMEOUT_MS = 15e3;
|
|
10459
|
-
var SITE_CRAWL_GRAPH_SEED_SCALE = 100;
|
|
10460
|
-
var graphSourcePage = alias(siteCrawlPages, "layout_graph_source_page");
|
|
10461
|
-
var graphTargetPage = alias(siteCrawlPages, "layout_graph_target_page");
|
|
10462
|
-
var FORCE_ATLAS_SETTINGS = Object.freeze({
|
|
10463
|
-
adjustSizes: false,
|
|
10464
|
-
barnesHutOptimize: true,
|
|
10465
|
-
barnesHutTheta: 0.5,
|
|
10466
|
-
edgeWeightInfluence: 0.5,
|
|
10467
|
-
gravity: 0.2,
|
|
10468
|
-
linLogMode: true,
|
|
10469
|
-
outboundAttractionDistribution: true,
|
|
10470
|
-
scalingRatio: 2,
|
|
10471
|
-
slowDown: 10,
|
|
10472
|
-
strongGravityMode: false
|
|
10473
|
-
});
|
|
10474
|
-
var RENDER_MAX_NODE_SIZE = 14;
|
|
10475
|
-
var RENDER_HALF_EXTENT_PX = 420;
|
|
10476
|
-
var NODE_SIZE_EXTENT_FRACTION = RENDER_MAX_NODE_SIZE / RENDER_HALF_EXTENT_PX;
|
|
10477
|
-
function siteCrawlGraphNodeSize(extent, nodeCount) {
|
|
10478
|
-
if (!Number.isFinite(extent) || extent <= 0 || nodeCount <= 0) return 0;
|
|
10479
|
-
const byExtent = extent * NODE_SIZE_EXTENT_FRACTION;
|
|
10480
|
-
const byArea = Math.sqrt(NOVERLAP_SETTINGS.maxPackingFraction * (2 * extent) ** 2 / nodeCount) / 2;
|
|
10481
|
-
return Math.min(byExtent, byArea);
|
|
10482
|
-
}
|
|
10483
|
-
function siteCrawlGraphExtent(positions) {
|
|
10484
|
-
if (positions.length === 0) return 0;
|
|
10485
|
-
const centerX = positions.reduce((sum, p) => sum + p.x, 0) / positions.length;
|
|
10486
|
-
const centerY = positions.reduce((sum, p) => sum + p.y, 0) / positions.length;
|
|
10487
|
-
let extent = 0;
|
|
10488
|
-
for (const position of positions) {
|
|
10489
|
-
extent = Math.max(extent, Math.abs(position.x - centerX), Math.abs(position.y - centerY));
|
|
10490
|
-
}
|
|
10491
|
-
return extent;
|
|
10492
|
-
}
|
|
10493
|
-
var NOVERLAP_SETTINGS = Object.freeze({
|
|
10494
|
-
/** Half a node of clear space between two touching nodes. */
|
|
10495
|
-
marginFactor: 0.5,
|
|
10496
|
-
expansion: 1.1,
|
|
10497
|
-
ratio: 1,
|
|
10498
|
-
speed: 3,
|
|
10499
|
-
/**
|
|
10500
|
-
* Node discs may claim at most this share of the graph's bounding box. At
|
|
10501
|
-
* the 20k cap the extent-derived size asks for spacing the graph physically
|
|
10502
|
-
* does not have, and noverlap either runs for minutes or overflows its
|
|
10503
|
-
* collision set. This bound keeps the pass solvable at every graph size,
|
|
10504
|
-
* and it only binds on graphs far too dense to read node-by-node anyway.
|
|
10505
|
-
*/
|
|
10506
|
-
maxPackingFraction: 0.15,
|
|
10507
|
-
/** Cells hold a handful of nodes each, which is what keeps the pass linear. */
|
|
10508
|
-
gridSizeMin: 10,
|
|
10509
|
-
gridSizeMax: 200,
|
|
10510
|
-
iterationLadder: [
|
|
10511
|
-
{ maxNodes: 1e3, iterations: 120 },
|
|
10512
|
-
{ maxNodes: 5e3, iterations: 60 }
|
|
10513
|
-
],
|
|
10514
|
-
iterationsFloor: 20
|
|
10515
|
-
});
|
|
10516
|
-
function noverlapGridSize(nodeCount) {
|
|
10517
|
-
return Math.max(
|
|
10518
|
-
NOVERLAP_SETTINGS.gridSizeMin,
|
|
10519
|
-
Math.min(NOVERLAP_SETTINGS.gridSizeMax, Math.ceil(Math.sqrt(Math.max(1, nodeCount))))
|
|
10520
|
-
);
|
|
10521
|
-
}
|
|
10522
|
-
function noverlapMaxIterations(nodeCount) {
|
|
10523
|
-
for (const step of NOVERLAP_SETTINGS.iterationLadder) {
|
|
10524
|
-
if (nodeCount <= step.maxNodes) return step.iterations;
|
|
10525
|
-
}
|
|
10526
|
-
return NOVERLAP_SETTINGS.iterationsFloor;
|
|
10527
|
-
}
|
|
10528
|
-
var COMPONENT_PACKING_SPACING_FACTOR = 3;
|
|
10529
|
-
function siteCrawlGraphLayoutSettingsFingerprint(settings) {
|
|
10530
|
-
return crypto18.createHash("sha256").update(JSON.stringify(settings)).digest("hex").slice(0, 8);
|
|
10531
|
-
}
|
|
10532
|
-
var SITE_CRAWL_GRAPH_LAYOUT_ALGORITHM = "site-health-fa2-v5";
|
|
10533
|
-
var SITE_CRAWL_GRAPH_LAYOUT_VERSION = `${SITE_CRAWL_GRAPH_LAYOUT_ALGORITHM}-${siteCrawlGraphLayoutSettingsFingerprint({
|
|
10534
|
-
forceAtlas: FORCE_ATLAS_SETTINGS,
|
|
10535
|
-
noverlap: NOVERLAP_SETTINGS,
|
|
10536
|
-
nodeSizeExtentFraction: NODE_SIZE_EXTENT_FRACTION,
|
|
10537
|
-
componentPackingSpacingFactor: COMPONENT_PACKING_SPACING_FACTOR,
|
|
10538
|
-
templateLinkRatioThreshold: TEMPLATE_LINK_RATIO_THRESHOLD
|
|
10539
|
-
})}`;
|
|
10540
|
-
var WORKER_SOURCE = String.raw`
|
|
10541
|
-
const {parentPort, workerData} = require('node:worker_threads');
|
|
10542
|
-
try {
|
|
10543
|
-
const graphologyModule = require(workerData.graphologyPath);
|
|
10544
|
-
const forceAtlasModule = require(workerData.forceAtlasPath);
|
|
10545
|
-
const noverlapModule = require(workerData.noverlapPath);
|
|
10546
|
-
const MultiDirectedGraph = graphologyModule.MultiDirectedGraph;
|
|
10547
|
-
const forceAtlas2 = forceAtlasModule.default || forceAtlasModule;
|
|
10548
|
-
const noverlap = noverlapModule.default || noverlapModule;
|
|
10549
|
-
const graph = new MultiDirectedGraph({allowSelfLoops: true});
|
|
10550
|
-
for (const node of workerData.input.nodes) {
|
|
10551
|
-
graph.addNode(node.nodeKey, {x: node.x, y: node.y});
|
|
10552
|
-
}
|
|
10553
|
-
for (const edge of workerData.input.edges) {
|
|
10554
|
-
graph.addDirectedEdgeWithKey(edge.edgeKey, edge.sourceNodeKey, edge.targetNodeKey, {
|
|
10555
|
-
weight: 1 + Math.log2(Math.max(1, edge.occurrences)),
|
|
10556
|
-
});
|
|
10557
|
-
}
|
|
10558
|
-
if (graph.order > 1 && graph.size > 0) {
|
|
10559
|
-
forceAtlas2.assign(graph, {
|
|
10560
|
-
iterations: workerData.input.iterations,
|
|
10561
|
-
settings: workerData.settings,
|
|
10562
|
-
});
|
|
10563
|
-
}
|
|
10564
|
-
let nodeSize = 0;
|
|
10565
|
-
// Anti-overlap runs on the spatialization ForceAtlas2 just produced. Node
|
|
10566
|
-
// size is uniform and derived from the graph's own extent, so the pass is
|
|
10567
|
-
// scale invariant and stays deterministic for a fixed input.
|
|
10568
|
-
if (graph.order > 1) {
|
|
10569
|
-
let sumX = 0;
|
|
10570
|
-
let sumY = 0;
|
|
10571
|
-
graph.forEachNode(function (nodeKey, attributes) {
|
|
10572
|
-
sumX += attributes.x;
|
|
10573
|
-
sumY += attributes.y;
|
|
10574
|
-
});
|
|
10575
|
-
const centerX = sumX / graph.order;
|
|
10576
|
-
const centerY = sumY / graph.order;
|
|
10577
|
-
let extent = 0;
|
|
10578
|
-
graph.forEachNode(function (nodeKey, attributes) {
|
|
10579
|
-
extent = Math.max(extent, Math.abs(attributes.x - centerX), Math.abs(attributes.y - centerY));
|
|
10580
|
-
});
|
|
10581
|
-
if (Number.isFinite(extent) && extent > 0) {
|
|
10582
|
-
// Whichever bound is tighter: one render-sized node, or the largest disc
|
|
10583
|
-
// that still leaves the graph mostly empty space at this node count.
|
|
10584
|
-
// Mirrors siteCrawlGraphNodeSize; a test asserts the two agree.
|
|
10585
|
-
const byExtent = extent * workerData.nodeSizeExtentFraction;
|
|
10586
|
-
const byArea = Math.sqrt(
|
|
10587
|
-
(workerData.noverlap.maxPackingFraction * Math.pow(2 * extent, 2)) / graph.order,
|
|
10588
|
-
) / 2;
|
|
10589
|
-
nodeSize = Math.min(byExtent, byArea);
|
|
10590
|
-
graph.forEachNode(function (nodeKey) {
|
|
10591
|
-
graph.setNodeAttribute(nodeKey, 'size', nodeSize);
|
|
10592
|
-
});
|
|
10593
|
-
noverlap.assign(graph, {
|
|
10594
|
-
maxIterations: workerData.noverlap.maxIterations,
|
|
10595
|
-
settings: {
|
|
10596
|
-
gridSize: workerData.noverlap.gridSize,
|
|
10597
|
-
margin: nodeSize * workerData.noverlap.marginFactor,
|
|
10598
|
-
expansion: workerData.noverlap.expansion,
|
|
10599
|
-
ratio: workerData.noverlap.ratio,
|
|
10600
|
-
speed: workerData.noverlap.speed,
|
|
10601
|
-
},
|
|
10602
|
-
});
|
|
10603
|
-
}
|
|
10604
|
-
}
|
|
10605
|
-
const positions = [];
|
|
10606
|
-
graph.forEachNode((nodeKey, attributes) => {
|
|
10607
|
-
positions.push({nodeKey, x: attributes.x, y: attributes.y});
|
|
10608
|
-
});
|
|
10609
|
-
parentPort.postMessage({ok: true, positions, nodeSize});
|
|
10610
|
-
} catch (error) {
|
|
10611
|
-
parentPort.postMessage({
|
|
10612
|
-
ok: false,
|
|
10613
|
-
error: error && typeof error.message === 'string' ? error.message : String(error),
|
|
10614
|
-
});
|
|
10615
|
-
}
|
|
10616
|
-
`;
|
|
10617
|
-
var GraphLayoutTimeoutError = class extends Error {
|
|
10618
|
-
name = "GraphLayoutTimeoutError";
|
|
10619
|
-
};
|
|
10620
|
-
function hashUnit(value) {
|
|
10621
|
-
let hash = 2166136261;
|
|
10622
|
-
for (let i = 0; i < value.length; i++) {
|
|
10623
|
-
hash ^= value.charCodeAt(i);
|
|
10624
|
-
hash = Math.imul(hash, 16777619);
|
|
10625
|
-
}
|
|
10626
|
-
return (hash >>> 0) / 4294967296;
|
|
10627
|
-
}
|
|
10628
|
-
function pathSection(path9) {
|
|
10629
|
-
return path9.split("/").filter(Boolean)[0] ?? "__root__";
|
|
10630
|
-
}
|
|
10631
|
-
function seedSiteCrawlGraphNodes(nodes, rootNodeKey, priorPositions) {
|
|
10632
|
-
return [...nodes].sort((a, b) => a.sampleRank - b.sampleRank || a.nodeKey.localeCompare(b.nodeKey)).map((node) => {
|
|
10633
|
-
const prior = priorPositions?.get(node.nodeKey);
|
|
10634
|
-
if (prior && Number.isFinite(prior.x) && Number.isFinite(prior.y)) {
|
|
10635
|
-
return {
|
|
10636
|
-
...node,
|
|
10637
|
-
x: prior.x * SITE_CRAWL_GRAPH_SEED_SCALE,
|
|
10638
|
-
y: prior.y * SITE_CRAWL_GRAPH_SEED_SCALE
|
|
10639
|
-
};
|
|
10640
|
-
}
|
|
10641
|
-
if (node.nodeKey === rootNodeKey) return { ...node, x: 0, y: 0 };
|
|
10642
|
-
const section = pathSection(node.path);
|
|
10643
|
-
const sectionAngle = hashUnit(`section:${section}`) * Math.PI * 2;
|
|
10644
|
-
const sectionRadius = section === "__root__" ? 24 : 100;
|
|
10645
|
-
const clusterX = Math.cos(sectionAngle) * sectionRadius;
|
|
10646
|
-
const clusterY = Math.sin(sectionAngle) * sectionRadius;
|
|
10647
|
-
const localAngle = hashUnit(`node-angle:${node.nodeKey}`) * Math.PI * 2;
|
|
10648
|
-
const depth = Math.max(0, Math.min(12, node.depth ?? 4));
|
|
10649
|
-
const localRadius = 4 + depth * 2.5 + Math.sqrt(hashUnit(`node-radius:${node.nodeKey}`)) * 14;
|
|
10650
|
-
return {
|
|
10651
|
-
...node,
|
|
10652
|
-
x: clusterX + Math.cos(localAngle) * localRadius,
|
|
10653
|
-
y: clusterY + Math.sin(localAngle) * localRadius
|
|
10654
|
-
};
|
|
10655
|
-
});
|
|
10656
|
-
}
|
|
10657
|
-
function adaptiveForceAtlasIterations(nodeCount, edgeCount) {
|
|
10658
|
-
if (nodeCount <= 100 && edgeCount <= 500) return 100;
|
|
10659
|
-
if (nodeCount <= 500 && edgeCount <= 2500) return 70;
|
|
10660
|
-
if (nodeCount <= 2e3 && edgeCount <= 1e4) return 40;
|
|
10661
|
-
if (nodeCount <= 5e3 && edgeCount <= 2e4) return 24;
|
|
10662
|
-
if (nodeCount <= 1e4 && edgeCount <= 35e3) return 12;
|
|
10663
|
-
return 6;
|
|
10664
|
-
}
|
|
10665
|
-
async function computeForceAtlasPositions(input, options = {}) {
|
|
10666
|
-
if (input.nodes.length <= 1 || input.edges.length === 0) {
|
|
10667
|
-
const positions = input.nodes.map(({ nodeKey, x, y }) => ({ nodeKey, x, y }));
|
|
10668
|
-
return { positions, nodeSize: siteCrawlGraphNodeSize(siteCrawlGraphExtent(positions), positions.length) };
|
|
10669
|
-
}
|
|
10670
|
-
const requireFromPackage = createRequire3(import.meta.url);
|
|
10671
|
-
const worker = new Worker(WORKER_SOURCE, {
|
|
10672
|
-
eval: true,
|
|
10673
|
-
workerData: {
|
|
10674
|
-
graphologyPath: requireFromPackage.resolve("graphology"),
|
|
10675
|
-
forceAtlasPath: requireFromPackage.resolve("graphology-layout-forceatlas2"),
|
|
10676
|
-
noverlapPath: requireFromPackage.resolve("graphology-layout-noverlap"),
|
|
10677
|
-
input,
|
|
10678
|
-
settings: FORCE_ATLAS_SETTINGS,
|
|
10679
|
-
noverlap: {
|
|
10680
|
-
...NOVERLAP_SETTINGS,
|
|
10681
|
-
gridSize: noverlapGridSize(input.nodes.length),
|
|
10682
|
-
maxIterations: noverlapMaxIterations(input.nodes.length)
|
|
10683
|
-
},
|
|
10684
|
-
nodeSizeExtentFraction: NODE_SIZE_EXTENT_FRACTION
|
|
10685
|
-
}
|
|
10686
|
-
});
|
|
10687
|
-
const timeoutMs = options.timeoutMs ?? SITE_CRAWL_GRAPH_LAYOUT_TIMEOUT_MS;
|
|
10688
|
-
return await new Promise((resolve, reject) => {
|
|
10689
|
-
let settled = false;
|
|
10690
|
-
const finish = (result) => {
|
|
10691
|
-
if (settled) return;
|
|
10692
|
-
settled = true;
|
|
10693
|
-
clearTimeout(timer);
|
|
10694
|
-
options.signal?.removeEventListener("abort", onAbort);
|
|
10695
|
-
void worker.terminate();
|
|
10696
|
-
if (result.error) reject(result.error instanceof Error ? result.error : new Error("Graph layout failed with a non-Error reason", { cause: result.error }));
|
|
10697
|
-
else resolve({ positions: result.positions ?? [], nodeSize: result.nodeSize ?? 0 });
|
|
10698
|
-
};
|
|
10699
|
-
const onAbort = () => finish({ error: options.signal?.reason ?? new DOMException("Layout aborted", "AbortError") });
|
|
10700
|
-
const timer = setTimeout(() => finish({ error: new GraphLayoutTimeoutError(`Graph layout exceeded ${timeoutMs}ms`) }), timeoutMs);
|
|
10701
|
-
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
10702
|
-
if (options.signal?.aborted) return onAbort();
|
|
10703
|
-
worker.once("message", (message) => {
|
|
10704
|
-
if (!message.ok) finish({ error: new Error(message.error ?? "Graph layout worker failed") });
|
|
10705
|
-
else finish({ positions: message.positions, nodeSize: message.nodeSize });
|
|
10706
|
-
});
|
|
10707
|
-
worker.once("error", (error) => finish({ error }));
|
|
10708
|
-
worker.once("exit", (code) => {
|
|
10709
|
-
if (code !== 0 && !settled) finish({ error: new Error(`Graph layout worker exited with code ${code}`) });
|
|
10710
|
-
});
|
|
10711
|
-
});
|
|
10712
|
-
}
|
|
10713
|
-
function siteCrawlGraphComponents(nodeKeys, edges) {
|
|
10714
|
-
const parent = new Map(nodeKeys.map((key) => [key, key]));
|
|
10715
|
-
const find = (key) => {
|
|
10716
|
-
let root = key;
|
|
10717
|
-
while (parent.get(root) !== root) root = parent.get(root);
|
|
10718
|
-
let cursor = key;
|
|
10719
|
-
while (parent.get(cursor) !== root) {
|
|
10720
|
-
const next = parent.get(cursor);
|
|
10721
|
-
parent.set(cursor, root);
|
|
10722
|
-
cursor = next;
|
|
10723
|
-
}
|
|
10724
|
-
return root;
|
|
10725
|
-
};
|
|
10726
|
-
for (const edge of edges) {
|
|
10727
|
-
if (!parent.has(edge.sourceNodeKey) || !parent.has(edge.targetNodeKey)) continue;
|
|
10728
|
-
const sourceRoot = find(edge.sourceNodeKey);
|
|
10729
|
-
const targetRoot = find(edge.targetNodeKey);
|
|
10730
|
-
if (sourceRoot === targetRoot) continue;
|
|
10731
|
-
if (sourceRoot < targetRoot) parent.set(targetRoot, sourceRoot);
|
|
10732
|
-
else parent.set(sourceRoot, targetRoot);
|
|
10733
|
-
}
|
|
10734
|
-
const members = /* @__PURE__ */ new Map();
|
|
10735
|
-
for (const key of [...nodeKeys].sort()) {
|
|
10736
|
-
const root = find(key);
|
|
10737
|
-
const group = members.get(root);
|
|
10738
|
-
if (group) group.push(key);
|
|
10739
|
-
else members.set(root, [key]);
|
|
10740
|
-
}
|
|
10741
|
-
return [...members.values()].sort((left, right) => right.length - left.length || (left[0] < right[0] ? -1 : left[0] > right[0] ? 1 : 0));
|
|
10742
|
-
}
|
|
10743
|
-
function describeComponent(nodeKeys, byKey) {
|
|
10744
|
-
let sumX = 0;
|
|
10745
|
-
let sumY = 0;
|
|
10746
|
-
for (const nodeKey of nodeKeys) {
|
|
10747
|
-
const position = byKey.get(nodeKey);
|
|
10748
|
-
sumX += position.x;
|
|
10749
|
-
sumY += position.y;
|
|
10750
|
-
}
|
|
10751
|
-
const centerX = sumX / nodeKeys.length;
|
|
10752
|
-
const centerY = sumY / nodeKeys.length;
|
|
10753
|
-
let radius = 0;
|
|
10754
|
-
for (const nodeKey of nodeKeys) {
|
|
10755
|
-
const position = byKey.get(nodeKey);
|
|
10756
|
-
radius = Math.max(radius, Math.hypot(position.x - centerX, position.y - centerY));
|
|
10757
|
-
}
|
|
10758
|
-
return { nodeKeys, centerX, centerY, radius };
|
|
10759
|
-
}
|
|
10760
|
-
function packSiteCrawlGraphComponents(positions, edges, nodeSpacing) {
|
|
10761
|
-
const components = siteCrawlGraphComponents(positions.map((position) => position.nodeKey), edges);
|
|
10762
|
-
if (components.length <= 1) return [...positions];
|
|
10763
|
-
const byKey = new Map(positions.map((position) => [position.nodeKey, position]));
|
|
10764
|
-
if (positions.some((position) => !Number.isFinite(position.x) || !Number.isFinite(position.y))) {
|
|
10765
|
-
return [...positions];
|
|
10766
|
-
}
|
|
10767
|
-
const described = components.map((nodeKeys) => describeComponent(nodeKeys, byKey));
|
|
10768
|
-
const [anchor, ...rest] = described;
|
|
10769
|
-
const spacing = Math.max(nodeSpacing, Number.EPSILON) * COMPONENT_PACKING_SPACING_FACTOR;
|
|
10770
|
-
const ordered = [...rest].sort((left, right) => right.radius - left.radius || (left.nodeKeys[0] < right.nodeKeys[0] ? -1 : left.nodeKeys[0] > right.nodeKeys[0] ? 1 : 0));
|
|
10771
|
-
const moved = /* @__PURE__ */ new Map();
|
|
10772
|
-
let ringRadius = anchor.radius + spacing;
|
|
10773
|
-
let ringPitch = 0;
|
|
10774
|
-
let placedOnRing = 0;
|
|
10775
|
-
let ringCapacity = 0;
|
|
10776
|
-
for (const component of ordered) {
|
|
10777
|
-
if (placedOnRing >= ringCapacity) {
|
|
10778
|
-
ringPitch = 2 * component.radius + spacing;
|
|
10779
|
-
ringRadius += ringPitch / 2 + (placedOnRing === 0 ? 0 : ringPitch / 2);
|
|
10780
|
-
ringCapacity = Math.max(1, Math.floor(2 * Math.PI * ringRadius / ringPitch));
|
|
10781
|
-
placedOnRing = 0;
|
|
10782
|
-
}
|
|
10783
|
-
const angle = placedOnRing / ringCapacity * Math.PI * 2;
|
|
10784
|
-
placedOnRing += 1;
|
|
10785
|
-
const targetX = anchor.centerX + Math.cos(angle) * ringRadius;
|
|
10786
|
-
const targetY = anchor.centerY + Math.sin(angle) * ringRadius;
|
|
10787
|
-
const shiftX = targetX - component.centerX;
|
|
10788
|
-
const shiftY = targetY - component.centerY;
|
|
10789
|
-
for (const nodeKey of component.nodeKeys) {
|
|
10790
|
-
const position = byKey.get(nodeKey);
|
|
10791
|
-
moved.set(nodeKey, { nodeKey, x: position.x + shiftX, y: position.y + shiftY });
|
|
10792
|
-
}
|
|
10793
|
-
}
|
|
10794
|
-
return positions.map((position) => moved.get(position.nodeKey) ?? position);
|
|
10795
|
-
}
|
|
10796
|
-
function normalizePositions(seededNodes, computed, rootNodeKey) {
|
|
10797
|
-
const byKey = new Map(computed.map((position) => [position.nodeKey, position]));
|
|
10798
|
-
if (byKey.size !== seededNodes.length) throw new Error("Graph layout returned an incomplete node set");
|
|
10799
|
-
const root = rootNodeKey ? byKey.get(rootNodeKey) : null;
|
|
10800
|
-
const centerX = root?.x ?? computed.reduce((sum, position) => sum + position.x, 0) / Math.max(1, computed.length);
|
|
10801
|
-
const centerY = root?.y ?? computed.reduce((sum, position) => sum + position.y, 0) / Math.max(1, computed.length);
|
|
10802
|
-
if (!Number.isFinite(centerX) || !Number.isFinite(centerY)) throw new Error("Graph layout returned a non-finite center");
|
|
10803
|
-
let scale = 0;
|
|
10804
|
-
for (const position of computed) {
|
|
10805
|
-
if (!Number.isFinite(position.x) || !Number.isFinite(position.y)) throw new Error("Graph layout returned non-finite coordinates");
|
|
10806
|
-
scale = Math.max(scale, Math.abs(position.x - centerX), Math.abs(position.y - centerY));
|
|
10807
|
-
}
|
|
10808
|
-
scale = scale > 0 ? scale : 1;
|
|
10809
|
-
return seededNodes.map((node) => {
|
|
10810
|
-
const position = byKey.get(node.nodeKey);
|
|
10811
|
-
if (!position) throw new Error(`Graph layout omitted ${node.nodeKey}`);
|
|
10812
|
-
return {
|
|
10813
|
-
nodeKey: node.nodeKey,
|
|
10814
|
-
sampleRank: node.sampleRank,
|
|
10815
|
-
x: Math.round((position.x - centerX) / scale * 1e6) / 1e6,
|
|
10816
|
-
y: Math.round((position.y - centerY) / scale * 1e6) / 1e6
|
|
10817
|
-
};
|
|
10818
|
-
});
|
|
10819
|
-
}
|
|
10820
|
-
async function layoutSiteCrawlGraphInput(input, options = {}) {
|
|
10821
|
-
options.signal?.throwIfAborted();
|
|
10822
|
-
if (input.nodes.length === 0) {
|
|
10823
|
-
return {
|
|
10824
|
-
state: "unavailable",
|
|
10825
|
-
failureCode: "empty-crawl",
|
|
10826
|
-
totalNodes: input.totalNodes,
|
|
10827
|
-
totalEdges: input.totalEdges,
|
|
10828
|
-
totalTemplateEdges: input.totalTemplateEdges,
|
|
10829
|
-
nodeCount: 0,
|
|
10830
|
-
edgeCount: 0,
|
|
10831
|
-
nodes: [],
|
|
10832
|
-
edges: []
|
|
10833
|
-
};
|
|
10834
|
-
}
|
|
10835
|
-
const layoutEdges = input.edges.filter((edge) => !edge.isTemplate);
|
|
10836
|
-
const seeded = seedSiteCrawlGraphNodes(input.nodes, input.rootNodeKey, input.priorPositions);
|
|
10837
|
-
const timeoutMs = options.timeoutMs ?? SITE_CRAWL_GRAPH_LAYOUT_TIMEOUT_MS;
|
|
10838
|
-
const deadline = Date.now() + timeoutMs;
|
|
10839
|
-
const assertWithinBudget = () => {
|
|
10840
|
-
options.signal?.throwIfAborted();
|
|
10841
|
-
if (Date.now() > deadline) throw new GraphLayoutTimeoutError(`Graph layout exceeded ${timeoutMs}ms`);
|
|
10842
|
-
};
|
|
10843
|
-
try {
|
|
10844
|
-
const compute = options.computePositions ?? computeForceAtlasPositions;
|
|
10845
|
-
const { positions, nodeSize } = await compute({
|
|
10846
|
-
nodes: seeded,
|
|
10847
|
-
edges: layoutEdges,
|
|
10848
|
-
iterations: adaptiveForceAtlasIterations(input.nodes.length, layoutEdges.length)
|
|
10849
|
-
}, { signal: options.signal, timeoutMs: Math.max(1, deadline - Date.now()) });
|
|
10850
|
-
assertWithinBudget();
|
|
10851
|
-
const framed = packSiteCrawlGraphComponents(positions, layoutEdges, nodeSize);
|
|
10852
|
-
assertWithinBudget();
|
|
10853
|
-
return {
|
|
10854
|
-
state: "ready",
|
|
10855
|
-
layoutVersion: SITE_CRAWL_GRAPH_LAYOUT_VERSION,
|
|
10856
|
-
totalNodes: input.totalNodes,
|
|
10857
|
-
totalEdges: input.totalEdges,
|
|
10858
|
-
totalTemplateEdges: input.totalTemplateEdges,
|
|
10859
|
-
nodeCount: input.nodes.length,
|
|
10860
|
-
edgeCount: input.edges.length,
|
|
10861
|
-
templateLinksExcluded: true,
|
|
10862
|
-
nodes: normalizePositions(seeded, framed, input.rootNodeKey),
|
|
10863
|
-
edges: input.edges.map((edge, sampleRank) => ({ ...edge, sampleRank }))
|
|
10864
|
-
};
|
|
10865
|
-
} catch (error) {
|
|
10866
|
-
if (options.signal?.aborted) options.signal.throwIfAborted();
|
|
10867
|
-
if (error instanceof DOMException && error.name === "AbortError") throw error;
|
|
10868
|
-
return {
|
|
10869
|
-
state: "unavailable",
|
|
10870
|
-
failureCode: error instanceof GraphLayoutTimeoutError ? "layout-timeout" : "layout-error",
|
|
10871
|
-
totalNodes: input.totalNodes,
|
|
10872
|
-
totalEdges: input.totalEdges,
|
|
10873
|
-
totalTemplateEdges: input.totalTemplateEdges,
|
|
10874
|
-
nodeCount: 0,
|
|
10875
|
-
edgeCount: 0,
|
|
10876
|
-
nodes: [],
|
|
10877
|
-
edges: []
|
|
10878
|
-
};
|
|
10879
|
-
}
|
|
10880
|
-
}
|
|
10881
|
-
function loadPriorSiteCrawlGraphPositions(db, scope) {
|
|
10882
|
-
const prior = db.select({
|
|
10883
|
-
runId: siteCrawlSnapshots.runId,
|
|
10884
|
-
attemptId: siteCrawlSnapshots.attemptId
|
|
10885
|
-
}).from(siteCrawlSnapshots).innerJoin(siteCrawlGraphLayouts, and11(
|
|
10886
|
-
eq13(siteCrawlGraphLayouts.projectId, siteCrawlSnapshots.projectId),
|
|
10887
|
-
eq13(siteCrawlGraphLayouts.runId, siteCrawlSnapshots.runId),
|
|
10888
|
-
eq13(siteCrawlGraphLayouts.attemptId, siteCrawlSnapshots.attemptId)
|
|
10889
|
-
)).where(and11(
|
|
10890
|
-
eq13(siteCrawlSnapshots.projectId, scope.projectId),
|
|
10891
|
-
ne2(siteCrawlSnapshots.runId, scope.runId),
|
|
10892
|
-
eq13(siteCrawlSnapshots.complete, true),
|
|
10893
|
-
eq13(siteCrawlGraphLayouts.state, "ready"),
|
|
10894
|
-
eq13(siteCrawlGraphLayouts.layoutVersion, SITE_CRAWL_GRAPH_LAYOUT_VERSION)
|
|
10895
|
-
)).orderBy(desc6(siteCrawlSnapshots.createdAt), desc6(siteCrawlSnapshots.id)).limit(1).get();
|
|
10896
|
-
if (!prior?.attemptId) return /* @__PURE__ */ new Map();
|
|
10897
|
-
return new Map(db.select({
|
|
10898
|
-
nodeKey: siteCrawlGraphNodes.nodeKey,
|
|
10899
|
-
x: siteCrawlGraphNodes.x,
|
|
10900
|
-
y: siteCrawlGraphNodes.y
|
|
10901
|
-
}).from(siteCrawlGraphNodes).where(and11(
|
|
10902
|
-
eq13(siteCrawlGraphNodes.projectId, scope.projectId),
|
|
10903
|
-
eq13(siteCrawlGraphNodes.runId, prior.runId),
|
|
10904
|
-
eq13(siteCrawlGraphNodes.attemptId, prior.attemptId)
|
|
10905
|
-
)).all().map((node) => [node.nodeKey, { x: node.x, y: node.y }]));
|
|
10906
|
-
}
|
|
10907
|
-
async function prepareSiteCrawlGraphLayout(db, scope, options = {}) {
|
|
10908
|
-
let totalNodes = 0;
|
|
10909
|
-
let totalEdges = 0;
|
|
10910
|
-
let totalTemplateEdges = 0;
|
|
10911
|
-
try {
|
|
10912
|
-
const assertNotAborted = () => options.signal?.throwIfAborted();
|
|
10913
|
-
assertNotAborted();
|
|
10914
|
-
const pageScope = [
|
|
10915
|
-
eq13(siteCrawlPages.projectId, scope.projectId),
|
|
10916
|
-
eq13(siteCrawlPages.runId, scope.runId),
|
|
10917
|
-
eq13(siteCrawlPages.attemptId, scope.attemptId)
|
|
10918
|
-
];
|
|
10919
|
-
const edgeScope = [
|
|
10920
|
-
eq13(siteCrawlEdges.projectId, scope.projectId),
|
|
10921
|
-
eq13(siteCrawlEdges.runId, scope.runId),
|
|
10922
|
-
eq13(siteCrawlEdges.attemptId, scope.attemptId),
|
|
10923
|
-
eq13(siteCrawlEdges.internal, true),
|
|
10924
|
-
eq13(siteCrawlEdges.relation, "anchor")
|
|
10925
|
-
];
|
|
10926
|
-
totalNodes = db.select({ value: count() }).from(siteCrawlPages).where(and11(...pageScope)).get()?.value ?? 0;
|
|
10927
|
-
assertNotAborted();
|
|
10928
|
-
const countGraphCompatibleEdges = (...extra) => db.select({ value: count() }).from(siteCrawlEdges).innerJoin(graphSourcePage, and11(
|
|
10929
|
-
eq13(graphSourcePage.projectId, scope.projectId),
|
|
10930
|
-
eq13(graphSourcePage.runId, scope.runId),
|
|
10931
|
-
eq13(graphSourcePage.attemptId, scope.attemptId),
|
|
10932
|
-
eq13(graphSourcePage.nodeKey, siteCrawlEdges.sourceNodeKey)
|
|
10933
|
-
)).innerJoin(graphTargetPage, and11(
|
|
10934
|
-
eq13(graphTargetPage.projectId, scope.projectId),
|
|
10935
|
-
eq13(graphTargetPage.runId, scope.runId),
|
|
10936
|
-
eq13(graphTargetPage.attemptId, scope.attemptId),
|
|
10937
|
-
eq13(graphTargetPage.nodeKey, siteCrawlEdges.targetNodeKey)
|
|
10938
|
-
)).where(and11(...edgeScope, ...extra)).get()?.value ?? 0;
|
|
10939
|
-
totalEdges = countGraphCompatibleEdges();
|
|
10940
|
-
assertNotAborted();
|
|
10941
|
-
totalTemplateEdges = countGraphCompatibleEdges(eq13(siteCrawlEdges.isTemplate, true));
|
|
10942
|
-
assertNotAborted();
|
|
10943
|
-
const graphSeedColumns = {
|
|
10944
|
-
nodeKey: siteCrawlPages.nodeKey,
|
|
10945
|
-
path: siteCrawlPages.path,
|
|
10946
|
-
depth: siteCrawlPages.depth
|
|
10947
|
-
};
|
|
10948
|
-
const rootRows = db.select(graphSeedColumns).from(siteCrawlPages).where(and11(...pageScope, eq13(siteCrawlPages.url, scope.rootUrl))).orderBy(asc(siteCrawlPages.nodeKey)).limit(1).all();
|
|
10949
|
-
assertNotAborted();
|
|
10950
|
-
const retainedRows = [...rootRows];
|
|
10951
|
-
if (retainedRows.length < SITE_CRAWL_GRAPH_MAX_NODES) {
|
|
10952
|
-
retainedRows.push(...db.select(graphSeedColumns).from(siteCrawlPages).where(and11(...pageScope, eq13(siteCrawlPages.depth, 0), ne2(siteCrawlPages.url, scope.rootUrl))).orderBy(asc(siteCrawlPages.nodeKey)).limit(SITE_CRAWL_GRAPH_MAX_NODES - retainedRows.length).all());
|
|
10953
|
-
}
|
|
10954
|
-
assertNotAborted();
|
|
10955
|
-
if (retainedRows.length < SITE_CRAWL_GRAPH_MAX_NODES) {
|
|
10956
|
-
retainedRows.push(...db.select(graphSeedColumns).from(siteCrawlPages).where(and11(
|
|
10957
|
-
...pageScope,
|
|
10958
|
-
or(isNull(siteCrawlPages.depth), ne2(siteCrawlPages.depth, 0)),
|
|
10959
|
-
ne2(siteCrawlPages.url, scope.rootUrl)
|
|
10960
|
-
)).orderBy(desc6(siteCrawlPages.linkScoreNormalized), asc(siteCrawlPages.nodeKey)).limit(SITE_CRAWL_GRAPH_MAX_NODES - retainedRows.length).all());
|
|
10961
|
-
}
|
|
10962
|
-
assertNotAborted();
|
|
10963
|
-
const nodes = retainedRows.map((row, sampleRank) => ({
|
|
10964
|
-
nodeKey: row.nodeKey,
|
|
10965
|
-
path: row.path,
|
|
10966
|
-
depth: row.depth,
|
|
10967
|
-
sampleRank
|
|
10968
|
-
}));
|
|
10969
|
-
const rootNodeKey = rootRows[0]?.nodeKey ?? null;
|
|
10970
|
-
const priorPositions = loadPriorSiteCrawlGraphPositions(db, scope);
|
|
10971
|
-
assertNotAborted();
|
|
10972
|
-
const selectedKeys = JSON.stringify(nodes.map((node) => node.nodeKey));
|
|
10973
|
-
const edgeRows = nodes.length === 0 ? [] : db.all(sql5`
|
|
10974
|
-
WITH selected(node_key) AS (SELECT value FROM json_each(${selectedKeys}))
|
|
10975
|
-
SELECT
|
|
10976
|
-
edge.edge_key AS edgeKey,
|
|
10977
|
-
edge.source_node_key AS sourceNodeKey,
|
|
10978
|
-
edge.target_node_key AS targetNodeKey,
|
|
10979
|
-
edge.followable AS followable,
|
|
10980
|
-
edge.occurrences AS occurrences,
|
|
10981
|
-
edge.is_template AS isTemplate
|
|
10982
|
-
FROM site_crawl_edges AS edge
|
|
10983
|
-
INNER JOIN selected AS source ON source.node_key = edge.source_node_key
|
|
10984
|
-
INNER JOIN selected AS target ON target.node_key = edge.target_node_key
|
|
10985
|
-
WHERE edge.project_id = ${scope.projectId}
|
|
10986
|
-
AND edge.run_id = ${scope.runId}
|
|
10987
|
-
AND edge.attempt_id = ${scope.attemptId}
|
|
10988
|
-
AND edge.internal = 1
|
|
10989
|
-
AND edge.relation = 'anchor'
|
|
10990
|
-
ORDER BY edge.occurrences DESC, edge.edge_key ASC
|
|
10991
|
-
LIMIT ${SITE_CRAWL_GRAPH_MAX_EDGES}
|
|
10992
|
-
`);
|
|
10993
|
-
assertNotAborted();
|
|
10994
|
-
return await layoutSiteCrawlGraphInput({
|
|
10995
|
-
totalNodes,
|
|
10996
|
-
totalEdges,
|
|
10997
|
-
totalTemplateEdges,
|
|
10998
|
-
nodes,
|
|
10999
|
-
edges: edgeRows.map((row) => ({ ...row, isTemplate: Boolean(row.isTemplate) })),
|
|
11000
|
-
rootNodeKey,
|
|
11001
|
-
priorPositions
|
|
11002
|
-
}, options);
|
|
11003
|
-
} catch (error) {
|
|
11004
|
-
if (options.signal?.aborted) options.signal.throwIfAborted();
|
|
11005
|
-
if (error instanceof DOMException && error.name === "AbortError") throw error;
|
|
11006
|
-
return {
|
|
11007
|
-
state: "unavailable",
|
|
11008
|
-
failureCode: "layout-error",
|
|
11009
|
-
totalNodes,
|
|
11010
|
-
totalEdges,
|
|
11011
|
-
totalTemplateEdges,
|
|
11012
|
-
nodeCount: 0,
|
|
11013
|
-
edgeCount: 0,
|
|
11014
|
-
nodes: [],
|
|
11015
|
-
edges: []
|
|
11016
|
-
};
|
|
11017
|
-
}
|
|
11018
|
-
}
|
|
11019
|
-
function persistSiteCrawlGraphLayout(db, scope, layout, now = (/* @__PURE__ */ new Date()).toISOString()) {
|
|
11020
|
-
db.transaction((tx) => {
|
|
11021
|
-
tx.delete(siteCrawlGraphLayouts).where(and11(
|
|
11022
|
-
eq13(siteCrawlGraphLayouts.projectId, scope.projectId),
|
|
11023
|
-
eq13(siteCrawlGraphLayouts.runId, scope.runId),
|
|
11024
|
-
eq13(siteCrawlGraphLayouts.attemptId, scope.attemptId)
|
|
11025
|
-
)).run();
|
|
11026
|
-
tx.insert(siteCrawlGraphLayouts).values({
|
|
11027
|
-
id: crypto18.randomUUID(),
|
|
11028
|
-
projectId: scope.projectId,
|
|
11029
|
-
runId: scope.runId,
|
|
11030
|
-
attemptId: scope.attemptId,
|
|
11031
|
-
state: layout.state,
|
|
11032
|
-
layoutVersion: layout.state === "ready" ? layout.layoutVersion : null,
|
|
11033
|
-
failureCode: layout.state === "unavailable" ? layout.failureCode : null,
|
|
11034
|
-
totalNodes: layout.totalNodes,
|
|
11035
|
-
totalEdges: layout.totalEdges,
|
|
11036
|
-
totalTemplateEdges: layout.totalTemplateEdges,
|
|
11037
|
-
nodeCount: layout.nodeCount,
|
|
11038
|
-
edgeCount: layout.edgeCount,
|
|
11039
|
-
templateLinksExcluded: layout.state === "ready",
|
|
11040
|
-
createdAt: now,
|
|
11041
|
-
updatedAt: now
|
|
11042
|
-
}).run();
|
|
11043
|
-
if (layout.state !== "ready") return;
|
|
11044
|
-
const batchSize = 250;
|
|
11045
|
-
for (let offset = 0; offset < layout.nodes.length; offset += batchSize) {
|
|
11046
|
-
tx.insert(siteCrawlGraphNodes).values(layout.nodes.slice(offset, offset + batchSize).map((node) => ({
|
|
11047
|
-
id: crypto18.randomUUID(),
|
|
11048
|
-
projectId: scope.projectId,
|
|
11049
|
-
runId: scope.runId,
|
|
11050
|
-
attemptId: scope.attemptId,
|
|
11051
|
-
nodeKey: node.nodeKey,
|
|
11052
|
-
sampleRank: node.sampleRank,
|
|
11053
|
-
x: node.x,
|
|
11054
|
-
y: node.y,
|
|
11055
|
-
createdAt: now
|
|
11056
|
-
}))).run();
|
|
11057
|
-
}
|
|
11058
|
-
for (let offset = 0; offset < layout.edges.length; offset += batchSize) {
|
|
11059
|
-
tx.insert(siteCrawlGraphEdges).values(layout.edges.slice(offset, offset + batchSize).map((edge) => ({
|
|
11060
|
-
id: crypto18.randomUUID(),
|
|
11061
|
-
projectId: scope.projectId,
|
|
11062
|
-
runId: scope.runId,
|
|
11063
|
-
attemptId: scope.attemptId,
|
|
11064
|
-
edgeKey: edge.edgeKey,
|
|
11065
|
-
sampleRank: edge.sampleRank,
|
|
11066
|
-
sourceNodeKey: edge.sourceNodeKey,
|
|
11067
|
-
targetNodeKey: edge.targetNodeKey,
|
|
11068
|
-
followable: edge.followable,
|
|
11069
|
-
occurrences: edge.occurrences,
|
|
11070
|
-
isTemplate: edge.isTemplate,
|
|
11071
|
-
createdAt: now
|
|
11072
|
-
}))).run();
|
|
11073
|
-
}
|
|
11074
|
-
});
|
|
11075
|
-
}
|
|
11076
|
-
function deleteSiteCrawlGraphLayout(db, scope) {
|
|
11077
|
-
db.delete(siteCrawlGraphLayouts).where(and11(
|
|
11078
|
-
eq13(siteCrawlGraphLayouts.projectId, scope.projectId),
|
|
11079
|
-
eq13(siteCrawlGraphLayouts.runId, scope.runId),
|
|
11080
|
-
eq13(siteCrawlGraphLayouts.attemptId, scope.attemptId)
|
|
11081
|
-
)).run();
|
|
11082
|
-
}
|
|
11083
|
-
|
|
11084
10453
|
// src/site-crawl-template-links.ts
|
|
11085
|
-
import { and as
|
|
10454
|
+
import { and as and11, asc, eq as eq13, gt, inArray as inArray5 } from "drizzle-orm";
|
|
11086
10455
|
var TEMPLATE_LINK_READ_BATCH = 2e3;
|
|
11087
10456
|
var TEMPLATE_LINK_WRITE_BATCH = 500;
|
|
11088
10457
|
function placementOccurrencesOf(row) {
|
|
@@ -11104,12 +10473,12 @@ function* streamAttemptEdges(db, scope) {
|
|
|
11104
10473
|
placementNavigationOccurrences: siteCrawlEdges.placementNavigationOccurrences,
|
|
11105
10474
|
placementContentOccurrences: siteCrawlEdges.placementContentOccurrences,
|
|
11106
10475
|
placementUnknownOccurrences: siteCrawlEdges.placementUnknownOccurrences
|
|
11107
|
-
}).from(siteCrawlEdges).where(
|
|
11108
|
-
|
|
11109
|
-
|
|
11110
|
-
|
|
10476
|
+
}).from(siteCrawlEdges).where(and11(
|
|
10477
|
+
eq13(siteCrawlEdges.projectId, scope.projectId),
|
|
10478
|
+
eq13(siteCrawlEdges.runId, scope.runId),
|
|
10479
|
+
eq13(siteCrawlEdges.attemptId, scope.attemptId),
|
|
11111
10480
|
gt(siteCrawlEdges.edgeKey, after)
|
|
11112
|
-
)).orderBy(
|
|
10481
|
+
)).orderBy(asc(siteCrawlEdges.edgeKey)).limit(TEMPLATE_LINK_READ_BATCH).all();
|
|
11113
10482
|
if (rows.length === 0) return;
|
|
11114
10483
|
yield rows.map((row) => ({
|
|
11115
10484
|
edgeKey: row.edgeKey,
|
|
@@ -11132,10 +10501,10 @@ function writeGroupKey(isTemplate, templateRatio) {
|
|
|
11132
10501
|
return `${isTemplate ? "1" : "0"}${templateRatio ?? ""}`;
|
|
11133
10502
|
}
|
|
11134
10503
|
function classifySiteCrawlTemplateLinks(db, scope, pagesFetched, placementRulesetVersion) {
|
|
11135
|
-
const scopeFilter =
|
|
11136
|
-
|
|
11137
|
-
|
|
11138
|
-
|
|
10504
|
+
const scopeFilter = and11(
|
|
10505
|
+
eq13(siteCrawlEdges.projectId, scope.projectId),
|
|
10506
|
+
eq13(siteCrawlEdges.runId, scope.runId),
|
|
10507
|
+
eq13(siteCrawlEdges.attemptId, scope.attemptId)
|
|
11139
10508
|
);
|
|
11140
10509
|
const placementAvailable = templateLinkPlacementAvailable(placementRulesetVersion);
|
|
11141
10510
|
const ubiquityAvailable = templateLinkUbiquityAvailable(pagesFetched);
|
|
@@ -11170,7 +10539,7 @@ function classifySiteCrawlTemplateLinks(db, scope, pagesFetched, placementRulese
|
|
|
11170
10539
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
11171
10540
|
for (const group of edgeKeysByValue.values()) {
|
|
11172
10541
|
for (const chunk of chunked(group.edgeKeys)) {
|
|
11173
|
-
db.update(siteCrawlEdges).set({ isTemplate: group.isTemplate, templateRatio: group.templateRatio, updatedAt: now }).where(
|
|
10542
|
+
db.update(siteCrawlEdges).set({ isTemplate: group.isTemplate, templateRatio: group.templateRatio, updatedAt: now }).where(and11(scopeFilter, inArray5(siteCrawlEdges.edgeKey, chunk))).run();
|
|
11174
10543
|
}
|
|
11175
10544
|
}
|
|
11176
10545
|
return {
|
|
@@ -11349,7 +10718,7 @@ function trackSiteAuditOutcome(db, input) {
|
|
|
11349
10718
|
);
|
|
11350
10719
|
const pagesAudited = input.crawl?.pagesAudited ?? 0;
|
|
11351
10720
|
if (pagesAudited === 0 || input.trigger === RunTriggers.probe) return;
|
|
11352
|
-
const prior = db.select({ id: siteAuditSnapshots.id }).from(siteAuditSnapshots).where(
|
|
10721
|
+
const prior = db.select({ id: siteAuditSnapshots.id }).from(siteAuditSnapshots).where(and12(eq14(siteAuditSnapshots.projectId, input.projectId), ne2(siteAuditSnapshots.runId, input.runId))).limit(1).get();
|
|
11353
10722
|
if (prior) return;
|
|
11354
10723
|
trackEvent("activation.completed", {
|
|
11355
10724
|
flowVersion: ONBOARDING_FLOW_VERSION,
|
|
@@ -11363,13 +10732,13 @@ function trackSiteAuditOutcome(db, input) {
|
|
|
11363
10732
|
}
|
|
11364
10733
|
async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
11365
10734
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
11366
|
-
const claim = db.update(runs).set({ status: "running", startedAt }).where(
|
|
10735
|
+
const claim = db.update(runs).set({ status: "running", startedAt }).where(and12(eq14(runs.id, runId), eq14(runs.projectId, projectId), eq14(runs.status, "queued"))).run();
|
|
11367
10736
|
if (claim.changes === 0) return;
|
|
11368
10737
|
const startTime = Date.now();
|
|
11369
|
-
const trigger = db.select({ trigger: runs.trigger }).from(runs).where(
|
|
10738
|
+
const trigger = db.select({ trigger: runs.trigger }).from(runs).where(eq14(runs.id, runId)).get()?.trigger ?? null;
|
|
11370
10739
|
let canonicalDomain = null;
|
|
11371
|
-
const attemptNumber = (db.select({ value:
|
|
11372
|
-
const attemptId =
|
|
10740
|
+
const attemptNumber = (db.select({ value: sql5`coalesce(max(${siteCrawlAttempts.attemptNumber}), 0)` }).from(siteCrawlAttempts).where(eq14(siteCrawlAttempts.runId, runId)).get()?.value ?? 0) + 1;
|
|
10741
|
+
const attemptId = crypto18.randomUUID();
|
|
11373
10742
|
db.insert(siteCrawlAttempts).values({
|
|
11374
10743
|
id: attemptId,
|
|
11375
10744
|
projectId,
|
|
@@ -11403,22 +10772,22 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11403
10772
|
const nodeKey = nodeKeyByUrl.get(url);
|
|
11404
10773
|
if (!nodeKey) continue;
|
|
11405
10774
|
const scope = [
|
|
11406
|
-
|
|
11407
|
-
|
|
11408
|
-
|
|
10775
|
+
eq14(siteCrawlEdges.projectId, projectId),
|
|
10776
|
+
eq14(siteCrawlEdges.runId, runId),
|
|
10777
|
+
eq14(siteCrawlEdges.attemptId, attemptId)
|
|
11409
10778
|
];
|
|
11410
|
-
tx.update(siteCrawlEdges).set({ sourceNodeKey: nodeKey, updatedAt: now }).where(
|
|
10779
|
+
tx.update(siteCrawlEdges).set({ sourceNodeKey: nodeKey, updatedAt: now }).where(and12(
|
|
11411
10780
|
...scope,
|
|
11412
|
-
|
|
10781
|
+
eq14(siteCrawlEdges.sourceUrl, url)
|
|
11413
10782
|
)).run();
|
|
11414
|
-
tx.update(siteCrawlEdges).set({ targetNodeKey: nodeKey, updatedAt: now }).where(
|
|
10783
|
+
tx.update(siteCrawlEdges).set({ targetNodeKey: nodeKey, updatedAt: now }).where(and12(
|
|
11415
10784
|
...scope,
|
|
11416
|
-
|
|
10785
|
+
eq14(siteCrawlEdges.targetUrl, url)
|
|
11417
10786
|
)).run();
|
|
11418
10787
|
const pageScope = [
|
|
11419
|
-
|
|
11420
|
-
|
|
11421
|
-
|
|
10788
|
+
eq14(siteCrawlPages.projectId, projectId),
|
|
10789
|
+
eq14(siteCrawlPages.runId, runId),
|
|
10790
|
+
eq14(siteCrawlPages.attemptId, attemptId)
|
|
11422
10791
|
];
|
|
11423
10792
|
const canonicalTargets = tx.select({
|
|
11424
10793
|
id: siteCrawlPages.id,
|
|
@@ -11426,13 +10795,13 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11426
10795
|
fetchState: siteCrawlPages.fetchState,
|
|
11427
10796
|
indexabilityState: siteCrawlPages.indexabilityState,
|
|
11428
10797
|
indexabilityReasons: siteCrawlPages.indexabilityReasons
|
|
11429
|
-
}).from(siteCrawlPages).where(
|
|
10798
|
+
}).from(siteCrawlPages).where(and12(...pageScope, eq14(siteCrawlPages.canonicalUrl, url))).all();
|
|
11430
10799
|
for (const target of canonicalTargets) {
|
|
11431
10800
|
tx.update(siteCrawlPages).set({
|
|
11432
10801
|
canonicalNodeKey: nodeKey,
|
|
11433
10802
|
healthState: deriveSiteHealthState({ ...target, canonicalNodeKey: nodeKey }),
|
|
11434
10803
|
updatedAt: now
|
|
11435
|
-
}).where(
|
|
10804
|
+
}).where(eq14(siteCrawlPages.id, target.id)).run();
|
|
11436
10805
|
}
|
|
11437
10806
|
}
|
|
11438
10807
|
};
|
|
@@ -11447,7 +10816,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11447
10816
|
nodeKey: page.key
|
|
11448
10817
|
});
|
|
11449
10818
|
tx.insert(siteCrawlPages).values({
|
|
11450
|
-
id:
|
|
10819
|
+
id: crypto18.randomUUID(),
|
|
11451
10820
|
projectId,
|
|
11452
10821
|
runId,
|
|
11453
10822
|
attemptId,
|
|
@@ -11497,7 +10866,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11497
10866
|
discoveryProvenance: [{ discoveredFrom: page.provenance.discoveredFrom, sitemapSources: page.provenance.sitemapSources, root: page.provenance.root }],
|
|
11498
10867
|
sitemapMetadata: { sources: page.provenance.sitemapSources },
|
|
11499
10868
|
fetchState: page.state,
|
|
11500
|
-
fetchedAt: page.state === "discovered" || page.state === "robots-blocked" ? null :
|
|
10869
|
+
fetchedAt: page.state === "discovered" || page.state === "robots-blocked" ? null : sql5`coalesce(${siteCrawlPages.fetchedAt}, ${now})`,
|
|
11501
10870
|
httpStatus: page.statusCode,
|
|
11502
10871
|
contentType: page.contentType,
|
|
11503
10872
|
finalUrl: page.finalUrl,
|
|
@@ -11540,7 +10909,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11540
10909
|
if (isSelfLink(edge.from, edge.to)) return;
|
|
11541
10910
|
observedEdges.set(edge.key, edge);
|
|
11542
10911
|
tx.insert(siteCrawlEdges).values({
|
|
11543
|
-
id:
|
|
10912
|
+
id: crypto18.randomUUID(),
|
|
11544
10913
|
projectId,
|
|
11545
10914
|
runId,
|
|
11546
10915
|
attemptId,
|
|
@@ -11588,20 +10957,20 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11588
10957
|
linkScoreRaw: metrics.linkScoreRaw,
|
|
11589
10958
|
linkScoreNormalized: metrics.linkScore,
|
|
11590
10959
|
updatedAt: now
|
|
11591
|
-
}).where(
|
|
11592
|
-
|
|
11593
|
-
|
|
11594
|
-
|
|
11595
|
-
|
|
10960
|
+
}).where(and12(
|
|
10961
|
+
eq14(siteCrawlPages.projectId, projectId),
|
|
10962
|
+
eq14(siteCrawlPages.runId, runId),
|
|
10963
|
+
eq14(siteCrawlPages.attemptId, attemptId),
|
|
10964
|
+
eq14(siteCrawlPages.nodeKey, key)
|
|
11596
10965
|
)).run();
|
|
11597
10966
|
};
|
|
11598
10967
|
const persistEvent = async (event) => {
|
|
11599
10968
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
11600
10969
|
db.transaction((tx) => {
|
|
11601
|
-
const receipt = tx.select({ checksum: siteCrawlEventReceipts.checksum }).from(siteCrawlEventReceipts).where(
|
|
11602
|
-
|
|
11603
|
-
|
|
11604
|
-
|
|
10970
|
+
const receipt = tx.select({ checksum: siteCrawlEventReceipts.checksum }).from(siteCrawlEventReceipts).where(and12(
|
|
10971
|
+
eq14(siteCrawlEventReceipts.attemptId, attemptId),
|
|
10972
|
+
eq14(siteCrawlEventReceipts.sequence, event.sequence),
|
|
10973
|
+
eq14(siteCrawlEventReceipts.batchId, event.batchId)
|
|
11605
10974
|
)).get();
|
|
11606
10975
|
if (receipt) {
|
|
11607
10976
|
if (receipt.checksum !== event.checksum) {
|
|
@@ -11610,7 +10979,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11610
10979
|
return;
|
|
11611
10980
|
}
|
|
11612
10981
|
tx.insert(siteCrawlEventReceipts).values({
|
|
11613
|
-
id:
|
|
10982
|
+
id: crypto18.randomUUID(),
|
|
11614
10983
|
projectId,
|
|
11615
10984
|
runId,
|
|
11616
10985
|
attemptId,
|
|
@@ -11630,22 +10999,22 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11630
10999
|
lastEventChecksum: event.checksum,
|
|
11631
11000
|
pagesDiscovered: progress.pagesDiscovered,
|
|
11632
11001
|
pagesFetched: progress.pagesFetched,
|
|
11633
|
-
pagesEligible: event.type === "summary" ? observedPages.size === 0 ? 0 : [...observedPages.values()].filter(isInventoryEligible).length :
|
|
11634
|
-
pagesErrored: event.type === "summary" ? observedErrorCount(observedPages.values()) :
|
|
11002
|
+
pagesEligible: event.type === "summary" ? observedPages.size === 0 ? 0 : [...observedPages.values()].filter(isInventoryEligible).length : sql5`${siteCrawlAttempts.pagesEligible}`,
|
|
11003
|
+
pagesErrored: event.type === "summary" ? observedErrorCount(observedPages.values()) : sql5`${siteCrawlAttempts.pagesErrored}`,
|
|
11635
11004
|
edgesDiscovered: progress.edgesObserved,
|
|
11636
11005
|
updatedAt: now
|
|
11637
|
-
}).where(
|
|
11006
|
+
}).where(eq14(siteCrawlAttempts.id, attemptId)).run();
|
|
11638
11007
|
} else {
|
|
11639
11008
|
tx.update(siteCrawlAttempts).set({
|
|
11640
11009
|
lastEventSequence: event.sequence,
|
|
11641
11010
|
lastEventChecksum: event.checksum,
|
|
11642
11011
|
updatedAt: now
|
|
11643
|
-
}).where(
|
|
11012
|
+
}).where(eq14(siteCrawlAttempts.id, attemptId)).run();
|
|
11644
11013
|
}
|
|
11645
11014
|
});
|
|
11646
11015
|
};
|
|
11647
11016
|
try {
|
|
11648
|
-
const project = db.select().from(projects).where(
|
|
11017
|
+
const project = db.select().from(projects).where(eq14(projects.id, projectId)).get();
|
|
11649
11018
|
if (!project) throw new Error(`Project not found: ${projectId}`);
|
|
11650
11019
|
canonicalDomain = project.canonicalDomain;
|
|
11651
11020
|
const homepageUrl = toHomepageUrl(project.canonicalDomain);
|
|
@@ -11742,11 +11111,11 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11742
11111
|
}
|
|
11743
11112
|
}
|
|
11744
11113
|
const published = db.transaction((tx) => {
|
|
11745
|
-
const claim2 = tx.update(runs).set({ status: terminalStatus, finishedAt }).where(
|
|
11114
|
+
const claim2 = tx.update(runs).set({ status: terminalStatus, finishedAt }).where(and12(eq14(runs.id, runId), eq14(runs.status, "running"))).run();
|
|
11746
11115
|
if (claim2.changes === 0) return false;
|
|
11747
11116
|
if (hasAuditedPages) {
|
|
11748
11117
|
tx.insert(siteAuditSnapshots).values({
|
|
11749
|
-
id:
|
|
11118
|
+
id: crypto18.randomUUID(),
|
|
11750
11119
|
projectId,
|
|
11751
11120
|
runId,
|
|
11752
11121
|
sitemapUrl: opts.sitemapUrl ?? new URL("/sitemap.xml", crawlRootUrl).href,
|
|
@@ -11764,7 +11133,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11764
11133
|
for (const page of observedPages.values()) {
|
|
11765
11134
|
if (!isLegacyScorecardPage(page)) continue;
|
|
11766
11135
|
tx.insert(siteAuditPages).values({
|
|
11767
|
-
id:
|
|
11136
|
+
id: crypto18.randomUUID(),
|
|
11768
11137
|
projectId,
|
|
11769
11138
|
runId,
|
|
11770
11139
|
url: page.finalUrl ?? page.requestedUrl,
|
|
@@ -11777,7 +11146,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11777
11146
|
}
|
|
11778
11147
|
}
|
|
11779
11148
|
tx.insert(siteCrawlSnapshots).values({
|
|
11780
|
-
id:
|
|
11149
|
+
id: crypto18.randomUUID(),
|
|
11781
11150
|
projectId,
|
|
11782
11151
|
runId,
|
|
11783
11152
|
attemptId,
|
|
@@ -11823,7 +11192,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11823
11192
|
if (opts.checkDeadLinks) {
|
|
11824
11193
|
for (const finding of deadLinkFindings) {
|
|
11825
11194
|
tx.insert(siteCrawlFindings).values({
|
|
11826
|
-
id:
|
|
11195
|
+
id: crypto18.randomUUID(),
|
|
11827
11196
|
projectId,
|
|
11828
11197
|
runId,
|
|
11829
11198
|
attemptId,
|
|
@@ -11849,7 +11218,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11849
11218
|
edgesDiscovered: crawlSummary.edgesObserved,
|
|
11850
11219
|
finishedAt,
|
|
11851
11220
|
updatedAt: finishedAt
|
|
11852
|
-
}).where(
|
|
11221
|
+
}).where(eq14(siteCrawlAttempts.id, attemptId)).run();
|
|
11853
11222
|
return true;
|
|
11854
11223
|
});
|
|
11855
11224
|
if (!published) {
|
|
@@ -11862,7 +11231,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11862
11231
|
error: describeError(cleanupError)
|
|
11863
11232
|
});
|
|
11864
11233
|
}
|
|
11865
|
-
const current = db.select({ status: runs.status }).from(runs).where(
|
|
11234
|
+
const current = db.select({ status: runs.status }).from(runs).where(eq14(runs.id, runId)).get();
|
|
11866
11235
|
if (current?.status === "cancelled") throw new SiteAuditCancelledError(`Site audit cancelled before publication: ${runId}`);
|
|
11867
11236
|
throw new Error(`Site audit run was no longer running before publication: ${runId}`);
|
|
11868
11237
|
}
|
|
@@ -11897,12 +11266,12 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11897
11266
|
error: message,
|
|
11898
11267
|
finishedAt,
|
|
11899
11268
|
updatedAt: finishedAt
|
|
11900
|
-
}).where(
|
|
11269
|
+
}).where(eq14(siteCrawlAttempts.id, attemptId)).run();
|
|
11901
11270
|
tx.update(runs).set({
|
|
11902
11271
|
status: cancelled ? "cancelled" : "failed",
|
|
11903
11272
|
error: message,
|
|
11904
11273
|
finishedAt
|
|
11905
|
-
}).where(
|
|
11274
|
+
}).where(and12(eq14(runs.id, runId), eq14(runs.status, "running"))).run();
|
|
11906
11275
|
});
|
|
11907
11276
|
log13.error("failed", { runId, projectId, cancelled, error: message });
|
|
11908
11277
|
trackSiteAuditOutcome(db, {
|
|
@@ -11918,7 +11287,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11918
11287
|
}
|
|
11919
11288
|
|
|
11920
11289
|
// src/commands/backfill.ts
|
|
11921
|
-
import { and as
|
|
11290
|
+
import { and as and13, eq as eq15, inArray as inArray6, isNull, sql as sql6 } from "drizzle-orm";
|
|
11922
11291
|
var SNAPSHOT_BATCH_SIZE = 500;
|
|
11923
11292
|
async function backfillAnswerVisibilityCommand(opts) {
|
|
11924
11293
|
const config = loadConfig();
|
|
@@ -11926,7 +11295,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
11926
11295
|
migrate(db);
|
|
11927
11296
|
const projectFilter = opts?.project?.trim();
|
|
11928
11297
|
const isDryRun = opts?.dryRun === true;
|
|
11929
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
11298
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq15(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
11930
11299
|
let examined = 0;
|
|
11931
11300
|
let updated = 0;
|
|
11932
11301
|
let wouldUpdate = 0;
|
|
@@ -11934,10 +11303,10 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
11934
11303
|
let reparsed = 0;
|
|
11935
11304
|
let providerErrors = 0;
|
|
11936
11305
|
if (scopedProjects.length > 0) {
|
|
11937
|
-
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
11938
|
-
|
|
11306
|
+
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(and13(
|
|
11307
|
+
eq15(runs.kind, RunKinds["answer-visibility"]),
|
|
11939
11308
|
inArray6(runs.projectId, scopedProjects.map((project) => project.id))
|
|
11940
|
-
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
11309
|
+
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(eq15(runs.kind, RunKinds["answer-visibility"])).all();
|
|
11941
11310
|
const runIdsByProject = /* @__PURE__ */ new Map();
|
|
11942
11311
|
for (const run of runRows) {
|
|
11943
11312
|
const existing = runIdsByProject.get(run.projectId);
|
|
@@ -11945,7 +11314,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
11945
11314
|
else runIdsByProject.set(run.projectId, [run.id]);
|
|
11946
11315
|
}
|
|
11947
11316
|
for (const project of scopedProjects) {
|
|
11948
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
11317
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq15(competitors.projectId, project.id)).all().map((row) => row.domain);
|
|
11949
11318
|
const runIds = runIdsByProject.get(project.id) ?? [];
|
|
11950
11319
|
if (runIds.length === 0) continue;
|
|
11951
11320
|
const projectDomains = effectiveDomains({
|
|
@@ -12038,7 +11407,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
12038
11407
|
} else {
|
|
12039
11408
|
db.transaction((tx) => {
|
|
12040
11409
|
for (const update of pendingUpdates) {
|
|
12041
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
11410
|
+
tx.update(querySnapshots).set(update.patch).where(eq15(querySnapshots.id, update.id)).run();
|
|
12042
11411
|
}
|
|
12043
11412
|
});
|
|
12044
11413
|
updated += pendingUpdates.length;
|
|
@@ -12087,13 +11456,13 @@ No DB writes performed. Re-run without --dry-run to apply.`);
|
|
|
12087
11456
|
function backfillNormalizedPaths(db, opts) {
|
|
12088
11457
|
const baseConditions = [];
|
|
12089
11458
|
if (opts?.projectId) {
|
|
12090
|
-
baseConditions.push(
|
|
11459
|
+
baseConditions.push(eq15(gaTrafficSnapshots.projectId, opts.projectId));
|
|
12091
11460
|
}
|
|
12092
11461
|
const rows = db.select({
|
|
12093
11462
|
id: gaTrafficSnapshots.id,
|
|
12094
11463
|
landingPage: gaTrafficSnapshots.landingPage,
|
|
12095
11464
|
landingPageNormalized: gaTrafficSnapshots.landingPageNormalized
|
|
12096
|
-
}).from(gaTrafficSnapshots).where(baseConditions.length > 0 ?
|
|
11465
|
+
}).from(gaTrafficSnapshots).where(baseConditions.length > 0 ? and13(...baseConditions) : void 0).all();
|
|
12097
11466
|
let updated = 0;
|
|
12098
11467
|
let unchanged = 0;
|
|
12099
11468
|
if (rows.length > 0) {
|
|
@@ -12108,7 +11477,7 @@ function backfillNormalizedPaths(db, opts) {
|
|
|
12108
11477
|
unchanged++;
|
|
12109
11478
|
continue;
|
|
12110
11479
|
}
|
|
12111
|
-
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(
|
|
11480
|
+
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(eq15(gaTrafficSnapshots.id, row.id)).run();
|
|
12112
11481
|
updated++;
|
|
12113
11482
|
}
|
|
12114
11483
|
});
|
|
@@ -12122,7 +11491,7 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
12122
11491
|
const projectFilter = opts?.project?.trim();
|
|
12123
11492
|
let projectId;
|
|
12124
11493
|
if (projectFilter) {
|
|
12125
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
11494
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq15(projects.name, projectFilter)).get();
|
|
12126
11495
|
if (!project) {
|
|
12127
11496
|
const result2 = {
|
|
12128
11497
|
project: projectFilter,
|
|
@@ -12159,13 +11528,13 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
12159
11528
|
function backfillAiReferralPaths(db, opts) {
|
|
12160
11529
|
const baseConditions = [];
|
|
12161
11530
|
if (opts?.projectId) {
|
|
12162
|
-
baseConditions.push(
|
|
11531
|
+
baseConditions.push(eq15(gaAiReferrals.projectId, opts.projectId));
|
|
12163
11532
|
}
|
|
12164
11533
|
const rows = db.select({
|
|
12165
11534
|
id: gaAiReferrals.id,
|
|
12166
11535
|
landingPage: gaAiReferrals.landingPage,
|
|
12167
11536
|
landingPageNormalized: gaAiReferrals.landingPageNormalized
|
|
12168
|
-
}).from(gaAiReferrals).where(baseConditions.length > 0 ?
|
|
11537
|
+
}).from(gaAiReferrals).where(baseConditions.length > 0 ? and13(...baseConditions) : void 0).all();
|
|
12169
11538
|
let updated = 0;
|
|
12170
11539
|
let unchanged = 0;
|
|
12171
11540
|
if (rows.length > 0) {
|
|
@@ -12180,7 +11549,7 @@ function backfillAiReferralPaths(db, opts) {
|
|
|
12180
11549
|
unchanged++;
|
|
12181
11550
|
continue;
|
|
12182
11551
|
}
|
|
12183
|
-
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(
|
|
11552
|
+
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(eq15(gaAiReferrals.id, row.id)).run();
|
|
12184
11553
|
updated++;
|
|
12185
11554
|
}
|
|
12186
11555
|
});
|
|
@@ -12194,7 +11563,7 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
12194
11563
|
const projectFilter = opts?.project?.trim();
|
|
12195
11564
|
let projectId;
|
|
12196
11565
|
if (projectFilter) {
|
|
12197
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
11566
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq15(projects.name, projectFilter)).get();
|
|
12198
11567
|
if (!project) {
|
|
12199
11568
|
const result2 = {
|
|
12200
11569
|
project: projectFilter,
|
|
@@ -12230,10 +11599,10 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
12230
11599
|
}
|
|
12231
11600
|
function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
12232
11601
|
const isDryRun = opts?.dryRun === true;
|
|
12233
|
-
const project = db.select().from(projects).where(
|
|
11602
|
+
const project = db.select().from(projects).where(eq15(projects.id, projectId)).get();
|
|
12234
11603
|
if (!project) return { examined: 0, updated: 0, mentioned: 0 };
|
|
12235
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
12236
|
-
const runRows = db.select({ id: runs.id }).from(runs).where(
|
|
11604
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq15(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
11605
|
+
const runRows = db.select({ id: runs.id }).from(runs).where(and13(eq15(runs.kind, RunKinds["answer-visibility"]), eq15(runs.projectId, projectId))).all();
|
|
12237
11606
|
const runIds = runRows.map((r) => r.id);
|
|
12238
11607
|
let examined = 0;
|
|
12239
11608
|
let updated = 0;
|
|
@@ -12307,7 +11676,7 @@ function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
|
12307
11676
|
} else {
|
|
12308
11677
|
db.transaction((tx) => {
|
|
12309
11678
|
for (const update of pendingUpdates) {
|
|
12310
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
11679
|
+
tx.update(querySnapshots).set(update.patch).where(eq15(querySnapshots.id, update.id)).run();
|
|
12311
11680
|
}
|
|
12312
11681
|
});
|
|
12313
11682
|
updated += pendingUpdates.length;
|
|
@@ -12322,7 +11691,7 @@ async function backfillAnswerMentionsCommand(opts) {
|
|
|
12322
11691
|
migrate(db);
|
|
12323
11692
|
const projectFilter = opts?.project?.trim();
|
|
12324
11693
|
const isDryRun = opts?.dryRun === true;
|
|
12325
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
11694
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq15(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
12326
11695
|
let examined = 0;
|
|
12327
11696
|
let updated = 0;
|
|
12328
11697
|
let wouldUpdate = 0;
|
|
@@ -12382,7 +11751,7 @@ function readStoredGroundingSources(rawResponse) {
|
|
|
12382
11751
|
return result;
|
|
12383
11752
|
}
|
|
12384
11753
|
async function backfillInsightsCommand(project, opts) {
|
|
12385
|
-
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-
|
|
11754
|
+
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-SJO5E7WN.js");
|
|
12386
11755
|
const config = loadConfig();
|
|
12387
11756
|
const db = createClient(config.database);
|
|
12388
11757
|
migrate(db);
|
|
@@ -12541,7 +11910,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
12541
11910
|
const config = loadConfig();
|
|
12542
11911
|
const db = createClient(config.database);
|
|
12543
11912
|
migrate(db);
|
|
12544
|
-
const project = db.select().from(projects).where(
|
|
11913
|
+
const project = db.select().from(projects).where(eq15(projects.name, opts.project)).get();
|
|
12545
11914
|
if (!project) {
|
|
12546
11915
|
throw new Error(`Project "${opts.project}" not found`);
|
|
12547
11916
|
}
|
|
@@ -12552,8 +11921,8 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
12552
11921
|
process.stderr.write(`Recovering orphan snapshot attribution for "${project.name}"${mode}...
|
|
12553
11922
|
`);
|
|
12554
11923
|
}
|
|
12555
|
-
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(
|
|
12556
|
-
|
|
11924
|
+
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(and13(
|
|
11925
|
+
eq15(auditLog.projectId, project.id),
|
|
12557
11926
|
inArray6(auditLog.action, ["keywords.appended", "keywords.deleted", "queries.appended", "queries.deleted", "queries.replaced"])
|
|
12558
11927
|
)).orderBy(auditLog.createdAt).all();
|
|
12559
11928
|
const history = replayQueryAuditLog(events);
|
|
@@ -12561,10 +11930,10 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
12561
11930
|
runId: runs.id,
|
|
12562
11931
|
createdAt: runs.createdAt,
|
|
12563
11932
|
location: runs.location
|
|
12564
|
-
}).from(runs).innerJoin(querySnapshots,
|
|
12565
|
-
|
|
12566
|
-
|
|
12567
|
-
|
|
11933
|
+
}).from(runs).innerJoin(querySnapshots, eq15(querySnapshots.runId, runs.id)).where(and13(
|
|
11934
|
+
eq15(runs.projectId, project.id),
|
|
11935
|
+
isNull(querySnapshots.queryId),
|
|
11936
|
+
isNull(querySnapshots.queryText)
|
|
12568
11937
|
)).groupBy(runs.id).orderBy(runs.createdAt).all();
|
|
12569
11938
|
const result = {
|
|
12570
11939
|
project: project.name,
|
|
@@ -12584,10 +11953,10 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
12584
11953
|
provider: querySnapshots.provider,
|
|
12585
11954
|
createdAt: querySnapshots.createdAt,
|
|
12586
11955
|
answerText: querySnapshots.answerText
|
|
12587
|
-
}).from(querySnapshots).where(
|
|
12588
|
-
|
|
12589
|
-
|
|
12590
|
-
|
|
11956
|
+
}).from(querySnapshots).where(and13(
|
|
11957
|
+
eq15(querySnapshots.runId, run.runId),
|
|
11958
|
+
isNull(querySnapshots.queryId),
|
|
11959
|
+
isNull(querySnapshots.queryText)
|
|
12591
11960
|
)).orderBy(querySnapshots.provider, querySnapshots.createdAt).all();
|
|
12592
11961
|
const byProvider = /* @__PURE__ */ new Map();
|
|
12593
11962
|
for (const s of orphanSnaps) {
|
|
@@ -12651,7 +12020,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
12651
12020
|
if (!isDryRun && updates.length > 0) {
|
|
12652
12021
|
db.transaction((tx) => {
|
|
12653
12022
|
for (const u of updates) {
|
|
12654
|
-
tx.update(querySnapshots).set({ queryText: u.queryText }).where(
|
|
12023
|
+
tx.update(querySnapshots).set({ queryText: u.queryText }).where(eq15(querySnapshots.id, u.id)).run();
|
|
12655
12024
|
}
|
|
12656
12025
|
});
|
|
12657
12026
|
}
|
|
@@ -12725,7 +12094,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
12725
12094
|
const projectFilter = opts?.project?.trim();
|
|
12726
12095
|
const isDryRun = opts?.dryRun === true;
|
|
12727
12096
|
const isJson = isMachineFormat(opts?.format);
|
|
12728
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
12097
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq15(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
12729
12098
|
if (scopedProjects.length === 0) {
|
|
12730
12099
|
if (projectFilter && !isJson) {
|
|
12731
12100
|
process.stderr.write(`No project named "${projectFilter}".
|
|
@@ -12750,8 +12119,8 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
12750
12119
|
dryRun: isDryRun,
|
|
12751
12120
|
byBot: {}
|
|
12752
12121
|
};
|
|
12753
|
-
const unknownCountRow = db.select({ n:
|
|
12754
|
-
|
|
12122
|
+
const unknownCountRow = db.select({ n: sql6`count(*)` }).from(rawEventSamples).where(and13(
|
|
12123
|
+
eq15(rawEventSamples.eventType, "unknown"),
|
|
12755
12124
|
inArray6(rawEventSamples.projectId, projectIds)
|
|
12756
12125
|
)).get();
|
|
12757
12126
|
result.unknownBefore = Number(unknownCountRow?.n ?? 0);
|
|
@@ -12763,8 +12132,8 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
12763
12132
|
userAgent: rawEventSamples.userAgent,
|
|
12764
12133
|
pathNormalized: rawEventSamples.pathNormalized,
|
|
12765
12134
|
status: rawEventSamples.status
|
|
12766
|
-
}).from(rawEventSamples).where(
|
|
12767
|
-
|
|
12135
|
+
}).from(rawEventSamples).where(and13(
|
|
12136
|
+
eq15(rawEventSamples.eventType, "unknown"),
|
|
12768
12137
|
inArray6(rawEventSamples.projectId, projectIds)
|
|
12769
12138
|
)).all();
|
|
12770
12139
|
result.examined = unknownSamples.length;
|
|
@@ -12808,7 +12177,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
12808
12177
|
const tsHourIso = tsHour.toISOString();
|
|
12809
12178
|
const status = snap.status ?? 200;
|
|
12810
12179
|
db.transaction((tx) => {
|
|
12811
|
-
tx.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(
|
|
12180
|
+
tx.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(eq15(rawEventSamples.id, snap.id)).run();
|
|
12812
12181
|
if (userFetch) {
|
|
12813
12182
|
tx.insert(aiUserFetchEventsHourly).values({
|
|
12814
12183
|
projectId: snap.projectId,
|
|
@@ -12834,7 +12203,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
12834
12203
|
aiUserFetchEventsHourly.status
|
|
12835
12204
|
],
|
|
12836
12205
|
set: {
|
|
12837
|
-
hits:
|
|
12206
|
+
hits: sql6`${aiUserFetchEventsHourly.hits} + 1`,
|
|
12838
12207
|
updatedAt: now
|
|
12839
12208
|
}
|
|
12840
12209
|
}).run();
|
|
@@ -12863,7 +12232,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
12863
12232
|
crawlerEventsHourly.status
|
|
12864
12233
|
],
|
|
12865
12234
|
set: {
|
|
12866
|
-
hits:
|
|
12235
|
+
hits: sql6`${crawlerEventsHourly.hits} + 1`,
|
|
12867
12236
|
updatedAt: now
|
|
12868
12237
|
}
|
|
12869
12238
|
}).run();
|
|
@@ -12871,8 +12240,8 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
12871
12240
|
});
|
|
12872
12241
|
}
|
|
12873
12242
|
if (!isDryRun) {
|
|
12874
|
-
const afterRow = db.select({ n:
|
|
12875
|
-
|
|
12243
|
+
const afterRow = db.select({ n: sql6`count(*)` }).from(rawEventSamples).where(and13(
|
|
12244
|
+
eq15(rawEventSamples.eventType, "unknown"),
|
|
12876
12245
|
inArray6(rawEventSamples.projectId, projectIds)
|
|
12877
12246
|
)).get();
|
|
12878
12247
|
result.unknownAfter = Number(afterRow?.n ?? 0);
|
|
@@ -12896,8 +12265,8 @@ Traffic classification ${result.dryRun ? "preview" : "recovery"} complete.`);
|
|
|
12896
12265
|
if (result.reclassified > 0) {
|
|
12897
12266
|
console.log(` By bot:`);
|
|
12898
12267
|
const sorted = Object.entries(result.byBot).sort(([, a], [, b]) => b - a);
|
|
12899
|
-
for (const [bot,
|
|
12900
|
-
console.log(` ${
|
|
12268
|
+
for (const [bot, count] of sorted) {
|
|
12269
|
+
console.log(` ${count.toString().padStart(5)} ${bot}`);
|
|
12901
12270
|
}
|
|
12902
12271
|
}
|
|
12903
12272
|
if (result.dryRun) {
|
|
@@ -13018,14 +12387,14 @@ function createProviderModelCatalog(registry) {
|
|
|
13018
12387
|
}
|
|
13019
12388
|
|
|
13020
12389
|
// src/scheduler.ts
|
|
13021
|
-
import
|
|
12390
|
+
import crypto19 from "crypto";
|
|
13022
12391
|
import cron from "node-cron";
|
|
13023
|
-
import { and as
|
|
12392
|
+
import { and as and14, eq as eq16, inArray as inArray7, notExists, sql as sql7 } from "drizzle-orm";
|
|
13024
12393
|
var log14 = createLogger("Scheduler");
|
|
13025
12394
|
var DEFAULT_HEALTH_CRON = "0 */6 * * *";
|
|
13026
12395
|
function ensureDefaultHealthSchedule(db, projectId, now = (/* @__PURE__ */ new Date()).toISOString()) {
|
|
13027
12396
|
const result = db.insert(schedules).values({
|
|
13028
|
-
id:
|
|
12397
|
+
id: crypto19.randomUUID(),
|
|
13029
12398
|
projectId,
|
|
13030
12399
|
kind: SchedulableRunKinds.doctor,
|
|
13031
12400
|
cronExpr: DEFAULT_HEALTH_CRON,
|
|
@@ -13071,9 +12440,9 @@ var Scheduler = class {
|
|
|
13071
12440
|
*/
|
|
13072
12441
|
ensureHealthSchedules() {
|
|
13073
12442
|
const projectsWithoutHealth = this.db.select({ id: projects.id }).from(projects).where(notExists(
|
|
13074
|
-
this.db.select({ one:
|
|
13075
|
-
|
|
13076
|
-
|
|
12443
|
+
this.db.select({ one: sql7`1` }).from(schedules).where(and14(
|
|
12444
|
+
eq16(schedules.projectId, projects.id),
|
|
12445
|
+
eq16(schedules.kind, SchedulableRunKinds.doctor)
|
|
13077
12446
|
))
|
|
13078
12447
|
)).all();
|
|
13079
12448
|
if (projectsWithoutHealth.length === 0) return;
|
|
@@ -13116,7 +12485,7 @@ var Scheduler = class {
|
|
|
13116
12485
|
start() {
|
|
13117
12486
|
this.ensureHealthSchedules();
|
|
13118
12487
|
this.ensureQueryBaskets();
|
|
13119
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
12488
|
+
const allSchedules = this.db.select().from(schedules).where(eq16(schedules.enabled, true)).all();
|
|
13120
12489
|
for (const schedule of allSchedules) {
|
|
13121
12490
|
const missedRunAt = schedule.nextRunAt;
|
|
13122
12491
|
const recurrence = scheduleRecurrence(schedule);
|
|
@@ -13150,7 +12519,7 @@ var Scheduler = class {
|
|
|
13150
12519
|
this.stopTask(key, existing, "Stopped");
|
|
13151
12520
|
this.tasks.delete(key);
|
|
13152
12521
|
}
|
|
13153
|
-
const schedule = this.db.select().from(schedules).where(
|
|
12522
|
+
const schedule = this.db.select().from(schedules).where(and14(eq16(schedules.projectId, projectId), eq16(schedules.kind, kind))).get();
|
|
13154
12523
|
if (schedule && schedule.enabled) {
|
|
13155
12524
|
this.registerCronTask(schedule);
|
|
13156
12525
|
}
|
|
@@ -13178,12 +12547,12 @@ var Scheduler = class {
|
|
|
13178
12547
|
/** Runtime timing writes must not restore a version already issued to a settings caller. */
|
|
13179
12548
|
updateScheduleTiming(scheduleId, timing) {
|
|
13180
12549
|
this.db.transaction((tx) => {
|
|
13181
|
-
const current = tx.select({ updatedAt: schedules.updatedAt }).from(schedules).where(
|
|
12550
|
+
const current = tx.select({ updatedAt: schedules.updatedAt }).from(schedules).where(eq16(schedules.id, scheduleId)).get();
|
|
13182
12551
|
if (!current) return;
|
|
13183
12552
|
tx.update(schedules).set({
|
|
13184
12553
|
...timing,
|
|
13185
12554
|
updatedAt: nextScheduleUpdatedAt(current.updatedAt)
|
|
13186
|
-
}).where(
|
|
12555
|
+
}).where(eq16(schedules.id, scheduleId)).run();
|
|
13187
12556
|
});
|
|
13188
12557
|
}
|
|
13189
12558
|
registerCronTask(schedule, options = {}) {
|
|
@@ -13244,7 +12613,7 @@ var Scheduler = class {
|
|
|
13244
12613
|
};
|
|
13245
12614
|
const tick = () => {
|
|
13246
12615
|
if (stopped) return;
|
|
13247
|
-
const current = this.db.select().from(schedules).where(
|
|
12616
|
+
const current = this.db.select().from(schedules).where(eq16(schedules.id, scheduleId)).get();
|
|
13248
12617
|
if (!current || !current.enabled || !scheduleRecurrence(current)) {
|
|
13249
12618
|
this.remove(projectId, kind);
|
|
13250
12619
|
return;
|
|
@@ -13262,7 +12631,7 @@ var Scheduler = class {
|
|
|
13262
12631
|
this.triggerRun(scheduleId, projectId, kind);
|
|
13263
12632
|
}
|
|
13264
12633
|
}
|
|
13265
|
-
arm(this.db.select({ nextRunAt: schedules.nextRunAt }).from(schedules).where(
|
|
12634
|
+
arm(this.db.select({ nextRunAt: schedules.nextRunAt }).from(schedules).where(eq16(schedules.id, scheduleId)).get()?.nextRunAt);
|
|
13266
12635
|
};
|
|
13267
12636
|
this.tasks.set(key, task);
|
|
13268
12637
|
let registeredNextRunAt = schedule.nextRunAt;
|
|
@@ -13277,7 +12646,7 @@ var Scheduler = class {
|
|
|
13277
12646
|
/** Atomically advance a due callback-only calendar occurrence before dispatch. */
|
|
13278
12647
|
claimCalendarOccurrence(schedule, dueAt, now) {
|
|
13279
12648
|
return this.db.transaction((tx) => {
|
|
13280
|
-
const current = tx.select().from(schedules).where(
|
|
12649
|
+
const current = tx.select().from(schedules).where(eq16(schedules.id, schedule.id)).get();
|
|
13281
12650
|
if (!current || !current.enabled || current.updatedAt !== schedule.updatedAt || current.nextRunAt !== dueAt) return false;
|
|
13282
12651
|
const recurrence = scheduleRecurrence(current);
|
|
13283
12652
|
if (!recurrence) return false;
|
|
@@ -13286,11 +12655,11 @@ var Scheduler = class {
|
|
|
13286
12655
|
const result = tx.update(schedules).set({
|
|
13287
12656
|
nextRunAt,
|
|
13288
12657
|
updatedAt: nextScheduleUpdatedAt(current.updatedAt)
|
|
13289
|
-
}).where(
|
|
13290
|
-
|
|
13291
|
-
|
|
13292
|
-
|
|
13293
|
-
|
|
12658
|
+
}).where(and14(
|
|
12659
|
+
eq16(schedules.id, current.id),
|
|
12660
|
+
eq16(schedules.enabled, true),
|
|
12661
|
+
eq16(schedules.nextRunAt, dueAt),
|
|
12662
|
+
eq16(schedules.updatedAt, schedule.updatedAt)
|
|
13294
12663
|
)).run();
|
|
13295
12664
|
return result.changes === 1;
|
|
13296
12665
|
});
|
|
@@ -13298,7 +12667,7 @@ var Scheduler = class {
|
|
|
13298
12667
|
triggerRun(scheduleId, projectId, kind, claimedOccurrence) {
|
|
13299
12668
|
try {
|
|
13300
12669
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
13301
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
12670
|
+
const currentSchedule = this.db.select().from(schedules).where(eq16(schedules.id, scheduleId)).get();
|
|
13302
12671
|
if (!currentSchedule || !currentSchedule.enabled) {
|
|
13303
12672
|
log14.warn("schedule.stale", { scheduleId, projectId, kind, msg: "schedule no longer exists or is disabled" });
|
|
13304
12673
|
this.remove(projectId, kind);
|
|
@@ -13314,7 +12683,7 @@ var Scheduler = class {
|
|
|
13314
12683
|
timezone: currentSchedule.timezone,
|
|
13315
12684
|
recurrence
|
|
13316
12685
|
});
|
|
13317
|
-
const project = this.db.select().from(projects).where(
|
|
12686
|
+
const project = this.db.select().from(projects).where(eq16(projects.id, projectId)).get();
|
|
13318
12687
|
if (!project) {
|
|
13319
12688
|
log14.error("project.not-found", { projectId, kind, msg: "skipping scheduled run" });
|
|
13320
12689
|
this.remove(projectId, kind);
|
|
@@ -13343,7 +12712,7 @@ var Scheduler = class {
|
|
|
13343
12712
|
log14.warn("gbp-sync.no-callback", { scheduleId, projectId, msg: "host did not register onGbpSyncRequested" });
|
|
13344
12713
|
return;
|
|
13345
12714
|
}
|
|
13346
|
-
const runId2 =
|
|
12715
|
+
const runId2 = crypto19.randomUUID();
|
|
13347
12716
|
this.db.insert(runs).values({
|
|
13348
12717
|
id: runId2,
|
|
13349
12718
|
projectId,
|
|
@@ -13365,9 +12734,9 @@ var Scheduler = class {
|
|
|
13365
12734
|
log14.warn("ads-sync.no-callback", { scheduleId, projectId, msg: "host did not register onAdsSyncRequested" });
|
|
13366
12735
|
return;
|
|
13367
12736
|
}
|
|
13368
|
-
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(
|
|
13369
|
-
|
|
13370
|
-
|
|
12737
|
+
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(and14(
|
|
12738
|
+
eq16(runs.projectId, projectId),
|
|
12739
|
+
eq16(runs.kind, RunKinds["ads-sync"]),
|
|
13371
12740
|
inArray7(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
13372
12741
|
)).get();
|
|
13373
12742
|
if (activeAdsRun) {
|
|
@@ -13375,7 +12744,7 @@ var Scheduler = class {
|
|
|
13375
12744
|
this.updateScheduleTiming(currentSchedule.id, { nextRunAt });
|
|
13376
12745
|
return;
|
|
13377
12746
|
}
|
|
13378
|
-
const runId2 =
|
|
12747
|
+
const runId2 = crypto19.randomUUID();
|
|
13379
12748
|
this.db.insert(runs).values({
|
|
13380
12749
|
id: runId2,
|
|
13381
12750
|
projectId,
|
|
@@ -13436,9 +12805,9 @@ var Scheduler = class {
|
|
|
13436
12805
|
log14.warn("site-audit.no-callback", { scheduleId, projectId, msg: "host did not register onSiteAuditRequested" });
|
|
13437
12806
|
return;
|
|
13438
12807
|
}
|
|
13439
|
-
const active = this.db.select({ id: runs.id }).from(runs).where(
|
|
13440
|
-
|
|
13441
|
-
|
|
12808
|
+
const active = this.db.select({ id: runs.id }).from(runs).where(and14(
|
|
12809
|
+
eq16(runs.projectId, projectId),
|
|
12810
|
+
eq16(runs.kind, RunKinds["site-audit"]),
|
|
13442
12811
|
inArray7(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
13443
12812
|
)).get();
|
|
13444
12813
|
if (active) {
|
|
@@ -13446,7 +12815,7 @@ var Scheduler = class {
|
|
|
13446
12815
|
this.updateScheduleTiming(currentSchedule.id, { nextRunAt });
|
|
13447
12816
|
return;
|
|
13448
12817
|
}
|
|
13449
|
-
const runId2 =
|
|
12818
|
+
const runId2 = crypto19.randomUUID();
|
|
13450
12819
|
const effectiveRequest = normalizeSiteAuditRunRequest({});
|
|
13451
12820
|
this.db.transaction((tx) => {
|
|
13452
12821
|
tx.insert(runs).values({
|
|
@@ -13560,8 +12929,8 @@ async function refreshAllIntegrations(client, projectName) {
|
|
|
13560
12929
|
}
|
|
13561
12930
|
|
|
13562
12931
|
// src/notifier.ts
|
|
13563
|
-
import { eq as
|
|
13564
|
-
import
|
|
12932
|
+
import { eq as eq17, desc as desc6, and as and15, inArray as inArray8, or } from "drizzle-orm";
|
|
12933
|
+
import crypto20 from "crypto";
|
|
13565
12934
|
var log16 = createLogger("Notifier");
|
|
13566
12935
|
var MATERIAL_MAGNITUDE_DELTA = 20;
|
|
13567
12936
|
var SEVERITY_RANK = { low: 0, medium: 1, high: 2, critical: 3 };
|
|
@@ -13588,18 +12957,18 @@ var Notifier = class {
|
|
|
13588
12957
|
/** Called after a run completes (success, partial, or failed). */
|
|
13589
12958
|
async onRunCompleted(runId, projectId) {
|
|
13590
12959
|
log16.info("run.completed", { runId, projectId });
|
|
13591
|
-
const notifs = this.db.select().from(notifications).where(
|
|
12960
|
+
const notifs = this.db.select().from(notifications).where(eq17(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
13592
12961
|
if (notifs.length === 0) {
|
|
13593
12962
|
log16.info("notifications.none-enabled", { projectId });
|
|
13594
12963
|
return;
|
|
13595
12964
|
}
|
|
13596
12965
|
log16.info("notifications.found", { projectId, count: notifs.length });
|
|
13597
|
-
const run = this.db.select().from(runs).where(
|
|
12966
|
+
const run = this.db.select().from(runs).where(eq17(runs.id, runId)).get();
|
|
13598
12967
|
if (!run) {
|
|
13599
12968
|
log16.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
13600
12969
|
return;
|
|
13601
12970
|
}
|
|
13602
|
-
const project = this.db.select().from(projects).where(
|
|
12971
|
+
const project = this.db.select().from(projects).where(eq17(projects.id, projectId)).get();
|
|
13603
12972
|
if (!project) {
|
|
13604
12973
|
log16.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
13605
12974
|
return;
|
|
@@ -13652,7 +13021,7 @@ var Notifier = class {
|
|
|
13652
13021
|
* inferred from the absence of a webhook.
|
|
13653
13022
|
*/
|
|
13654
13023
|
async onHealthChecked(projectId, report) {
|
|
13655
|
-
const project = this.db.select().from(projects).where(
|
|
13024
|
+
const project = this.db.select().from(projects).where(eq17(projects.id, projectId)).get();
|
|
13656
13025
|
if (!project) {
|
|
13657
13026
|
log16.error("project.not-found", { projectId, msg: "skipping health notification" });
|
|
13658
13027
|
return null;
|
|
@@ -13683,7 +13052,7 @@ var Notifier = class {
|
|
|
13683
13052
|
const status = noSignal ? "warn" : worst ? worst.status : "ok";
|
|
13684
13053
|
const code = noSignal ? "health.no-signal" : worst?.code ?? "health.ok";
|
|
13685
13054
|
const summary = noSignal ? `No health check produced a result (${report.checks.length} skipped) \u2014 health is unknown, not confirmed.` : worst?.summary ?? `All ${graded.length} health check(s) passing.`;
|
|
13686
|
-
const previous = this.db.select().from(doctorHealthState).where(
|
|
13055
|
+
const previous = this.db.select().from(doctorHealthState).where(eq17(doctorHealthState.projectId, projectId)).get();
|
|
13687
13056
|
const previousStatus = previous?.status ?? null;
|
|
13688
13057
|
let event = null;
|
|
13689
13058
|
if (previous === void 0) {
|
|
@@ -13698,13 +13067,13 @@ var Notifier = class {
|
|
|
13698
13067
|
if (previous === void 0) {
|
|
13699
13068
|
this.db.insert(doctorHealthState).values({ ...observation, notifiedAt: null }).run();
|
|
13700
13069
|
} else {
|
|
13701
|
-
this.db.update(doctorHealthState).set(observation).where(
|
|
13070
|
+
this.db.update(doctorHealthState).set(observation).where(eq17(doctorHealthState.projectId, projectId)).run();
|
|
13702
13071
|
}
|
|
13703
13072
|
if (!event) {
|
|
13704
13073
|
log16.info("health.unchanged", { projectId, status, code });
|
|
13705
13074
|
return null;
|
|
13706
13075
|
}
|
|
13707
|
-
const notifs = this.db.select().from(notifications).where(
|
|
13076
|
+
const notifs = this.db.select().from(notifications).where(eq17(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
13708
13077
|
const payload = {
|
|
13709
13078
|
source: "canonry",
|
|
13710
13079
|
event,
|
|
@@ -13728,7 +13097,7 @@ var Notifier = class {
|
|
|
13728
13097
|
delivered += 1;
|
|
13729
13098
|
}
|
|
13730
13099
|
if (delivered > 0) {
|
|
13731
|
-
this.db.update(doctorHealthState).set({ notifiedAt: now }).where(
|
|
13100
|
+
this.db.update(doctorHealthState).set({ notifiedAt: now }).where(eq17(doctorHealthState.projectId, projectId)).run();
|
|
13732
13101
|
}
|
|
13733
13102
|
log16.info("health.notified", { projectId, event, status, code, subscribers: notifs.length, delivered });
|
|
13734
13103
|
return event;
|
|
@@ -13751,7 +13120,7 @@ var Notifier = class {
|
|
|
13751
13120
|
*/
|
|
13752
13121
|
isNewInsightNews(projectId, insight) {
|
|
13753
13122
|
const key = insightNotifyKey(projectId, insight);
|
|
13754
|
-
const previous = this.db.select().from(insightNotifyState).where(
|
|
13123
|
+
const previous = this.db.select().from(insightNotifyState).where(eq17(insightNotifyState.key, key)).get();
|
|
13755
13124
|
if (!previous) return true;
|
|
13756
13125
|
if (severityRank(insight.severity) > severityRank(previous.severity)) return true;
|
|
13757
13126
|
const magnitude = insightMagnitude(insight);
|
|
@@ -13773,9 +13142,9 @@ var Notifier = class {
|
|
|
13773
13142
|
magnitude: insightMagnitude(insight),
|
|
13774
13143
|
notifiedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
13775
13144
|
};
|
|
13776
|
-
const existing = this.db.select().from(insightNotifyState).where(
|
|
13145
|
+
const existing = this.db.select().from(insightNotifyState).where(eq17(insightNotifyState.key, key)).get();
|
|
13777
13146
|
if (existing) {
|
|
13778
|
-
this.db.update(insightNotifyState).set(row).where(
|
|
13147
|
+
this.db.update(insightNotifyState).set(row).where(eq17(insightNotifyState.key, key)).run();
|
|
13779
13148
|
} else {
|
|
13780
13149
|
this.db.insert(insightNotifyState).values(row).run();
|
|
13781
13150
|
}
|
|
@@ -13789,11 +13158,11 @@ var Notifier = class {
|
|
|
13789
13158
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
13790
13159
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
13791
13160
|
if (insightEvents.length === 0) return;
|
|
13792
|
-
const notifs = this.db.select().from(notifications).where(
|
|
13161
|
+
const notifs = this.db.select().from(notifications).where(eq17(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
13793
13162
|
if (notifs.length === 0) return;
|
|
13794
|
-
const run = this.db.select().from(runs).where(
|
|
13163
|
+
const run = this.db.select().from(runs).where(eq17(runs.id, runId)).get();
|
|
13795
13164
|
if (!run) return;
|
|
13796
|
-
const project = this.db.select().from(projects).where(
|
|
13165
|
+
const project = this.db.select().from(projects).where(eq17(projects.id, projectId)).get();
|
|
13797
13166
|
if (!project) return;
|
|
13798
13167
|
for (const notif of notifs) {
|
|
13799
13168
|
const config = notif.config;
|
|
@@ -13824,7 +13193,7 @@ var Notifier = class {
|
|
|
13824
13193
|
}
|
|
13825
13194
|
}
|
|
13826
13195
|
computeTransitions(runId, projectId) {
|
|
13827
|
-
const thisRun = this.db.select().from(runs).where(
|
|
13196
|
+
const thisRun = this.db.select().from(runs).where(eq17(runs.id, runId)).get();
|
|
13828
13197
|
if (!thisRun) return [];
|
|
13829
13198
|
const completeness = measurementRunCompleteness(this.db, runId);
|
|
13830
13199
|
if (completeness.planned && !completeness.complete) {
|
|
@@ -13835,10 +13204,10 @@ var Notifier = class {
|
|
|
13835
13204
|
});
|
|
13836
13205
|
return [];
|
|
13837
13206
|
}
|
|
13838
|
-
const groupSiblings = this.db.select().from(runs).where(
|
|
13839
|
-
|
|
13840
|
-
|
|
13841
|
-
|
|
13207
|
+
const groupSiblings = this.db.select().from(runs).where(and15(
|
|
13208
|
+
eq17(runs.projectId, projectId),
|
|
13209
|
+
eq17(runs.kind, thisRun.kind),
|
|
13210
|
+
eq17(runs.createdAt, thisRun.createdAt)
|
|
13842
13211
|
)).all();
|
|
13843
13212
|
const stillPending = groupSiblings.some((r) => r.status === "queued" || r.status === "running");
|
|
13844
13213
|
if (stillPending) return [];
|
|
@@ -13854,19 +13223,19 @@ var Notifier = class {
|
|
|
13854
13223
|
return candidate.id > best.id ? candidate : best;
|
|
13855
13224
|
});
|
|
13856
13225
|
if (winner.id !== runId) return [];
|
|
13857
|
-
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
13226
|
+
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(eq17(projects.id, projectId)).get();
|
|
13858
13227
|
const locationCount = Math.max(
|
|
13859
13228
|
1,
|
|
13860
13229
|
(projectLocations?.locations ?? []).length
|
|
13861
13230
|
);
|
|
13862
13231
|
const RECENT_FETCH_LIMIT = Math.max(8, locationCount * 4);
|
|
13863
13232
|
const recentRuns = this.db.select().from(runs).where(
|
|
13864
|
-
|
|
13865
|
-
|
|
13866
|
-
|
|
13867
|
-
|
|
13233
|
+
and15(
|
|
13234
|
+
eq17(runs.projectId, projectId),
|
|
13235
|
+
eq17(runs.kind, thisRun.kind),
|
|
13236
|
+
or(eq17(runs.status, "completed"), eq17(runs.status, "partial"))
|
|
13868
13237
|
)
|
|
13869
|
-
).orderBy(
|
|
13238
|
+
).orderBy(desc6(runs.createdAt), desc6(runs.id)).limit(RECENT_FETCH_LIMIT).all();
|
|
13870
13239
|
const groups = groupRunsByCreatedAt(recentRuns);
|
|
13871
13240
|
const currentGroupIdx = groups.findIndex((g) => g[0]?.createdAt === thisRun.createdAt);
|
|
13872
13241
|
if (currentGroupIdx < 0) return [];
|
|
@@ -13881,7 +13250,7 @@ var Notifier = class {
|
|
|
13881
13250
|
provider: querySnapshots.provider,
|
|
13882
13251
|
location: querySnapshots.location,
|
|
13883
13252
|
citationState: querySnapshots.citationState
|
|
13884
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
13253
|
+
}).from(querySnapshots).leftJoin(queries, eq17(querySnapshots.queryId, queries.id)).where(inArray8(querySnapshots.runId, currentRunIds)).all();
|
|
13885
13254
|
const previousSnapshots = this.db.select({
|
|
13886
13255
|
queryId: querySnapshots.queryId,
|
|
13887
13256
|
provider: querySnapshots.provider,
|
|
@@ -13951,7 +13320,7 @@ var Notifier = class {
|
|
|
13951
13320
|
}
|
|
13952
13321
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
13953
13322
|
this.db.insert(auditLog).values({
|
|
13954
|
-
id:
|
|
13323
|
+
id: crypto20.randomUUID(),
|
|
13955
13324
|
projectId,
|
|
13956
13325
|
actor: "scheduler",
|
|
13957
13326
|
action: `notification.${status}`,
|
|
@@ -13964,7 +13333,7 @@ var Notifier = class {
|
|
|
13964
13333
|
};
|
|
13965
13334
|
|
|
13966
13335
|
// src/run-coordinator.ts
|
|
13967
|
-
import { eq as
|
|
13336
|
+
import { eq as eq18 } from "drizzle-orm";
|
|
13968
13337
|
var log17 = createLogger("RunCoordinator");
|
|
13969
13338
|
var RunCoordinator = class {
|
|
13970
13339
|
constructor(db, notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
@@ -13980,7 +13349,7 @@ var RunCoordinator = class {
|
|
|
13980
13349
|
onInsightsGenerated;
|
|
13981
13350
|
onAeroEvent;
|
|
13982
13351
|
async onRunCompleted(runId, projectId) {
|
|
13983
|
-
const runRow = this.db.select().from(runs).where(
|
|
13352
|
+
const runRow = this.db.select().from(runs).where(eq18(runs.id, runId)).get();
|
|
13984
13353
|
const kind = runRow?.kind ?? RunKinds["answer-visibility"];
|
|
13985
13354
|
if (runRow?.trigger === RunTriggers.probe) {
|
|
13986
13355
|
log17.info("probe.skip-side-effects", { runId, projectId, kind });
|
|
@@ -14071,7 +13440,7 @@ var RunCoordinator = class {
|
|
|
14071
13440
|
* so the Aero queue is never starved of a follow-up.
|
|
14072
13441
|
*/
|
|
14073
13442
|
buildDiscoveryAeroContext(runId, projectId, status, error) {
|
|
14074
|
-
const session = this.db.select().from(discoverySessions).where(
|
|
13443
|
+
const session = this.db.select().from(discoverySessions).where(eq18(discoverySessions.runId, runId)).get();
|
|
14075
13444
|
const competitorMap = session ? session.competitorMap : [];
|
|
14076
13445
|
return {
|
|
14077
13446
|
kind: RunKinds["aeo-discover-probe"],
|
|
@@ -14113,8 +13482,8 @@ function analysisResultFromInsights(insights2) {
|
|
|
14113
13482
|
}
|
|
14114
13483
|
|
|
14115
13484
|
// src/agent/session-registry.ts
|
|
14116
|
-
import
|
|
14117
|
-
import { eq as
|
|
13485
|
+
import crypto24 from "crypto";
|
|
13486
|
+
import { eq as eq20 } from "drizzle-orm";
|
|
14118
13487
|
|
|
14119
13488
|
// src/agent/session.ts
|
|
14120
13489
|
import fs7 from "fs";
|
|
@@ -14546,9 +13915,9 @@ function trimNestedArrays(full) {
|
|
|
14546
13915
|
const priorCount = typeof omitted[key] === "number" ? omitted[key] : 0;
|
|
14547
13916
|
owner.__truncated = true;
|
|
14548
13917
|
owner.__omittedRowsByField = omitted;
|
|
14549
|
-
const keep = (
|
|
14550
|
-
owner[key] = rows.slice(0,
|
|
14551
|
-
omitted[key] = priorCount + rows.length -
|
|
13918
|
+
const keep = (count) => {
|
|
13919
|
+
owner[key] = rows.slice(0, count);
|
|
13920
|
+
omitted[key] = priorCount + rows.length - count;
|
|
14552
13921
|
return serializeResult(copy);
|
|
14553
13922
|
};
|
|
14554
13923
|
out = keep(0);
|
|
@@ -14923,7 +14292,7 @@ function buildAeroStateTools(ctx, opts = {}) {
|
|
|
14923
14292
|
}
|
|
14924
14293
|
|
|
14925
14294
|
// src/agent/llm-usage.ts
|
|
14926
|
-
import
|
|
14295
|
+
import crypto21 from "crypto";
|
|
14927
14296
|
var AeroLlmUsageFeatures = {
|
|
14928
14297
|
turn: "aero.turn"
|
|
14929
14298
|
};
|
|
@@ -14942,7 +14311,7 @@ function recordLlmUsageEvent(args) {
|
|
|
14942
14311
|
const usage = args.message.usage;
|
|
14943
14312
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
14944
14313
|
args.db.insert(llmUsageEvents).values({
|
|
14945
|
-
id:
|
|
14314
|
+
id: crypto21.randomUUID(),
|
|
14946
14315
|
projectId: args.projectId,
|
|
14947
14316
|
runId: args.runId,
|
|
14948
14317
|
agentSessionId: args.agentSessionId,
|
|
@@ -15018,7 +14387,7 @@ function splitAeroAnthropicSystemCachePayload(payload, model) {
|
|
|
15018
14387
|
}
|
|
15019
14388
|
|
|
15020
14389
|
// src/agent/tool-usage.ts
|
|
15021
|
-
import
|
|
14390
|
+
import crypto22 from "crypto";
|
|
15022
14391
|
var AeroToolEventStatuses = {
|
|
15023
14392
|
success: "success",
|
|
15024
14393
|
error: "error"
|
|
@@ -15081,7 +14450,7 @@ function createAeroToolUsageHooks(args) {
|
|
|
15081
14450
|
function recordAgentToolEvent(args) {
|
|
15082
14451
|
try {
|
|
15083
14452
|
args.db.insert(agentToolEvents).values({
|
|
15084
|
-
id:
|
|
14453
|
+
id: crypto22.randomUUID(),
|
|
15085
14454
|
projectId: args.projectId,
|
|
15086
14455
|
agentSessionId: args.agentSessionId,
|
|
15087
14456
|
toolCallId: args.toolCallId,
|
|
@@ -15308,8 +14677,8 @@ async function loadExternalMcpTools(servers, opts = {}) {
|
|
|
15308
14677
|
}
|
|
15309
14678
|
|
|
15310
14679
|
// src/agent/memory-store.ts
|
|
15311
|
-
import
|
|
15312
|
-
import { and as
|
|
14680
|
+
import crypto23 from "crypto";
|
|
14681
|
+
import { and as and16, desc as desc7, eq as eq19, like, sql as sql8 } from "drizzle-orm";
|
|
15313
14682
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
15314
14683
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
15315
14684
|
function rowToDto(row) {
|
|
@@ -15323,7 +14692,7 @@ function rowToDto(row) {
|
|
|
15323
14692
|
};
|
|
15324
14693
|
}
|
|
15325
14694
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
15326
|
-
const query = db.select().from(agentMemory).where(
|
|
14695
|
+
const query = db.select().from(agentMemory).where(eq19(agentMemory.projectId, projectId)).orderBy(desc7(agentMemory.updatedAt));
|
|
15327
14696
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
15328
14697
|
return rows.map(rowToDto);
|
|
15329
14698
|
}
|
|
@@ -15337,7 +14706,7 @@ function upsertMemoryEntry(db, args) {
|
|
|
15337
14706
|
throw new Error(`memory key prefix "${COMPACTION_KEY_PREFIX}" is reserved for compaction notes`);
|
|
15338
14707
|
}
|
|
15339
14708
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
15340
|
-
const id =
|
|
14709
|
+
const id = crypto23.randomUUID();
|
|
15341
14710
|
db.insert(agentMemory).values({
|
|
15342
14711
|
id,
|
|
15343
14712
|
projectId: args.projectId,
|
|
@@ -15354,12 +14723,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
15354
14723
|
updatedAt: now
|
|
15355
14724
|
}
|
|
15356
14725
|
}).run();
|
|
15357
|
-
const row = db.select().from(agentMemory).where(
|
|
14726
|
+
const row = db.select().from(agentMemory).where(and16(eq19(agentMemory.projectId, args.projectId), eq19(agentMemory.key, args.key))).get();
|
|
15358
14727
|
if (!row) throw new Error("memory upsert produced no row");
|
|
15359
14728
|
return rowToDto(row);
|
|
15360
14729
|
}
|
|
15361
14730
|
function deleteMemoryEntry(db, projectId, key) {
|
|
15362
|
-
const result = db.delete(agentMemory).where(
|
|
14731
|
+
const result = db.delete(agentMemory).where(and16(eq19(agentMemory.projectId, projectId), eq19(agentMemory.key, key))).run();
|
|
15363
14732
|
const changes = result.changes ?? 0;
|
|
15364
14733
|
return changes > 0;
|
|
15365
14734
|
}
|
|
@@ -15374,7 +14743,7 @@ function writeCompactionNote(db, args) {
|
|
|
15374
14743
|
}
|
|
15375
14744
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
15376
14745
|
const key = `${COMPACTION_KEY_PREFIX}${args.sessionId}:${now}`;
|
|
15377
|
-
const id =
|
|
14746
|
+
const id = crypto23.randomUUID();
|
|
15378
14747
|
let inserted;
|
|
15379
14748
|
db.transaction((tx) => {
|
|
15380
14749
|
tx.insert(agentMemory).values({
|
|
@@ -15388,16 +14757,16 @@ function writeCompactionNote(db, args) {
|
|
|
15388
14757
|
}).run();
|
|
15389
14758
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
15390
14759
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
15391
|
-
|
|
15392
|
-
|
|
14760
|
+
and16(
|
|
14761
|
+
eq19(agentMemory.projectId, args.projectId),
|
|
15393
14762
|
like(agentMemory.key, `${sessionPrefix}%`)
|
|
15394
14763
|
)
|
|
15395
|
-
).orderBy(
|
|
14764
|
+
).orderBy(desc7(agentMemory.updatedAt)).all();
|
|
15396
14765
|
const stale = existing.slice(COMPACTION_NOTES_PER_SESSION).map((r) => r.id);
|
|
15397
14766
|
if (stale.length > 0) {
|
|
15398
|
-
tx.delete(agentMemory).where(
|
|
14767
|
+
tx.delete(agentMemory).where(sql8`${agentMemory.id} IN (${sql8.join(stale.map((s) => sql8`${s}`), sql8`, `)})`).run();
|
|
15399
14768
|
}
|
|
15400
|
-
const row = tx.select().from(agentMemory).where(
|
|
14769
|
+
const row = tx.select().from(agentMemory).where(and16(eq19(agentMemory.projectId, args.projectId), eq19(agentMemory.key, key))).get();
|
|
15401
14770
|
if (row) inserted = rowToDto(row);
|
|
15402
14771
|
});
|
|
15403
14772
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -15628,7 +14997,7 @@ var SessionRegistry = class {
|
|
|
15628
14997
|
modelProvider: effectiveProvider,
|
|
15629
14998
|
modelId: effectiveModelId,
|
|
15630
14999
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
15631
|
-
}).where(
|
|
15000
|
+
}).where(eq20(agentSessions.projectId, projectId)).run();
|
|
15632
15001
|
}
|
|
15633
15002
|
const agent2 = createAeroSession({
|
|
15634
15003
|
projectName,
|
|
@@ -15656,7 +15025,7 @@ var SessionRegistry = class {
|
|
|
15656
15025
|
return agent2;
|
|
15657
15026
|
}
|
|
15658
15027
|
const { provider, modelId } = resolveSessionProviderAndModel(this.opts.config, preferences);
|
|
15659
|
-
const sessionId =
|
|
15028
|
+
const sessionId = crypto24.randomUUID();
|
|
15660
15029
|
const agent = createAeroSession({
|
|
15661
15030
|
projectName,
|
|
15662
15031
|
client: this.opts.client,
|
|
@@ -15868,7 +15237,7 @@ ${lines.join("\n")}
|
|
|
15868
15237
|
modelProvider: nextProvider,
|
|
15869
15238
|
modelId: nextModelId,
|
|
15870
15239
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
15871
|
-
}).where(
|
|
15240
|
+
}).where(eq20(agentSessions.projectId, projectId)).run();
|
|
15872
15241
|
}
|
|
15873
15242
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
15874
15243
|
save(projectName) {
|
|
@@ -16032,17 +15401,17 @@ ${lines.join("\n")}
|
|
|
16032
15401
|
return id;
|
|
16033
15402
|
}
|
|
16034
15403
|
tryResolveProjectId(projectName) {
|
|
16035
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
15404
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq20(projects.name, projectName)).get();
|
|
16036
15405
|
return row?.id;
|
|
16037
15406
|
}
|
|
16038
15407
|
loadRow(projectId) {
|
|
16039
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
15408
|
+
const row = this.opts.db.select().from(agentSessions).where(eq20(agentSessions.projectId, projectId)).get();
|
|
16040
15409
|
return row ?? null;
|
|
16041
15410
|
}
|
|
16042
15411
|
insertRow(params) {
|
|
16043
15412
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
16044
15413
|
this.opts.db.insert(agentSessions).values({
|
|
16045
|
-
id: params.id ??
|
|
15414
|
+
id: params.id ?? crypto24.randomUUID(),
|
|
16046
15415
|
projectId: params.projectId,
|
|
16047
15416
|
systemPrompt: params.systemPrompt,
|
|
16048
15417
|
modelProvider: params.provider ?? params.modelProvider ?? AgentProviderIds.claude,
|
|
@@ -16055,7 +15424,7 @@ ${lines.join("\n")}
|
|
|
16055
15424
|
}
|
|
16056
15425
|
updateRow(projectId, patch) {
|
|
16057
15426
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
16058
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
15427
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq20(agentSessions.projectId, projectId)).run();
|
|
16059
15428
|
}
|
|
16060
15429
|
/**
|
|
16061
15430
|
* Store a refreshed prompt snapshot without touching `updatedAt`, which the
|
|
@@ -16064,20 +15433,20 @@ ${lines.join("\n")}
|
|
|
16064
15433
|
* recently used right after an upgrade.
|
|
16065
15434
|
*/
|
|
16066
15435
|
persistPromptSnapshot(projectId, systemPrompt) {
|
|
16067
|
-
this.opts.db.update(agentSessions).set({ systemPrompt }).where(
|
|
15436
|
+
this.opts.db.update(agentSessions).set({ systemPrompt }).where(eq20(agentSessions.projectId, projectId)).run();
|
|
16068
15437
|
}
|
|
16069
15438
|
};
|
|
16070
15439
|
|
|
16071
15440
|
// src/mcp-http.ts
|
|
16072
|
-
import
|
|
16073
|
-
import { eq as
|
|
15441
|
+
import crypto25 from "crypto";
|
|
15442
|
+
import { eq as eq21 } from "drizzle-orm";
|
|
16074
15443
|
import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
|
|
16075
15444
|
import { isInitializeRequest } from "@modelcontextprotocol/sdk/types.js";
|
|
16076
15445
|
var SESSION_IDLE_MS = 10 * 60 * 1e3;
|
|
16077
15446
|
var SWEEP_INTERVAL_MS = 60 * 1e3;
|
|
16078
15447
|
function mintSessionKey(db, scopes, userId) {
|
|
16079
|
-
const raw = `cnry_${
|
|
16080
|
-
const id =
|
|
15448
|
+
const raw = `cnry_${crypto25.randomBytes(24).toString("hex")}`;
|
|
15449
|
+
const id = crypto25.randomUUID();
|
|
16081
15450
|
db.insert(apiKeys).values({
|
|
16082
15451
|
id,
|
|
16083
15452
|
name: `mcp-session:${userId}`,
|
|
@@ -16091,7 +15460,7 @@ function mintSessionKey(db, scopes, userId) {
|
|
|
16091
15460
|
}
|
|
16092
15461
|
function revokeSessionKey(db, id) {
|
|
16093
15462
|
try {
|
|
16094
|
-
db.update(apiKeys).set({ revokedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
15463
|
+
db.update(apiKeys).set({ revokedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq21(apiKeys.id, id)).run();
|
|
16095
15464
|
} catch {
|
|
16096
15465
|
}
|
|
16097
15466
|
}
|
|
@@ -16161,7 +15530,7 @@ function registerMcpHttpRoutes(scope, opts) {
|
|
|
16161
15530
|
clientName: `canonry-mcp/${PACKAGE_VERSION}`,
|
|
16162
15531
|
surface: "mcp-http",
|
|
16163
15532
|
// Correlation only: this is never accepted as caller identity or authority.
|
|
16164
|
-
actorSession:
|
|
15533
|
+
actorSession: crypto25.randomUUID()
|
|
16165
15534
|
});
|
|
16166
15535
|
let server;
|
|
16167
15536
|
try {
|
|
@@ -16178,7 +15547,7 @@ function registerMcpHttpRoutes(scope, opts) {
|
|
|
16178
15547
|
throw error;
|
|
16179
15548
|
}
|
|
16180
15549
|
const transport = new StreamableHTTPServerTransport({
|
|
16181
|
-
sessionIdGenerator: () =>
|
|
15550
|
+
sessionIdGenerator: () => crypto25.randomUUID(),
|
|
16182
15551
|
onsessioninitialized: (id) => {
|
|
16183
15552
|
sessions.set(id, { transport, close, authorizationId, segmentId: segment.id, lastSeenAt: now() });
|
|
16184
15553
|
}
|
|
@@ -16274,9 +15643,9 @@ function registerMcpHttpRoutes(scope, opts) {
|
|
|
16274
15643
|
}
|
|
16275
15644
|
|
|
16276
15645
|
// src/agent/agent-routes.ts
|
|
16277
|
-
import { eq as
|
|
15646
|
+
import { eq as eq22 } from "drizzle-orm";
|
|
16278
15647
|
function resolveProject(db, name) {
|
|
16279
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
15648
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq22(projects.name, name)).get();
|
|
16280
15649
|
if (!row) throw notFound("project", name);
|
|
16281
15650
|
return row;
|
|
16282
15651
|
}
|
|
@@ -16285,7 +15654,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
16285
15654
|
"/projects/:name/agent/transcript",
|
|
16286
15655
|
async (request) => {
|
|
16287
15656
|
const project = resolveProject(opts.db, request.params.name);
|
|
16288
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
15657
|
+
const row = opts.db.select().from(agentSessions).where(eq22(agentSessions.projectId, project.id)).get();
|
|
16289
15658
|
if (!row) {
|
|
16290
15659
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
16291
15660
|
}
|
|
@@ -16309,7 +15678,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
16309
15678
|
async (request) => {
|
|
16310
15679
|
const project = resolveProject(opts.db, request.params.name);
|
|
16311
15680
|
opts.sessionRegistry.reset(project.name);
|
|
16312
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
15681
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq22(agentSessions.projectId, project.id)).run();
|
|
16313
15682
|
return { status: "reset" };
|
|
16314
15683
|
}
|
|
16315
15684
|
);
|
|
@@ -17132,7 +16501,7 @@ function buildSnapshotSummary(companyName, queries2, providers, queryResults, au
|
|
|
17132
16501
|
competitorCounts.set(competitor, (competitorCounts.get(competitor) ?? 0) + 1);
|
|
17133
16502
|
}
|
|
17134
16503
|
}
|
|
17135
|
-
const topCompetitors = [...competitorCounts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 10).map(([name,
|
|
16504
|
+
const topCompetitors = [...competitorCounts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 10).map(([name, count]) => ({ name, count }));
|
|
17136
16505
|
const defaultMeaning = totalComparisons > 0 ? `${companyName} was mentioned in ${mentionCount}/${totalComparisons} successful provider-query response${totalComparisons === 1 ? "" : "s"} across ${queries2.length} category queries.` : `No successful provider responses were returned across ${queries2.length} category queries.`;
|
|
17137
16506
|
const failureNote = failedComparisons > 0 ? [
|
|
17138
16507
|
`${failedComparisons} provider response${failedComparisons === 1 ? "" : "s"} failed and ${failedComparisons === 1 ? "was" : "were"} excluded from visibility totals.`
|
|
@@ -17267,20 +16636,20 @@ function clipText(value, length) {
|
|
|
17267
16636
|
}
|
|
17268
16637
|
|
|
17269
16638
|
// src/research-runner.ts
|
|
17270
|
-
import { and as
|
|
16639
|
+
import { and as and17, eq as eq23, inArray as inArray9, sql as sql9 } from "drizzle-orm";
|
|
17271
16640
|
var unfinishedResearchQueryStatuses = [ResearchQueryStatuses.queued, ResearchQueryStatuses.running];
|
|
17272
16641
|
function finalResearchRunStatus(completed, failed) {
|
|
17273
16642
|
if (failed === 0) return ResearchRunStatuses.completed;
|
|
17274
16643
|
return completed > 0 ? ResearchRunStatuses.partial : ResearchRunStatuses.failed;
|
|
17275
16644
|
}
|
|
17276
16645
|
async function executeResearchRun(db, registry, runId, projectId) {
|
|
17277
|
-
const run = db.select().from(researchRuns).where(
|
|
16646
|
+
const run = db.select().from(researchRuns).where(and17(eq23(researchRuns.id, runId), eq23(researchRuns.projectId, projectId))).get();
|
|
17278
16647
|
if (!run || run.status !== ResearchRunStatuses.queued) return;
|
|
17279
|
-
const project = db.select().from(projects).where(
|
|
16648
|
+
const project = db.select().from(projects).where(eq23(projects.id, projectId)).get();
|
|
17280
16649
|
if (!project) return;
|
|
17281
16650
|
const provider = registry.get(run.provider);
|
|
17282
16651
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
17283
|
-
const claim = db.update(researchRuns).set({ status: ResearchRunStatuses.running, startedAt: now }).where(
|
|
16652
|
+
const claim = db.update(researchRuns).set({ status: ResearchRunStatuses.running, startedAt: now }).where(and17(eq23(researchRuns.id, runId), eq23(researchRuns.status, ResearchRunStatuses.queued))).run();
|
|
17284
16653
|
if (claim.changes !== 1) return;
|
|
17285
16654
|
let reserved = 0;
|
|
17286
16655
|
let dispatched = 0;
|
|
@@ -17304,8 +16673,8 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
17304
16673
|
}
|
|
17305
16674
|
reserved = run.totalQueries;
|
|
17306
16675
|
reservation = { scope, period };
|
|
17307
|
-
const rows = db.select().from(researchRunQueries).where(
|
|
17308
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
16676
|
+
const rows = db.select().from(researchRunQueries).where(eq23(researchRunQueries.researchRunId, runId)).orderBy(researchRunQueries.position).all();
|
|
16677
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq23(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
17309
16678
|
const domains = effectiveDomains(project);
|
|
17310
16679
|
const brands = effectiveBrandNames(project);
|
|
17311
16680
|
const config = { ...provider.config, model: run.resolvedModel };
|
|
@@ -17313,7 +16682,7 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
17313
16682
|
await mapWithConcurrency(rows, Math.max(1, provider.config.quotaPolicy.maxConcurrency), async (row) => {
|
|
17314
16683
|
try {
|
|
17315
16684
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
17316
|
-
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.running, startedAt }).where(
|
|
16685
|
+
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.running, startedAt }).where(and17(eq23(researchRunQueries.id, row.id), eq23(researchRunQueries.status, ResearchQueryStatuses.queued))).run();
|
|
17317
16686
|
const raw = await gate.run(async () => {
|
|
17318
16687
|
dispatched++;
|
|
17319
16688
|
return provider.adapter.executeTrackedQuery({ query: row.queryText, canonicalDomains: domains, competitorDomains, ...run.location ? { location: run.location } : {} }, config);
|
|
@@ -17340,7 +16709,7 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
17340
16709
|
citationState: determineCitationState(normalized, domains),
|
|
17341
16710
|
rawResponse: raw.rawResponse,
|
|
17342
16711
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
17343
|
-
}).where(
|
|
16712
|
+
}).where(and17(eq23(researchRunQueries.id, row.id), eq23(researchRunQueries.status, ResearchQueryStatuses.running))).run();
|
|
17344
16713
|
if (completed.changes === 1) incrementResearchProgress(db, runId, "completedQueries");
|
|
17345
16714
|
} catch (error) {
|
|
17346
16715
|
try {
|
|
@@ -17364,18 +16733,18 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
17364
16733
|
}
|
|
17365
16734
|
}
|
|
17366
16735
|
function incrementResearchProgress(db, runId, column) {
|
|
17367
|
-
db.update(researchRuns).set({ [column]:
|
|
16736
|
+
db.update(researchRuns).set({ [column]: sql9`${researchRuns[column]} + 1` }).where(eq23(researchRuns.id, runId)).run();
|
|
17368
16737
|
}
|
|
17369
16738
|
function markResearchQueryFailed(db, runId, queryId, error) {
|
|
17370
16739
|
const message = describeError(error);
|
|
17371
|
-
const failed = db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error: message, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
16740
|
+
const failed = db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error: message, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and17(eq23(researchRunQueries.id, queryId), inArray9(researchRunQueries.status, unfinishedResearchQueryStatuses))).run();
|
|
17372
16741
|
if (failed.changes === 1) incrementResearchProgress(db, runId, "failedQueries");
|
|
17373
16742
|
}
|
|
17374
16743
|
function markUnfinishedResearchQueriesFailed(db, runId, error) {
|
|
17375
|
-
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
16744
|
+
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and17(eq23(researchRunQueries.researchRunId, runId), inArray9(researchRunQueries.status, unfinishedResearchQueryStatuses))).run();
|
|
17376
16745
|
}
|
|
17377
16746
|
function finalizeResearchRun(db, runId, fatalError) {
|
|
17378
|
-
const rows = db.select({ status: researchRunQueries.status }).from(researchRunQueries).where(
|
|
16747
|
+
const rows = db.select({ status: researchRunQueries.status }).from(researchRunQueries).where(eq23(researchRunQueries.researchRunId, runId)).all();
|
|
17379
16748
|
const completed = rows.filter((row) => row.status === ResearchQueryStatuses.completed).length;
|
|
17380
16749
|
const failed = rows.filter((row) => row.status === ResearchQueryStatuses.failed).length;
|
|
17381
16750
|
db.update(researchRuns).set({
|
|
@@ -17384,11 +16753,11 @@ function finalizeResearchRun(db, runId, fatalError) {
|
|
|
17384
16753
|
failedQueries: failed,
|
|
17385
16754
|
error: fatalError ?? (failed === rows.length ? "Every research query failed." : null),
|
|
17386
16755
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
17387
|
-
}).where(
|
|
16756
|
+
}).where(eq23(researchRuns.id, runId)).run();
|
|
17388
16757
|
}
|
|
17389
16758
|
|
|
17390
16759
|
// src/google-marketing-sync.ts
|
|
17391
|
-
import { and as
|
|
16760
|
+
import { and as and18, eq as eq24, isNull as isNull2 } from "drizzle-orm";
|
|
17392
16761
|
function safeErrorMessage(error) {
|
|
17393
16762
|
const raw = describeError(error);
|
|
17394
16763
|
return raw.replace(/(["']?authorization["']?\s*[:=]\s*["']?bearer\s+)[^\s,"'}]+/gi, "$1[redacted]").replace(/(["']?(?:developer|access|refresh|client)[_-]?(?:token|secret)["']?\s*[:=]\s*["']?)[^\s,"'}]+/gi, "$1[redacted]").replace(/[\r\n\t]+/g, " ").slice(0, 1e3);
|
|
@@ -17411,10 +16780,10 @@ function assertSnapshotContext(metadata, expected) {
|
|
|
17411
16780
|
}
|
|
17412
16781
|
}
|
|
17413
16782
|
function requireSyncRun(db, runId, projectId, kind) {
|
|
17414
|
-
const run = db.select().from(runs).where(
|
|
17415
|
-
|
|
17416
|
-
|
|
17417
|
-
|
|
16783
|
+
const run = db.select().from(runs).where(and18(
|
|
16784
|
+
eq24(runs.id, runId),
|
|
16785
|
+
eq24(runs.projectId, projectId),
|
|
16786
|
+
eq24(runs.kind, kind)
|
|
17418
16787
|
)).get();
|
|
17419
16788
|
if (!run) throw new Error(`Run ${runId} is not a ${kind} run for this project.`);
|
|
17420
16789
|
if (run.status !== RunStatuses.queued) {
|
|
@@ -17427,19 +16796,19 @@ function markFailed(db, runId, projectId, kind, expectedStatus, error) {
|
|
|
17427
16796
|
status: RunStatuses.failed,
|
|
17428
16797
|
error: serializeRunError({ message: safeErrorMessage(error) }),
|
|
17429
16798
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
17430
|
-
}).where(
|
|
17431
|
-
|
|
17432
|
-
|
|
17433
|
-
|
|
17434
|
-
|
|
16799
|
+
}).where(and18(
|
|
16800
|
+
eq24(runs.id, runId),
|
|
16801
|
+
eq24(runs.projectId, projectId),
|
|
16802
|
+
eq24(runs.kind, kind),
|
|
16803
|
+
eq24(runs.status, expectedStatus)
|
|
17435
16804
|
)).run();
|
|
17436
16805
|
}
|
|
17437
16806
|
function claimSyncRun(db, runId, projectId, kind, startedAt) {
|
|
17438
|
-
const claim = db.update(runs).set({ status: RunStatuses.running, startedAt, error: null }).where(
|
|
17439
|
-
|
|
17440
|
-
|
|
17441
|
-
|
|
17442
|
-
|
|
16807
|
+
const claim = db.update(runs).set({ status: RunStatuses.running, startedAt, error: null }).where(and18(
|
|
16808
|
+
eq24(runs.id, runId),
|
|
16809
|
+
eq24(runs.projectId, projectId),
|
|
16810
|
+
eq24(runs.kind, kind),
|
|
16811
|
+
eq24(runs.status, RunStatuses.queued)
|
|
17443
16812
|
)).run();
|
|
17444
16813
|
return claim.changes === 1;
|
|
17445
16814
|
}
|
|
@@ -17447,9 +16816,9 @@ async function executeGoogleAdsMarketingSync(db, runtime, runId, projectId) {
|
|
|
17447
16816
|
let claimed = false;
|
|
17448
16817
|
try {
|
|
17449
16818
|
requireSyncRun(db, runId, projectId, RunKinds["google-ads-sync"]);
|
|
17450
|
-
const project = db.select().from(projects).where(
|
|
16819
|
+
const project = db.select().from(projects).where(eq24(projects.id, projectId)).get();
|
|
17451
16820
|
if (!project) throw new Error(`Project ${projectId} was not found.`);
|
|
17452
|
-
const connection = db.select().from(googleAdsConnections).where(
|
|
16821
|
+
const connection = db.select().from(googleAdsConnections).where(eq24(googleAdsConnections.projectId, projectId)).get();
|
|
17453
16822
|
if (!connection?.selectedCustomerId) {
|
|
17454
16823
|
throw new Error("Google Ads needs a selected customer before it can sync.");
|
|
17455
16824
|
}
|
|
@@ -17500,11 +16869,11 @@ async function executeGoogleAdsMarketingSync(db, runtime, runId, projectId) {
|
|
|
17500
16869
|
status: RunStatuses.completed,
|
|
17501
16870
|
finishedAt,
|
|
17502
16871
|
error: null
|
|
17503
|
-
}).where(
|
|
17504
|
-
|
|
17505
|
-
|
|
17506
|
-
|
|
17507
|
-
|
|
16872
|
+
}).where(and18(
|
|
16873
|
+
eq24(runs.id, runId),
|
|
16874
|
+
eq24(runs.projectId, projectId),
|
|
16875
|
+
eq24(runs.kind, RunKinds["google-ads-sync"]),
|
|
16876
|
+
eq24(runs.status, RunStatuses.running)
|
|
17508
16877
|
)).run();
|
|
17509
16878
|
if (terminalClaim.changes !== 1) {
|
|
17510
16879
|
throw new Error(`Run ${runId} was no longer running when Google Ads evidence was ready.`);
|
|
@@ -17521,12 +16890,12 @@ async function executeGoogleAdsMarketingSync(db, runtime, runId, projectId) {
|
|
|
17521
16890
|
lastMetricsSnapshotAt: result.metrics?.metadata.capturedAt ?? connection.lastMetricsSnapshotAt,
|
|
17522
16891
|
lastMetricsSnapshotId: result.metrics?.metadata.id ?? connection.lastMetricsSnapshotId,
|
|
17523
16892
|
updatedAt: finishedAt
|
|
17524
|
-
}).where(
|
|
17525
|
-
|
|
17526
|
-
|
|
17527
|
-
|
|
17528
|
-
connection.selectedLoginCustomerId === null ?
|
|
17529
|
-
|
|
16893
|
+
}).where(and18(
|
|
16894
|
+
eq24(googleAdsConnections.id, connection.id),
|
|
16895
|
+
eq24(googleAdsConnections.projectId, projectId),
|
|
16896
|
+
eq24(googleAdsConnections.selectedCustomerId, connection.selectedCustomerId),
|
|
16897
|
+
connection.selectedLoginCustomerId === null ? isNull2(googleAdsConnections.selectedLoginCustomerId) : eq24(googleAdsConnections.selectedLoginCustomerId, connection.selectedLoginCustomerId),
|
|
16898
|
+
eq24(googleAdsConnections.selectionGeneration, connection.selectionGeneration)
|
|
17530
16899
|
)).run();
|
|
17531
16900
|
if (connectionClaim.changes !== 1) {
|
|
17532
16901
|
throw new Error("Google Ads selection changed while its evidence was being collected.");
|
|
@@ -17553,9 +16922,9 @@ async function executeGtmMarketingSync(db, runtime, runId, projectId) {
|
|
|
17553
16922
|
let claimed = false;
|
|
17554
16923
|
try {
|
|
17555
16924
|
requireSyncRun(db, runId, projectId, RunKinds["gtm-sync"]);
|
|
17556
|
-
const project = db.select().from(projects).where(
|
|
16925
|
+
const project = db.select().from(projects).where(eq24(projects.id, projectId)).get();
|
|
17557
16926
|
if (!project) throw new Error(`Project ${projectId} was not found.`);
|
|
17558
|
-
const connection = db.select().from(gtmConnections).where(
|
|
16927
|
+
const connection = db.select().from(gtmConnections).where(eq24(gtmConnections.projectId, projectId)).get();
|
|
17559
16928
|
if (!connection?.selectedAccountId || !connection.selectedContainerId) {
|
|
17560
16929
|
throw new Error("GTM needs a selected account and container before it can sync.");
|
|
17561
16930
|
}
|
|
@@ -17590,11 +16959,11 @@ async function executeGtmMarketingSync(db, runtime, runId, projectId) {
|
|
|
17590
16959
|
status: RunStatuses.completed,
|
|
17591
16960
|
finishedAt,
|
|
17592
16961
|
error: null
|
|
17593
|
-
}).where(
|
|
17594
|
-
|
|
17595
|
-
|
|
17596
|
-
|
|
17597
|
-
|
|
16962
|
+
}).where(and18(
|
|
16963
|
+
eq24(runs.id, runId),
|
|
16964
|
+
eq24(runs.projectId, projectId),
|
|
16965
|
+
eq24(runs.kind, RunKinds["gtm-sync"]),
|
|
16966
|
+
eq24(runs.status, RunStatuses.running)
|
|
17598
16967
|
)).run();
|
|
17599
16968
|
if (terminalClaim.changes !== 1) {
|
|
17600
16969
|
throw new Error(`Run ${runId} was no longer running when GTM evidence was ready.`);
|
|
@@ -17608,13 +16977,13 @@ async function executeGtmMarketingSync(db, runtime, runId, projectId) {
|
|
|
17608
16977
|
lastSnapshotAt: snapshot.metadata.capturedAt,
|
|
17609
16978
|
lastSnapshotId: snapshot.metadata.id,
|
|
17610
16979
|
updatedAt: finishedAt
|
|
17611
|
-
}).where(
|
|
17612
|
-
|
|
17613
|
-
|
|
17614
|
-
|
|
17615
|
-
|
|
17616
|
-
connection.selectedWorkspaceId === null ?
|
|
17617
|
-
|
|
16980
|
+
}).where(and18(
|
|
16981
|
+
eq24(gtmConnections.id, connection.id),
|
|
16982
|
+
eq24(gtmConnections.projectId, projectId),
|
|
16983
|
+
eq24(gtmConnections.selectedAccountId, connection.selectedAccountId),
|
|
16984
|
+
eq24(gtmConnections.selectedContainerId, connection.selectedContainerId),
|
|
16985
|
+
connection.selectedWorkspaceId === null ? isNull2(gtmConnections.selectedWorkspaceId) : eq24(gtmConnections.selectedWorkspaceId, connection.selectedWorkspaceId),
|
|
16986
|
+
eq24(gtmConnections.selectionGeneration, connection.selectionGeneration)
|
|
17618
16987
|
)).run();
|
|
17619
16988
|
if (connectionClaim.changes !== 1) {
|
|
17620
16989
|
throw new Error("GTM selection changed while its evidence was being collected.");
|
|
@@ -17635,7 +17004,7 @@ async function executeGtmMarketingSync(db, runtime, runId, projectId) {
|
|
|
17635
17004
|
}
|
|
17636
17005
|
|
|
17637
17006
|
// src/server.ts
|
|
17638
|
-
var _require3 =
|
|
17007
|
+
var _require3 = createRequire3(import.meta.url);
|
|
17639
17008
|
var { version: PKG_VERSION2 } = _require3("../package.json");
|
|
17640
17009
|
var log21 = createLogger("Server");
|
|
17641
17010
|
var runtimeStartupByServer = /* @__PURE__ */ new WeakMap();
|
|
@@ -17735,14 +17104,14 @@ function summarizeProviderConfig(config) {
|
|
|
17735
17104
|
};
|
|
17736
17105
|
}
|
|
17737
17106
|
function hashApiKey2(key) {
|
|
17738
|
-
return
|
|
17107
|
+
return crypto26.createHash("sha256").update(key).digest("hex");
|
|
17739
17108
|
}
|
|
17740
17109
|
var DASHBOARD_SCRYPT_KEYLEN = 64;
|
|
17741
17110
|
var DASHBOARD_SCRYPT_COST = 1 << 15;
|
|
17742
17111
|
var DASHBOARD_SCRYPT_MAXMEM = 64 * 1024 * 1024;
|
|
17743
17112
|
function hashDashboardPassword(password) {
|
|
17744
|
-
const salt =
|
|
17745
|
-
const derived =
|
|
17113
|
+
const salt = crypto26.randomBytes(16);
|
|
17114
|
+
const derived = crypto26.scryptSync(password, salt, DASHBOARD_SCRYPT_KEYLEN, {
|
|
17746
17115
|
N: DASHBOARD_SCRYPT_COST,
|
|
17747
17116
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
17748
17117
|
});
|
|
@@ -17763,14 +17132,14 @@ function verifyDashboardPassword(password, storedHash) {
|
|
|
17763
17132
|
} catch {
|
|
17764
17133
|
return { ok: false, needsRehash: false };
|
|
17765
17134
|
}
|
|
17766
|
-
const derived =
|
|
17135
|
+
const derived = crypto26.scryptSync(password, salt, expected.length, {
|
|
17767
17136
|
N: DASHBOARD_SCRYPT_COST,
|
|
17768
17137
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
17769
17138
|
});
|
|
17770
17139
|
if (derived.length !== expected.length)
|
|
17771
17140
|
return { ok: false, needsRehash: false };
|
|
17772
17141
|
return {
|
|
17773
|
-
ok:
|
|
17142
|
+
ok: crypto26.timingSafeEqual(derived, expected),
|
|
17774
17143
|
needsRehash: false
|
|
17775
17144
|
};
|
|
17776
17145
|
}
|
|
@@ -17779,7 +17148,7 @@ function verifyDashboardPassword(password, storedHash) {
|
|
|
17779
17148
|
const expected = Buffer.from(storedHash, "hex");
|
|
17780
17149
|
if (candidate.length !== expected.length)
|
|
17781
17150
|
return { ok: false, needsRehash: false };
|
|
17782
|
-
const ok =
|
|
17151
|
+
const ok = crypto26.timingSafeEqual(candidate, expected);
|
|
17783
17152
|
return { ok, needsRehash: ok };
|
|
17784
17153
|
}
|
|
17785
17154
|
return { ok: false, needsRehash: false };
|
|
@@ -17915,7 +17284,7 @@ function cloneGtmConfig(config) {
|
|
|
17915
17284
|
function createGoogleMarketingConfigCredentialStore(input) {
|
|
17916
17285
|
const persistConfigPatch = input.saveConfigPatch ?? saveConfigPatch;
|
|
17917
17286
|
const env = input.env ?? process.env;
|
|
17918
|
-
const randomUUID2 = input.randomUUID ??
|
|
17287
|
+
const randomUUID2 = input.randomUUID ?? crypto26.randomUUID;
|
|
17919
17288
|
const snapshot = () => ({
|
|
17920
17289
|
googleAds: cloneGoogleAdsConfig(input.config.googleAds),
|
|
17921
17290
|
gtm: cloneGtmConfig(input.config.gtm)
|
|
@@ -18018,7 +17387,7 @@ async function createServer(opts) {
|
|
|
18018
17387
|
const trustProxy = resolveTrustProxy(process.env.CANONRY_TRUST_PROXY);
|
|
18019
17388
|
const app = Fastify({
|
|
18020
17389
|
loggerInstance: logger,
|
|
18021
|
-
genReqId: () =>
|
|
17390
|
+
genReqId: () => crypto26.randomUUID(),
|
|
18022
17391
|
trustProxy
|
|
18023
17392
|
});
|
|
18024
17393
|
const registry = new ProviderRegistry();
|
|
@@ -18107,7 +17476,7 @@ async function createServer(opts) {
|
|
|
18107
17476
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
18108
17477
|
async (ctx) => {
|
|
18109
17478
|
if (!sessionRegistry) return;
|
|
18110
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
17479
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq25(projects.id, ctx.projectId)).get();
|
|
18111
17480
|
if (!project) return;
|
|
18112
17481
|
let content;
|
|
18113
17482
|
if (ctx.kind === RunKinds["aeo-discover-probe"]) {
|
|
@@ -18553,7 +17922,7 @@ async function createServer(opts) {
|
|
|
18553
17922
|
void (async () => {
|
|
18554
17923
|
try {
|
|
18555
17924
|
const report = await schedulerClient.runDoctor({ project: projectName });
|
|
18556
|
-
const project = opts.db.select().from(projects).where(
|
|
17925
|
+
const project = opts.db.select().from(projects).where(eq25(projects.name, projectName)).get();
|
|
18557
17926
|
if (!project) {
|
|
18558
17927
|
app.log.warn({ projectName }, "doctor schedule fired for an unknown project");
|
|
18559
17928
|
return;
|
|
@@ -18581,9 +17950,9 @@ async function createServer(opts) {
|
|
|
18581
17950
|
});
|
|
18582
17951
|
if (!probed) return;
|
|
18583
17952
|
const alreadySynced = opts.db.select().from(ccReleaseSyncs).where(
|
|
18584
|
-
|
|
18585
|
-
|
|
18586
|
-
|
|
17953
|
+
and19(
|
|
17954
|
+
eq25(ccReleaseSyncs.release, probed.release),
|
|
17955
|
+
eq25(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)
|
|
18587
17956
|
)
|
|
18588
17957
|
).limit(1).get();
|
|
18589
17958
|
if (alreadySynced) {
|
|
@@ -18764,7 +18133,7 @@ async function createServer(opts) {
|
|
|
18764
18133
|
return removed;
|
|
18765
18134
|
}
|
|
18766
18135
|
};
|
|
18767
|
-
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ??
|
|
18136
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto26.randomBytes(32).toString("hex");
|
|
18768
18137
|
const googleConnectionStore = {
|
|
18769
18138
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
18770
18139
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -18854,11 +18223,11 @@ async function createServer(opts) {
|
|
|
18854
18223
|
})();
|
|
18855
18224
|
if (opts.config.apiKey) {
|
|
18856
18225
|
const keyHash = hashApiKey2(opts.config.apiKey);
|
|
18857
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
18226
|
+
const existing = opts.db.select().from(apiKeys).where(eq25(apiKeys.keyHash, keyHash)).get();
|
|
18858
18227
|
if (!existing) {
|
|
18859
18228
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
18860
18229
|
opts.db.insert(apiKeys).values({
|
|
18861
|
-
id: `key_${
|
|
18230
|
+
id: `key_${crypto26.randomBytes(8).toString("hex")}`,
|
|
18862
18231
|
name: "default",
|
|
18863
18232
|
keyHash,
|
|
18864
18233
|
keyPrefix: prefix,
|
|
@@ -18882,7 +18251,7 @@ async function createServer(opts) {
|
|
|
18882
18251
|
};
|
|
18883
18252
|
const createSession = (apiKeyId) => {
|
|
18884
18253
|
pruneExpiredSessions();
|
|
18885
|
-
const sessionId =
|
|
18254
|
+
const sessionId = crypto26.randomBytes(32).toString("hex");
|
|
18886
18255
|
sessions.set(sessionId, {
|
|
18887
18256
|
apiKeyId,
|
|
18888
18257
|
expiresAt: Date.now() + SESSION_TTL_MS
|
|
@@ -18906,7 +18275,7 @@ async function createServer(opts) {
|
|
|
18906
18275
|
};
|
|
18907
18276
|
const getDefaultApiKey = () => {
|
|
18908
18277
|
if (!opts.config.apiKey) return void 0;
|
|
18909
|
-
return opts.db.select().from(apiKeys).where(
|
|
18278
|
+
return opts.db.select().from(apiKeys).where(eq25(apiKeys.keyHash, hashApiKey2(opts.config.apiKey))).get();
|
|
18910
18279
|
};
|
|
18911
18280
|
const createPasswordSession = (reply) => {
|
|
18912
18281
|
const key = getDefaultApiKey();
|
|
@@ -18930,7 +18299,7 @@ async function createServer(opts) {
|
|
|
18930
18299
|
if (!header) return false;
|
|
18931
18300
|
const parts = header.split(" ");
|
|
18932
18301
|
if (parts.length !== 2 || parts[0] !== "Bearer") return false;
|
|
18933
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
18302
|
+
const key = opts.db.select().from(apiKeys).where(eq25(apiKeys.keyHash, hashApiKey2(parts[1]))).get();
|
|
18934
18303
|
return Boolean(key && !key.revokedAt);
|
|
18935
18304
|
};
|
|
18936
18305
|
const namedAccountsInUse = () => anyUsersExist(opts.db);
|
|
@@ -19020,12 +18389,12 @@ async function createServer(opts) {
|
|
|
19020
18389
|
return reply.send({ authenticated: true });
|
|
19021
18390
|
}
|
|
19022
18391
|
if (apiKey) {
|
|
19023
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
18392
|
+
const key = opts.db.select().from(apiKeys).where(eq25(apiKeys.keyHash, hashApiKey2(apiKey))).get();
|
|
19024
18393
|
if (!key || key.revokedAt) {
|
|
19025
18394
|
const err2 = authInvalid();
|
|
19026
18395
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
19027
18396
|
}
|
|
19028
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
18397
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq25(apiKeys.id, key.id)).run();
|
|
19029
18398
|
const sessionId = createSession(key.id);
|
|
19030
18399
|
reply.header(
|
|
19031
18400
|
"set-cookie",
|
|
@@ -19187,8 +18556,8 @@ async function createServer(opts) {
|
|
|
19187
18556
|
getAgentProviderSummary: () => buildAgentProvidersResponse(opts.config).providers,
|
|
19188
18557
|
getGoogleMarketingDoctorInput: (ctx) => {
|
|
19189
18558
|
if (!ctx.project) return null;
|
|
19190
|
-
const googleAds = opts.db.select().from(googleAdsConnections).where(
|
|
19191
|
-
const gtm = opts.db.select().from(gtmConnections).where(
|
|
18559
|
+
const googleAds = opts.db.select().from(googleAdsConnections).where(eq25(googleAdsConnections.projectId, ctx.project.id)).get();
|
|
18560
|
+
const gtm = opts.db.select().from(gtmConnections).where(eq25(gtmConnections.projectId, ctx.project.id)).get();
|
|
19192
18561
|
const googleAdsCredential = getGoogleAdsConnection(opts.config, ctx.project.id);
|
|
19193
18562
|
const gtmCredential = getGtmConnection(opts.config, ctx.project.id);
|
|
19194
18563
|
const googleAdsConnected = Boolean(googleAdsCredential?.accessToken);
|
|
@@ -19312,7 +18681,7 @@ async function createServer(opts) {
|
|
|
19312
18681
|
deps: {
|
|
19313
18682
|
enqueueAutoExtract: ({ projectId, release: r }) => {
|
|
19314
18683
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
19315
|
-
const runId =
|
|
18684
|
+
const runId = crypto26.randomUUID();
|
|
19316
18685
|
opts.db.insert(runs).values({
|
|
19317
18686
|
id: runId,
|
|
19318
18687
|
projectId,
|
|
@@ -19439,7 +18808,7 @@ async function createServer(opts) {
|
|
|
19439
18808
|
...inspectOpts,
|
|
19440
18809
|
config: opts.config
|
|
19441
18810
|
}).then(() => {
|
|
19442
|
-
const finished = opts.db.select({ status: runs.status }).from(runs).where(
|
|
18811
|
+
const finished = opts.db.select({ status: runs.status }).from(runs).where(eq25(runs.id, runId)).get();
|
|
19443
18812
|
if (finished?.status === RunStatuses.completed || finished?.status === RunStatuses.partial) {
|
|
19444
18813
|
return maybeRefreshGscCoverage(
|
|
19445
18814
|
opts.db,
|
|
@@ -19549,7 +18918,7 @@ async function createServer(opts) {
|
|
|
19549
18918
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
19550
18919
|
opts.db.insert(auditLog).values(
|
|
19551
18920
|
targetProjectIds.map((projectId) => ({
|
|
19552
|
-
id:
|
|
18921
|
+
id: crypto26.randomUUID(),
|
|
19553
18922
|
projectId,
|
|
19554
18923
|
actor: auditContext?.actor ?? "api",
|
|
19555
18924
|
userAgent: auditContext?.userAgent ?? null,
|
|
@@ -19697,7 +19066,7 @@ async function createServer(opts) {
|
|
|
19697
19066
|
}
|
|
19698
19067
|
];
|
|
19699
19068
|
},
|
|
19700
|
-
onGenerateQueries: async (providerName,
|
|
19069
|
+
onGenerateQueries: async (providerName, count, project) => {
|
|
19701
19070
|
const provider = registry.get(providerName);
|
|
19702
19071
|
if (!provider)
|
|
19703
19072
|
throw new Error(`Provider "${providerName}" is not configured`);
|
|
@@ -19709,10 +19078,10 @@ async function createServer(opts) {
|
|
|
19709
19078
|
language: project.language,
|
|
19710
19079
|
existingQueries: project.existingQueries,
|
|
19711
19080
|
siteText,
|
|
19712
|
-
count
|
|
19081
|
+
count
|
|
19713
19082
|
});
|
|
19714
19083
|
const raw = await provider.adapter.generateText(prompt, provider.config);
|
|
19715
|
-
return parseQueryResponse(raw,
|
|
19084
|
+
return parseQueryResponse(raw, count);
|
|
19716
19085
|
},
|
|
19717
19086
|
onSnapshotRequested: async (input) => {
|
|
19718
19087
|
return snapshotService.createReport(input);
|
|
@@ -19892,7 +19261,7 @@ async function createServer(opts) {
|
|
|
19892
19261
|
if (runtimeStartupSettled) return;
|
|
19893
19262
|
try {
|
|
19894
19263
|
scheduler.start();
|
|
19895
|
-
for (const run of opts.db.select({ id: researchRuns.id, projectId: researchRuns.projectId }).from(researchRuns).where(
|
|
19264
|
+
for (const run of opts.db.select({ id: researchRuns.id, projectId: researchRuns.projectId }).from(researchRuns).where(eq25(researchRuns.status, ResearchRunStatuses.queued)).all()) {
|
|
19896
19265
|
dispatchResearchRun(run.id, run.projectId);
|
|
19897
19266
|
}
|
|
19898
19267
|
runtimeStartupSettled = true;
|
|
@@ -19945,7 +19314,7 @@ function buildQueryGenerationPrompt(ctx) {
|
|
|
19945
19314
|
);
|
|
19946
19315
|
return lines.join("\n");
|
|
19947
19316
|
}
|
|
19948
|
-
function parseQueryResponse(raw,
|
|
19317
|
+
function parseQueryResponse(raw, count) {
|
|
19949
19318
|
const seen = /* @__PURE__ */ new Set();
|
|
19950
19319
|
const results = [];
|
|
19951
19320
|
for (const line of raw.split("\n")) {
|
|
@@ -19961,7 +19330,7 @@ function parseQueryResponse(raw, count2) {
|
|
|
19961
19330
|
if (seen.has(key)) continue;
|
|
19962
19331
|
seen.add(key);
|
|
19963
19332
|
results.push(cleaned);
|
|
19964
|
-
if (results.length >=
|
|
19333
|
+
if (results.length >= count) break;
|
|
19965
19334
|
}
|
|
19966
19335
|
return results;
|
|
19967
19336
|
}
|