@canonry/canonry 4.158.0 → 4.160.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/THIRD_PARTY_NOTICES.md +102 -0
- package/assets/THIRD_PARTY_NOTICES.md +102 -0
- package/assets/agent-workspace/skills/aero/SKILL.md +16 -1
- package/assets/agent-workspace/skills/canonry/SKILL.md +17 -5
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +67 -0
- package/assets/agent-workspace/skills/canonry/references/server-side-traffic.md +231 -62
- package/assets/assets/{AuditHistoryPanel-McQfjoU9.js → AuditHistoryPanel-AtgoG2AH.js} +1 -1
- package/assets/assets/{BacklinksPage-Da8y1Wv8.js → BacklinksPage-DrOSF7Mu.js} +1 -1
- package/assets/assets/{HistoryPage-9MJfhRyT.js → HistoryPage-CGQXleJc.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-BiNAHEJW.js → MeasurementPropertyPage-C7D2OgIQ.js} +1 -1
- package/assets/assets/ProjectPage-Lp1VjNvL.js +12 -0
- package/assets/assets/{RunRow-DCG2LBlS.js → RunRow-BdzKKODo.js} +1 -1
- package/assets/assets/{RunsPage-ByWA_FSk.js → RunsPage-BU96L07S.js} +1 -1
- package/assets/assets/{SettingsPage-DdSeKAKF.js → SettingsPage-LEca0kDH.js} +1 -1
- package/assets/assets/SiteHealthSection-CKM4DUfE.js +4 -0
- package/assets/assets/{TrafficPage-5u8rPcBt.js → TrafficPage-tU9wl1F_.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-Dm-fK87u.js → TrafficSourceDetailPage-DZTpnU9f.js} +1 -1
- package/assets/assets/extract-error-message-BAyN-ftk.js +1 -0
- package/assets/assets/geist-cyrillic-ext-wght-normal-DjL33-gN.woff2 +0 -0
- package/assets/assets/geist-cyrillic-wght-normal-BEAKL7Jp.woff2 +0 -0
- package/assets/assets/geist-latin-ext-wght-normal-DC-KSUi6.woff2 +0 -0
- package/assets/assets/geist-latin-wght-normal-BgDaEnEv.woff2 +0 -0
- package/assets/assets/geist-mono-cyrillic-ext-wght-normal-X_5orZeX.woff2 +0 -0
- package/assets/assets/geist-mono-cyrillic-wght-normal-DiZS0aHC.woff2 +0 -0
- package/assets/assets/geist-mono-latin-ext-wght-normal-Bwz-egvJ.woff2 +0 -0
- package/assets/assets/geist-mono-latin-wght-normal-XN7g48iV.woff2 +0 -0
- package/assets/assets/geist-mono-symbols2-wght-normal-CO5SzqOn.woff2 +0 -0
- package/assets/assets/geist-mono-vietnamese-wght-normal-DadHysG0.woff2 +0 -0
- package/assets/assets/geist-vietnamese-wght-normal-6IgcOCM7.woff2 +0 -0
- package/assets/assets/index-CACRaOmZ.css +1 -0
- package/assets/assets/index-Wqv7kvs-.js +83 -0
- package/assets/assets/{react-sigma_core.esm.min-ComIyDKi.js → react-sigma_core.esm.min-DlZOAc_w.js} +2 -2
- package/assets/assets/{vendor-lucide-BtYdosYk.js → vendor-lucide-nYKv-BB7.js} +1 -1
- package/assets/assets/{vendor-markdown-D274_wsI.js → vendor-markdown-Bn7fRwrx.js} +1 -1
- package/assets/assets/{vendor-radix-BxMXSwbC.js → vendor-radix-4UOaVWza.js} +1 -1
- package/assets/assets/{vendor-recharts-C4ktw_aE.js → vendor-recharts-D8rt5lKn.js} +1 -1
- package/assets/assets/{vendor-tanstack-pFxw0eGP.js → vendor-tanstack-Dock0eH1.js} +1 -1
- package/assets/index.html +7 -7
- package/dist/{chunk-DEVAEC2S.js → chunk-7M7T5QXS.js} +290 -16
- package/dist/{chunk-LK6YFHG4.js → chunk-HQAVEK42.js} +65 -15
- package/dist/{chunk-YP4K2KXH.js → chunk-X7CYY5Q7.js} +2687 -686
- package/dist/{chunk-4VZP64GZ.js → chunk-XGESWHFP.js} +660 -274
- package/dist/cli.js +333 -117
- package/dist/index.d.ts +37 -14
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-LQ6ZRWMF.js → intelligence-service-6XQMM466.js} +2 -2
- package/dist/mcp.js +2 -2
- package/package.json +11 -9
- package/assets/assets/ProjectPage-mWSMjXrR.js +0 -15
- package/assets/assets/extract-error-message-CtxkAd6E.js +0 -1
- package/assets/assets/index-DUfg50Pl.js +0 -83
- package/assets/assets/index-cXb3mfeQ.css +0 -1
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
loadConfig,
|
|
11
11
|
loadConfigRaw,
|
|
12
12
|
saveConfigPatch
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-HQAVEK42.js";
|
|
14
14
|
import {
|
|
15
15
|
CC_CACHE_DIR,
|
|
16
16
|
DUCKDB_SPEC,
|
|
@@ -117,6 +117,7 @@ import {
|
|
|
117
117
|
researchRunQueries,
|
|
118
118
|
researchRuns,
|
|
119
119
|
resolveDestination,
|
|
120
|
+
resolveMeasurementSitemapTarget,
|
|
120
121
|
resolveTrustProxy,
|
|
121
122
|
resolveVercelSyncDeadlineMs,
|
|
122
123
|
resolveWebhookTarget,
|
|
@@ -136,7 +137,7 @@ import {
|
|
|
136
137
|
siteCrawlSnapshots,
|
|
137
138
|
toAlertView,
|
|
138
139
|
usageCounters
|
|
139
|
-
} from "./chunk-
|
|
140
|
+
} from "./chunk-X7CYY5Q7.js";
|
|
140
141
|
import {
|
|
141
142
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
142
143
|
AGENT_PROVIDER_IDS,
|
|
@@ -169,7 +170,9 @@ import {
|
|
|
169
170
|
SITE_CRAWL_GRAPH_MAX_NODES,
|
|
170
171
|
SKILL_MANIFEST_FILENAME,
|
|
171
172
|
SchedulableRunKinds,
|
|
173
|
+
SiteHealthTemplateDetections,
|
|
172
174
|
SkillsClients,
|
|
175
|
+
TEMPLATE_LINK_RATIO_THRESHOLD,
|
|
173
176
|
TrafficEventConfidences,
|
|
174
177
|
TrafficEventKinds,
|
|
175
178
|
TrafficEvidenceKinds,
|
|
@@ -188,12 +191,15 @@ import {
|
|
|
188
191
|
bucketOnboardingCount,
|
|
189
192
|
buildRunErrorFromMessages,
|
|
190
193
|
classifySkillFile,
|
|
194
|
+
cloudflareQueueNameSchema,
|
|
191
195
|
coerceSkillManifest,
|
|
192
196
|
computeBacklinkSummaryMetrics,
|
|
193
197
|
contentActionLabel,
|
|
194
198
|
contentBriefDtoSchema,
|
|
199
|
+
createTemplateLinkPairIndex,
|
|
195
200
|
deriveCitedUrlCandidates,
|
|
196
201
|
deriveIndexCoverage,
|
|
202
|
+
deriveSiteHealthState,
|
|
197
203
|
determineAnswerMentioned,
|
|
198
204
|
dollarsToMicros,
|
|
199
205
|
effectiveBrandNames,
|
|
@@ -211,6 +217,8 @@ import {
|
|
|
211
217
|
isBrowserProvider,
|
|
212
218
|
isGhostTelemetryEvent,
|
|
213
219
|
isRetryableHttpError,
|
|
220
|
+
isSelfLink,
|
|
221
|
+
isTemplateLinkRatio,
|
|
214
222
|
isVertexGroundingRedirect,
|
|
215
223
|
mapWithConcurrency,
|
|
216
224
|
normalizeIdTokens,
|
|
@@ -219,6 +227,7 @@ import {
|
|
|
219
227
|
normalizeSiteAuditRunRequest,
|
|
220
228
|
normalizeUrlPath,
|
|
221
229
|
notFound,
|
|
230
|
+
observeTemplateLinkEdges,
|
|
222
231
|
parseMeasurementRunManifestV1,
|
|
223
232
|
parseOriginList,
|
|
224
233
|
providerError,
|
|
@@ -231,12 +240,14 @@ import {
|
|
|
231
240
|
skillsClientSchema,
|
|
232
241
|
splitList,
|
|
233
242
|
startOfDayHourInTimeZone,
|
|
243
|
+
templateLinkDetection,
|
|
244
|
+
templateLinkRatio,
|
|
234
245
|
textContainsBrandAlias,
|
|
235
246
|
textContainsDomain,
|
|
236
247
|
validationError,
|
|
237
248
|
winnabilityClassLabel,
|
|
238
249
|
withRetry
|
|
239
|
-
} from "./chunk-
|
|
250
|
+
} from "./chunk-7M7T5QXS.js";
|
|
240
251
|
|
|
241
252
|
// src/telemetry.ts
|
|
242
253
|
import crypto from "crypto";
|
|
@@ -571,7 +582,7 @@ import crypto25 from "crypto";
|
|
|
571
582
|
import fs8 from "fs";
|
|
572
583
|
import path9 from "path";
|
|
573
584
|
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
574
|
-
import { and as
|
|
585
|
+
import { and as and18, eq as eq24 } from "drizzle-orm";
|
|
575
586
|
import Fastify from "fastify";
|
|
576
587
|
import os5 from "os";
|
|
577
588
|
|
|
@@ -3177,29 +3188,33 @@ function ensureConnections6(config) {
|
|
|
3177
3188
|
if (!config.cloudflareTraffic.connections) config.cloudflareTraffic.connections = [];
|
|
3178
3189
|
return config.cloudflareTraffic.connections;
|
|
3179
3190
|
}
|
|
3180
|
-
function
|
|
3191
|
+
function normalizeCloudflareTrafficConnection(connection) {
|
|
3181
3192
|
const deliveryMode = connection.deliveryMode;
|
|
3182
|
-
if (deliveryMode
|
|
3183
|
-
|
|
3193
|
+
if (deliveryMode === "queue-pull") {
|
|
3194
|
+
const queue = connection;
|
|
3195
|
+
if (typeof queue.apiToken !== "string" || queue.apiToken.length === 0 || typeof queue.accountId !== "string" || queue.accountId.length === 0 || typeof queue.queueId !== "string" || queue.queueId.length === 0 || typeof queue.queueName !== "string" || !cloudflareQueueNameSchema.safeParse(queue.queueName).success || typeof queue.retentionSeconds !== "number" || !Number.isInteger(queue.retentionSeconds) || queue.retentionSeconds < 60 || queue.retentionSeconds > 1209600) {
|
|
3196
|
+
throw new Error("Invalid Cloudflare queue-pull credential configuration");
|
|
3197
|
+
}
|
|
3198
|
+
return connection;
|
|
3184
3199
|
}
|
|
3185
3200
|
if (deliveryMode === "direct-push") return connection;
|
|
3186
|
-
|
|
3201
|
+
if (deliveryMode === void 0) {
|
|
3202
|
+
return { ...connection, deliveryMode: "direct-push" };
|
|
3203
|
+
}
|
|
3204
|
+
throw unsupportedDeliveryMode(deliveryMode);
|
|
3187
3205
|
}
|
|
3188
3206
|
function getCloudflareTrafficConnection(config, projectName) {
|
|
3189
3207
|
const connection = (config.cloudflareTraffic?.connections ?? []).find((c) => c.projectName === projectName);
|
|
3190
|
-
return connection ?
|
|
3208
|
+
return connection ? normalizeCloudflareTrafficConnection(connection) : void 0;
|
|
3191
3209
|
}
|
|
3192
3210
|
function getCloudflareTrafficConnectionBySourceId(config, sourceId) {
|
|
3193
3211
|
const connection = (config.cloudflareTraffic?.connections ?? []).find((c) => c.sourceId === sourceId);
|
|
3194
|
-
return connection ?
|
|
3212
|
+
return connection ? normalizeCloudflareTrafficConnection(connection) : void 0;
|
|
3195
3213
|
}
|
|
3196
3214
|
function upsertCloudflareTrafficConnection(config, connection) {
|
|
3197
|
-
|
|
3198
|
-
if (deliveryMode !== "direct-push") {
|
|
3199
|
-
throw unsupportedDeliveryMode(deliveryMode);
|
|
3200
|
-
}
|
|
3215
|
+
normalizeCloudflareTrafficConnection(connection);
|
|
3201
3216
|
const connections = ensureConnections6(config);
|
|
3202
|
-
const index = connections.findIndex((c) => c.
|
|
3217
|
+
const index = connections.findIndex((c) => c.sourceId === connection.sourceId);
|
|
3203
3218
|
if (index === -1) {
|
|
3204
3219
|
connections.push(connection);
|
|
3205
3220
|
return connection;
|
|
@@ -3207,6 +3222,16 @@ function upsertCloudflareTrafficConnection(config, connection) {
|
|
|
3207
3222
|
connections[index] = connection;
|
|
3208
3223
|
return connection;
|
|
3209
3224
|
}
|
|
3225
|
+
function removeCloudflareTrafficConnectionBySourceId(config, sourceId) {
|
|
3226
|
+
const connections = config.cloudflareTraffic?.connections;
|
|
3227
|
+
if (!connections?.length) return false;
|
|
3228
|
+
const next = connections.filter((c) => c.sourceId !== sourceId);
|
|
3229
|
+
if (next.length === connections.length) return false;
|
|
3230
|
+
if (!config.cloudflareTraffic) return false;
|
|
3231
|
+
config.cloudflareTraffic.connections = next;
|
|
3232
|
+
if (next.length === 0) delete config.cloudflareTraffic;
|
|
3233
|
+
return true;
|
|
3234
|
+
}
|
|
3210
3235
|
function removeCloudflareTrafficConnection(config, projectName) {
|
|
3211
3236
|
const connections = config.cloudflareTraffic?.connections;
|
|
3212
3237
|
if (!connections?.length) return false;
|
|
@@ -4169,12 +4194,12 @@ var JobRunner = class {
|
|
|
4169
4194
|
error: errorMessage
|
|
4170
4195
|
}).where(eq2(runs.id, runId)).run();
|
|
4171
4196
|
this.flushProviderUsage(providerDispatchCounts, providerReservations);
|
|
4172
|
-
const
|
|
4197
|
+
const abortReason2 = classifyRunAbortReason(errorMessage);
|
|
4173
4198
|
const phases = buildPhases({ startTime, providerCallStart, providerCallEnd });
|
|
4174
|
-
if (
|
|
4199
|
+
if (abortReason2) {
|
|
4175
4200
|
const domainHash = hashDomain(executionContext.canonicalDomain ?? null);
|
|
4176
4201
|
trackEvent("run.aborted", {
|
|
4177
|
-
reason:
|
|
4202
|
+
reason: abortReason2,
|
|
4178
4203
|
providerCount: executionContext.providerCount,
|
|
4179
4204
|
providers: executionContext.providers,
|
|
4180
4205
|
queryCount: executionContext.queryCount,
|
|
@@ -6995,9 +7020,193 @@ function buildDiscoveryInsightTitle(input) {
|
|
|
6995
7020
|
|
|
6996
7021
|
// src/execute-site-audit.ts
|
|
6997
7022
|
import crypto17 from "crypto";
|
|
6998
|
-
import { and as
|
|
7023
|
+
import { and as and12, eq as eq15, sql as sql6 } from "drizzle-orm";
|
|
6999
7024
|
import { runSiteCrawl } from "@canonry/aeo-audit";
|
|
7000
7025
|
|
|
7026
|
+
// src/site-audit-root.ts
|
|
7027
|
+
import http from "http";
|
|
7028
|
+
import https from "https";
|
|
7029
|
+
var DEFAULT_TIMEOUT_MS = 1e4;
|
|
7030
|
+
var MAX_REDIRECTS = 5;
|
|
7031
|
+
var USER_AGENT = "Canonry/1.0 (site-health)";
|
|
7032
|
+
async function resolveSiteAuditRootUrl(rawUrl, options) {
|
|
7033
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
7034
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) throw new Error("Site audit root timeout must be positive");
|
|
7035
|
+
const requested = parseHttpUrl2(rawUrl);
|
|
7036
|
+
let current = requested;
|
|
7037
|
+
const redirects = [];
|
|
7038
|
+
const deadlineAt = Date.now() + timeoutMs;
|
|
7039
|
+
const transport = options.transport ?? requestPinnedSiteAuditRoot;
|
|
7040
|
+
for (; ; ) {
|
|
7041
|
+
throwIfAborted(options.signal);
|
|
7042
|
+
const checked = await beforeDeadline(
|
|
7043
|
+
() => options.resolveTarget(current.href),
|
|
7044
|
+
deadlineAt,
|
|
7045
|
+
timeoutMs,
|
|
7046
|
+
options.signal
|
|
7047
|
+
);
|
|
7048
|
+
if (!checked.ok) throw new Error(`Site audit root URL rejected: ${checked.message}`);
|
|
7049
|
+
const checkedUrl = parseHttpUrl2(checked.target.url.href);
|
|
7050
|
+
if (checkedUrl.href !== current.href) {
|
|
7051
|
+
throw new Error("Site audit root target resolver returned a different URL than it validated");
|
|
7052
|
+
}
|
|
7053
|
+
const response = await beforeDeadline(
|
|
7054
|
+
() => transport(checked.target, { timeoutMs: remainingTime(deadlineAt, timeoutMs), signal: options.signal }),
|
|
7055
|
+
deadlineAt,
|
|
7056
|
+
timeoutMs,
|
|
7057
|
+
options.signal
|
|
7058
|
+
);
|
|
7059
|
+
if (!isRedirect(response.status)) {
|
|
7060
|
+
return { requestedUrl: requested.href, effectiveUrl: current.href, redirects };
|
|
7061
|
+
}
|
|
7062
|
+
if (redirects.length >= MAX_REDIRECTS) {
|
|
7063
|
+
throw new Error(`Site audit root has more than ${MAX_REDIRECTS} redirects`);
|
|
7064
|
+
}
|
|
7065
|
+
const location = firstHeader(response.headers.location);
|
|
7066
|
+
if (!location) throw new Error("Site audit root redirect response has no Location header");
|
|
7067
|
+
let next;
|
|
7068
|
+
try {
|
|
7069
|
+
next = parseHttpUrl2(new URL(location, current).href);
|
|
7070
|
+
} catch {
|
|
7071
|
+
throw new Error("Site audit root redirect has an invalid Location header");
|
|
7072
|
+
}
|
|
7073
|
+
assertAllowedRedirect(current, next);
|
|
7074
|
+
redirects.push({ status: response.status, from: current.href, to: next.href });
|
|
7075
|
+
current = next;
|
|
7076
|
+
}
|
|
7077
|
+
}
|
|
7078
|
+
function parseHttpUrl2(rawUrl) {
|
|
7079
|
+
let url;
|
|
7080
|
+
try {
|
|
7081
|
+
url = new URL(rawUrl);
|
|
7082
|
+
} catch {
|
|
7083
|
+
throw new Error("Site audit root must be a valid URL");
|
|
7084
|
+
}
|
|
7085
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
7086
|
+
throw new Error("Site audit root must use HTTP or HTTPS");
|
|
7087
|
+
}
|
|
7088
|
+
if (url.username || url.password) throw new Error("Site audit root must not include credentials");
|
|
7089
|
+
url.hash = "";
|
|
7090
|
+
return url;
|
|
7091
|
+
}
|
|
7092
|
+
function canonicalHostAlias(hostname) {
|
|
7093
|
+
return hostname.toLowerCase().replace(/\.$/, "").replace(/^www\./, "");
|
|
7094
|
+
}
|
|
7095
|
+
function assertAllowedRedirect(current, next) {
|
|
7096
|
+
if (current.protocol === "https:" && next.protocol === "http:") {
|
|
7097
|
+
throw new Error("Site audit root redirect would downgrade HTTPS to HTTP");
|
|
7098
|
+
}
|
|
7099
|
+
if (current.port !== next.port) throw new Error("Site audit root redirect changes port");
|
|
7100
|
+
if (canonicalHostAlias(current.hostname) !== canonicalHostAlias(next.hostname)) {
|
|
7101
|
+
throw new Error("Site audit root redirect points outside the approved site");
|
|
7102
|
+
}
|
|
7103
|
+
}
|
|
7104
|
+
function isRedirect(status) {
|
|
7105
|
+
return status >= 300 && status < 400;
|
|
7106
|
+
}
|
|
7107
|
+
function firstHeader(value) {
|
|
7108
|
+
return Array.isArray(value) ? value[0] : value;
|
|
7109
|
+
}
|
|
7110
|
+
function remainingTime(deadlineAt, timeoutMs) {
|
|
7111
|
+
const remaining = deadlineAt - Date.now();
|
|
7112
|
+
if (remaining <= 0) throw new Error(`Site audit root resolution exceeded its ${timeoutMs}ms deadline`);
|
|
7113
|
+
return remaining;
|
|
7114
|
+
}
|
|
7115
|
+
async function beforeDeadline(operation, deadlineAt, timeoutMs, signal) {
|
|
7116
|
+
throwIfAborted(signal);
|
|
7117
|
+
const remaining = remainingTime(deadlineAt, timeoutMs);
|
|
7118
|
+
let timer;
|
|
7119
|
+
let abortListener;
|
|
7120
|
+
try {
|
|
7121
|
+
const value = await Promise.race([
|
|
7122
|
+
operation(),
|
|
7123
|
+
new Promise((_resolve, reject) => {
|
|
7124
|
+
timer = setTimeout(
|
|
7125
|
+
() => reject(new Error(`Site audit root resolution exceeded its ${timeoutMs}ms deadline`)),
|
|
7126
|
+
remaining
|
|
7127
|
+
);
|
|
7128
|
+
}),
|
|
7129
|
+
new Promise((_resolve, reject) => {
|
|
7130
|
+
if (!signal) return;
|
|
7131
|
+
abortListener = () => reject(abortReason(signal));
|
|
7132
|
+
signal.addEventListener("abort", abortListener, { once: true });
|
|
7133
|
+
if (signal.aborted) abortListener();
|
|
7134
|
+
})
|
|
7135
|
+
]);
|
|
7136
|
+
remainingTime(deadlineAt, timeoutMs);
|
|
7137
|
+
return value;
|
|
7138
|
+
} finally {
|
|
7139
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
7140
|
+
if (abortListener) signal?.removeEventListener("abort", abortListener);
|
|
7141
|
+
}
|
|
7142
|
+
}
|
|
7143
|
+
function abortReason(signal) {
|
|
7144
|
+
return signal.reason instanceof Error ? signal.reason : new DOMException("Operation aborted", "AbortError");
|
|
7145
|
+
}
|
|
7146
|
+
function throwIfAborted(signal) {
|
|
7147
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
7148
|
+
}
|
|
7149
|
+
function stripIpv6Brackets(hostname) {
|
|
7150
|
+
return hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
|
|
7151
|
+
}
|
|
7152
|
+
function buildSiteAuditRootRequestOptions(target) {
|
|
7153
|
+
const secure = target.url.protocol === "https:";
|
|
7154
|
+
const requestOptions = {
|
|
7155
|
+
hostname: target.address,
|
|
7156
|
+
family: target.family,
|
|
7157
|
+
port: target.url.port ? Number(target.url.port) : secure ? 443 : 80,
|
|
7158
|
+
method: "GET",
|
|
7159
|
+
path: `${target.url.pathname}${target.url.search}`,
|
|
7160
|
+
headers: {
|
|
7161
|
+
Host: target.url.host,
|
|
7162
|
+
Accept: "text/html,*/*;q=0.1",
|
|
7163
|
+
"User-Agent": USER_AGENT
|
|
7164
|
+
}
|
|
7165
|
+
};
|
|
7166
|
+
if (secure) requestOptions.servername = stripIpv6Brackets(target.url.hostname);
|
|
7167
|
+
return requestOptions;
|
|
7168
|
+
}
|
|
7169
|
+
async function requestPinnedSiteAuditRoot(target, options) {
|
|
7170
|
+
throwIfAborted(options.signal);
|
|
7171
|
+
const secure = target.url.protocol === "https:";
|
|
7172
|
+
const requestOptions = buildSiteAuditRootRequestOptions(target);
|
|
7173
|
+
return await new Promise((resolve, reject) => {
|
|
7174
|
+
let settled = false;
|
|
7175
|
+
let abortListener;
|
|
7176
|
+
const cleanup = () => {
|
|
7177
|
+
clearTimeout(timer);
|
|
7178
|
+
if (abortListener) options.signal?.removeEventListener("abort", abortListener);
|
|
7179
|
+
};
|
|
7180
|
+
const succeed = (response) => {
|
|
7181
|
+
if (settled) return;
|
|
7182
|
+
settled = true;
|
|
7183
|
+
cleanup();
|
|
7184
|
+
resolve(response);
|
|
7185
|
+
};
|
|
7186
|
+
const fail = (error) => {
|
|
7187
|
+
if (settled) return;
|
|
7188
|
+
settled = true;
|
|
7189
|
+
cleanup();
|
|
7190
|
+
reject(error);
|
|
7191
|
+
};
|
|
7192
|
+
const request = (secure ? https.request : http.request)(requestOptions, (response) => {
|
|
7193
|
+
response.destroy();
|
|
7194
|
+
succeed({ status: response.statusCode ?? 0, headers: response.headers });
|
|
7195
|
+
});
|
|
7196
|
+
const timer = setTimeout(
|
|
7197
|
+
() => request.destroy(new Error(`Site audit root request timed out after ${options.timeoutMs}ms`)),
|
|
7198
|
+
options.timeoutMs
|
|
7199
|
+
);
|
|
7200
|
+
if (options.signal) {
|
|
7201
|
+
abortListener = () => request.destroy(abortReason(options.signal));
|
|
7202
|
+
options.signal.addEventListener("abort", abortListener, { once: true });
|
|
7203
|
+
if (options.signal.aborted) abortListener();
|
|
7204
|
+
}
|
|
7205
|
+
request.on("error", fail);
|
|
7206
|
+
request.end();
|
|
7207
|
+
});
|
|
7208
|
+
}
|
|
7209
|
+
|
|
7001
7210
|
// src/site-crawl-graph-layout.ts
|
|
7002
7211
|
import crypto16 from "crypto";
|
|
7003
7212
|
import { createRequire as createRequire3 } from "module";
|
|
@@ -7023,6 +7232,22 @@ var FORCE_ATLAS_SETTINGS = Object.freeze({
|
|
|
7023
7232
|
var RENDER_MAX_NODE_SIZE = 14;
|
|
7024
7233
|
var RENDER_HALF_EXTENT_PX = 420;
|
|
7025
7234
|
var NODE_SIZE_EXTENT_FRACTION = RENDER_MAX_NODE_SIZE / RENDER_HALF_EXTENT_PX;
|
|
7235
|
+
function siteCrawlGraphNodeSize(extent, nodeCount) {
|
|
7236
|
+
if (!Number.isFinite(extent) || extent <= 0 || nodeCount <= 0) return 0;
|
|
7237
|
+
const byExtent = extent * NODE_SIZE_EXTENT_FRACTION;
|
|
7238
|
+
const byArea = Math.sqrt(NOVERLAP_SETTINGS.maxPackingFraction * (2 * extent) ** 2 / nodeCount) / 2;
|
|
7239
|
+
return Math.min(byExtent, byArea);
|
|
7240
|
+
}
|
|
7241
|
+
function siteCrawlGraphExtent(positions) {
|
|
7242
|
+
if (positions.length === 0) return 0;
|
|
7243
|
+
const centerX = positions.reduce((sum, p) => sum + p.x, 0) / positions.length;
|
|
7244
|
+
const centerY = positions.reduce((sum, p) => sum + p.y, 0) / positions.length;
|
|
7245
|
+
let extent = 0;
|
|
7246
|
+
for (const position of positions) {
|
|
7247
|
+
extent = Math.max(extent, Math.abs(position.x - centerX), Math.abs(position.y - centerY));
|
|
7248
|
+
}
|
|
7249
|
+
return extent;
|
|
7250
|
+
}
|
|
7026
7251
|
var NOVERLAP_SETTINGS = Object.freeze({
|
|
7027
7252
|
/** Half a node of clear space between two touching nodes. */
|
|
7028
7253
|
marginFactor: 0.5,
|
|
@@ -7058,26 +7283,17 @@ function noverlapMaxIterations(nodeCount) {
|
|
|
7058
7283
|
}
|
|
7059
7284
|
return NOVERLAP_SETTINGS.iterationsFloor;
|
|
7060
7285
|
}
|
|
7061
|
-
var
|
|
7062
|
-
var OUTLIER_RING_OUTER_FACTOR = 1.75;
|
|
7063
|
-
var OUTLIER_RING_MAX_RINGS = 4;
|
|
7064
|
-
var OUTLIER_RING_TARGET_PER_RING = 16;
|
|
7065
|
-
var OUTLIER_RING_MIN_RADIUS = 1;
|
|
7286
|
+
var COMPONENT_PACKING_SPACING_FACTOR = 3;
|
|
7066
7287
|
function siteCrawlGraphLayoutSettingsFingerprint(settings) {
|
|
7067
7288
|
return crypto16.createHash("sha256").update(JSON.stringify(settings)).digest("hex").slice(0, 8);
|
|
7068
7289
|
}
|
|
7069
|
-
var SITE_CRAWL_GRAPH_LAYOUT_ALGORITHM = "site-health-fa2-
|
|
7290
|
+
var SITE_CRAWL_GRAPH_LAYOUT_ALGORITHM = "site-health-fa2-v4";
|
|
7070
7291
|
var SITE_CRAWL_GRAPH_LAYOUT_VERSION = `${SITE_CRAWL_GRAPH_LAYOUT_ALGORITHM}-${siteCrawlGraphLayoutSettingsFingerprint({
|
|
7071
7292
|
forceAtlas: FORCE_ATLAS_SETTINGS,
|
|
7072
7293
|
noverlap: NOVERLAP_SETTINGS,
|
|
7073
7294
|
nodeSizeExtentFraction: NODE_SIZE_EXTENT_FRACTION,
|
|
7074
|
-
|
|
7075
|
-
|
|
7076
|
-
outer: OUTLIER_RING_OUTER_FACTOR,
|
|
7077
|
-
maxRings: OUTLIER_RING_MAX_RINGS,
|
|
7078
|
-
targetPerRing: OUTLIER_RING_TARGET_PER_RING,
|
|
7079
|
-
minRadius: OUTLIER_RING_MIN_RADIUS
|
|
7080
|
-
}
|
|
7295
|
+
componentPackingSpacingFactor: COMPONENT_PACKING_SPACING_FACTOR,
|
|
7296
|
+
templateLinkRatioThreshold: TEMPLATE_LINK_RATIO_THRESHOLD
|
|
7081
7297
|
})}`;
|
|
7082
7298
|
var WORKER_SOURCE = String.raw`
|
|
7083
7299
|
const {parentPort, workerData} = require('node:worker_threads');
|
|
@@ -7103,6 +7319,7 @@ try {
|
|
|
7103
7319
|
settings: workerData.settings,
|
|
7104
7320
|
});
|
|
7105
7321
|
}
|
|
7322
|
+
let nodeSize = 0;
|
|
7106
7323
|
// Anti-overlap runs on the spatialization ForceAtlas2 just produced. Node
|
|
7107
7324
|
// size is uniform and derived from the graph's own extent, so the pass is
|
|
7108
7325
|
// scale invariant and stays deterministic for a fixed input.
|
|
@@ -7122,11 +7339,12 @@ try {
|
|
|
7122
7339
|
if (Number.isFinite(extent) && extent > 0) {
|
|
7123
7340
|
// Whichever bound is tighter: one render-sized node, or the largest disc
|
|
7124
7341
|
// that still leaves the graph mostly empty space at this node count.
|
|
7342
|
+
// Mirrors siteCrawlGraphNodeSize; a test asserts the two agree.
|
|
7125
7343
|
const byExtent = extent * workerData.nodeSizeExtentFraction;
|
|
7126
7344
|
const byArea = Math.sqrt(
|
|
7127
7345
|
(workerData.noverlap.maxPackingFraction * Math.pow(2 * extent, 2)) / graph.order,
|
|
7128
7346
|
) / 2;
|
|
7129
|
-
|
|
7347
|
+
nodeSize = Math.min(byExtent, byArea);
|
|
7130
7348
|
graph.forEachNode(function (nodeKey) {
|
|
7131
7349
|
graph.setNodeAttribute(nodeKey, 'size', nodeSize);
|
|
7132
7350
|
});
|
|
@@ -7146,7 +7364,7 @@ try {
|
|
|
7146
7364
|
graph.forEachNode((nodeKey, attributes) => {
|
|
7147
7365
|
positions.push({nodeKey, x: attributes.x, y: attributes.y});
|
|
7148
7366
|
});
|
|
7149
|
-
parentPort.postMessage({ok: true, positions});
|
|
7367
|
+
parentPort.postMessage({ok: true, positions, nodeSize});
|
|
7150
7368
|
} catch (error) {
|
|
7151
7369
|
parentPort.postMessage({
|
|
7152
7370
|
ok: false,
|
|
@@ -7204,7 +7422,8 @@ function adaptiveForceAtlasIterations(nodeCount, edgeCount) {
|
|
|
7204
7422
|
}
|
|
7205
7423
|
async function computeForceAtlasPositions(input, options = {}) {
|
|
7206
7424
|
if (input.nodes.length <= 1 || input.edges.length === 0) {
|
|
7207
|
-
|
|
7425
|
+
const positions = input.nodes.map(({ nodeKey, x, y }) => ({ nodeKey, x, y }));
|
|
7426
|
+
return { positions, nodeSize: siteCrawlGraphNodeSize(siteCrawlGraphExtent(positions), positions.length) };
|
|
7208
7427
|
}
|
|
7209
7428
|
const requireFromPackage = createRequire3(import.meta.url);
|
|
7210
7429
|
const worker = new Worker(WORKER_SOURCE, {
|
|
@@ -7233,7 +7452,7 @@ async function computeForceAtlasPositions(input, options = {}) {
|
|
|
7233
7452
|
options.signal?.removeEventListener("abort", onAbort);
|
|
7234
7453
|
void worker.terminate();
|
|
7235
7454
|
if (result.error) reject(result.error instanceof Error ? result.error : new Error("Graph layout failed with a non-Error reason", { cause: result.error }));
|
|
7236
|
-
else resolve(result.positions ?? []);
|
|
7455
|
+
else resolve({ positions: result.positions ?? [], nodeSize: result.nodeSize ?? 0 });
|
|
7237
7456
|
};
|
|
7238
7457
|
const onAbort = () => finish({ error: options.signal?.reason ?? new DOMException("Layout aborted", "AbortError") });
|
|
7239
7458
|
const timer = setTimeout(() => finish({ error: new GraphLayoutTimeoutError(`Graph layout exceeded ${timeoutMs}ms`) }), timeoutMs);
|
|
@@ -7241,7 +7460,7 @@ async function computeForceAtlasPositions(input, options = {}) {
|
|
|
7241
7460
|
if (options.signal?.aborted) return onAbort();
|
|
7242
7461
|
worker.once("message", (message) => {
|
|
7243
7462
|
if (!message.ok) finish({ error: new Error(message.error ?? "Graph layout worker failed") });
|
|
7244
|
-
else finish({ positions: message.positions });
|
|
7463
|
+
else finish({ positions: message.positions, nodeSize: message.nodeSize });
|
|
7245
7464
|
});
|
|
7246
7465
|
worker.once("error", (error) => finish({ error }));
|
|
7247
7466
|
worker.once("exit", (code) => {
|
|
@@ -7277,48 +7496,60 @@ function siteCrawlGraphComponents(nodeKeys, edges) {
|
|
|
7277
7496
|
if (group) group.push(key);
|
|
7278
7497
|
else members.set(root, [key]);
|
|
7279
7498
|
}
|
|
7280
|
-
|
|
7281
|
-
|
|
7282
|
-
|
|
7283
|
-
|
|
7284
|
-
|
|
7285
|
-
|
|
7286
|
-
const
|
|
7287
|
-
|
|
7288
|
-
|
|
7289
|
-
|
|
7290
|
-
const
|
|
7291
|
-
|
|
7292
|
-
|
|
7293
|
-
|
|
7294
|
-
|
|
7295
|
-
|
|
7296
|
-
|
|
7297
|
-
|
|
7298
|
-
|
|
7299
|
-
|
|
7300
|
-
const
|
|
7301
|
-
|
|
7302
|
-
|
|
7303
|
-
)
|
|
7304
|
-
|
|
7305
|
-
|
|
7306
|
-
|
|
7307
|
-
|
|
7308
|
-
const
|
|
7309
|
-
|
|
7310
|
-
|
|
7311
|
-
|
|
7312
|
-
|
|
7313
|
-
|
|
7314
|
-
|
|
7315
|
-
|
|
7316
|
-
|
|
7317
|
-
|
|
7318
|
-
|
|
7319
|
-
|
|
7320
|
-
|
|
7321
|
-
|
|
7499
|
+
return [...members.values()].sort((left, right) => right.length - left.length || (left[0] < right[0] ? -1 : left[0] > right[0] ? 1 : 0));
|
|
7500
|
+
}
|
|
7501
|
+
function describeComponent(nodeKeys, byKey) {
|
|
7502
|
+
let sumX = 0;
|
|
7503
|
+
let sumY = 0;
|
|
7504
|
+
for (const nodeKey of nodeKeys) {
|
|
7505
|
+
const position = byKey.get(nodeKey);
|
|
7506
|
+
sumX += position.x;
|
|
7507
|
+
sumY += position.y;
|
|
7508
|
+
}
|
|
7509
|
+
const centerX = sumX / nodeKeys.length;
|
|
7510
|
+
const centerY = sumY / nodeKeys.length;
|
|
7511
|
+
let radius = 0;
|
|
7512
|
+
for (const nodeKey of nodeKeys) {
|
|
7513
|
+
const position = byKey.get(nodeKey);
|
|
7514
|
+
radius = Math.max(radius, Math.hypot(position.x - centerX, position.y - centerY));
|
|
7515
|
+
}
|
|
7516
|
+
return { nodeKeys, centerX, centerY, radius };
|
|
7517
|
+
}
|
|
7518
|
+
function packSiteCrawlGraphComponents(positions, edges, nodeSpacing) {
|
|
7519
|
+
const components = siteCrawlGraphComponents(positions.map((position) => position.nodeKey), edges);
|
|
7520
|
+
if (components.length <= 1) return [...positions];
|
|
7521
|
+
const byKey = new Map(positions.map((position) => [position.nodeKey, position]));
|
|
7522
|
+
if (positions.some((position) => !Number.isFinite(position.x) || !Number.isFinite(position.y))) {
|
|
7523
|
+
return [...positions];
|
|
7524
|
+
}
|
|
7525
|
+
const described = components.map((nodeKeys) => describeComponent(nodeKeys, byKey));
|
|
7526
|
+
const [anchor, ...rest] = described;
|
|
7527
|
+
const spacing = Math.max(nodeSpacing, Number.EPSILON) * COMPONENT_PACKING_SPACING_FACTOR;
|
|
7528
|
+
const ordered = [...rest].sort((left, right) => right.radius - left.radius || (left.nodeKeys[0] < right.nodeKeys[0] ? -1 : left.nodeKeys[0] > right.nodeKeys[0] ? 1 : 0));
|
|
7529
|
+
const moved = /* @__PURE__ */ new Map();
|
|
7530
|
+
let ringRadius = anchor.radius + spacing;
|
|
7531
|
+
let ringPitch = 0;
|
|
7532
|
+
let placedOnRing = 0;
|
|
7533
|
+
let ringCapacity = 0;
|
|
7534
|
+
for (const component of ordered) {
|
|
7535
|
+
if (placedOnRing >= ringCapacity) {
|
|
7536
|
+
ringPitch = 2 * component.radius + spacing;
|
|
7537
|
+
ringRadius += ringPitch / 2 + (placedOnRing === 0 ? 0 : ringPitch / 2);
|
|
7538
|
+
ringCapacity = Math.max(1, Math.floor(2 * Math.PI * ringRadius / ringPitch));
|
|
7539
|
+
placedOnRing = 0;
|
|
7540
|
+
}
|
|
7541
|
+
const angle = placedOnRing / ringCapacity * Math.PI * 2;
|
|
7542
|
+
placedOnRing += 1;
|
|
7543
|
+
const targetX = anchor.centerX + Math.cos(angle) * ringRadius;
|
|
7544
|
+
const targetY = anchor.centerY + Math.sin(angle) * ringRadius;
|
|
7545
|
+
const shiftX = targetX - component.centerX;
|
|
7546
|
+
const shiftY = targetY - component.centerY;
|
|
7547
|
+
for (const nodeKey of component.nodeKeys) {
|
|
7548
|
+
const position = byKey.get(nodeKey);
|
|
7549
|
+
moved.set(nodeKey, { nodeKey, x: position.x + shiftX, y: position.y + shiftY });
|
|
7550
|
+
}
|
|
7551
|
+
}
|
|
7552
|
+
return positions.map((position) => moved.get(position.nodeKey) ?? position);
|
|
7322
7553
|
}
|
|
7323
7554
|
function normalizePositions(seededNodes, computed, rootNodeKey) {
|
|
7324
7555
|
const byKey = new Map(computed.map((position) => [position.nodeKey, position]));
|
|
@@ -7352,29 +7583,40 @@ async function layoutSiteCrawlGraphInput(input, options = {}) {
|
|
|
7352
7583
|
failureCode: "empty-crawl",
|
|
7353
7584
|
totalNodes: input.totalNodes,
|
|
7354
7585
|
totalEdges: input.totalEdges,
|
|
7586
|
+
totalTemplateEdges: input.totalTemplateEdges,
|
|
7355
7587
|
nodeCount: 0,
|
|
7356
7588
|
edgeCount: 0,
|
|
7357
7589
|
nodes: [],
|
|
7358
7590
|
edges: []
|
|
7359
7591
|
};
|
|
7360
7592
|
}
|
|
7593
|
+
const layoutEdges = input.edges.filter((edge) => !edge.isTemplate);
|
|
7361
7594
|
const seeded = seedSiteCrawlGraphNodes(input.nodes, input.rootNodeKey, input.priorPositions);
|
|
7595
|
+
const timeoutMs = options.timeoutMs ?? SITE_CRAWL_GRAPH_LAYOUT_TIMEOUT_MS;
|
|
7596
|
+
const deadline = Date.now() + timeoutMs;
|
|
7597
|
+
const assertWithinBudget = () => {
|
|
7598
|
+
options.signal?.throwIfAborted();
|
|
7599
|
+
if (Date.now() > deadline) throw new GraphLayoutTimeoutError(`Graph layout exceeded ${timeoutMs}ms`);
|
|
7600
|
+
};
|
|
7362
7601
|
try {
|
|
7363
7602
|
const compute = options.computePositions ?? computeForceAtlasPositions;
|
|
7364
|
-
const positions = await compute({
|
|
7603
|
+
const { positions, nodeSize } = await compute({
|
|
7365
7604
|
nodes: seeded,
|
|
7366
|
-
edges:
|
|
7367
|
-
iterations: adaptiveForceAtlasIterations(input.nodes.length,
|
|
7368
|
-
}, { signal: options.signal, timeoutMs:
|
|
7369
|
-
|
|
7370
|
-
const framed =
|
|
7605
|
+
edges: layoutEdges,
|
|
7606
|
+
iterations: adaptiveForceAtlasIterations(input.nodes.length, layoutEdges.length)
|
|
7607
|
+
}, { signal: options.signal, timeoutMs: Math.max(1, deadline - Date.now()) });
|
|
7608
|
+
assertWithinBudget();
|
|
7609
|
+
const framed = packSiteCrawlGraphComponents(positions, layoutEdges, nodeSize);
|
|
7610
|
+
assertWithinBudget();
|
|
7371
7611
|
return {
|
|
7372
7612
|
state: "ready",
|
|
7373
7613
|
layoutVersion: SITE_CRAWL_GRAPH_LAYOUT_VERSION,
|
|
7374
7614
|
totalNodes: input.totalNodes,
|
|
7375
7615
|
totalEdges: input.totalEdges,
|
|
7616
|
+
totalTemplateEdges: input.totalTemplateEdges,
|
|
7376
7617
|
nodeCount: input.nodes.length,
|
|
7377
7618
|
edgeCount: input.edges.length,
|
|
7619
|
+
templateLinksExcluded: true,
|
|
7378
7620
|
nodes: normalizePositions(seeded, framed, input.rootNodeKey),
|
|
7379
7621
|
edges: input.edges.map((edge, sampleRank) => ({ ...edge, sampleRank }))
|
|
7380
7622
|
};
|
|
@@ -7386,6 +7628,7 @@ async function layoutSiteCrawlGraphInput(input, options = {}) {
|
|
|
7386
7628
|
failureCode: error instanceof GraphLayoutTimeoutError ? "layout-timeout" : "layout-error",
|
|
7387
7629
|
totalNodes: input.totalNodes,
|
|
7388
7630
|
totalEdges: input.totalEdges,
|
|
7631
|
+
totalTemplateEdges: input.totalTemplateEdges,
|
|
7389
7632
|
nodeCount: 0,
|
|
7390
7633
|
edgeCount: 0,
|
|
7391
7634
|
nodes: [],
|
|
@@ -7422,6 +7665,7 @@ function loadPriorSiteCrawlGraphPositions(db, scope) {
|
|
|
7422
7665
|
async function prepareSiteCrawlGraphLayout(db, scope, options = {}) {
|
|
7423
7666
|
let totalNodes = 0;
|
|
7424
7667
|
let totalEdges = 0;
|
|
7668
|
+
let totalTemplateEdges = 0;
|
|
7425
7669
|
try {
|
|
7426
7670
|
const assertNotAborted = () => options.signal?.throwIfAborted();
|
|
7427
7671
|
assertNotAborted();
|
|
@@ -7439,7 +7683,7 @@ async function prepareSiteCrawlGraphLayout(db, scope, options = {}) {
|
|
|
7439
7683
|
];
|
|
7440
7684
|
totalNodes = db.select({ value: count() }).from(siteCrawlPages).where(and10(...pageScope)).get()?.value ?? 0;
|
|
7441
7685
|
assertNotAborted();
|
|
7442
|
-
|
|
7686
|
+
const countGraphCompatibleEdges = (...extra) => db.select({ value: count() }).from(siteCrawlEdges).innerJoin(graphSourcePage, and10(
|
|
7443
7687
|
eq13(graphSourcePage.projectId, scope.projectId),
|
|
7444
7688
|
eq13(graphSourcePage.runId, scope.runId),
|
|
7445
7689
|
eq13(graphSourcePage.attemptId, scope.attemptId),
|
|
@@ -7449,7 +7693,10 @@ async function prepareSiteCrawlGraphLayout(db, scope, options = {}) {
|
|
|
7449
7693
|
eq13(graphTargetPage.runId, scope.runId),
|
|
7450
7694
|
eq13(graphTargetPage.attemptId, scope.attemptId),
|
|
7451
7695
|
eq13(graphTargetPage.nodeKey, siteCrawlEdges.targetNodeKey)
|
|
7452
|
-
)).where(and10(...edgeScope)).get()?.value ?? 0;
|
|
7696
|
+
)).where(and10(...edgeScope, ...extra)).get()?.value ?? 0;
|
|
7697
|
+
totalEdges = countGraphCompatibleEdges();
|
|
7698
|
+
assertNotAborted();
|
|
7699
|
+
totalTemplateEdges = countGraphCompatibleEdges(eq13(siteCrawlEdges.isTemplate, true));
|
|
7453
7700
|
assertNotAborted();
|
|
7454
7701
|
const graphSeedColumns = {
|
|
7455
7702
|
nodeKey: siteCrawlPages.nodeKey,
|
|
@@ -7488,7 +7735,8 @@ async function prepareSiteCrawlGraphLayout(db, scope, options = {}) {
|
|
|
7488
7735
|
edge.source_node_key AS sourceNodeKey,
|
|
7489
7736
|
edge.target_node_key AS targetNodeKey,
|
|
7490
7737
|
edge.followable AS followable,
|
|
7491
|
-
edge.occurrences AS occurrences
|
|
7738
|
+
edge.occurrences AS occurrences,
|
|
7739
|
+
edge.is_template AS isTemplate
|
|
7492
7740
|
FROM site_crawl_edges AS edge
|
|
7493
7741
|
INNER JOIN selected AS source ON source.node_key = edge.source_node_key
|
|
7494
7742
|
INNER JOIN selected AS target ON target.node_key = edge.target_node_key
|
|
@@ -7504,8 +7752,9 @@ async function prepareSiteCrawlGraphLayout(db, scope, options = {}) {
|
|
|
7504
7752
|
return await layoutSiteCrawlGraphInput({
|
|
7505
7753
|
totalNodes,
|
|
7506
7754
|
totalEdges,
|
|
7755
|
+
totalTemplateEdges,
|
|
7507
7756
|
nodes,
|
|
7508
|
-
edges: edgeRows,
|
|
7757
|
+
edges: edgeRows.map((row) => ({ ...row, isTemplate: Boolean(row.isTemplate) })),
|
|
7509
7758
|
rootNodeKey,
|
|
7510
7759
|
priorPositions
|
|
7511
7760
|
}, options);
|
|
@@ -7517,6 +7766,7 @@ async function prepareSiteCrawlGraphLayout(db, scope, options = {}) {
|
|
|
7517
7766
|
failureCode: "layout-error",
|
|
7518
7767
|
totalNodes,
|
|
7519
7768
|
totalEdges,
|
|
7769
|
+
totalTemplateEdges,
|
|
7520
7770
|
nodeCount: 0,
|
|
7521
7771
|
edgeCount: 0,
|
|
7522
7772
|
nodes: [],
|
|
@@ -7541,8 +7791,10 @@ function persistSiteCrawlGraphLayout(db, scope, layout, now = (/* @__PURE__ */ n
|
|
|
7541
7791
|
failureCode: layout.state === "unavailable" ? layout.failureCode : null,
|
|
7542
7792
|
totalNodes: layout.totalNodes,
|
|
7543
7793
|
totalEdges: layout.totalEdges,
|
|
7794
|
+
totalTemplateEdges: layout.totalTemplateEdges,
|
|
7544
7795
|
nodeCount: layout.nodeCount,
|
|
7545
7796
|
edgeCount: layout.edgeCount,
|
|
7797
|
+
templateLinksExcluded: layout.state === "ready",
|
|
7546
7798
|
createdAt: now,
|
|
7547
7799
|
updatedAt: now
|
|
7548
7800
|
}).run();
|
|
@@ -7573,6 +7825,7 @@ function persistSiteCrawlGraphLayout(db, scope, layout, now = (/* @__PURE__ */ n
|
|
|
7573
7825
|
targetNodeKey: edge.targetNodeKey,
|
|
7574
7826
|
followable: edge.followable,
|
|
7575
7827
|
occurrences: edge.occurrences,
|
|
7828
|
+
isTemplate: edge.isTemplate,
|
|
7576
7829
|
createdAt: now
|
|
7577
7830
|
}))).run();
|
|
7578
7831
|
}
|
|
@@ -7586,6 +7839,70 @@ function deleteSiteCrawlGraphLayout(db, scope) {
|
|
|
7586
7839
|
)).run();
|
|
7587
7840
|
}
|
|
7588
7841
|
|
|
7842
|
+
// src/site-crawl-template-links.ts
|
|
7843
|
+
import { and as and11, asc as asc2, eq as eq14, gt, inArray as inArray4 } from "drizzle-orm";
|
|
7844
|
+
var TEMPLATE_LINK_READ_BATCH = 2e3;
|
|
7845
|
+
var TEMPLATE_LINK_WRITE_BATCH = 500;
|
|
7846
|
+
function* streamAttemptEdges(db, scope) {
|
|
7847
|
+
let after = "";
|
|
7848
|
+
for (; ; ) {
|
|
7849
|
+
const rows = db.select({
|
|
7850
|
+
edgeKey: siteCrawlEdges.edgeKey,
|
|
7851
|
+
sourceNodeKey: siteCrawlEdges.sourceNodeKey,
|
|
7852
|
+
targetNodeKey: siteCrawlEdges.targetNodeKey,
|
|
7853
|
+
anchors: siteCrawlEdges.anchors
|
|
7854
|
+
}).from(siteCrawlEdges).where(and11(
|
|
7855
|
+
eq14(siteCrawlEdges.projectId, scope.projectId),
|
|
7856
|
+
eq14(siteCrawlEdges.runId, scope.runId),
|
|
7857
|
+
eq14(siteCrawlEdges.attemptId, scope.attemptId),
|
|
7858
|
+
gt(siteCrawlEdges.edgeKey, after)
|
|
7859
|
+
)).orderBy(asc2(siteCrawlEdges.edgeKey)).limit(TEMPLATE_LINK_READ_BATCH).all();
|
|
7860
|
+
if (rows.length === 0) return;
|
|
7861
|
+
yield rows;
|
|
7862
|
+
after = rows[rows.length - 1].edgeKey;
|
|
7863
|
+
if (rows.length < TEMPLATE_LINK_READ_BATCH) return;
|
|
7864
|
+
}
|
|
7865
|
+
}
|
|
7866
|
+
function* chunked(values, size = TEMPLATE_LINK_WRITE_BATCH) {
|
|
7867
|
+
for (let offset = 0; offset < values.length; offset += size) {
|
|
7868
|
+
yield values.slice(offset, offset + size);
|
|
7869
|
+
}
|
|
7870
|
+
}
|
|
7871
|
+
function classifySiteCrawlTemplateLinks(db, scope, pagesFetched) {
|
|
7872
|
+
const scopeFilter = and11(
|
|
7873
|
+
eq14(siteCrawlEdges.projectId, scope.projectId),
|
|
7874
|
+
eq14(siteCrawlEdges.runId, scope.runId),
|
|
7875
|
+
eq14(siteCrawlEdges.attemptId, scope.attemptId)
|
|
7876
|
+
);
|
|
7877
|
+
const detection = templateLinkDetection(pagesFetched);
|
|
7878
|
+
db.update(siteCrawlEdges).set({ isTemplate: false, templateRatio: null }).where(scopeFilter).run();
|
|
7879
|
+
if (detection !== SiteHealthTemplateDetections.applied) {
|
|
7880
|
+
return { detection, templateEdgeCount: 0 };
|
|
7881
|
+
}
|
|
7882
|
+
const index = createTemplateLinkPairIndex();
|
|
7883
|
+
for (const batch of streamAttemptEdges(db, scope)) observeTemplateLinkEdges(index, batch);
|
|
7884
|
+
const edgeKeysByRatio = /* @__PURE__ */ new Map();
|
|
7885
|
+
let templateEdgeCount = 0;
|
|
7886
|
+
for (const batch of streamAttemptEdges(db, scope)) {
|
|
7887
|
+
for (const edge of batch) {
|
|
7888
|
+
const ratio = templateLinkRatio(index, pagesFetched, edge);
|
|
7889
|
+
if (ratio == null) continue;
|
|
7890
|
+
const group = edgeKeysByRatio.get(ratio);
|
|
7891
|
+
if (group) group.push(edge.edgeKey);
|
|
7892
|
+
else edgeKeysByRatio.set(ratio, [edge.edgeKey]);
|
|
7893
|
+
if (isTemplateLinkRatio(ratio)) templateEdgeCount += 1;
|
|
7894
|
+
}
|
|
7895
|
+
}
|
|
7896
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7897
|
+
for (const [ratio, edgeKeys] of edgeKeysByRatio) {
|
|
7898
|
+
const isTemplate = isTemplateLinkRatio(ratio);
|
|
7899
|
+
for (const chunk of chunked(edgeKeys)) {
|
|
7900
|
+
db.update(siteCrawlEdges).set({ isTemplate, templateRatio: ratio, updatedAt: now }).where(and11(scopeFilter, inArray4(siteCrawlEdges.edgeKey, chunk))).run();
|
|
7901
|
+
}
|
|
7902
|
+
}
|
|
7903
|
+
return { detection, templateEdgeCount };
|
|
7904
|
+
}
|
|
7905
|
+
|
|
7589
7906
|
// src/execute-site-audit.ts
|
|
7590
7907
|
var log12 = createLogger("SiteAudit");
|
|
7591
7908
|
var SiteAuditCancelledError = class extends Error {
|
|
@@ -7733,9 +8050,9 @@ function deadLinkCheckedCount(edges, pages) {
|
|
|
7733
8050
|
}
|
|
7734
8051
|
async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
7735
8052
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
7736
|
-
const claim = db.update(runs).set({ status: "running", startedAt }).where(
|
|
8053
|
+
const claim = db.update(runs).set({ status: "running", startedAt }).where(and12(eq15(runs.id, runId), eq15(runs.projectId, projectId), eq15(runs.status, "queued"))).run();
|
|
7737
8054
|
if (claim.changes === 0) return;
|
|
7738
|
-
const attemptNumber = (db.select({ value: sql6`coalesce(max(${siteCrawlAttempts.attemptNumber}), 0)` }).from(siteCrawlAttempts).where(
|
|
8055
|
+
const attemptNumber = (db.select({ value: sql6`coalesce(max(${siteCrawlAttempts.attemptNumber}), 0)` }).from(siteCrawlAttempts).where(eq15(siteCrawlAttempts.runId, runId)).get()?.value ?? 0) + 1;
|
|
7739
8056
|
const attemptId = crypto17.randomUUID();
|
|
7740
8057
|
db.insert(siteCrawlAttempts).values({
|
|
7741
8058
|
id: attemptId,
|
|
@@ -7770,28 +8087,49 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7770
8087
|
const nodeKey = nodeKeyByUrl.get(url);
|
|
7771
8088
|
if (!nodeKey) continue;
|
|
7772
8089
|
const scope = [
|
|
7773
|
-
|
|
7774
|
-
|
|
7775
|
-
|
|
8090
|
+
eq15(siteCrawlEdges.projectId, projectId),
|
|
8091
|
+
eq15(siteCrawlEdges.runId, runId),
|
|
8092
|
+
eq15(siteCrawlEdges.attemptId, attemptId)
|
|
7776
8093
|
];
|
|
7777
|
-
tx.update(siteCrawlEdges).set({ sourceNodeKey: nodeKey, updatedAt: now }).where(
|
|
8094
|
+
tx.update(siteCrawlEdges).set({ sourceNodeKey: nodeKey, updatedAt: now }).where(and12(
|
|
7778
8095
|
...scope,
|
|
7779
|
-
|
|
8096
|
+
eq15(siteCrawlEdges.sourceUrl, url)
|
|
7780
8097
|
)).run();
|
|
7781
|
-
tx.update(siteCrawlEdges).set({ targetNodeKey: nodeKey, updatedAt: now }).where(
|
|
8098
|
+
tx.update(siteCrawlEdges).set({ targetNodeKey: nodeKey, updatedAt: now }).where(and12(
|
|
7782
8099
|
...scope,
|
|
7783
|
-
|
|
7784
|
-
)).run();
|
|
7785
|
-
tx.update(siteCrawlPages).set({ canonicalNodeKey: nodeKey, updatedAt: now }).where(and11(
|
|
7786
|
-
eq14(siteCrawlPages.projectId, projectId),
|
|
7787
|
-
eq14(siteCrawlPages.runId, runId),
|
|
7788
|
-
eq14(siteCrawlPages.attemptId, attemptId),
|
|
7789
|
-
eq14(siteCrawlPages.canonicalUrl, url)
|
|
8100
|
+
eq15(siteCrawlEdges.targetUrl, url)
|
|
7790
8101
|
)).run();
|
|
8102
|
+
const pageScope = [
|
|
8103
|
+
eq15(siteCrawlPages.projectId, projectId),
|
|
8104
|
+
eq15(siteCrawlPages.runId, runId),
|
|
8105
|
+
eq15(siteCrawlPages.attemptId, attemptId)
|
|
8106
|
+
];
|
|
8107
|
+
const canonicalTargets = tx.select({
|
|
8108
|
+
id: siteCrawlPages.id,
|
|
8109
|
+
nodeKey: siteCrawlPages.nodeKey,
|
|
8110
|
+
fetchState: siteCrawlPages.fetchState,
|
|
8111
|
+
indexabilityState: siteCrawlPages.indexabilityState,
|
|
8112
|
+
indexabilityReasons: siteCrawlPages.indexabilityReasons
|
|
8113
|
+
}).from(siteCrawlPages).where(and12(...pageScope, eq15(siteCrawlPages.canonicalUrl, url))).all();
|
|
8114
|
+
for (const target of canonicalTargets) {
|
|
8115
|
+
tx.update(siteCrawlPages).set({
|
|
8116
|
+
canonicalNodeKey: nodeKey,
|
|
8117
|
+
healthState: deriveSiteHealthState({ ...target, canonicalNodeKey: nodeKey }),
|
|
8118
|
+
updatedAt: now
|
|
8119
|
+
}).where(eq15(siteCrawlPages.id, target.id)).run();
|
|
8120
|
+
}
|
|
7791
8121
|
}
|
|
7792
8122
|
};
|
|
7793
8123
|
const persistPage = (tx, page, now) => {
|
|
7794
8124
|
const audit = page.audit;
|
|
8125
|
+
const canonicalNodeKey = page.canonicalUrl ? nodeKeyByUrl.get(page.canonicalUrl) ?? null : null;
|
|
8126
|
+
const healthState = deriveSiteHealthState({
|
|
8127
|
+
fetchState: page.state,
|
|
8128
|
+
indexabilityState: page.indexability.state,
|
|
8129
|
+
indexabilityReasons: page.indexability.reasons,
|
|
8130
|
+
canonicalNodeKey,
|
|
8131
|
+
nodeKey: page.key
|
|
8132
|
+
});
|
|
7795
8133
|
tx.insert(siteCrawlPages).values({
|
|
7796
8134
|
id: crypto17.randomUUID(),
|
|
7797
8135
|
projectId,
|
|
@@ -7816,9 +8154,10 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7816
8154
|
redirectChain: page.redirectChain.map((hop) => hop.to),
|
|
7817
8155
|
directives: { metaRobots: page.metaRobots, xRobots: page.xRobots },
|
|
7818
8156
|
canonicalUrl: page.canonicalUrl,
|
|
7819
|
-
canonicalNodeKey
|
|
8157
|
+
canonicalNodeKey,
|
|
7820
8158
|
indexabilityState: page.indexability.state,
|
|
7821
8159
|
indexabilityReasons: page.indexability.reasons,
|
|
8160
|
+
healthState,
|
|
7822
8161
|
auditState: pageAuditState(page),
|
|
7823
8162
|
auditScore: audit?.overallScore ?? null,
|
|
7824
8163
|
auditFields: audit ? crawlAuditFields(audit) : {},
|
|
@@ -7849,9 +8188,10 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7849
8188
|
redirectChain: page.redirectChain.map((hop) => hop.to),
|
|
7850
8189
|
directives: { metaRobots: page.metaRobots, xRobots: page.xRobots },
|
|
7851
8190
|
canonicalUrl: page.canonicalUrl,
|
|
7852
|
-
canonicalNodeKey
|
|
8191
|
+
canonicalNodeKey,
|
|
7853
8192
|
indexabilityState: page.indexability.state,
|
|
7854
8193
|
indexabilityReasons: page.indexability.reasons,
|
|
8194
|
+
healthState,
|
|
7855
8195
|
auditState: pageAuditState(page),
|
|
7856
8196
|
auditScore: audit?.overallScore ?? null,
|
|
7857
8197
|
auditFields: audit ? crawlAuditFields(audit) : {},
|
|
@@ -7874,6 +8214,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7874
8214
|
bindReferences(tx, changedUrls, now);
|
|
7875
8215
|
};
|
|
7876
8216
|
const persistEdge = (tx, edge, now) => {
|
|
8217
|
+
if (isSelfLink(edge.from, edge.to)) return;
|
|
7877
8218
|
observedEdges.set(edge.key, edge);
|
|
7878
8219
|
tx.insert(siteCrawlEdges).values({
|
|
7879
8220
|
id: crypto17.randomUUID(),
|
|
@@ -7922,20 +8263,20 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7922
8263
|
linkScoreRaw: metrics.linkScoreRaw,
|
|
7923
8264
|
linkScoreNormalized: metrics.linkScore,
|
|
7924
8265
|
updatedAt: now
|
|
7925
|
-
}).where(
|
|
7926
|
-
|
|
7927
|
-
|
|
7928
|
-
|
|
7929
|
-
|
|
8266
|
+
}).where(and12(
|
|
8267
|
+
eq15(siteCrawlPages.projectId, projectId),
|
|
8268
|
+
eq15(siteCrawlPages.runId, runId),
|
|
8269
|
+
eq15(siteCrawlPages.attemptId, attemptId),
|
|
8270
|
+
eq15(siteCrawlPages.nodeKey, key)
|
|
7930
8271
|
)).run();
|
|
7931
8272
|
};
|
|
7932
8273
|
const persistEvent = async (event) => {
|
|
7933
8274
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7934
8275
|
db.transaction((tx) => {
|
|
7935
|
-
const receipt = tx.select({ checksum: siteCrawlEventReceipts.checksum }).from(siteCrawlEventReceipts).where(
|
|
7936
|
-
|
|
7937
|
-
|
|
7938
|
-
|
|
8276
|
+
const receipt = tx.select({ checksum: siteCrawlEventReceipts.checksum }).from(siteCrawlEventReceipts).where(and12(
|
|
8277
|
+
eq15(siteCrawlEventReceipts.attemptId, attemptId),
|
|
8278
|
+
eq15(siteCrawlEventReceipts.sequence, event.sequence),
|
|
8279
|
+
eq15(siteCrawlEventReceipts.batchId, event.batchId)
|
|
7939
8280
|
)).get();
|
|
7940
8281
|
if (receipt) {
|
|
7941
8282
|
if (receipt.checksum !== event.checksum) {
|
|
@@ -7968,26 +8309,39 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7968
8309
|
pagesErrored: event.type === "summary" ? observedErrorCount(observedPages.values()) : sql6`${siteCrawlAttempts.pagesErrored}`,
|
|
7969
8310
|
edgesDiscovered: progress.edgesObserved,
|
|
7970
8311
|
updatedAt: now
|
|
7971
|
-
}).where(
|
|
8312
|
+
}).where(eq15(siteCrawlAttempts.id, attemptId)).run();
|
|
7972
8313
|
} else {
|
|
7973
8314
|
tx.update(siteCrawlAttempts).set({
|
|
7974
8315
|
lastEventSequence: event.sequence,
|
|
7975
8316
|
lastEventChecksum: event.checksum,
|
|
7976
8317
|
updatedAt: now
|
|
7977
|
-
}).where(
|
|
8318
|
+
}).where(eq15(siteCrawlAttempts.id, attemptId)).run();
|
|
7978
8319
|
}
|
|
7979
8320
|
});
|
|
7980
8321
|
};
|
|
7981
8322
|
try {
|
|
7982
|
-
const project = db.select().from(projects).where(
|
|
8323
|
+
const project = db.select().from(projects).where(eq15(projects.id, projectId)).get();
|
|
7983
8324
|
if (!project) throw new Error(`Project not found: ${projectId}`);
|
|
7984
8325
|
const homepageUrl = toHomepageUrl(project.canonicalDomain);
|
|
7985
8326
|
const maxPages = clampSiteAuditLimit(opts.maxPages ?? opts.limit);
|
|
7986
8327
|
const maxEdges = clampSiteAuditEdgeLimit(opts.maxEdges);
|
|
7987
|
-
await
|
|
8328
|
+
const resolvedRoot = await resolveSiteAuditRootUrl(homepageUrl, {
|
|
8329
|
+
resolveTarget: resolveMeasurementSitemapTarget,
|
|
8330
|
+
signal: opts.signal
|
|
8331
|
+
});
|
|
8332
|
+
const crawlRootUrl = resolvedRoot.effectiveUrl;
|
|
7988
8333
|
if (opts.sitemapUrl) await assertSiteAuditUrlAllowed(opts.sitemapUrl, "sitemapUrl");
|
|
7989
|
-
log12.info("start", {
|
|
7990
|
-
|
|
8334
|
+
log12.info("start", {
|
|
8335
|
+
runId,
|
|
8336
|
+
projectId,
|
|
8337
|
+
homepageUrl,
|
|
8338
|
+
crawlRootUrl,
|
|
8339
|
+
redirects: resolvedRoot.redirects.length,
|
|
8340
|
+
maxPages,
|
|
8341
|
+
maxEdges,
|
|
8342
|
+
maxDepth: opts.maxDepth ?? null
|
|
8343
|
+
});
|
|
8344
|
+
const report = await runSiteCrawl(crawlRootUrl, {
|
|
7991
8345
|
mode: "summary",
|
|
7992
8346
|
sitemapUrl: opts.sitemapUrl,
|
|
7993
8347
|
maxPages,
|
|
@@ -8002,6 +8356,17 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
8002
8356
|
if (crawlSummary.complete && !hasAuditedPages) {
|
|
8003
8357
|
throw emptyCompleteCrawlError(crawlSummary.rootUrl, crawlSummary.finalRootUrl, crawlSummary.pagesObserved);
|
|
8004
8358
|
}
|
|
8359
|
+
const templateLinks = classifySiteCrawlTemplateLinks(
|
|
8360
|
+
db,
|
|
8361
|
+
{ projectId, runId, attemptId },
|
|
8362
|
+
crawlSummary.pagesFetched
|
|
8363
|
+
);
|
|
8364
|
+
log12.info("template-links", {
|
|
8365
|
+
runId,
|
|
8366
|
+
projectId,
|
|
8367
|
+
detection: templateLinks.detection,
|
|
8368
|
+
templateEdges: templateLinks.templateEdgeCount
|
|
8369
|
+
});
|
|
8005
8370
|
const graphLayout = await prepareSiteCrawlGraphLayout(db, {
|
|
8006
8371
|
projectId,
|
|
8007
8372
|
runId,
|
|
@@ -8029,6 +8394,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
8029
8394
|
failureCode: "layout-error",
|
|
8030
8395
|
totalNodes: graphLayout.totalNodes,
|
|
8031
8396
|
totalEdges: graphLayout.totalEdges,
|
|
8397
|
+
totalTemplateEdges: graphLayout.totalTemplateEdges,
|
|
8032
8398
|
nodeCount: 0,
|
|
8033
8399
|
edgeCount: 0,
|
|
8034
8400
|
nodes: [],
|
|
@@ -8043,14 +8409,14 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
8043
8409
|
}
|
|
8044
8410
|
}
|
|
8045
8411
|
const published = db.transaction((tx) => {
|
|
8046
|
-
const claim2 = tx.update(runs).set({ status: terminalStatus, finishedAt }).where(
|
|
8412
|
+
const claim2 = tx.update(runs).set({ status: terminalStatus, finishedAt }).where(and12(eq15(runs.id, runId), eq15(runs.status, "running"))).run();
|
|
8047
8413
|
if (claim2.changes === 0) return false;
|
|
8048
8414
|
if (hasAuditedPages) {
|
|
8049
8415
|
tx.insert(siteAuditSnapshots).values({
|
|
8050
8416
|
id: crypto17.randomUUID(),
|
|
8051
8417
|
projectId,
|
|
8052
8418
|
runId,
|
|
8053
|
-
sitemapUrl: opts.sitemapUrl ??
|
|
8419
|
+
sitemapUrl: opts.sitemapUrl ?? new URL("/sitemap.xml", crawlRootUrl).href,
|
|
8054
8420
|
auditedAt: crawlSummary.completedAt,
|
|
8055
8421
|
aggregateScore: crawlSummary.auditRollup.aggregateScore ?? 0,
|
|
8056
8422
|
pagesDiscovered: crawlSummary.pagesDiscovered,
|
|
@@ -8082,7 +8448,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
8082
8448
|
projectId,
|
|
8083
8449
|
runId,
|
|
8084
8450
|
attemptId,
|
|
8085
|
-
requestedRootUrl:
|
|
8451
|
+
requestedRootUrl: resolvedRoot.requestedUrl,
|
|
8086
8452
|
rootUrl: crawlSummary.finalRootUrl ?? crawlSummary.rootUrl,
|
|
8087
8453
|
crawlSchemaVersion: crawlSummary.crawlSchemaVersion,
|
|
8088
8454
|
engineVersion: crawlSummary.engineVersion,
|
|
@@ -8113,6 +8479,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
8113
8479
|
deadLinkState: report.deadLinks.state,
|
|
8114
8480
|
deadLinksChecked,
|
|
8115
8481
|
deadLinksFound,
|
|
8482
|
+
templateDetection: templateLinks.detection,
|
|
8116
8483
|
createdAt: finishedAt,
|
|
8117
8484
|
updatedAt: finishedAt
|
|
8118
8485
|
}).run();
|
|
@@ -8145,7 +8512,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
8145
8512
|
edgesDiscovered: crawlSummary.edgesObserved,
|
|
8146
8513
|
finishedAt,
|
|
8147
8514
|
updatedAt: finishedAt
|
|
8148
|
-
}).where(
|
|
8515
|
+
}).where(eq15(siteCrawlAttempts.id, attemptId)).run();
|
|
8149
8516
|
return true;
|
|
8150
8517
|
});
|
|
8151
8518
|
if (!published) {
|
|
@@ -8158,7 +8525,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
8158
8525
|
error: cleanupError instanceof Error ? cleanupError.message : String(cleanupError)
|
|
8159
8526
|
});
|
|
8160
8527
|
}
|
|
8161
|
-
const current = db.select({ status: runs.status }).from(runs).where(
|
|
8528
|
+
const current = db.select({ status: runs.status }).from(runs).where(eq15(runs.id, runId)).get();
|
|
8162
8529
|
if (current?.status === "cancelled") throw new SiteAuditCancelledError(`Site audit cancelled before publication: ${runId}`);
|
|
8163
8530
|
throw new Error(`Site audit run was no longer running before publication: ${runId}`);
|
|
8164
8531
|
}
|
|
@@ -8173,12 +8540,12 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
8173
8540
|
error: message,
|
|
8174
8541
|
finishedAt,
|
|
8175
8542
|
updatedAt: finishedAt
|
|
8176
|
-
}).where(
|
|
8543
|
+
}).where(eq15(siteCrawlAttempts.id, attemptId)).run();
|
|
8177
8544
|
tx.update(runs).set({
|
|
8178
8545
|
status: cancelled ? "cancelled" : "failed",
|
|
8179
8546
|
error: message,
|
|
8180
8547
|
finishedAt
|
|
8181
|
-
}).where(
|
|
8548
|
+
}).where(and12(eq15(runs.id, runId), eq15(runs.status, "running"))).run();
|
|
8182
8549
|
});
|
|
8183
8550
|
log12.error("failed", { runId, projectId, cancelled, error: message });
|
|
8184
8551
|
throw error;
|
|
@@ -8186,7 +8553,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
8186
8553
|
}
|
|
8187
8554
|
|
|
8188
8555
|
// src/commands/backfill.ts
|
|
8189
|
-
import { and as
|
|
8556
|
+
import { and as and13, eq as eq16, inArray as inArray5, isNull as isNull2, sql as sql7 } from "drizzle-orm";
|
|
8190
8557
|
var SNAPSHOT_BATCH_SIZE = 500;
|
|
8191
8558
|
async function backfillAnswerVisibilityCommand(opts) {
|
|
8192
8559
|
const config = loadConfig();
|
|
@@ -8194,7 +8561,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
8194
8561
|
migrate(db);
|
|
8195
8562
|
const projectFilter = opts?.project?.trim();
|
|
8196
8563
|
const isDryRun = opts?.dryRun === true;
|
|
8197
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
8564
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq16(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
8198
8565
|
let examined = 0;
|
|
8199
8566
|
let updated = 0;
|
|
8200
8567
|
let wouldUpdate = 0;
|
|
@@ -8202,10 +8569,10 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
8202
8569
|
let reparsed = 0;
|
|
8203
8570
|
let providerErrors = 0;
|
|
8204
8571
|
if (scopedProjects.length > 0) {
|
|
8205
|
-
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
8206
|
-
|
|
8207
|
-
|
|
8208
|
-
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
8572
|
+
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(and13(
|
|
8573
|
+
eq16(runs.kind, RunKinds["answer-visibility"]),
|
|
8574
|
+
inArray5(runs.projectId, scopedProjects.map((project) => project.id))
|
|
8575
|
+
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(eq16(runs.kind, RunKinds["answer-visibility"])).all();
|
|
8209
8576
|
const runIdsByProject = /* @__PURE__ */ new Map();
|
|
8210
8577
|
for (const run of runRows) {
|
|
8211
8578
|
const existing = runIdsByProject.get(run.projectId);
|
|
@@ -8213,7 +8580,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
8213
8580
|
else runIdsByProject.set(run.projectId, [run.id]);
|
|
8214
8581
|
}
|
|
8215
8582
|
for (const project of scopedProjects) {
|
|
8216
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
8583
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq16(competitors.projectId, project.id)).all().map((row) => row.domain);
|
|
8217
8584
|
const runIds = runIdsByProject.get(project.id) ?? [];
|
|
8218
8585
|
if (runIds.length === 0) continue;
|
|
8219
8586
|
const projectDomains = effectiveDomains({
|
|
@@ -8236,7 +8603,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
8236
8603
|
competitorOverlap: querySnapshots.competitorOverlap,
|
|
8237
8604
|
recommendedCompetitors: querySnapshots.recommendedCompetitors,
|
|
8238
8605
|
rawResponse: querySnapshots.rawResponse
|
|
8239
|
-
}).from(querySnapshots).where(
|
|
8606
|
+
}).from(querySnapshots).where(inArray5(querySnapshots.runId, batchRunIds)).all();
|
|
8240
8607
|
const pendingUpdates = [];
|
|
8241
8608
|
for (const snapshot of snapshotRows) {
|
|
8242
8609
|
examined++;
|
|
@@ -8306,7 +8673,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
8306
8673
|
} else {
|
|
8307
8674
|
db.transaction((tx) => {
|
|
8308
8675
|
for (const update of pendingUpdates) {
|
|
8309
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
8676
|
+
tx.update(querySnapshots).set(update.patch).where(eq16(querySnapshots.id, update.id)).run();
|
|
8310
8677
|
}
|
|
8311
8678
|
});
|
|
8312
8679
|
updated += pendingUpdates.length;
|
|
@@ -8355,13 +8722,13 @@ No DB writes performed. Re-run without --dry-run to apply.`);
|
|
|
8355
8722
|
function backfillNormalizedPaths(db, opts) {
|
|
8356
8723
|
const baseConditions = [];
|
|
8357
8724
|
if (opts?.projectId) {
|
|
8358
|
-
baseConditions.push(
|
|
8725
|
+
baseConditions.push(eq16(gaTrafficSnapshots.projectId, opts.projectId));
|
|
8359
8726
|
}
|
|
8360
8727
|
const rows = db.select({
|
|
8361
8728
|
id: gaTrafficSnapshots.id,
|
|
8362
8729
|
landingPage: gaTrafficSnapshots.landingPage,
|
|
8363
8730
|
landingPageNormalized: gaTrafficSnapshots.landingPageNormalized
|
|
8364
|
-
}).from(gaTrafficSnapshots).where(baseConditions.length > 0 ?
|
|
8731
|
+
}).from(gaTrafficSnapshots).where(baseConditions.length > 0 ? and13(...baseConditions) : void 0).all();
|
|
8365
8732
|
let updated = 0;
|
|
8366
8733
|
let unchanged = 0;
|
|
8367
8734
|
if (rows.length > 0) {
|
|
@@ -8376,7 +8743,7 @@ function backfillNormalizedPaths(db, opts) {
|
|
|
8376
8743
|
unchanged++;
|
|
8377
8744
|
continue;
|
|
8378
8745
|
}
|
|
8379
|
-
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(
|
|
8746
|
+
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(eq16(gaTrafficSnapshots.id, row.id)).run();
|
|
8380
8747
|
updated++;
|
|
8381
8748
|
}
|
|
8382
8749
|
});
|
|
@@ -8390,7 +8757,7 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
8390
8757
|
const projectFilter = opts?.project?.trim();
|
|
8391
8758
|
let projectId;
|
|
8392
8759
|
if (projectFilter) {
|
|
8393
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
8760
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq16(projects.name, projectFilter)).get();
|
|
8394
8761
|
if (!project) {
|
|
8395
8762
|
const result2 = {
|
|
8396
8763
|
project: projectFilter,
|
|
@@ -8427,13 +8794,13 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
8427
8794
|
function backfillAiReferralPaths(db, opts) {
|
|
8428
8795
|
const baseConditions = [];
|
|
8429
8796
|
if (opts?.projectId) {
|
|
8430
|
-
baseConditions.push(
|
|
8797
|
+
baseConditions.push(eq16(gaAiReferrals.projectId, opts.projectId));
|
|
8431
8798
|
}
|
|
8432
8799
|
const rows = db.select({
|
|
8433
8800
|
id: gaAiReferrals.id,
|
|
8434
8801
|
landingPage: gaAiReferrals.landingPage,
|
|
8435
8802
|
landingPageNormalized: gaAiReferrals.landingPageNormalized
|
|
8436
|
-
}).from(gaAiReferrals).where(baseConditions.length > 0 ?
|
|
8803
|
+
}).from(gaAiReferrals).where(baseConditions.length > 0 ? and13(...baseConditions) : void 0).all();
|
|
8437
8804
|
let updated = 0;
|
|
8438
8805
|
let unchanged = 0;
|
|
8439
8806
|
if (rows.length > 0) {
|
|
@@ -8448,7 +8815,7 @@ function backfillAiReferralPaths(db, opts) {
|
|
|
8448
8815
|
unchanged++;
|
|
8449
8816
|
continue;
|
|
8450
8817
|
}
|
|
8451
|
-
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(
|
|
8818
|
+
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(eq16(gaAiReferrals.id, row.id)).run();
|
|
8452
8819
|
updated++;
|
|
8453
8820
|
}
|
|
8454
8821
|
});
|
|
@@ -8462,7 +8829,7 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
8462
8829
|
const projectFilter = opts?.project?.trim();
|
|
8463
8830
|
let projectId;
|
|
8464
8831
|
if (projectFilter) {
|
|
8465
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
8832
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq16(projects.name, projectFilter)).get();
|
|
8466
8833
|
if (!project) {
|
|
8467
8834
|
const result2 = {
|
|
8468
8835
|
project: projectFilter,
|
|
@@ -8498,10 +8865,10 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
8498
8865
|
}
|
|
8499
8866
|
function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
8500
8867
|
const isDryRun = opts?.dryRun === true;
|
|
8501
|
-
const project = db.select().from(projects).where(
|
|
8868
|
+
const project = db.select().from(projects).where(eq16(projects.id, projectId)).get();
|
|
8502
8869
|
if (!project) return { examined: 0, updated: 0, mentioned: 0 };
|
|
8503
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
8504
|
-
const runRows = db.select({ id: runs.id }).from(runs).where(
|
|
8870
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq16(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
8871
|
+
const runRows = db.select({ id: runs.id }).from(runs).where(and13(eq16(runs.kind, RunKinds["answer-visibility"]), eq16(runs.projectId, projectId))).all();
|
|
8505
8872
|
const runIds = runRows.map((r) => r.id);
|
|
8506
8873
|
let examined = 0;
|
|
8507
8874
|
let updated = 0;
|
|
@@ -8529,7 +8896,7 @@ function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
|
8529
8896
|
competitorOverlap: querySnapshots.competitorOverlap,
|
|
8530
8897
|
recommendedCompetitors: querySnapshots.recommendedCompetitors,
|
|
8531
8898
|
rawResponse: querySnapshots.rawResponse
|
|
8532
|
-
}).from(querySnapshots).where(
|
|
8899
|
+
}).from(querySnapshots).where(inArray5(querySnapshots.runId, batchRunIds)).all();
|
|
8533
8900
|
const pendingUpdates = [];
|
|
8534
8901
|
for (const snapshot of snapshotRows) {
|
|
8535
8902
|
examined++;
|
|
@@ -8575,7 +8942,7 @@ function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
|
8575
8942
|
} else {
|
|
8576
8943
|
db.transaction((tx) => {
|
|
8577
8944
|
for (const update of pendingUpdates) {
|
|
8578
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
8945
|
+
tx.update(querySnapshots).set(update.patch).where(eq16(querySnapshots.id, update.id)).run();
|
|
8579
8946
|
}
|
|
8580
8947
|
});
|
|
8581
8948
|
updated += pendingUpdates.length;
|
|
@@ -8590,7 +8957,7 @@ async function backfillAnswerMentionsCommand(opts) {
|
|
|
8590
8957
|
migrate(db);
|
|
8591
8958
|
const projectFilter = opts?.project?.trim();
|
|
8592
8959
|
const isDryRun = opts?.dryRun === true;
|
|
8593
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
8960
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq16(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
8594
8961
|
let examined = 0;
|
|
8595
8962
|
let updated = 0;
|
|
8596
8963
|
let wouldUpdate = 0;
|
|
@@ -8650,7 +9017,7 @@ function readStoredGroundingSources(rawResponse) {
|
|
|
8650
9017
|
return result;
|
|
8651
9018
|
}
|
|
8652
9019
|
async function backfillInsightsCommand(project, opts) {
|
|
8653
|
-
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-
|
|
9020
|
+
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-6XQMM466.js");
|
|
8654
9021
|
const config = loadConfig();
|
|
8655
9022
|
const db = createClient(config.database);
|
|
8656
9023
|
migrate(db);
|
|
@@ -8809,7 +9176,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
8809
9176
|
const config = loadConfig();
|
|
8810
9177
|
const db = createClient(config.database);
|
|
8811
9178
|
migrate(db);
|
|
8812
|
-
const project = db.select().from(projects).where(
|
|
9179
|
+
const project = db.select().from(projects).where(eq16(projects.name, opts.project)).get();
|
|
8813
9180
|
if (!project) {
|
|
8814
9181
|
throw new Error(`Project "${opts.project}" not found`);
|
|
8815
9182
|
}
|
|
@@ -8820,17 +9187,17 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
8820
9187
|
process.stderr.write(`Recovering orphan snapshot attribution for "${project.name}"${mode}...
|
|
8821
9188
|
`);
|
|
8822
9189
|
}
|
|
8823
|
-
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(
|
|
8824
|
-
|
|
8825
|
-
|
|
9190
|
+
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(and13(
|
|
9191
|
+
eq16(auditLog.projectId, project.id),
|
|
9192
|
+
inArray5(auditLog.action, ["keywords.appended", "keywords.deleted", "queries.appended", "queries.deleted", "queries.replaced"])
|
|
8826
9193
|
)).orderBy(auditLog.createdAt).all();
|
|
8827
9194
|
const history = replayQueryAuditLog(events);
|
|
8828
9195
|
const orphanRuns = db.select({
|
|
8829
9196
|
runId: runs.id,
|
|
8830
9197
|
createdAt: runs.createdAt,
|
|
8831
9198
|
location: runs.location
|
|
8832
|
-
}).from(runs).innerJoin(querySnapshots,
|
|
8833
|
-
|
|
9199
|
+
}).from(runs).innerJoin(querySnapshots, eq16(querySnapshots.runId, runs.id)).where(and13(
|
|
9200
|
+
eq16(runs.projectId, project.id),
|
|
8834
9201
|
isNull2(querySnapshots.queryId),
|
|
8835
9202
|
isNull2(querySnapshots.queryText)
|
|
8836
9203
|
)).groupBy(runs.id).orderBy(runs.createdAt).all();
|
|
@@ -8852,8 +9219,8 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
8852
9219
|
provider: querySnapshots.provider,
|
|
8853
9220
|
createdAt: querySnapshots.createdAt,
|
|
8854
9221
|
answerText: querySnapshots.answerText
|
|
8855
|
-
}).from(querySnapshots).where(
|
|
8856
|
-
|
|
9222
|
+
}).from(querySnapshots).where(and13(
|
|
9223
|
+
eq16(querySnapshots.runId, run.runId),
|
|
8857
9224
|
isNull2(querySnapshots.queryId),
|
|
8858
9225
|
isNull2(querySnapshots.queryText)
|
|
8859
9226
|
)).orderBy(querySnapshots.provider, querySnapshots.createdAt).all();
|
|
@@ -8919,7 +9286,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
8919
9286
|
if (!isDryRun && updates.length > 0) {
|
|
8920
9287
|
db.transaction((tx) => {
|
|
8921
9288
|
for (const u of updates) {
|
|
8922
|
-
tx.update(querySnapshots).set({ queryText: u.queryText }).where(
|
|
9289
|
+
tx.update(querySnapshots).set({ queryText: u.queryText }).where(eq16(querySnapshots.id, u.id)).run();
|
|
8923
9290
|
}
|
|
8924
9291
|
});
|
|
8925
9292
|
}
|
|
@@ -8993,7 +9360,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
8993
9360
|
const projectFilter = opts?.project?.trim();
|
|
8994
9361
|
const isDryRun = opts?.dryRun === true;
|
|
8995
9362
|
const isJson = isMachineFormat(opts?.format);
|
|
8996
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
9363
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq16(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
8997
9364
|
if (scopedProjects.length === 0) {
|
|
8998
9365
|
if (projectFilter && !isJson) {
|
|
8999
9366
|
process.stderr.write(`No project named "${projectFilter}".
|
|
@@ -9018,9 +9385,9 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
9018
9385
|
dryRun: isDryRun,
|
|
9019
9386
|
byBot: {}
|
|
9020
9387
|
};
|
|
9021
|
-
const unknownCountRow = db.select({ n: sql7`count(*)` }).from(rawEventSamples).where(
|
|
9022
|
-
|
|
9023
|
-
|
|
9388
|
+
const unknownCountRow = db.select({ n: sql7`count(*)` }).from(rawEventSamples).where(and13(
|
|
9389
|
+
eq16(rawEventSamples.eventType, "unknown"),
|
|
9390
|
+
inArray5(rawEventSamples.projectId, projectIds)
|
|
9024
9391
|
)).get();
|
|
9025
9392
|
result.unknownBefore = Number(unknownCountRow?.n ?? 0);
|
|
9026
9393
|
const unknownSamples = db.select({
|
|
@@ -9031,9 +9398,9 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
9031
9398
|
userAgent: rawEventSamples.userAgent,
|
|
9032
9399
|
pathNormalized: rawEventSamples.pathNormalized,
|
|
9033
9400
|
status: rawEventSamples.status
|
|
9034
|
-
}).from(rawEventSamples).where(
|
|
9035
|
-
|
|
9036
|
-
|
|
9401
|
+
}).from(rawEventSamples).where(and13(
|
|
9402
|
+
eq16(rawEventSamples.eventType, "unknown"),
|
|
9403
|
+
inArray5(rawEventSamples.projectId, projectIds)
|
|
9037
9404
|
)).all();
|
|
9038
9405
|
result.examined = unknownSamples.length;
|
|
9039
9406
|
if (unknownSamples.length === 0) {
|
|
@@ -9071,7 +9438,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
9071
9438
|
result.reclassified++;
|
|
9072
9439
|
result.byBot[classified.botId] = (result.byBot[classified.botId] ?? 0) + 1;
|
|
9073
9440
|
if (isDryRun) continue;
|
|
9074
|
-
db.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(
|
|
9441
|
+
db.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(eq16(rawEventSamples.id, snap.id)).run();
|
|
9075
9442
|
const tsHour = new Date(snap.ts);
|
|
9076
9443
|
tsHour.setUTCMinutes(0, 0, 0);
|
|
9077
9444
|
if (userFetch) {
|
|
@@ -9135,9 +9502,9 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
9135
9502
|
}
|
|
9136
9503
|
}
|
|
9137
9504
|
if (!isDryRun) {
|
|
9138
|
-
const afterRow = db.select({ n: sql7`count(*)` }).from(rawEventSamples).where(
|
|
9139
|
-
|
|
9140
|
-
|
|
9505
|
+
const afterRow = db.select({ n: sql7`count(*)` }).from(rawEventSamples).where(and13(
|
|
9506
|
+
eq16(rawEventSamples.eventType, "unknown"),
|
|
9507
|
+
inArray5(rawEventSamples.projectId, projectIds)
|
|
9141
9508
|
)).get();
|
|
9142
9509
|
result.unknownAfter = Number(afterRow?.n ?? 0);
|
|
9143
9510
|
} else {
|
|
@@ -9570,7 +9937,7 @@ var ProviderRegistry = class {
|
|
|
9570
9937
|
// src/scheduler.ts
|
|
9571
9938
|
import crypto19 from "crypto";
|
|
9572
9939
|
import cron from "node-cron";
|
|
9573
|
-
import { and as
|
|
9940
|
+
import { and as and14, eq as eq17, inArray as inArray6, notExists, sql as sql8 } from "drizzle-orm";
|
|
9574
9941
|
var log13 = createLogger("Scheduler");
|
|
9575
9942
|
var DEFAULT_HEALTH_CRON = "0 */6 * * *";
|
|
9576
9943
|
function taskKey(projectId, kind) {
|
|
@@ -9602,9 +9969,9 @@ var Scheduler = class {
|
|
|
9602
9969
|
*/
|
|
9603
9970
|
ensureHealthSchedules() {
|
|
9604
9971
|
const projectsWithoutHealth = this.db.select({ id: projects.id }).from(projects).where(notExists(
|
|
9605
|
-
this.db.select({ one: sql8`1` }).from(schedules).where(
|
|
9606
|
-
|
|
9607
|
-
|
|
9972
|
+
this.db.select({ one: sql8`1` }).from(schedules).where(and14(
|
|
9973
|
+
eq17(schedules.projectId, projects.id),
|
|
9974
|
+
eq17(schedules.kind, SchedulableRunKinds.doctor)
|
|
9608
9975
|
))
|
|
9609
9976
|
)).all();
|
|
9610
9977
|
if (projectsWithoutHealth.length === 0) return;
|
|
@@ -9655,7 +10022,7 @@ var Scheduler = class {
|
|
|
9655
10022
|
start() {
|
|
9656
10023
|
this.ensureHealthSchedules();
|
|
9657
10024
|
this.ensureQueryBaskets();
|
|
9658
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
10025
|
+
const allSchedules = this.db.select().from(schedules).where(eq17(schedules.enabled, true)).all();
|
|
9659
10026
|
for (const schedule of allSchedules) {
|
|
9660
10027
|
const missedRunAt = schedule.nextRunAt;
|
|
9661
10028
|
this.registerCronTask(schedule);
|
|
@@ -9685,7 +10052,7 @@ var Scheduler = class {
|
|
|
9685
10052
|
this.stopTask(key, existing, "Stopped");
|
|
9686
10053
|
this.tasks.delete(key);
|
|
9687
10054
|
}
|
|
9688
|
-
const schedule = this.db.select().from(schedules).where(
|
|
10055
|
+
const schedule = this.db.select().from(schedules).where(and14(eq17(schedules.projectId, projectId), eq17(schedules.kind, kind))).get();
|
|
9689
10056
|
if (schedule && schedule.enabled) {
|
|
9690
10057
|
this.registerCronTask(schedule);
|
|
9691
10058
|
}
|
|
@@ -9726,21 +10093,21 @@ var Scheduler = class {
|
|
|
9726
10093
|
this.db.update(schedules).set({
|
|
9727
10094
|
nextRunAt: nextRunFromCron(cronExpr, timezone),
|
|
9728
10095
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
9729
|
-
}).where(
|
|
10096
|
+
}).where(eq17(schedules.id, scheduleId)).run();
|
|
9730
10097
|
const label = schedule.preset ?? cronExpr;
|
|
9731
10098
|
log13.info("cron.registered", { projectId, kind, schedule: label, timezone });
|
|
9732
10099
|
}
|
|
9733
10100
|
triggerRun(scheduleId, projectId, kind) {
|
|
9734
10101
|
try {
|
|
9735
10102
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9736
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
10103
|
+
const currentSchedule = this.db.select().from(schedules).where(eq17(schedules.id, scheduleId)).get();
|
|
9737
10104
|
if (!currentSchedule || !currentSchedule.enabled) {
|
|
9738
10105
|
log13.warn("schedule.stale", { scheduleId, projectId, kind, msg: "schedule no longer exists or is disabled" });
|
|
9739
10106
|
this.remove(projectId, kind);
|
|
9740
10107
|
return;
|
|
9741
10108
|
}
|
|
9742
10109
|
const nextRunAt = nextRunFromCron(currentSchedule.cronExpr, currentSchedule.timezone);
|
|
9743
|
-
const project = this.db.select().from(projects).where(
|
|
10110
|
+
const project = this.db.select().from(projects).where(eq17(projects.id, projectId)).get();
|
|
9744
10111
|
if (!project) {
|
|
9745
10112
|
log13.error("project.not-found", { projectId, kind, msg: "skipping scheduled run" });
|
|
9746
10113
|
this.remove(projectId, kind);
|
|
@@ -9760,7 +10127,7 @@ var Scheduler = class {
|
|
|
9760
10127
|
lastRunAt: now,
|
|
9761
10128
|
nextRunAt,
|
|
9762
10129
|
updatedAt: now
|
|
9763
|
-
}).where(
|
|
10130
|
+
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9764
10131
|
log13.info("traffic-sync.triggered", { projectName: project.name, sourceId });
|
|
9765
10132
|
this.callbacks.onTrafficSyncRequested(project.name, sourceId);
|
|
9766
10133
|
return;
|
|
@@ -9783,7 +10150,7 @@ var Scheduler = class {
|
|
|
9783
10150
|
lastRunAt: now,
|
|
9784
10151
|
nextRunAt,
|
|
9785
10152
|
updatedAt: now
|
|
9786
|
-
}).where(
|
|
10153
|
+
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9787
10154
|
log13.info("gbp-sync.triggered", { runId: runId2, projectName: project.name });
|
|
9788
10155
|
this.callbacks.onGbpSyncRequested(runId2, projectId);
|
|
9789
10156
|
return;
|
|
@@ -9793,14 +10160,14 @@ var Scheduler = class {
|
|
|
9793
10160
|
log13.warn("ads-sync.no-callback", { scheduleId, projectId, msg: "host did not register onAdsSyncRequested" });
|
|
9794
10161
|
return;
|
|
9795
10162
|
}
|
|
9796
|
-
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(
|
|
9797
|
-
|
|
9798
|
-
|
|
9799
|
-
|
|
10163
|
+
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(and14(
|
|
10164
|
+
eq17(runs.projectId, projectId),
|
|
10165
|
+
eq17(runs.kind, RunKinds["ads-sync"]),
|
|
10166
|
+
inArray6(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
9800
10167
|
)).get();
|
|
9801
10168
|
if (activeAdsRun) {
|
|
9802
10169
|
log13.info("ads-sync.skipped-active", { projectName: project.name, activeRunId: activeAdsRun.id });
|
|
9803
|
-
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(
|
|
10170
|
+
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9804
10171
|
return;
|
|
9805
10172
|
}
|
|
9806
10173
|
const runId2 = crypto19.randomUUID();
|
|
@@ -9816,7 +10183,7 @@ var Scheduler = class {
|
|
|
9816
10183
|
lastRunAt: now,
|
|
9817
10184
|
nextRunAt,
|
|
9818
10185
|
updatedAt: now
|
|
9819
|
-
}).where(
|
|
10186
|
+
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9820
10187
|
log13.info("ads-sync.triggered", { runId: runId2, projectName: project.name });
|
|
9821
10188
|
this.callbacks.onAdsSyncRequested(runId2, projectId);
|
|
9822
10189
|
return;
|
|
@@ -9830,7 +10197,7 @@ var Scheduler = class {
|
|
|
9830
10197
|
lastRunAt: now,
|
|
9831
10198
|
nextRunAt,
|
|
9832
10199
|
updatedAt: now
|
|
9833
|
-
}).where(
|
|
10200
|
+
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9834
10201
|
log13.info("data-refresh.triggered", { projectName: project.name });
|
|
9835
10202
|
this.callbacks.onDataRefreshRequested(project.name);
|
|
9836
10203
|
return;
|
|
@@ -9844,7 +10211,7 @@ var Scheduler = class {
|
|
|
9844
10211
|
lastRunAt: now,
|
|
9845
10212
|
nextRunAt,
|
|
9846
10213
|
updatedAt: now
|
|
9847
|
-
}).where(
|
|
10214
|
+
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9848
10215
|
log13.info("doctor.triggered", { projectName: project.name });
|
|
9849
10216
|
this.callbacks.onDoctorRequested(project.name);
|
|
9850
10217
|
return;
|
|
@@ -9858,7 +10225,7 @@ var Scheduler = class {
|
|
|
9858
10225
|
lastRunAt: now,
|
|
9859
10226
|
nextRunAt,
|
|
9860
10227
|
updatedAt: now
|
|
9861
|
-
}).where(
|
|
10228
|
+
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9862
10229
|
log13.info("backlinks-sync.triggered", { projectName: project.name });
|
|
9863
10230
|
this.callbacks.onBacklinksSyncRequested(project.name);
|
|
9864
10231
|
return;
|
|
@@ -9868,14 +10235,14 @@ var Scheduler = class {
|
|
|
9868
10235
|
log13.warn("site-audit.no-callback", { scheduleId, projectId, msg: "host did not register onSiteAuditRequested" });
|
|
9869
10236
|
return;
|
|
9870
10237
|
}
|
|
9871
|
-
const active = this.db.select({ id: runs.id }).from(runs).where(
|
|
9872
|
-
|
|
9873
|
-
|
|
9874
|
-
|
|
10238
|
+
const active = this.db.select({ id: runs.id }).from(runs).where(and14(
|
|
10239
|
+
eq17(runs.projectId, projectId),
|
|
10240
|
+
eq17(runs.kind, RunKinds["site-audit"]),
|
|
10241
|
+
inArray6(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
9875
10242
|
)).get();
|
|
9876
10243
|
if (active) {
|
|
9877
10244
|
log13.info("site-audit.skipped-active", { projectName: project.name, activeRunId: active.id });
|
|
9878
|
-
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(
|
|
10245
|
+
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9879
10246
|
return;
|
|
9880
10247
|
}
|
|
9881
10248
|
const runId2 = crypto19.randomUUID();
|
|
@@ -9901,7 +10268,7 @@ var Scheduler = class {
|
|
|
9901
10268
|
lastRunAt: now,
|
|
9902
10269
|
nextRunAt,
|
|
9903
10270
|
updatedAt: now
|
|
9904
|
-
}).where(
|
|
10271
|
+
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9905
10272
|
log13.info("site-audit.triggered", { runId: runId2, projectName: project.name });
|
|
9906
10273
|
this.callbacks.onSiteAuditRequested(runId2, projectId);
|
|
9907
10274
|
return;
|
|
@@ -9934,7 +10301,7 @@ var Scheduler = class {
|
|
|
9934
10301
|
this.db.update(schedules).set({
|
|
9935
10302
|
nextRunAt,
|
|
9936
10303
|
updatedAt: now
|
|
9937
|
-
}).where(
|
|
10304
|
+
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9938
10305
|
return;
|
|
9939
10306
|
}
|
|
9940
10307
|
const runId = queueResult.runId;
|
|
@@ -9942,7 +10309,7 @@ var Scheduler = class {
|
|
|
9942
10309
|
lastRunAt: now,
|
|
9943
10310
|
nextRunAt,
|
|
9944
10311
|
updatedAt: now
|
|
9945
|
-
}).where(
|
|
10312
|
+
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9946
10313
|
log13.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
9947
10314
|
this.callbacks.onRunCreated(runId, projectId, providers, resolvedLocation);
|
|
9948
10315
|
} catch (err) {
|
|
@@ -9975,7 +10342,7 @@ async function refreshAllIntegrations(client, projectName) {
|
|
|
9975
10342
|
}
|
|
9976
10343
|
|
|
9977
10344
|
// src/notifier.ts
|
|
9978
|
-
import { eq as
|
|
10345
|
+
import { eq as eq18, desc as desc7, and as and15, inArray as inArray7, or as or2 } from "drizzle-orm";
|
|
9979
10346
|
import crypto20 from "crypto";
|
|
9980
10347
|
var log15 = createLogger("Notifier");
|
|
9981
10348
|
var Notifier = class {
|
|
@@ -9988,18 +10355,18 @@ var Notifier = class {
|
|
|
9988
10355
|
/** Called after a run completes (success, partial, or failed). */
|
|
9989
10356
|
async onRunCompleted(runId, projectId) {
|
|
9990
10357
|
log15.info("run.completed", { runId, projectId });
|
|
9991
|
-
const notifs = this.db.select().from(notifications).where(
|
|
10358
|
+
const notifs = this.db.select().from(notifications).where(eq18(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
9992
10359
|
if (notifs.length === 0) {
|
|
9993
10360
|
log15.info("notifications.none-enabled", { projectId });
|
|
9994
10361
|
return;
|
|
9995
10362
|
}
|
|
9996
10363
|
log15.info("notifications.found", { projectId, count: notifs.length });
|
|
9997
|
-
const run = this.db.select().from(runs).where(
|
|
10364
|
+
const run = this.db.select().from(runs).where(eq18(runs.id, runId)).get();
|
|
9998
10365
|
if (!run) {
|
|
9999
10366
|
log15.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
10000
10367
|
return;
|
|
10001
10368
|
}
|
|
10002
|
-
const project = this.db.select().from(projects).where(
|
|
10369
|
+
const project = this.db.select().from(projects).where(eq18(projects.id, projectId)).get();
|
|
10003
10370
|
if (!project) {
|
|
10004
10371
|
log15.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
10005
10372
|
return;
|
|
@@ -10052,7 +10419,7 @@ var Notifier = class {
|
|
|
10052
10419
|
* inferred from the absence of a webhook.
|
|
10053
10420
|
*/
|
|
10054
10421
|
async onHealthChecked(projectId, report) {
|
|
10055
|
-
const project = this.db.select().from(projects).where(
|
|
10422
|
+
const project = this.db.select().from(projects).where(eq18(projects.id, projectId)).get();
|
|
10056
10423
|
if (!project) {
|
|
10057
10424
|
log15.error("project.not-found", { projectId, msg: "skipping health notification" });
|
|
10058
10425
|
return null;
|
|
@@ -10083,7 +10450,7 @@ var Notifier = class {
|
|
|
10083
10450
|
const status = noSignal ? "warn" : worst ? worst.status : "ok";
|
|
10084
10451
|
const code = noSignal ? "health.no-signal" : worst?.code ?? "health.ok";
|
|
10085
10452
|
const summary = noSignal ? `No health check produced a result (${report.checks.length} skipped) \u2014 health is unknown, not confirmed.` : worst?.summary ?? `All ${graded.length} health check(s) passing.`;
|
|
10086
|
-
const previous = this.db.select().from(doctorHealthState).where(
|
|
10453
|
+
const previous = this.db.select().from(doctorHealthState).where(eq18(doctorHealthState.projectId, projectId)).get();
|
|
10087
10454
|
const previousStatus = previous?.status ?? null;
|
|
10088
10455
|
let event = null;
|
|
10089
10456
|
if (previous === void 0) {
|
|
@@ -10098,13 +10465,13 @@ var Notifier = class {
|
|
|
10098
10465
|
if (previous === void 0) {
|
|
10099
10466
|
this.db.insert(doctorHealthState).values({ ...observation, notifiedAt: null }).run();
|
|
10100
10467
|
} else {
|
|
10101
|
-
this.db.update(doctorHealthState).set(observation).where(
|
|
10468
|
+
this.db.update(doctorHealthState).set(observation).where(eq18(doctorHealthState.projectId, projectId)).run();
|
|
10102
10469
|
}
|
|
10103
10470
|
if (!event) {
|
|
10104
10471
|
log15.info("health.unchanged", { projectId, status, code });
|
|
10105
10472
|
return null;
|
|
10106
10473
|
}
|
|
10107
|
-
const notifs = this.db.select().from(notifications).where(
|
|
10474
|
+
const notifs = this.db.select().from(notifications).where(eq18(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
10108
10475
|
const payload = {
|
|
10109
10476
|
source: "canonry",
|
|
10110
10477
|
event,
|
|
@@ -10128,7 +10495,7 @@ var Notifier = class {
|
|
|
10128
10495
|
delivered += 1;
|
|
10129
10496
|
}
|
|
10130
10497
|
if (delivered > 0) {
|
|
10131
|
-
this.db.update(doctorHealthState).set({ notifiedAt: now }).where(
|
|
10498
|
+
this.db.update(doctorHealthState).set({ notifiedAt: now }).where(eq18(doctorHealthState.projectId, projectId)).run();
|
|
10132
10499
|
}
|
|
10133
10500
|
log15.info("health.notified", { projectId, event, status, code, subscribers: notifs.length, delivered });
|
|
10134
10501
|
return event;
|
|
@@ -10141,11 +10508,11 @@ var Notifier = class {
|
|
|
10141
10508
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
10142
10509
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
10143
10510
|
if (insightEvents.length === 0) return;
|
|
10144
|
-
const notifs = this.db.select().from(notifications).where(
|
|
10511
|
+
const notifs = this.db.select().from(notifications).where(eq18(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
10145
10512
|
if (notifs.length === 0) return;
|
|
10146
|
-
const run = this.db.select().from(runs).where(
|
|
10513
|
+
const run = this.db.select().from(runs).where(eq18(runs.id, runId)).get();
|
|
10147
10514
|
if (!run) return;
|
|
10148
|
-
const project = this.db.select().from(projects).where(
|
|
10515
|
+
const project = this.db.select().from(projects).where(eq18(projects.id, projectId)).get();
|
|
10149
10516
|
if (!project) return;
|
|
10150
10517
|
for (const notif of notifs) {
|
|
10151
10518
|
const config = notif.config;
|
|
@@ -10175,7 +10542,7 @@ var Notifier = class {
|
|
|
10175
10542
|
}
|
|
10176
10543
|
}
|
|
10177
10544
|
computeTransitions(runId, projectId) {
|
|
10178
|
-
const thisRun = this.db.select().from(runs).where(
|
|
10545
|
+
const thisRun = this.db.select().from(runs).where(eq18(runs.id, runId)).get();
|
|
10179
10546
|
if (!thisRun) return [];
|
|
10180
10547
|
const completeness = measurementRunCompleteness(this.db, runId);
|
|
10181
10548
|
if (completeness.planned && !completeness.complete) {
|
|
@@ -10186,10 +10553,10 @@ var Notifier = class {
|
|
|
10186
10553
|
});
|
|
10187
10554
|
return [];
|
|
10188
10555
|
}
|
|
10189
|
-
const groupSiblings = this.db.select().from(runs).where(
|
|
10190
|
-
|
|
10191
|
-
|
|
10192
|
-
|
|
10556
|
+
const groupSiblings = this.db.select().from(runs).where(and15(
|
|
10557
|
+
eq18(runs.projectId, projectId),
|
|
10558
|
+
eq18(runs.kind, thisRun.kind),
|
|
10559
|
+
eq18(runs.createdAt, thisRun.createdAt)
|
|
10193
10560
|
)).all();
|
|
10194
10561
|
const stillPending = groupSiblings.some((r) => r.status === "queued" || r.status === "running");
|
|
10195
10562
|
if (stillPending) return [];
|
|
@@ -10205,17 +10572,17 @@ var Notifier = class {
|
|
|
10205
10572
|
return candidate.id > best.id ? candidate : best;
|
|
10206
10573
|
});
|
|
10207
10574
|
if (winner.id !== runId) return [];
|
|
10208
|
-
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
10575
|
+
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(eq18(projects.id, projectId)).get();
|
|
10209
10576
|
const locationCount = Math.max(
|
|
10210
10577
|
1,
|
|
10211
10578
|
(projectLocations?.locations ?? []).length
|
|
10212
10579
|
);
|
|
10213
10580
|
const RECENT_FETCH_LIMIT = Math.max(8, locationCount * 4);
|
|
10214
10581
|
const recentRuns = this.db.select().from(runs).where(
|
|
10215
|
-
|
|
10216
|
-
|
|
10217
|
-
|
|
10218
|
-
or2(
|
|
10582
|
+
and15(
|
|
10583
|
+
eq18(runs.projectId, projectId),
|
|
10584
|
+
eq18(runs.kind, thisRun.kind),
|
|
10585
|
+
or2(eq18(runs.status, "completed"), eq18(runs.status, "partial"))
|
|
10219
10586
|
)
|
|
10220
10587
|
).orderBy(desc7(runs.createdAt), desc7(runs.id)).limit(RECENT_FETCH_LIMIT).all();
|
|
10221
10588
|
const groups = groupRunsByCreatedAt(recentRuns);
|
|
@@ -10232,13 +10599,13 @@ var Notifier = class {
|
|
|
10232
10599
|
provider: querySnapshots.provider,
|
|
10233
10600
|
location: querySnapshots.location,
|
|
10234
10601
|
citationState: querySnapshots.citationState
|
|
10235
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
10602
|
+
}).from(querySnapshots).leftJoin(queries, eq18(querySnapshots.queryId, queries.id)).where(inArray7(querySnapshots.runId, currentRunIds)).all();
|
|
10236
10603
|
const previousSnapshots = this.db.select({
|
|
10237
10604
|
queryId: querySnapshots.queryId,
|
|
10238
10605
|
provider: querySnapshots.provider,
|
|
10239
10606
|
location: querySnapshots.location,
|
|
10240
10607
|
citationState: querySnapshots.citationState
|
|
10241
|
-
}).from(querySnapshots).where(
|
|
10608
|
+
}).from(querySnapshots).where(inArray7(querySnapshots.runId, previousRunIds)).all();
|
|
10242
10609
|
const prevMap = /* @__PURE__ */ new Map();
|
|
10243
10610
|
for (const s of previousSnapshots) {
|
|
10244
10611
|
if (s.queryId == null) continue;
|
|
@@ -10315,7 +10682,7 @@ var Notifier = class {
|
|
|
10315
10682
|
};
|
|
10316
10683
|
|
|
10317
10684
|
// src/run-coordinator.ts
|
|
10318
|
-
import { eq as
|
|
10685
|
+
import { eq as eq19 } from "drizzle-orm";
|
|
10319
10686
|
var log16 = createLogger("RunCoordinator");
|
|
10320
10687
|
var RunCoordinator = class {
|
|
10321
10688
|
constructor(db, notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
@@ -10331,7 +10698,7 @@ var RunCoordinator = class {
|
|
|
10331
10698
|
onInsightsGenerated;
|
|
10332
10699
|
onAeroEvent;
|
|
10333
10700
|
async onRunCompleted(runId, projectId) {
|
|
10334
|
-
const runRow = this.db.select().from(runs).where(
|
|
10701
|
+
const runRow = this.db.select().from(runs).where(eq19(runs.id, runId)).get();
|
|
10335
10702
|
const kind = runRow?.kind ?? RunKinds["answer-visibility"];
|
|
10336
10703
|
if (runRow?.trigger === RunTriggers.probe) {
|
|
10337
10704
|
log16.info("probe.skip-side-effects", { runId, projectId, kind });
|
|
@@ -10422,7 +10789,7 @@ var RunCoordinator = class {
|
|
|
10422
10789
|
* so the Aero queue is never starved of a follow-up.
|
|
10423
10790
|
*/
|
|
10424
10791
|
buildDiscoveryAeroContext(runId, projectId, status, error) {
|
|
10425
|
-
const session = this.db.select().from(discoverySessions).where(
|
|
10792
|
+
const session = this.db.select().from(discoverySessions).where(eq19(discoverySessions.runId, runId)).get();
|
|
10426
10793
|
const competitorMap = session ? session.competitorMap : [];
|
|
10427
10794
|
return {
|
|
10428
10795
|
kind: RunKinds["aeo-discover-probe"],
|
|
@@ -10465,7 +10832,7 @@ function analysisResultFromInsights(insights2) {
|
|
|
10465
10832
|
|
|
10466
10833
|
// src/agent/session-registry.ts
|
|
10467
10834
|
import crypto24 from "crypto";
|
|
10468
|
-
import { eq as
|
|
10835
|
+
import { eq as eq21 } from "drizzle-orm";
|
|
10469
10836
|
|
|
10470
10837
|
// src/agent/session.ts
|
|
10471
10838
|
import fs7 from "fs";
|
|
@@ -11594,7 +11961,7 @@ async function loadExternalMcpTools(servers, opts = {}) {
|
|
|
11594
11961
|
|
|
11595
11962
|
// src/agent/memory-store.ts
|
|
11596
11963
|
import crypto23 from "crypto";
|
|
11597
|
-
import { and as
|
|
11964
|
+
import { and as and16, desc as desc8, eq as eq20, like, sql as sql9 } from "drizzle-orm";
|
|
11598
11965
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
11599
11966
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
11600
11967
|
function rowToDto(row) {
|
|
@@ -11608,7 +11975,7 @@ function rowToDto(row) {
|
|
|
11608
11975
|
};
|
|
11609
11976
|
}
|
|
11610
11977
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
11611
|
-
const query = db.select().from(agentMemory).where(
|
|
11978
|
+
const query = db.select().from(agentMemory).where(eq20(agentMemory.projectId, projectId)).orderBy(desc8(agentMemory.updatedAt));
|
|
11612
11979
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
11613
11980
|
return rows.map(rowToDto);
|
|
11614
11981
|
}
|
|
@@ -11639,12 +12006,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
11639
12006
|
updatedAt: now
|
|
11640
12007
|
}
|
|
11641
12008
|
}).run();
|
|
11642
|
-
const row = db.select().from(agentMemory).where(
|
|
12009
|
+
const row = db.select().from(agentMemory).where(and16(eq20(agentMemory.projectId, args.projectId), eq20(agentMemory.key, args.key))).get();
|
|
11643
12010
|
if (!row) throw new Error("memory upsert produced no row");
|
|
11644
12011
|
return rowToDto(row);
|
|
11645
12012
|
}
|
|
11646
12013
|
function deleteMemoryEntry(db, projectId, key) {
|
|
11647
|
-
const result = db.delete(agentMemory).where(
|
|
12014
|
+
const result = db.delete(agentMemory).where(and16(eq20(agentMemory.projectId, projectId), eq20(agentMemory.key, key))).run();
|
|
11648
12015
|
const changes = result.changes ?? 0;
|
|
11649
12016
|
return changes > 0;
|
|
11650
12017
|
}
|
|
@@ -11673,8 +12040,8 @@ function writeCompactionNote(db, args) {
|
|
|
11673
12040
|
}).run();
|
|
11674
12041
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
11675
12042
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
11676
|
-
|
|
11677
|
-
|
|
12043
|
+
and16(
|
|
12044
|
+
eq20(agentMemory.projectId, args.projectId),
|
|
11678
12045
|
like(agentMemory.key, `${sessionPrefix}%`)
|
|
11679
12046
|
)
|
|
11680
12047
|
).orderBy(desc8(agentMemory.updatedAt)).all();
|
|
@@ -11682,7 +12049,7 @@ function writeCompactionNote(db, args) {
|
|
|
11682
12049
|
if (stale.length > 0) {
|
|
11683
12050
|
tx.delete(agentMemory).where(sql9`${agentMemory.id} IN (${sql9.join(stale.map((s) => sql9`${s}`), sql9`, `)})`).run();
|
|
11684
12051
|
}
|
|
11685
|
-
const row = tx.select().from(agentMemory).where(
|
|
12052
|
+
const row = tx.select().from(agentMemory).where(and16(eq20(agentMemory.projectId, args.projectId), eq20(agentMemory.key, key))).get();
|
|
11686
12053
|
if (row) inserted = rowToDto(row);
|
|
11687
12054
|
});
|
|
11688
12055
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -11909,7 +12276,7 @@ var SessionRegistry = class {
|
|
|
11909
12276
|
modelProvider: effectiveProvider,
|
|
11910
12277
|
modelId: effectiveModelId,
|
|
11911
12278
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
11912
|
-
}).where(
|
|
12279
|
+
}).where(eq21(agentSessions.projectId, projectId)).run();
|
|
11913
12280
|
}
|
|
11914
12281
|
const agent2 = createAeroSession({
|
|
11915
12282
|
projectName,
|
|
@@ -12143,7 +12510,7 @@ ${lines.join("\n")}
|
|
|
12143
12510
|
modelProvider: nextProvider,
|
|
12144
12511
|
modelId: nextModelId,
|
|
12145
12512
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
12146
|
-
}).where(
|
|
12513
|
+
}).where(eq21(agentSessions.projectId, projectId)).run();
|
|
12147
12514
|
}
|
|
12148
12515
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
12149
12516
|
save(projectName) {
|
|
@@ -12307,11 +12674,11 @@ ${lines.join("\n")}
|
|
|
12307
12674
|
return id;
|
|
12308
12675
|
}
|
|
12309
12676
|
tryResolveProjectId(projectName) {
|
|
12310
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
12677
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq21(projects.name, projectName)).get();
|
|
12311
12678
|
return row?.id;
|
|
12312
12679
|
}
|
|
12313
12680
|
loadRow(projectId) {
|
|
12314
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
12681
|
+
const row = this.opts.db.select().from(agentSessions).where(eq21(agentSessions.projectId, projectId)).get();
|
|
12315
12682
|
return row ?? null;
|
|
12316
12683
|
}
|
|
12317
12684
|
insertRow(params) {
|
|
@@ -12330,14 +12697,14 @@ ${lines.join("\n")}
|
|
|
12330
12697
|
}
|
|
12331
12698
|
updateRow(projectId, patch) {
|
|
12332
12699
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
12333
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
12700
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq21(agentSessions.projectId, projectId)).run();
|
|
12334
12701
|
}
|
|
12335
12702
|
};
|
|
12336
12703
|
|
|
12337
12704
|
// src/agent/agent-routes.ts
|
|
12338
|
-
import { eq as
|
|
12705
|
+
import { eq as eq22 } from "drizzle-orm";
|
|
12339
12706
|
function resolveProject(db, name) {
|
|
12340
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
12707
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq22(projects.name, name)).get();
|
|
12341
12708
|
if (!row) throw notFound("project", name);
|
|
12342
12709
|
return row;
|
|
12343
12710
|
}
|
|
@@ -12346,7 +12713,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
12346
12713
|
"/projects/:name/agent/transcript",
|
|
12347
12714
|
async (request) => {
|
|
12348
12715
|
const project = resolveProject(opts.db, request.params.name);
|
|
12349
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
12716
|
+
const row = opts.db.select().from(agentSessions).where(eq22(agentSessions.projectId, project.id)).get();
|
|
12350
12717
|
if (!row) {
|
|
12351
12718
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
12352
12719
|
}
|
|
@@ -12370,7 +12737,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
12370
12737
|
async (request) => {
|
|
12371
12738
|
const project = resolveProject(opts.db, request.params.name);
|
|
12372
12739
|
opts.sessionRegistry.reset(project.name);
|
|
12373
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
12740
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq22(agentSessions.projectId, project.id)).run();
|
|
12374
12741
|
return { status: "reset" };
|
|
12375
12742
|
}
|
|
12376
12743
|
);
|
|
@@ -12697,16 +13064,16 @@ Your previous reply was not valid JSON. Return ONLY a JSON object with exactly t
|
|
|
12697
13064
|
import { runAeoAudit } from "@canonry/aeo-audit";
|
|
12698
13065
|
|
|
12699
13066
|
// src/site-fetch.ts
|
|
12700
|
-
import
|
|
13067
|
+
import https2 from "https";
|
|
12701
13068
|
var FETCH_TIMEOUT_MS2 = 1e4;
|
|
12702
13069
|
var MAX_TEXT_LENGTH = 4e3;
|
|
12703
13070
|
var MAX_BODY_BYTES = 512e3;
|
|
12704
|
-
var
|
|
13071
|
+
var USER_AGENT2 = "Canonry/1.0 (site-analysis)";
|
|
12705
13072
|
function fetchWithPinnedAddress(target) {
|
|
12706
13073
|
return new Promise((resolve) => {
|
|
12707
13074
|
const port = target.url.port ? Number(target.url.port) : 443;
|
|
12708
13075
|
const path10 = target.url.pathname + target.url.search;
|
|
12709
|
-
const req =
|
|
13076
|
+
const req = https2.request(
|
|
12710
13077
|
{
|
|
12711
13078
|
hostname: target.address,
|
|
12712
13079
|
family: target.family,
|
|
@@ -12718,7 +13085,7 @@ function fetchWithPinnedAddress(target) {
|
|
|
12718
13085
|
// SNI for TLS
|
|
12719
13086
|
headers: {
|
|
12720
13087
|
Host: target.url.host,
|
|
12721
|
-
"User-Agent":
|
|
13088
|
+
"User-Agent": USER_AGENT2,
|
|
12722
13089
|
Accept: "text/html"
|
|
12723
13090
|
}
|
|
12724
13091
|
},
|
|
@@ -13313,20 +13680,20 @@ function clipText(value, length) {
|
|
|
13313
13680
|
}
|
|
13314
13681
|
|
|
13315
13682
|
// src/research-runner.ts
|
|
13316
|
-
import { and as
|
|
13683
|
+
import { and as and17, eq as eq23, inArray as inArray8, sql as sql10 } from "drizzle-orm";
|
|
13317
13684
|
var unfinishedResearchQueryStatuses = [ResearchQueryStatuses.queued, ResearchQueryStatuses.running];
|
|
13318
13685
|
function finalResearchRunStatus(completed, failed) {
|
|
13319
13686
|
if (failed === 0) return ResearchRunStatuses.completed;
|
|
13320
13687
|
return completed > 0 ? ResearchRunStatuses.partial : ResearchRunStatuses.failed;
|
|
13321
13688
|
}
|
|
13322
13689
|
async function executeResearchRun(db, registry, runId, projectId) {
|
|
13323
|
-
const run = db.select().from(researchRuns).where(
|
|
13690
|
+
const run = db.select().from(researchRuns).where(and17(eq23(researchRuns.id, runId), eq23(researchRuns.projectId, projectId))).get();
|
|
13324
13691
|
if (!run || run.status !== ResearchRunStatuses.queued) return;
|
|
13325
|
-
const project = db.select().from(projects).where(
|
|
13692
|
+
const project = db.select().from(projects).where(eq23(projects.id, projectId)).get();
|
|
13326
13693
|
if (!project) return;
|
|
13327
13694
|
const provider = registry.get(run.provider);
|
|
13328
13695
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
13329
|
-
const claim = db.update(researchRuns).set({ status: ResearchRunStatuses.running, startedAt: now }).where(
|
|
13696
|
+
const claim = db.update(researchRuns).set({ status: ResearchRunStatuses.running, startedAt: now }).where(and17(eq23(researchRuns.id, runId), eq23(researchRuns.status, ResearchRunStatuses.queued))).run();
|
|
13330
13697
|
if (claim.changes !== 1) return;
|
|
13331
13698
|
let reserved = 0;
|
|
13332
13699
|
let dispatched = 0;
|
|
@@ -13350,8 +13717,8 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
13350
13717
|
}
|
|
13351
13718
|
reserved = run.totalQueries;
|
|
13352
13719
|
reservation = { scope, period };
|
|
13353
|
-
const rows = db.select().from(researchRunQueries).where(
|
|
13354
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
13720
|
+
const rows = db.select().from(researchRunQueries).where(eq23(researchRunQueries.researchRunId, runId)).orderBy(researchRunQueries.position).all();
|
|
13721
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq23(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
13355
13722
|
const domains = effectiveDomains(project);
|
|
13356
13723
|
const brands = effectiveBrandNames(project);
|
|
13357
13724
|
const config = { ...provider.config, model: run.resolvedModel };
|
|
@@ -13359,7 +13726,7 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
13359
13726
|
await mapWithConcurrency(rows, Math.max(1, provider.config.quotaPolicy.maxConcurrency), async (row) => {
|
|
13360
13727
|
try {
|
|
13361
13728
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
13362
|
-
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.running, startedAt }).where(
|
|
13729
|
+
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.running, startedAt }).where(and17(eq23(researchRunQueries.id, row.id), eq23(researchRunQueries.status, ResearchQueryStatuses.queued))).run();
|
|
13363
13730
|
const raw = await gate.run(async () => {
|
|
13364
13731
|
dispatched++;
|
|
13365
13732
|
return provider.adapter.executeTrackedQuery({ query: row.queryText, canonicalDomains: domains, competitorDomains, ...run.location ? { location: run.location } : {} }, config);
|
|
@@ -13386,7 +13753,7 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
13386
13753
|
citationState: determineCitationState(normalized, domains),
|
|
13387
13754
|
rawResponse: raw.rawResponse,
|
|
13388
13755
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
13389
|
-
}).where(
|
|
13756
|
+
}).where(and17(eq23(researchRunQueries.id, row.id), eq23(researchRunQueries.status, ResearchQueryStatuses.running))).run();
|
|
13390
13757
|
if (completed.changes === 1) incrementResearchProgress(db, runId, "completedQueries");
|
|
13391
13758
|
} catch (error) {
|
|
13392
13759
|
try {
|
|
@@ -13410,18 +13777,18 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
13410
13777
|
}
|
|
13411
13778
|
}
|
|
13412
13779
|
function incrementResearchProgress(db, runId, column) {
|
|
13413
|
-
db.update(researchRuns).set({ [column]: sql10`${researchRuns[column]} + 1` }).where(
|
|
13780
|
+
db.update(researchRuns).set({ [column]: sql10`${researchRuns[column]} + 1` }).where(eq23(researchRuns.id, runId)).run();
|
|
13414
13781
|
}
|
|
13415
13782
|
function markResearchQueryFailed(db, runId, queryId, error) {
|
|
13416
13783
|
const message = error instanceof Error ? error.message : String(error);
|
|
13417
|
-
const failed = db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error: message, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
13784
|
+
const failed = db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error: message, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and17(eq23(researchRunQueries.id, queryId), inArray8(researchRunQueries.status, unfinishedResearchQueryStatuses))).run();
|
|
13418
13785
|
if (failed.changes === 1) incrementResearchProgress(db, runId, "failedQueries");
|
|
13419
13786
|
}
|
|
13420
13787
|
function markUnfinishedResearchQueriesFailed(db, runId, error) {
|
|
13421
|
-
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
13788
|
+
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and17(eq23(researchRunQueries.researchRunId, runId), inArray8(researchRunQueries.status, unfinishedResearchQueryStatuses))).run();
|
|
13422
13789
|
}
|
|
13423
13790
|
function finalizeResearchRun(db, runId, fatalError) {
|
|
13424
|
-
const rows = db.select({ status: researchRunQueries.status }).from(researchRunQueries).where(
|
|
13791
|
+
const rows = db.select({ status: researchRunQueries.status }).from(researchRunQueries).where(eq23(researchRunQueries.researchRunId, runId)).all();
|
|
13425
13792
|
const completed = rows.filter((row) => row.status === ResearchQueryStatuses.completed).length;
|
|
13426
13793
|
const failed = rows.filter((row) => row.status === ResearchQueryStatuses.failed).length;
|
|
13427
13794
|
db.update(researchRuns).set({
|
|
@@ -13430,7 +13797,7 @@ function finalizeResearchRun(db, runId, fatalError) {
|
|
|
13430
13797
|
failedQueries: failed,
|
|
13431
13798
|
error: fatalError ?? (failed === rows.length ? "Every research query failed." : null),
|
|
13432
13799
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
13433
|
-
}).where(
|
|
13800
|
+
}).where(eq23(researchRuns.id, runId)).run();
|
|
13434
13801
|
}
|
|
13435
13802
|
|
|
13436
13803
|
// src/server.ts
|
|
@@ -13468,6 +13835,13 @@ function resolveDashboardShowResourceLinks(env, config) {
|
|
|
13468
13835
|
function resolveDashboardShowUpdateNotification(env, config) {
|
|
13469
13836
|
return parseBooleanEnv(env.CANONRY_DASHBOARD_SHOW_UPDATE_NOTIFICATION) ?? config.dashboard?.showUpdateNotification ?? true;
|
|
13470
13837
|
}
|
|
13838
|
+
function parseDashboardOnboardingMode(value) {
|
|
13839
|
+
const normalized = value?.trim().toLowerCase();
|
|
13840
|
+
return normalized === "legacy" || normalized === "platform" || normalized === "auto" ? normalized : void 0;
|
|
13841
|
+
}
|
|
13842
|
+
function resolveDashboardOnboardingMode(env, config) {
|
|
13843
|
+
return parseDashboardOnboardingMode(env.CANONRY_ONBOARDING_MODE) ?? config.dashboard?.onboardingMode;
|
|
13844
|
+
}
|
|
13471
13845
|
var API_ADAPTERS = [
|
|
13472
13846
|
geminiAdapter,
|
|
13473
13847
|
openaiAdapter,
|
|
@@ -13745,7 +14119,7 @@ async function createServer(opts) {
|
|
|
13745
14119
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
13746
14120
|
async (ctx) => {
|
|
13747
14121
|
if (!sessionRegistry) return;
|
|
13748
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
14122
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq24(projects.id, ctx.projectId)).get();
|
|
13749
14123
|
if (!project) return;
|
|
13750
14124
|
let content;
|
|
13751
14125
|
if (ctx.kind === RunKinds["aeo-discover-probe"]) {
|
|
@@ -14095,7 +14469,7 @@ async function createServer(opts) {
|
|
|
14095
14469
|
void (async () => {
|
|
14096
14470
|
try {
|
|
14097
14471
|
const report = await aeroClient.runDoctor({ project: projectName });
|
|
14098
|
-
const project = opts.db.select().from(projects).where(
|
|
14472
|
+
const project = opts.db.select().from(projects).where(eq24(projects.name, projectName)).get();
|
|
14099
14473
|
if (!project) {
|
|
14100
14474
|
app.log.warn({ projectName }, "doctor schedule fired for an unknown project");
|
|
14101
14475
|
return;
|
|
@@ -14123,9 +14497,9 @@ async function createServer(opts) {
|
|
|
14123
14497
|
});
|
|
14124
14498
|
if (!probed) return;
|
|
14125
14499
|
const alreadySynced = opts.db.select().from(ccReleaseSyncs).where(
|
|
14126
|
-
|
|
14127
|
-
|
|
14128
|
-
|
|
14500
|
+
and18(
|
|
14501
|
+
eq24(ccReleaseSyncs.release, probed.release),
|
|
14502
|
+
eq24(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)
|
|
14129
14503
|
)
|
|
14130
14504
|
).limit(1).get();
|
|
14131
14505
|
if (alreadySynced) {
|
|
@@ -14296,6 +14670,14 @@ async function createServer(opts) {
|
|
|
14296
14670
|
const removed = removeCloudflareTrafficConnection(opts.config, projectName);
|
|
14297
14671
|
if (removed) saveConfigPatch(opts.config);
|
|
14298
14672
|
return removed;
|
|
14673
|
+
},
|
|
14674
|
+
deleteConnectionBySourceId: (sourceId) => {
|
|
14675
|
+
const removed = removeCloudflareTrafficConnectionBySourceId(
|
|
14676
|
+
opts.config,
|
|
14677
|
+
sourceId
|
|
14678
|
+
);
|
|
14679
|
+
if (removed) saveConfigPatch(opts.config);
|
|
14680
|
+
return removed;
|
|
14299
14681
|
}
|
|
14300
14682
|
};
|
|
14301
14683
|
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto25.randomBytes(32).toString("hex");
|
|
@@ -14365,6 +14747,7 @@ async function createServer(opts) {
|
|
|
14365
14747
|
const dashboardRequirePassword = resolveDashboardRequirePassword(process.env, opts.config);
|
|
14366
14748
|
const dashboardShowResourceLinks = resolveDashboardShowResourceLinks(process.env, opts.config);
|
|
14367
14749
|
const dashboardShowUpdateNotification = resolveDashboardShowUpdateNotification(process.env, opts.config);
|
|
14750
|
+
const dashboardOnboardingMode = resolveDashboardOnboardingMode(process.env, opts.config);
|
|
14368
14751
|
app.log.info(
|
|
14369
14752
|
{ dashboardRequirePassword },
|
|
14370
14753
|
"Dashboard password gate resolved"
|
|
@@ -14373,7 +14756,7 @@ async function createServer(opts) {
|
|
|
14373
14756
|
const googlePublicUrl = resolveGooglePublicUrl(opts.config, basePath);
|
|
14374
14757
|
if (opts.config.apiKey) {
|
|
14375
14758
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
14376
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
14759
|
+
const existing = opts.db.select().from(apiKeys).where(eq24(apiKeys.keyHash, keyHash)).get();
|
|
14377
14760
|
if (!existing) {
|
|
14378
14761
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
14379
14762
|
opts.db.insert(apiKeys).values({
|
|
@@ -14425,7 +14808,7 @@ async function createServer(opts) {
|
|
|
14425
14808
|
};
|
|
14426
14809
|
const getDefaultApiKey = () => {
|
|
14427
14810
|
if (!opts.config.apiKey) return void 0;
|
|
14428
|
-
return opts.db.select().from(apiKeys).where(
|
|
14811
|
+
return opts.db.select().from(apiKeys).where(eq24(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
14429
14812
|
};
|
|
14430
14813
|
const createPasswordSession = (reply) => {
|
|
14431
14814
|
const key = getDefaultApiKey();
|
|
@@ -14449,7 +14832,7 @@ async function createServer(opts) {
|
|
|
14449
14832
|
if (!header) return false;
|
|
14450
14833
|
const parts = header.split(" ");
|
|
14451
14834
|
if (parts.length !== 2 || parts[0] !== "Bearer") return false;
|
|
14452
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
14835
|
+
const key = opts.db.select().from(apiKeys).where(eq24(apiKeys.keyHash, hashApiKey(parts[1]))).get();
|
|
14453
14836
|
return Boolean(key && !key.revokedAt);
|
|
14454
14837
|
};
|
|
14455
14838
|
const namedAccountsInUse = () => anyUsersExist(opts.db);
|
|
@@ -14539,12 +14922,12 @@ async function createServer(opts) {
|
|
|
14539
14922
|
return reply.send({ authenticated: true });
|
|
14540
14923
|
}
|
|
14541
14924
|
if (apiKey) {
|
|
14542
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
14925
|
+
const key = opts.db.select().from(apiKeys).where(eq24(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
14543
14926
|
if (!key || key.revokedAt) {
|
|
14544
14927
|
const err2 = authInvalid();
|
|
14545
14928
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
14546
14929
|
}
|
|
14547
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
14930
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq24(apiKeys.id, key.id)).run();
|
|
14548
14931
|
const sessionId = createSession(key.id);
|
|
14549
14932
|
reply.header(
|
|
14550
14933
|
"set-cookie",
|
|
@@ -14886,7 +15269,7 @@ async function createServer(opts) {
|
|
|
14886
15269
|
...inspectOpts,
|
|
14887
15270
|
config: opts.config
|
|
14888
15271
|
}).then(() => {
|
|
14889
|
-
const finished = opts.db.select({ status: runs.status }).from(runs).where(
|
|
15272
|
+
const finished = opts.db.select({ status: runs.status }).from(runs).where(eq24(runs.id, runId)).get();
|
|
14890
15273
|
if (finished?.status === RunStatuses.completed || finished?.status === RunStatuses.partial) {
|
|
14891
15274
|
return maybeRefreshGscCoverage(
|
|
14892
15275
|
opts.db,
|
|
@@ -15163,6 +15546,9 @@ async function createServer(opts) {
|
|
|
15163
15546
|
const clientConfig = {};
|
|
15164
15547
|
if (basePath) clientConfig.basePath = basePath;
|
|
15165
15548
|
const dashboardConfig = {};
|
|
15549
|
+
if (dashboardOnboardingMode) {
|
|
15550
|
+
dashboardConfig.onboardingMode = dashboardOnboardingMode;
|
|
15551
|
+
}
|
|
15166
15552
|
if (!dashboardShowResourceLinks) {
|
|
15167
15553
|
dashboardConfig.showResourceLinks = false;
|
|
15168
15554
|
}
|
|
@@ -15264,7 +15650,7 @@ async function createServer(opts) {
|
|
|
15264
15650
|
}
|
|
15265
15651
|
checkLatestVersionForServer();
|
|
15266
15652
|
scheduler.start();
|
|
15267
|
-
for (const run of opts.db.select({ id: researchRuns.id, projectId: researchRuns.projectId }).from(researchRuns).where(
|
|
15653
|
+
for (const run of opts.db.select({ id: researchRuns.id, projectId: researchRuns.projectId }).from(researchRuns).where(eq24(researchRuns.status, ResearchRunStatuses.queued)).all()) {
|
|
15268
15654
|
dispatchResearchRun(run.id, run.projectId);
|
|
15269
15655
|
}
|
|
15270
15656
|
app.addHook("onClose", async () => {
|
|
@@ -15347,7 +15733,7 @@ export {
|
|
|
15347
15733
|
backfillInsightsCommand,
|
|
15348
15734
|
backfillSnapshotAttributionCommand,
|
|
15349
15735
|
backfillTrafficClassificationCommand,
|
|
15350
|
-
|
|
15736
|
+
getCloudflareTrafficConnectionBySourceId,
|
|
15351
15737
|
setGoogleAuthConfig,
|
|
15352
15738
|
installSkills,
|
|
15353
15739
|
listSkills,
|