@canonry/canonry 4.157.0 → 4.159.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/THIRD_PARTY_NOTICES.md +102 -0
- package/assets/THIRD_PARTY_NOTICES.md +102 -0
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +1 -0
- package/assets/assets/{AuditHistoryPanel-D12lHs6n.js → AuditHistoryPanel-BiloGcQA.js} +1 -1
- package/assets/assets/{BacklinksPage-D2ScFYru.js → BacklinksPage-CPM1O2bC.js} +1 -1
- package/assets/assets/{HistoryPage-jZ_I4rwX.js → HistoryPage-1YOjtdJy.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-BIdQjXJT.js → MeasurementPropertyPage-CVDTY_gC.js} +1 -1
- package/assets/assets/ProjectPage-DnGXMx1n.js +12 -0
- package/assets/assets/{RunRow-Q2X0NR74.js → RunRow-fIaadH4h.js} +1 -1
- package/assets/assets/{RunsPage-Cw7YVsbJ.js → RunsPage-DfifdM5z.js} +1 -1
- package/assets/assets/{SettingsPage-bFl2EXOD.js → SettingsPage-efA2EEmS.js} +1 -1
- package/assets/assets/SiteHealthSection-BROtQGfu.js +4 -0
- package/assets/assets/{TrafficPage-DJjhk7Jw.js → TrafficPage-Wk43Bm_q.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-CGKwIu9A.js → TrafficSourceDetailPage-BhBvoLWu.js} +1 -1
- package/assets/assets/extract-error-message-YGzZFvWs.js +1 -0
- package/assets/assets/geist-cyrillic-ext-wght-normal-DjL33-gN.woff2 +0 -0
- package/assets/assets/geist-cyrillic-wght-normal-BEAKL7Jp.woff2 +0 -0
- package/assets/assets/geist-latin-ext-wght-normal-DC-KSUi6.woff2 +0 -0
- package/assets/assets/geist-latin-wght-normal-BgDaEnEv.woff2 +0 -0
- package/assets/assets/geist-mono-cyrillic-ext-wght-normal-X_5orZeX.woff2 +0 -0
- package/assets/assets/geist-mono-cyrillic-wght-normal-DiZS0aHC.woff2 +0 -0
- package/assets/assets/geist-mono-latin-ext-wght-normal-Bwz-egvJ.woff2 +0 -0
- package/assets/assets/geist-mono-latin-wght-normal-XN7g48iV.woff2 +0 -0
- package/assets/assets/geist-mono-symbols2-wght-normal-CO5SzqOn.woff2 +0 -0
- package/assets/assets/geist-mono-vietnamese-wght-normal-DadHysG0.woff2 +0 -0
- package/assets/assets/geist-vietnamese-wght-normal-6IgcOCM7.woff2 +0 -0
- package/assets/assets/index-CACRaOmZ.css +1 -0
- package/assets/assets/index-DmsIZQAm.js +83 -0
- package/assets/assets/{react-sigma_core.esm.min-EzswMBkr.js → react-sigma_core.esm.min-DM9XptbM.js} +2 -2
- package/assets/assets/{vendor-lucide-BtYdosYk.js → vendor-lucide-nYKv-BB7.js} +1 -1
- package/assets/assets/{vendor-markdown-D274_wsI.js → vendor-markdown-Bn7fRwrx.js} +1 -1
- package/assets/assets/{vendor-radix-BxMXSwbC.js → vendor-radix-4UOaVWza.js} +1 -1
- package/assets/assets/{vendor-recharts-C4ktw_aE.js → vendor-recharts-D8rt5lKn.js} +1 -1
- package/assets/assets/{vendor-tanstack-pFxw0eGP.js → vendor-tanstack-Dock0eH1.js} +1 -1
- package/assets/index.html +7 -7
- package/dist/{chunk-MEBL4EJO.js → chunk-5AFWHKKI.js} +875 -38
- package/dist/{chunk-AEIKAWOW.js → chunk-BZPBVOAM.js} +753 -212
- package/dist/{chunk-KMTRO56O.js → chunk-ZEG43TY2.js} +45 -15
- package/dist/{chunk-TRRVBTU5.js → chunk-ZL3VY435.js} +264 -10
- package/dist/cli.js +42 -4
- package/dist/index.d.ts +9 -0
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-YUUPSJRC.js → intelligence-service-DHY5CYWX.js} +2 -2
- package/dist/mcp.js +2 -2
- package/package.json +13 -11
- package/assets/assets/ProjectPage-CLvLOOtj.js +0 -15
- package/assets/assets/extract-error-message-BHm98tXH.js +0 -1
- package/assets/assets/index-BxegEwic.css +0 -1
- package/assets/assets/index-VO6nKq5b.js +0 -83
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
loadConfig,
|
|
11
11
|
loadConfigRaw,
|
|
12
12
|
saveConfigPatch
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-ZEG43TY2.js";
|
|
14
14
|
import {
|
|
15
15
|
CC_CACHE_DIR,
|
|
16
16
|
DUCKDB_SPEC,
|
|
@@ -117,6 +117,7 @@ import {
|
|
|
117
117
|
researchRunQueries,
|
|
118
118
|
researchRuns,
|
|
119
119
|
resolveDestination,
|
|
120
|
+
resolveMeasurementSitemapTarget,
|
|
120
121
|
resolveTrustProxy,
|
|
121
122
|
resolveVercelSyncDeadlineMs,
|
|
122
123
|
resolveWebhookTarget,
|
|
@@ -136,7 +137,7 @@ import {
|
|
|
136
137
|
siteCrawlSnapshots,
|
|
137
138
|
toAlertView,
|
|
138
139
|
usageCounters
|
|
139
|
-
} from "./chunk-
|
|
140
|
+
} from "./chunk-5AFWHKKI.js";
|
|
140
141
|
import {
|
|
141
142
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
142
143
|
AGENT_PROVIDER_IDS,
|
|
@@ -169,7 +170,9 @@ import {
|
|
|
169
170
|
SITE_CRAWL_GRAPH_MAX_NODES,
|
|
170
171
|
SKILL_MANIFEST_FILENAME,
|
|
171
172
|
SchedulableRunKinds,
|
|
173
|
+
SiteHealthTemplateDetections,
|
|
172
174
|
SkillsClients,
|
|
175
|
+
TEMPLATE_LINK_RATIO_THRESHOLD,
|
|
173
176
|
TrafficEventConfidences,
|
|
174
177
|
TrafficEventKinds,
|
|
175
178
|
TrafficEvidenceKinds,
|
|
@@ -192,8 +195,10 @@ import {
|
|
|
192
195
|
computeBacklinkSummaryMetrics,
|
|
193
196
|
contentActionLabel,
|
|
194
197
|
contentBriefDtoSchema,
|
|
198
|
+
createTemplateLinkPairIndex,
|
|
195
199
|
deriveCitedUrlCandidates,
|
|
196
200
|
deriveIndexCoverage,
|
|
201
|
+
deriveSiteHealthState,
|
|
197
202
|
determineAnswerMentioned,
|
|
198
203
|
dollarsToMicros,
|
|
199
204
|
effectiveBrandNames,
|
|
@@ -211,6 +216,8 @@ import {
|
|
|
211
216
|
isBrowserProvider,
|
|
212
217
|
isGhostTelemetryEvent,
|
|
213
218
|
isRetryableHttpError,
|
|
219
|
+
isSelfLink,
|
|
220
|
+
isTemplateLinkRatio,
|
|
214
221
|
isVertexGroundingRedirect,
|
|
215
222
|
mapWithConcurrency,
|
|
216
223
|
normalizeIdTokens,
|
|
@@ -219,6 +226,7 @@ import {
|
|
|
219
226
|
normalizeSiteAuditRunRequest,
|
|
220
227
|
normalizeUrlPath,
|
|
221
228
|
notFound,
|
|
229
|
+
observeTemplateLinkEdges,
|
|
222
230
|
parseMeasurementRunManifestV1,
|
|
223
231
|
parseOriginList,
|
|
224
232
|
providerError,
|
|
@@ -231,12 +239,14 @@ import {
|
|
|
231
239
|
skillsClientSchema,
|
|
232
240
|
splitList,
|
|
233
241
|
startOfDayHourInTimeZone,
|
|
242
|
+
templateLinkDetection,
|
|
243
|
+
templateLinkRatio,
|
|
234
244
|
textContainsBrandAlias,
|
|
235
245
|
textContainsDomain,
|
|
236
246
|
validationError,
|
|
237
247
|
winnabilityClassLabel,
|
|
238
248
|
withRetry
|
|
239
|
-
} from "./chunk-
|
|
249
|
+
} from "./chunk-ZL3VY435.js";
|
|
240
250
|
|
|
241
251
|
// src/telemetry.ts
|
|
242
252
|
import crypto from "crypto";
|
|
@@ -571,7 +581,7 @@ import crypto25 from "crypto";
|
|
|
571
581
|
import fs8 from "fs";
|
|
572
582
|
import path9 from "path";
|
|
573
583
|
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
574
|
-
import { and as
|
|
584
|
+
import { and as and18, eq as eq24 } from "drizzle-orm";
|
|
575
585
|
import Fastify from "fastify";
|
|
576
586
|
import os5 from "os";
|
|
577
587
|
|
|
@@ -4169,12 +4179,12 @@ var JobRunner = class {
|
|
|
4169
4179
|
error: errorMessage
|
|
4170
4180
|
}).where(eq2(runs.id, runId)).run();
|
|
4171
4181
|
this.flushProviderUsage(providerDispatchCounts, providerReservations);
|
|
4172
|
-
const
|
|
4182
|
+
const abortReason2 = classifyRunAbortReason(errorMessage);
|
|
4173
4183
|
const phases = buildPhases({ startTime, providerCallStart, providerCallEnd });
|
|
4174
|
-
if (
|
|
4184
|
+
if (abortReason2) {
|
|
4175
4185
|
const domainHash = hashDomain(executionContext.canonicalDomain ?? null);
|
|
4176
4186
|
trackEvent("run.aborted", {
|
|
4177
|
-
reason:
|
|
4187
|
+
reason: abortReason2,
|
|
4178
4188
|
providerCount: executionContext.providerCount,
|
|
4179
4189
|
providers: executionContext.providers,
|
|
4180
4190
|
queryCount: executionContext.queryCount,
|
|
@@ -6995,16 +7005,199 @@ function buildDiscoveryInsightTitle(input) {
|
|
|
6995
7005
|
|
|
6996
7006
|
// src/execute-site-audit.ts
|
|
6997
7007
|
import crypto17 from "crypto";
|
|
6998
|
-
import { and as
|
|
7008
|
+
import { and as and12, eq as eq15, sql as sql6 } from "drizzle-orm";
|
|
6999
7009
|
import { runSiteCrawl } from "@canonry/aeo-audit";
|
|
7000
7010
|
|
|
7011
|
+
// src/site-audit-root.ts
|
|
7012
|
+
import http from "http";
|
|
7013
|
+
import https from "https";
|
|
7014
|
+
var DEFAULT_TIMEOUT_MS = 1e4;
|
|
7015
|
+
var MAX_REDIRECTS = 5;
|
|
7016
|
+
var USER_AGENT = "Canonry/1.0 (site-health)";
|
|
7017
|
+
async function resolveSiteAuditRootUrl(rawUrl, options) {
|
|
7018
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
7019
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) throw new Error("Site audit root timeout must be positive");
|
|
7020
|
+
const requested = parseHttpUrl2(rawUrl);
|
|
7021
|
+
let current = requested;
|
|
7022
|
+
const redirects = [];
|
|
7023
|
+
const deadlineAt = Date.now() + timeoutMs;
|
|
7024
|
+
const transport = options.transport ?? requestPinnedSiteAuditRoot;
|
|
7025
|
+
for (; ; ) {
|
|
7026
|
+
throwIfAborted(options.signal);
|
|
7027
|
+
const checked = await beforeDeadline(
|
|
7028
|
+
() => options.resolveTarget(current.href),
|
|
7029
|
+
deadlineAt,
|
|
7030
|
+
timeoutMs,
|
|
7031
|
+
options.signal
|
|
7032
|
+
);
|
|
7033
|
+
if (!checked.ok) throw new Error(`Site audit root URL rejected: ${checked.message}`);
|
|
7034
|
+
const checkedUrl = parseHttpUrl2(checked.target.url.href);
|
|
7035
|
+
if (checkedUrl.href !== current.href) {
|
|
7036
|
+
throw new Error("Site audit root target resolver returned a different URL than it validated");
|
|
7037
|
+
}
|
|
7038
|
+
const response = await beforeDeadline(
|
|
7039
|
+
() => transport(checked.target, { timeoutMs: remainingTime(deadlineAt, timeoutMs), signal: options.signal }),
|
|
7040
|
+
deadlineAt,
|
|
7041
|
+
timeoutMs,
|
|
7042
|
+
options.signal
|
|
7043
|
+
);
|
|
7044
|
+
if (!isRedirect(response.status)) {
|
|
7045
|
+
return { requestedUrl: requested.href, effectiveUrl: current.href, redirects };
|
|
7046
|
+
}
|
|
7047
|
+
if (redirects.length >= MAX_REDIRECTS) {
|
|
7048
|
+
throw new Error(`Site audit root has more than ${MAX_REDIRECTS} redirects`);
|
|
7049
|
+
}
|
|
7050
|
+
const location = firstHeader(response.headers.location);
|
|
7051
|
+
if (!location) throw new Error("Site audit root redirect response has no Location header");
|
|
7052
|
+
let next;
|
|
7053
|
+
try {
|
|
7054
|
+
next = parseHttpUrl2(new URL(location, current).href);
|
|
7055
|
+
} catch {
|
|
7056
|
+
throw new Error("Site audit root redirect has an invalid Location header");
|
|
7057
|
+
}
|
|
7058
|
+
assertAllowedRedirect(current, next);
|
|
7059
|
+
redirects.push({ status: response.status, from: current.href, to: next.href });
|
|
7060
|
+
current = next;
|
|
7061
|
+
}
|
|
7062
|
+
}
|
|
7063
|
+
function parseHttpUrl2(rawUrl) {
|
|
7064
|
+
let url;
|
|
7065
|
+
try {
|
|
7066
|
+
url = new URL(rawUrl);
|
|
7067
|
+
} catch {
|
|
7068
|
+
throw new Error("Site audit root must be a valid URL");
|
|
7069
|
+
}
|
|
7070
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
7071
|
+
throw new Error("Site audit root must use HTTP or HTTPS");
|
|
7072
|
+
}
|
|
7073
|
+
if (url.username || url.password) throw new Error("Site audit root must not include credentials");
|
|
7074
|
+
url.hash = "";
|
|
7075
|
+
return url;
|
|
7076
|
+
}
|
|
7077
|
+
function canonicalHostAlias(hostname) {
|
|
7078
|
+
return hostname.toLowerCase().replace(/\.$/, "").replace(/^www\./, "");
|
|
7079
|
+
}
|
|
7080
|
+
function assertAllowedRedirect(current, next) {
|
|
7081
|
+
if (current.protocol === "https:" && next.protocol === "http:") {
|
|
7082
|
+
throw new Error("Site audit root redirect would downgrade HTTPS to HTTP");
|
|
7083
|
+
}
|
|
7084
|
+
if (current.port !== next.port) throw new Error("Site audit root redirect changes port");
|
|
7085
|
+
if (canonicalHostAlias(current.hostname) !== canonicalHostAlias(next.hostname)) {
|
|
7086
|
+
throw new Error("Site audit root redirect points outside the approved site");
|
|
7087
|
+
}
|
|
7088
|
+
}
|
|
7089
|
+
function isRedirect(status) {
|
|
7090
|
+
return status >= 300 && status < 400;
|
|
7091
|
+
}
|
|
7092
|
+
function firstHeader(value) {
|
|
7093
|
+
return Array.isArray(value) ? value[0] : value;
|
|
7094
|
+
}
|
|
7095
|
+
function remainingTime(deadlineAt, timeoutMs) {
|
|
7096
|
+
const remaining = deadlineAt - Date.now();
|
|
7097
|
+
if (remaining <= 0) throw new Error(`Site audit root resolution exceeded its ${timeoutMs}ms deadline`);
|
|
7098
|
+
return remaining;
|
|
7099
|
+
}
|
|
7100
|
+
async function beforeDeadline(operation, deadlineAt, timeoutMs, signal) {
|
|
7101
|
+
throwIfAborted(signal);
|
|
7102
|
+
const remaining = remainingTime(deadlineAt, timeoutMs);
|
|
7103
|
+
let timer;
|
|
7104
|
+
let abortListener;
|
|
7105
|
+
try {
|
|
7106
|
+
const value = await Promise.race([
|
|
7107
|
+
operation(),
|
|
7108
|
+
new Promise((_resolve, reject) => {
|
|
7109
|
+
timer = setTimeout(
|
|
7110
|
+
() => reject(new Error(`Site audit root resolution exceeded its ${timeoutMs}ms deadline`)),
|
|
7111
|
+
remaining
|
|
7112
|
+
);
|
|
7113
|
+
}),
|
|
7114
|
+
new Promise((_resolve, reject) => {
|
|
7115
|
+
if (!signal) return;
|
|
7116
|
+
abortListener = () => reject(abortReason(signal));
|
|
7117
|
+
signal.addEventListener("abort", abortListener, { once: true });
|
|
7118
|
+
if (signal.aborted) abortListener();
|
|
7119
|
+
})
|
|
7120
|
+
]);
|
|
7121
|
+
remainingTime(deadlineAt, timeoutMs);
|
|
7122
|
+
return value;
|
|
7123
|
+
} finally {
|
|
7124
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
7125
|
+
if (abortListener) signal?.removeEventListener("abort", abortListener);
|
|
7126
|
+
}
|
|
7127
|
+
}
|
|
7128
|
+
function abortReason(signal) {
|
|
7129
|
+
return signal.reason instanceof Error ? signal.reason : new DOMException("Operation aborted", "AbortError");
|
|
7130
|
+
}
|
|
7131
|
+
function throwIfAborted(signal) {
|
|
7132
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
7133
|
+
}
|
|
7134
|
+
function stripIpv6Brackets(hostname) {
|
|
7135
|
+
return hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
|
|
7136
|
+
}
|
|
7137
|
+
function buildSiteAuditRootRequestOptions(target) {
|
|
7138
|
+
const secure = target.url.protocol === "https:";
|
|
7139
|
+
const requestOptions = {
|
|
7140
|
+
hostname: target.address,
|
|
7141
|
+
family: target.family,
|
|
7142
|
+
port: target.url.port ? Number(target.url.port) : secure ? 443 : 80,
|
|
7143
|
+
method: "GET",
|
|
7144
|
+
path: `${target.url.pathname}${target.url.search}`,
|
|
7145
|
+
headers: {
|
|
7146
|
+
Host: target.url.host,
|
|
7147
|
+
Accept: "text/html,*/*;q=0.1",
|
|
7148
|
+
"User-Agent": USER_AGENT
|
|
7149
|
+
}
|
|
7150
|
+
};
|
|
7151
|
+
if (secure) requestOptions.servername = stripIpv6Brackets(target.url.hostname);
|
|
7152
|
+
return requestOptions;
|
|
7153
|
+
}
|
|
7154
|
+
async function requestPinnedSiteAuditRoot(target, options) {
|
|
7155
|
+
throwIfAborted(options.signal);
|
|
7156
|
+
const secure = target.url.protocol === "https:";
|
|
7157
|
+
const requestOptions = buildSiteAuditRootRequestOptions(target);
|
|
7158
|
+
return await new Promise((resolve, reject) => {
|
|
7159
|
+
let settled = false;
|
|
7160
|
+
let abortListener;
|
|
7161
|
+
const cleanup = () => {
|
|
7162
|
+
clearTimeout(timer);
|
|
7163
|
+
if (abortListener) options.signal?.removeEventListener("abort", abortListener);
|
|
7164
|
+
};
|
|
7165
|
+
const succeed = (response) => {
|
|
7166
|
+
if (settled) return;
|
|
7167
|
+
settled = true;
|
|
7168
|
+
cleanup();
|
|
7169
|
+
resolve(response);
|
|
7170
|
+
};
|
|
7171
|
+
const fail = (error) => {
|
|
7172
|
+
if (settled) return;
|
|
7173
|
+
settled = true;
|
|
7174
|
+
cleanup();
|
|
7175
|
+
reject(error);
|
|
7176
|
+
};
|
|
7177
|
+
const request = (secure ? https.request : http.request)(requestOptions, (response) => {
|
|
7178
|
+
response.destroy();
|
|
7179
|
+
succeed({ status: response.statusCode ?? 0, headers: response.headers });
|
|
7180
|
+
});
|
|
7181
|
+
const timer = setTimeout(
|
|
7182
|
+
() => request.destroy(new Error(`Site audit root request timed out after ${options.timeoutMs}ms`)),
|
|
7183
|
+
options.timeoutMs
|
|
7184
|
+
);
|
|
7185
|
+
if (options.signal) {
|
|
7186
|
+
abortListener = () => request.destroy(abortReason(options.signal));
|
|
7187
|
+
options.signal.addEventListener("abort", abortListener, { once: true });
|
|
7188
|
+
if (options.signal.aborted) abortListener();
|
|
7189
|
+
}
|
|
7190
|
+
request.on("error", fail);
|
|
7191
|
+
request.end();
|
|
7192
|
+
});
|
|
7193
|
+
}
|
|
7194
|
+
|
|
7001
7195
|
// src/site-crawl-graph-layout.ts
|
|
7002
7196
|
import crypto16 from "crypto";
|
|
7003
7197
|
import { createRequire as createRequire3 } from "module";
|
|
7004
7198
|
import { Worker } from "worker_threads";
|
|
7005
7199
|
import { and as and10, asc, count, desc as desc6, eq as eq13, isNull, ne as ne2, or, sql as sql5 } from "drizzle-orm";
|
|
7006
7200
|
import { alias } from "drizzle-orm/sqlite-core";
|
|
7007
|
-
var SITE_CRAWL_GRAPH_LAYOUT_VERSION = "site-health-fa2-v1";
|
|
7008
7201
|
var SITE_CRAWL_GRAPH_LAYOUT_TIMEOUT_MS = 15e3;
|
|
7009
7202
|
var SITE_CRAWL_GRAPH_SEED_SCALE = 100;
|
|
7010
7203
|
var graphSourcePage = alias(siteCrawlPages, "layout_graph_source_page");
|
|
@@ -7014,20 +7207,88 @@ var FORCE_ATLAS_SETTINGS = Object.freeze({
|
|
|
7014
7207
|
barnesHutOptimize: true,
|
|
7015
7208
|
barnesHutTheta: 0.5,
|
|
7016
7209
|
edgeWeightInfluence: 0.5,
|
|
7017
|
-
gravity:
|
|
7018
|
-
linLogMode:
|
|
7019
|
-
outboundAttractionDistribution:
|
|
7020
|
-
scalingRatio:
|
|
7210
|
+
gravity: 0.2,
|
|
7211
|
+
linLogMode: true,
|
|
7212
|
+
outboundAttractionDistribution: true,
|
|
7213
|
+
scalingRatio: 2,
|
|
7021
7214
|
slowDown: 10,
|
|
7022
7215
|
strongGravityMode: false
|
|
7023
7216
|
});
|
|
7217
|
+
var RENDER_MAX_NODE_SIZE = 14;
|
|
7218
|
+
var RENDER_HALF_EXTENT_PX = 420;
|
|
7219
|
+
var NODE_SIZE_EXTENT_FRACTION = RENDER_MAX_NODE_SIZE / RENDER_HALF_EXTENT_PX;
|
|
7220
|
+
function siteCrawlGraphNodeSize(extent, nodeCount) {
|
|
7221
|
+
if (!Number.isFinite(extent) || extent <= 0 || nodeCount <= 0) return 0;
|
|
7222
|
+
const byExtent = extent * NODE_SIZE_EXTENT_FRACTION;
|
|
7223
|
+
const byArea = Math.sqrt(NOVERLAP_SETTINGS.maxPackingFraction * (2 * extent) ** 2 / nodeCount) / 2;
|
|
7224
|
+
return Math.min(byExtent, byArea);
|
|
7225
|
+
}
|
|
7226
|
+
function siteCrawlGraphExtent(positions) {
|
|
7227
|
+
if (positions.length === 0) return 0;
|
|
7228
|
+
const centerX = positions.reduce((sum, p) => sum + p.x, 0) / positions.length;
|
|
7229
|
+
const centerY = positions.reduce((sum, p) => sum + p.y, 0) / positions.length;
|
|
7230
|
+
let extent = 0;
|
|
7231
|
+
for (const position of positions) {
|
|
7232
|
+
extent = Math.max(extent, Math.abs(position.x - centerX), Math.abs(position.y - centerY));
|
|
7233
|
+
}
|
|
7234
|
+
return extent;
|
|
7235
|
+
}
|
|
7236
|
+
var NOVERLAP_SETTINGS = Object.freeze({
|
|
7237
|
+
/** Half a node of clear space between two touching nodes. */
|
|
7238
|
+
marginFactor: 0.5,
|
|
7239
|
+
expansion: 1.1,
|
|
7240
|
+
ratio: 1,
|
|
7241
|
+
speed: 3,
|
|
7242
|
+
/**
|
|
7243
|
+
* Node discs may claim at most this share of the graph's bounding box. At
|
|
7244
|
+
* the 20k cap the extent-derived size asks for spacing the graph physically
|
|
7245
|
+
* does not have, and noverlap either runs for minutes or overflows its
|
|
7246
|
+
* collision set. This bound keeps the pass solvable at every graph size,
|
|
7247
|
+
* and it only binds on graphs far too dense to read node-by-node anyway.
|
|
7248
|
+
*/
|
|
7249
|
+
maxPackingFraction: 0.15,
|
|
7250
|
+
/** Cells hold a handful of nodes each, which is what keeps the pass linear. */
|
|
7251
|
+
gridSizeMin: 10,
|
|
7252
|
+
gridSizeMax: 200,
|
|
7253
|
+
iterationLadder: [
|
|
7254
|
+
{ maxNodes: 1e3, iterations: 120 },
|
|
7255
|
+
{ maxNodes: 5e3, iterations: 60 }
|
|
7256
|
+
],
|
|
7257
|
+
iterationsFloor: 20
|
|
7258
|
+
});
|
|
7259
|
+
function noverlapGridSize(nodeCount) {
|
|
7260
|
+
return Math.max(
|
|
7261
|
+
NOVERLAP_SETTINGS.gridSizeMin,
|
|
7262
|
+
Math.min(NOVERLAP_SETTINGS.gridSizeMax, Math.ceil(Math.sqrt(Math.max(1, nodeCount))))
|
|
7263
|
+
);
|
|
7264
|
+
}
|
|
7265
|
+
function noverlapMaxIterations(nodeCount) {
|
|
7266
|
+
for (const step of NOVERLAP_SETTINGS.iterationLadder) {
|
|
7267
|
+
if (nodeCount <= step.maxNodes) return step.iterations;
|
|
7268
|
+
}
|
|
7269
|
+
return NOVERLAP_SETTINGS.iterationsFloor;
|
|
7270
|
+
}
|
|
7271
|
+
var COMPONENT_PACKING_SPACING_FACTOR = 3;
|
|
7272
|
+
function siteCrawlGraphLayoutSettingsFingerprint(settings) {
|
|
7273
|
+
return crypto16.createHash("sha256").update(JSON.stringify(settings)).digest("hex").slice(0, 8);
|
|
7274
|
+
}
|
|
7275
|
+
var SITE_CRAWL_GRAPH_LAYOUT_ALGORITHM = "site-health-fa2-v4";
|
|
7276
|
+
var SITE_CRAWL_GRAPH_LAYOUT_VERSION = `${SITE_CRAWL_GRAPH_LAYOUT_ALGORITHM}-${siteCrawlGraphLayoutSettingsFingerprint({
|
|
7277
|
+
forceAtlas: FORCE_ATLAS_SETTINGS,
|
|
7278
|
+
noverlap: NOVERLAP_SETTINGS,
|
|
7279
|
+
nodeSizeExtentFraction: NODE_SIZE_EXTENT_FRACTION,
|
|
7280
|
+
componentPackingSpacingFactor: COMPONENT_PACKING_SPACING_FACTOR,
|
|
7281
|
+
templateLinkRatioThreshold: TEMPLATE_LINK_RATIO_THRESHOLD
|
|
7282
|
+
})}`;
|
|
7024
7283
|
var WORKER_SOURCE = String.raw`
|
|
7025
7284
|
const {parentPort, workerData} = require('node:worker_threads');
|
|
7026
7285
|
try {
|
|
7027
7286
|
const graphologyModule = require(workerData.graphologyPath);
|
|
7028
7287
|
const forceAtlasModule = require(workerData.forceAtlasPath);
|
|
7288
|
+
const noverlapModule = require(workerData.noverlapPath);
|
|
7029
7289
|
const MultiDirectedGraph = graphologyModule.MultiDirectedGraph;
|
|
7030
7290
|
const forceAtlas2 = forceAtlasModule.default || forceAtlasModule;
|
|
7291
|
+
const noverlap = noverlapModule.default || noverlapModule;
|
|
7031
7292
|
const graph = new MultiDirectedGraph({allowSelfLoops: true});
|
|
7032
7293
|
for (const node of workerData.input.nodes) {
|
|
7033
7294
|
graph.addNode(node.nodeKey, {x: node.x, y: node.y});
|
|
@@ -7043,11 +7304,52 @@ try {
|
|
|
7043
7304
|
settings: workerData.settings,
|
|
7044
7305
|
});
|
|
7045
7306
|
}
|
|
7307
|
+
let nodeSize = 0;
|
|
7308
|
+
// Anti-overlap runs on the spatialization ForceAtlas2 just produced. Node
|
|
7309
|
+
// size is uniform and derived from the graph's own extent, so the pass is
|
|
7310
|
+
// scale invariant and stays deterministic for a fixed input.
|
|
7311
|
+
if (graph.order > 1) {
|
|
7312
|
+
let sumX = 0;
|
|
7313
|
+
let sumY = 0;
|
|
7314
|
+
graph.forEachNode(function (nodeKey, attributes) {
|
|
7315
|
+
sumX += attributes.x;
|
|
7316
|
+
sumY += attributes.y;
|
|
7317
|
+
});
|
|
7318
|
+
const centerX = sumX / graph.order;
|
|
7319
|
+
const centerY = sumY / graph.order;
|
|
7320
|
+
let extent = 0;
|
|
7321
|
+
graph.forEachNode(function (nodeKey, attributes) {
|
|
7322
|
+
extent = Math.max(extent, Math.abs(attributes.x - centerX), Math.abs(attributes.y - centerY));
|
|
7323
|
+
});
|
|
7324
|
+
if (Number.isFinite(extent) && extent > 0) {
|
|
7325
|
+
// Whichever bound is tighter: one render-sized node, or the largest disc
|
|
7326
|
+
// that still leaves the graph mostly empty space at this node count.
|
|
7327
|
+
// Mirrors siteCrawlGraphNodeSize; a test asserts the two agree.
|
|
7328
|
+
const byExtent = extent * workerData.nodeSizeExtentFraction;
|
|
7329
|
+
const byArea = Math.sqrt(
|
|
7330
|
+
(workerData.noverlap.maxPackingFraction * Math.pow(2 * extent, 2)) / graph.order,
|
|
7331
|
+
) / 2;
|
|
7332
|
+
nodeSize = Math.min(byExtent, byArea);
|
|
7333
|
+
graph.forEachNode(function (nodeKey) {
|
|
7334
|
+
graph.setNodeAttribute(nodeKey, 'size', nodeSize);
|
|
7335
|
+
});
|
|
7336
|
+
noverlap.assign(graph, {
|
|
7337
|
+
maxIterations: workerData.noverlap.maxIterations,
|
|
7338
|
+
settings: {
|
|
7339
|
+
gridSize: workerData.noverlap.gridSize,
|
|
7340
|
+
margin: nodeSize * workerData.noverlap.marginFactor,
|
|
7341
|
+
expansion: workerData.noverlap.expansion,
|
|
7342
|
+
ratio: workerData.noverlap.ratio,
|
|
7343
|
+
speed: workerData.noverlap.speed,
|
|
7344
|
+
},
|
|
7345
|
+
});
|
|
7346
|
+
}
|
|
7347
|
+
}
|
|
7046
7348
|
const positions = [];
|
|
7047
7349
|
graph.forEachNode((nodeKey, attributes) => {
|
|
7048
7350
|
positions.push({nodeKey, x: attributes.x, y: attributes.y});
|
|
7049
7351
|
});
|
|
7050
|
-
parentPort.postMessage({ok: true, positions});
|
|
7352
|
+
parentPort.postMessage({ok: true, positions, nodeSize});
|
|
7051
7353
|
} catch (error) {
|
|
7052
7354
|
parentPort.postMessage({
|
|
7053
7355
|
ok: false,
|
|
@@ -7105,7 +7407,8 @@ function adaptiveForceAtlasIterations(nodeCount, edgeCount) {
|
|
|
7105
7407
|
}
|
|
7106
7408
|
async function computeForceAtlasPositions(input, options = {}) {
|
|
7107
7409
|
if (input.nodes.length <= 1 || input.edges.length === 0) {
|
|
7108
|
-
|
|
7410
|
+
const positions = input.nodes.map(({ nodeKey, x, y }) => ({ nodeKey, x, y }));
|
|
7411
|
+
return { positions, nodeSize: siteCrawlGraphNodeSize(siteCrawlGraphExtent(positions), positions.length) };
|
|
7109
7412
|
}
|
|
7110
7413
|
const requireFromPackage = createRequire3(import.meta.url);
|
|
7111
7414
|
const worker = new Worker(WORKER_SOURCE, {
|
|
@@ -7113,8 +7416,15 @@ async function computeForceAtlasPositions(input, options = {}) {
|
|
|
7113
7416
|
workerData: {
|
|
7114
7417
|
graphologyPath: requireFromPackage.resolve("graphology"),
|
|
7115
7418
|
forceAtlasPath: requireFromPackage.resolve("graphology-layout-forceatlas2"),
|
|
7419
|
+
noverlapPath: requireFromPackage.resolve("graphology-layout-noverlap"),
|
|
7116
7420
|
input,
|
|
7117
|
-
settings: FORCE_ATLAS_SETTINGS
|
|
7421
|
+
settings: FORCE_ATLAS_SETTINGS,
|
|
7422
|
+
noverlap: {
|
|
7423
|
+
...NOVERLAP_SETTINGS,
|
|
7424
|
+
gridSize: noverlapGridSize(input.nodes.length),
|
|
7425
|
+
maxIterations: noverlapMaxIterations(input.nodes.length)
|
|
7426
|
+
},
|
|
7427
|
+
nodeSizeExtentFraction: NODE_SIZE_EXTENT_FRACTION
|
|
7118
7428
|
}
|
|
7119
7429
|
});
|
|
7120
7430
|
const timeoutMs = options.timeoutMs ?? SITE_CRAWL_GRAPH_LAYOUT_TIMEOUT_MS;
|
|
@@ -7127,7 +7437,7 @@ async function computeForceAtlasPositions(input, options = {}) {
|
|
|
7127
7437
|
options.signal?.removeEventListener("abort", onAbort);
|
|
7128
7438
|
void worker.terminate();
|
|
7129
7439
|
if (result.error) reject(result.error instanceof Error ? result.error : new Error("Graph layout failed with a non-Error reason", { cause: result.error }));
|
|
7130
|
-
else resolve(result.positions ?? []);
|
|
7440
|
+
else resolve({ positions: result.positions ?? [], nodeSize: result.nodeSize ?? 0 });
|
|
7131
7441
|
};
|
|
7132
7442
|
const onAbort = () => finish({ error: options.signal?.reason ?? new DOMException("Layout aborted", "AbortError") });
|
|
7133
7443
|
const timer = setTimeout(() => finish({ error: new GraphLayoutTimeoutError(`Graph layout exceeded ${timeoutMs}ms`) }), timeoutMs);
|
|
@@ -7135,7 +7445,7 @@ async function computeForceAtlasPositions(input, options = {}) {
|
|
|
7135
7445
|
if (options.signal?.aborted) return onAbort();
|
|
7136
7446
|
worker.once("message", (message) => {
|
|
7137
7447
|
if (!message.ok) finish({ error: new Error(message.error ?? "Graph layout worker failed") });
|
|
7138
|
-
else finish({ positions: message.positions });
|
|
7448
|
+
else finish({ positions: message.positions, nodeSize: message.nodeSize });
|
|
7139
7449
|
});
|
|
7140
7450
|
worker.once("error", (error) => finish({ error }));
|
|
7141
7451
|
worker.once("exit", (code) => {
|
|
@@ -7143,6 +7453,89 @@ async function computeForceAtlasPositions(input, options = {}) {
|
|
|
7143
7453
|
});
|
|
7144
7454
|
});
|
|
7145
7455
|
}
|
|
7456
|
+
function siteCrawlGraphComponents(nodeKeys, edges) {
|
|
7457
|
+
const parent = new Map(nodeKeys.map((key) => [key, key]));
|
|
7458
|
+
const find = (key) => {
|
|
7459
|
+
let root = key;
|
|
7460
|
+
while (parent.get(root) !== root) root = parent.get(root);
|
|
7461
|
+
let cursor = key;
|
|
7462
|
+
while (parent.get(cursor) !== root) {
|
|
7463
|
+
const next = parent.get(cursor);
|
|
7464
|
+
parent.set(cursor, root);
|
|
7465
|
+
cursor = next;
|
|
7466
|
+
}
|
|
7467
|
+
return root;
|
|
7468
|
+
};
|
|
7469
|
+
for (const edge of edges) {
|
|
7470
|
+
if (!parent.has(edge.sourceNodeKey) || !parent.has(edge.targetNodeKey)) continue;
|
|
7471
|
+
const sourceRoot = find(edge.sourceNodeKey);
|
|
7472
|
+
const targetRoot = find(edge.targetNodeKey);
|
|
7473
|
+
if (sourceRoot === targetRoot) continue;
|
|
7474
|
+
if (sourceRoot < targetRoot) parent.set(targetRoot, sourceRoot);
|
|
7475
|
+
else parent.set(sourceRoot, targetRoot);
|
|
7476
|
+
}
|
|
7477
|
+
const members = /* @__PURE__ */ new Map();
|
|
7478
|
+
for (const key of [...nodeKeys].sort()) {
|
|
7479
|
+
const root = find(key);
|
|
7480
|
+
const group = members.get(root);
|
|
7481
|
+
if (group) group.push(key);
|
|
7482
|
+
else members.set(root, [key]);
|
|
7483
|
+
}
|
|
7484
|
+
return [...members.values()].sort((left, right) => right.length - left.length || (left[0] < right[0] ? -1 : left[0] > right[0] ? 1 : 0));
|
|
7485
|
+
}
|
|
7486
|
+
function describeComponent(nodeKeys, byKey) {
|
|
7487
|
+
let sumX = 0;
|
|
7488
|
+
let sumY = 0;
|
|
7489
|
+
for (const nodeKey of nodeKeys) {
|
|
7490
|
+
const position = byKey.get(nodeKey);
|
|
7491
|
+
sumX += position.x;
|
|
7492
|
+
sumY += position.y;
|
|
7493
|
+
}
|
|
7494
|
+
const centerX = sumX / nodeKeys.length;
|
|
7495
|
+
const centerY = sumY / nodeKeys.length;
|
|
7496
|
+
let radius = 0;
|
|
7497
|
+
for (const nodeKey of nodeKeys) {
|
|
7498
|
+
const position = byKey.get(nodeKey);
|
|
7499
|
+
radius = Math.max(radius, Math.hypot(position.x - centerX, position.y - centerY));
|
|
7500
|
+
}
|
|
7501
|
+
return { nodeKeys, centerX, centerY, radius };
|
|
7502
|
+
}
|
|
7503
|
+
function packSiteCrawlGraphComponents(positions, edges, nodeSpacing) {
|
|
7504
|
+
const components = siteCrawlGraphComponents(positions.map((position) => position.nodeKey), edges);
|
|
7505
|
+
if (components.length <= 1) return [...positions];
|
|
7506
|
+
const byKey = new Map(positions.map((position) => [position.nodeKey, position]));
|
|
7507
|
+
if (positions.some((position) => !Number.isFinite(position.x) || !Number.isFinite(position.y))) {
|
|
7508
|
+
return [...positions];
|
|
7509
|
+
}
|
|
7510
|
+
const described = components.map((nodeKeys) => describeComponent(nodeKeys, byKey));
|
|
7511
|
+
const [anchor, ...rest] = described;
|
|
7512
|
+
const spacing = Math.max(nodeSpacing, Number.EPSILON) * COMPONENT_PACKING_SPACING_FACTOR;
|
|
7513
|
+
const ordered = [...rest].sort((left, right) => right.radius - left.radius || (left.nodeKeys[0] < right.nodeKeys[0] ? -1 : left.nodeKeys[0] > right.nodeKeys[0] ? 1 : 0));
|
|
7514
|
+
const moved = /* @__PURE__ */ new Map();
|
|
7515
|
+
let ringRadius = anchor.radius + spacing;
|
|
7516
|
+
let ringPitch = 0;
|
|
7517
|
+
let placedOnRing = 0;
|
|
7518
|
+
let ringCapacity = 0;
|
|
7519
|
+
for (const component of ordered) {
|
|
7520
|
+
if (placedOnRing >= ringCapacity) {
|
|
7521
|
+
ringPitch = 2 * component.radius + spacing;
|
|
7522
|
+
ringRadius += ringPitch / 2 + (placedOnRing === 0 ? 0 : ringPitch / 2);
|
|
7523
|
+
ringCapacity = Math.max(1, Math.floor(2 * Math.PI * ringRadius / ringPitch));
|
|
7524
|
+
placedOnRing = 0;
|
|
7525
|
+
}
|
|
7526
|
+
const angle = placedOnRing / ringCapacity * Math.PI * 2;
|
|
7527
|
+
placedOnRing += 1;
|
|
7528
|
+
const targetX = anchor.centerX + Math.cos(angle) * ringRadius;
|
|
7529
|
+
const targetY = anchor.centerY + Math.sin(angle) * ringRadius;
|
|
7530
|
+
const shiftX = targetX - component.centerX;
|
|
7531
|
+
const shiftY = targetY - component.centerY;
|
|
7532
|
+
for (const nodeKey of component.nodeKeys) {
|
|
7533
|
+
const position = byKey.get(nodeKey);
|
|
7534
|
+
moved.set(nodeKey, { nodeKey, x: position.x + shiftX, y: position.y + shiftY });
|
|
7535
|
+
}
|
|
7536
|
+
}
|
|
7537
|
+
return positions.map((position) => moved.get(position.nodeKey) ?? position);
|
|
7538
|
+
}
|
|
7146
7539
|
function normalizePositions(seededNodes, computed, rootNodeKey) {
|
|
7147
7540
|
const byKey = new Map(computed.map((position) => [position.nodeKey, position]));
|
|
7148
7541
|
if (byKey.size !== seededNodes.length) throw new Error("Graph layout returned an incomplete node set");
|
|
@@ -7175,29 +7568,41 @@ async function layoutSiteCrawlGraphInput(input, options = {}) {
|
|
|
7175
7568
|
failureCode: "empty-crawl",
|
|
7176
7569
|
totalNodes: input.totalNodes,
|
|
7177
7570
|
totalEdges: input.totalEdges,
|
|
7571
|
+
totalTemplateEdges: input.totalTemplateEdges,
|
|
7178
7572
|
nodeCount: 0,
|
|
7179
7573
|
edgeCount: 0,
|
|
7180
7574
|
nodes: [],
|
|
7181
7575
|
edges: []
|
|
7182
7576
|
};
|
|
7183
7577
|
}
|
|
7578
|
+
const layoutEdges = input.edges.filter((edge) => !edge.isTemplate);
|
|
7184
7579
|
const seeded = seedSiteCrawlGraphNodes(input.nodes, input.rootNodeKey, input.priorPositions);
|
|
7580
|
+
const timeoutMs = options.timeoutMs ?? SITE_CRAWL_GRAPH_LAYOUT_TIMEOUT_MS;
|
|
7581
|
+
const deadline = Date.now() + timeoutMs;
|
|
7582
|
+
const assertWithinBudget = () => {
|
|
7583
|
+
options.signal?.throwIfAborted();
|
|
7584
|
+
if (Date.now() > deadline) throw new GraphLayoutTimeoutError(`Graph layout exceeded ${timeoutMs}ms`);
|
|
7585
|
+
};
|
|
7185
7586
|
try {
|
|
7186
7587
|
const compute = options.computePositions ?? computeForceAtlasPositions;
|
|
7187
|
-
const positions = await compute({
|
|
7588
|
+
const { positions, nodeSize } = await compute({
|
|
7188
7589
|
nodes: seeded,
|
|
7189
|
-
edges:
|
|
7190
|
-
iterations: adaptiveForceAtlasIterations(input.nodes.length,
|
|
7191
|
-
}, { signal: options.signal, timeoutMs:
|
|
7192
|
-
|
|
7590
|
+
edges: layoutEdges,
|
|
7591
|
+
iterations: adaptiveForceAtlasIterations(input.nodes.length, layoutEdges.length)
|
|
7592
|
+
}, { signal: options.signal, timeoutMs: Math.max(1, deadline - Date.now()) });
|
|
7593
|
+
assertWithinBudget();
|
|
7594
|
+
const framed = packSiteCrawlGraphComponents(positions, layoutEdges, nodeSize);
|
|
7595
|
+
assertWithinBudget();
|
|
7193
7596
|
return {
|
|
7194
7597
|
state: "ready",
|
|
7195
7598
|
layoutVersion: SITE_CRAWL_GRAPH_LAYOUT_VERSION,
|
|
7196
7599
|
totalNodes: input.totalNodes,
|
|
7197
7600
|
totalEdges: input.totalEdges,
|
|
7601
|
+
totalTemplateEdges: input.totalTemplateEdges,
|
|
7198
7602
|
nodeCount: input.nodes.length,
|
|
7199
7603
|
edgeCount: input.edges.length,
|
|
7200
|
-
|
|
7604
|
+
templateLinksExcluded: true,
|
|
7605
|
+
nodes: normalizePositions(seeded, framed, input.rootNodeKey),
|
|
7201
7606
|
edges: input.edges.map((edge, sampleRank) => ({ ...edge, sampleRank }))
|
|
7202
7607
|
};
|
|
7203
7608
|
} catch (error) {
|
|
@@ -7208,6 +7613,7 @@ async function layoutSiteCrawlGraphInput(input, options = {}) {
|
|
|
7208
7613
|
failureCode: error instanceof GraphLayoutTimeoutError ? "layout-timeout" : "layout-error",
|
|
7209
7614
|
totalNodes: input.totalNodes,
|
|
7210
7615
|
totalEdges: input.totalEdges,
|
|
7616
|
+
totalTemplateEdges: input.totalTemplateEdges,
|
|
7211
7617
|
nodeCount: 0,
|
|
7212
7618
|
edgeCount: 0,
|
|
7213
7619
|
nodes: [],
|
|
@@ -7244,6 +7650,7 @@ function loadPriorSiteCrawlGraphPositions(db, scope) {
|
|
|
7244
7650
|
async function prepareSiteCrawlGraphLayout(db, scope, options = {}) {
|
|
7245
7651
|
let totalNodes = 0;
|
|
7246
7652
|
let totalEdges = 0;
|
|
7653
|
+
let totalTemplateEdges = 0;
|
|
7247
7654
|
try {
|
|
7248
7655
|
const assertNotAborted = () => options.signal?.throwIfAborted();
|
|
7249
7656
|
assertNotAborted();
|
|
@@ -7261,7 +7668,7 @@ async function prepareSiteCrawlGraphLayout(db, scope, options = {}) {
|
|
|
7261
7668
|
];
|
|
7262
7669
|
totalNodes = db.select({ value: count() }).from(siteCrawlPages).where(and10(...pageScope)).get()?.value ?? 0;
|
|
7263
7670
|
assertNotAborted();
|
|
7264
|
-
|
|
7671
|
+
const countGraphCompatibleEdges = (...extra) => db.select({ value: count() }).from(siteCrawlEdges).innerJoin(graphSourcePage, and10(
|
|
7265
7672
|
eq13(graphSourcePage.projectId, scope.projectId),
|
|
7266
7673
|
eq13(graphSourcePage.runId, scope.runId),
|
|
7267
7674
|
eq13(graphSourcePage.attemptId, scope.attemptId),
|
|
@@ -7271,7 +7678,10 @@ async function prepareSiteCrawlGraphLayout(db, scope, options = {}) {
|
|
|
7271
7678
|
eq13(graphTargetPage.runId, scope.runId),
|
|
7272
7679
|
eq13(graphTargetPage.attemptId, scope.attemptId),
|
|
7273
7680
|
eq13(graphTargetPage.nodeKey, siteCrawlEdges.targetNodeKey)
|
|
7274
|
-
)).where(and10(...edgeScope)).get()?.value ?? 0;
|
|
7681
|
+
)).where(and10(...edgeScope, ...extra)).get()?.value ?? 0;
|
|
7682
|
+
totalEdges = countGraphCompatibleEdges();
|
|
7683
|
+
assertNotAborted();
|
|
7684
|
+
totalTemplateEdges = countGraphCompatibleEdges(eq13(siteCrawlEdges.isTemplate, true));
|
|
7275
7685
|
assertNotAborted();
|
|
7276
7686
|
const graphSeedColumns = {
|
|
7277
7687
|
nodeKey: siteCrawlPages.nodeKey,
|
|
@@ -7310,7 +7720,8 @@ async function prepareSiteCrawlGraphLayout(db, scope, options = {}) {
|
|
|
7310
7720
|
edge.source_node_key AS sourceNodeKey,
|
|
7311
7721
|
edge.target_node_key AS targetNodeKey,
|
|
7312
7722
|
edge.followable AS followable,
|
|
7313
|
-
edge.occurrences AS occurrences
|
|
7723
|
+
edge.occurrences AS occurrences,
|
|
7724
|
+
edge.is_template AS isTemplate
|
|
7314
7725
|
FROM site_crawl_edges AS edge
|
|
7315
7726
|
INNER JOIN selected AS source ON source.node_key = edge.source_node_key
|
|
7316
7727
|
INNER JOIN selected AS target ON target.node_key = edge.target_node_key
|
|
@@ -7326,8 +7737,9 @@ async function prepareSiteCrawlGraphLayout(db, scope, options = {}) {
|
|
|
7326
7737
|
return await layoutSiteCrawlGraphInput({
|
|
7327
7738
|
totalNodes,
|
|
7328
7739
|
totalEdges,
|
|
7740
|
+
totalTemplateEdges,
|
|
7329
7741
|
nodes,
|
|
7330
|
-
edges: edgeRows,
|
|
7742
|
+
edges: edgeRows.map((row) => ({ ...row, isTemplate: Boolean(row.isTemplate) })),
|
|
7331
7743
|
rootNodeKey,
|
|
7332
7744
|
priorPositions
|
|
7333
7745
|
}, options);
|
|
@@ -7339,6 +7751,7 @@ async function prepareSiteCrawlGraphLayout(db, scope, options = {}) {
|
|
|
7339
7751
|
failureCode: "layout-error",
|
|
7340
7752
|
totalNodes,
|
|
7341
7753
|
totalEdges,
|
|
7754
|
+
totalTemplateEdges,
|
|
7342
7755
|
nodeCount: 0,
|
|
7343
7756
|
edgeCount: 0,
|
|
7344
7757
|
nodes: [],
|
|
@@ -7363,8 +7776,10 @@ function persistSiteCrawlGraphLayout(db, scope, layout, now = (/* @__PURE__ */ n
|
|
|
7363
7776
|
failureCode: layout.state === "unavailable" ? layout.failureCode : null,
|
|
7364
7777
|
totalNodes: layout.totalNodes,
|
|
7365
7778
|
totalEdges: layout.totalEdges,
|
|
7779
|
+
totalTemplateEdges: layout.totalTemplateEdges,
|
|
7366
7780
|
nodeCount: layout.nodeCount,
|
|
7367
7781
|
edgeCount: layout.edgeCount,
|
|
7782
|
+
templateLinksExcluded: layout.state === "ready",
|
|
7368
7783
|
createdAt: now,
|
|
7369
7784
|
updatedAt: now
|
|
7370
7785
|
}).run();
|
|
@@ -7395,6 +7810,7 @@ function persistSiteCrawlGraphLayout(db, scope, layout, now = (/* @__PURE__ */ n
|
|
|
7395
7810
|
targetNodeKey: edge.targetNodeKey,
|
|
7396
7811
|
followable: edge.followable,
|
|
7397
7812
|
occurrences: edge.occurrences,
|
|
7813
|
+
isTemplate: edge.isTemplate,
|
|
7398
7814
|
createdAt: now
|
|
7399
7815
|
}))).run();
|
|
7400
7816
|
}
|
|
@@ -7408,6 +7824,70 @@ function deleteSiteCrawlGraphLayout(db, scope) {
|
|
|
7408
7824
|
)).run();
|
|
7409
7825
|
}
|
|
7410
7826
|
|
|
7827
|
+
// src/site-crawl-template-links.ts
|
|
7828
|
+
import { and as and11, asc as asc2, eq as eq14, gt, inArray as inArray4 } from "drizzle-orm";
|
|
7829
|
+
var TEMPLATE_LINK_READ_BATCH = 2e3;
|
|
7830
|
+
var TEMPLATE_LINK_WRITE_BATCH = 500;
|
|
7831
|
+
function* streamAttemptEdges(db, scope) {
|
|
7832
|
+
let after = "";
|
|
7833
|
+
for (; ; ) {
|
|
7834
|
+
const rows = db.select({
|
|
7835
|
+
edgeKey: siteCrawlEdges.edgeKey,
|
|
7836
|
+
sourceNodeKey: siteCrawlEdges.sourceNodeKey,
|
|
7837
|
+
targetNodeKey: siteCrawlEdges.targetNodeKey,
|
|
7838
|
+
anchors: siteCrawlEdges.anchors
|
|
7839
|
+
}).from(siteCrawlEdges).where(and11(
|
|
7840
|
+
eq14(siteCrawlEdges.projectId, scope.projectId),
|
|
7841
|
+
eq14(siteCrawlEdges.runId, scope.runId),
|
|
7842
|
+
eq14(siteCrawlEdges.attemptId, scope.attemptId),
|
|
7843
|
+
gt(siteCrawlEdges.edgeKey, after)
|
|
7844
|
+
)).orderBy(asc2(siteCrawlEdges.edgeKey)).limit(TEMPLATE_LINK_READ_BATCH).all();
|
|
7845
|
+
if (rows.length === 0) return;
|
|
7846
|
+
yield rows;
|
|
7847
|
+
after = rows[rows.length - 1].edgeKey;
|
|
7848
|
+
if (rows.length < TEMPLATE_LINK_READ_BATCH) return;
|
|
7849
|
+
}
|
|
7850
|
+
}
|
|
7851
|
+
function* chunked(values, size = TEMPLATE_LINK_WRITE_BATCH) {
|
|
7852
|
+
for (let offset = 0; offset < values.length; offset += size) {
|
|
7853
|
+
yield values.slice(offset, offset + size);
|
|
7854
|
+
}
|
|
7855
|
+
}
|
|
7856
|
+
function classifySiteCrawlTemplateLinks(db, scope, pagesFetched) {
|
|
7857
|
+
const scopeFilter = and11(
|
|
7858
|
+
eq14(siteCrawlEdges.projectId, scope.projectId),
|
|
7859
|
+
eq14(siteCrawlEdges.runId, scope.runId),
|
|
7860
|
+
eq14(siteCrawlEdges.attemptId, scope.attemptId)
|
|
7861
|
+
);
|
|
7862
|
+
const detection = templateLinkDetection(pagesFetched);
|
|
7863
|
+
db.update(siteCrawlEdges).set({ isTemplate: false, templateRatio: null }).where(scopeFilter).run();
|
|
7864
|
+
if (detection !== SiteHealthTemplateDetections.applied) {
|
|
7865
|
+
return { detection, templateEdgeCount: 0 };
|
|
7866
|
+
}
|
|
7867
|
+
const index = createTemplateLinkPairIndex();
|
|
7868
|
+
for (const batch of streamAttemptEdges(db, scope)) observeTemplateLinkEdges(index, batch);
|
|
7869
|
+
const edgeKeysByRatio = /* @__PURE__ */ new Map();
|
|
7870
|
+
let templateEdgeCount = 0;
|
|
7871
|
+
for (const batch of streamAttemptEdges(db, scope)) {
|
|
7872
|
+
for (const edge of batch) {
|
|
7873
|
+
const ratio = templateLinkRatio(index, pagesFetched, edge);
|
|
7874
|
+
if (ratio == null) continue;
|
|
7875
|
+
const group = edgeKeysByRatio.get(ratio);
|
|
7876
|
+
if (group) group.push(edge.edgeKey);
|
|
7877
|
+
else edgeKeysByRatio.set(ratio, [edge.edgeKey]);
|
|
7878
|
+
if (isTemplateLinkRatio(ratio)) templateEdgeCount += 1;
|
|
7879
|
+
}
|
|
7880
|
+
}
|
|
7881
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7882
|
+
for (const [ratio, edgeKeys] of edgeKeysByRatio) {
|
|
7883
|
+
const isTemplate = isTemplateLinkRatio(ratio);
|
|
7884
|
+
for (const chunk of chunked(edgeKeys)) {
|
|
7885
|
+
db.update(siteCrawlEdges).set({ isTemplate, templateRatio: ratio, updatedAt: now }).where(and11(scopeFilter, inArray4(siteCrawlEdges.edgeKey, chunk))).run();
|
|
7886
|
+
}
|
|
7887
|
+
}
|
|
7888
|
+
return { detection, templateEdgeCount };
|
|
7889
|
+
}
|
|
7890
|
+
|
|
7411
7891
|
// src/execute-site-audit.ts
|
|
7412
7892
|
var log12 = createLogger("SiteAudit");
|
|
7413
7893
|
var SiteAuditCancelledError = class extends Error {
|
|
@@ -7555,9 +8035,9 @@ function deadLinkCheckedCount(edges, pages) {
|
|
|
7555
8035
|
}
|
|
7556
8036
|
async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
7557
8037
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
7558
|
-
const claim = db.update(runs).set({ status: "running", startedAt }).where(
|
|
8038
|
+
const claim = db.update(runs).set({ status: "running", startedAt }).where(and12(eq15(runs.id, runId), eq15(runs.projectId, projectId), eq15(runs.status, "queued"))).run();
|
|
7559
8039
|
if (claim.changes === 0) return;
|
|
7560
|
-
const attemptNumber = (db.select({ value: sql6`coalesce(max(${siteCrawlAttempts.attemptNumber}), 0)` }).from(siteCrawlAttempts).where(
|
|
8040
|
+
const attemptNumber = (db.select({ value: sql6`coalesce(max(${siteCrawlAttempts.attemptNumber}), 0)` }).from(siteCrawlAttempts).where(eq15(siteCrawlAttempts.runId, runId)).get()?.value ?? 0) + 1;
|
|
7561
8041
|
const attemptId = crypto17.randomUUID();
|
|
7562
8042
|
db.insert(siteCrawlAttempts).values({
|
|
7563
8043
|
id: attemptId,
|
|
@@ -7592,28 +8072,49 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7592
8072
|
const nodeKey = nodeKeyByUrl.get(url);
|
|
7593
8073
|
if (!nodeKey) continue;
|
|
7594
8074
|
const scope = [
|
|
7595
|
-
|
|
7596
|
-
|
|
7597
|
-
|
|
8075
|
+
eq15(siteCrawlEdges.projectId, projectId),
|
|
8076
|
+
eq15(siteCrawlEdges.runId, runId),
|
|
8077
|
+
eq15(siteCrawlEdges.attemptId, attemptId)
|
|
7598
8078
|
];
|
|
7599
|
-
tx.update(siteCrawlEdges).set({ sourceNodeKey: nodeKey, updatedAt: now }).where(
|
|
8079
|
+
tx.update(siteCrawlEdges).set({ sourceNodeKey: nodeKey, updatedAt: now }).where(and12(
|
|
7600
8080
|
...scope,
|
|
7601
|
-
|
|
8081
|
+
eq15(siteCrawlEdges.sourceUrl, url)
|
|
7602
8082
|
)).run();
|
|
7603
|
-
tx.update(siteCrawlEdges).set({ targetNodeKey: nodeKey, updatedAt: now }).where(
|
|
8083
|
+
tx.update(siteCrawlEdges).set({ targetNodeKey: nodeKey, updatedAt: now }).where(and12(
|
|
7604
8084
|
...scope,
|
|
7605
|
-
|
|
7606
|
-
)).run();
|
|
7607
|
-
tx.update(siteCrawlPages).set({ canonicalNodeKey: nodeKey, updatedAt: now }).where(and11(
|
|
7608
|
-
eq14(siteCrawlPages.projectId, projectId),
|
|
7609
|
-
eq14(siteCrawlPages.runId, runId),
|
|
7610
|
-
eq14(siteCrawlPages.attemptId, attemptId),
|
|
7611
|
-
eq14(siteCrawlPages.canonicalUrl, url)
|
|
8085
|
+
eq15(siteCrawlEdges.targetUrl, url)
|
|
7612
8086
|
)).run();
|
|
8087
|
+
const pageScope = [
|
|
8088
|
+
eq15(siteCrawlPages.projectId, projectId),
|
|
8089
|
+
eq15(siteCrawlPages.runId, runId),
|
|
8090
|
+
eq15(siteCrawlPages.attemptId, attemptId)
|
|
8091
|
+
];
|
|
8092
|
+
const canonicalTargets = tx.select({
|
|
8093
|
+
id: siteCrawlPages.id,
|
|
8094
|
+
nodeKey: siteCrawlPages.nodeKey,
|
|
8095
|
+
fetchState: siteCrawlPages.fetchState,
|
|
8096
|
+
indexabilityState: siteCrawlPages.indexabilityState,
|
|
8097
|
+
indexabilityReasons: siteCrawlPages.indexabilityReasons
|
|
8098
|
+
}).from(siteCrawlPages).where(and12(...pageScope, eq15(siteCrawlPages.canonicalUrl, url))).all();
|
|
8099
|
+
for (const target of canonicalTargets) {
|
|
8100
|
+
tx.update(siteCrawlPages).set({
|
|
8101
|
+
canonicalNodeKey: nodeKey,
|
|
8102
|
+
healthState: deriveSiteHealthState({ ...target, canonicalNodeKey: nodeKey }),
|
|
8103
|
+
updatedAt: now
|
|
8104
|
+
}).where(eq15(siteCrawlPages.id, target.id)).run();
|
|
8105
|
+
}
|
|
7613
8106
|
}
|
|
7614
8107
|
};
|
|
7615
8108
|
const persistPage = (tx, page, now) => {
|
|
7616
8109
|
const audit = page.audit;
|
|
8110
|
+
const canonicalNodeKey = page.canonicalUrl ? nodeKeyByUrl.get(page.canonicalUrl) ?? null : null;
|
|
8111
|
+
const healthState = deriveSiteHealthState({
|
|
8112
|
+
fetchState: page.state,
|
|
8113
|
+
indexabilityState: page.indexability.state,
|
|
8114
|
+
indexabilityReasons: page.indexability.reasons,
|
|
8115
|
+
canonicalNodeKey,
|
|
8116
|
+
nodeKey: page.key
|
|
8117
|
+
});
|
|
7617
8118
|
tx.insert(siteCrawlPages).values({
|
|
7618
8119
|
id: crypto17.randomUUID(),
|
|
7619
8120
|
projectId,
|
|
@@ -7638,9 +8139,10 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7638
8139
|
redirectChain: page.redirectChain.map((hop) => hop.to),
|
|
7639
8140
|
directives: { metaRobots: page.metaRobots, xRobots: page.xRobots },
|
|
7640
8141
|
canonicalUrl: page.canonicalUrl,
|
|
7641
|
-
canonicalNodeKey
|
|
8142
|
+
canonicalNodeKey,
|
|
7642
8143
|
indexabilityState: page.indexability.state,
|
|
7643
8144
|
indexabilityReasons: page.indexability.reasons,
|
|
8145
|
+
healthState,
|
|
7644
8146
|
auditState: pageAuditState(page),
|
|
7645
8147
|
auditScore: audit?.overallScore ?? null,
|
|
7646
8148
|
auditFields: audit ? crawlAuditFields(audit) : {},
|
|
@@ -7671,9 +8173,10 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7671
8173
|
redirectChain: page.redirectChain.map((hop) => hop.to),
|
|
7672
8174
|
directives: { metaRobots: page.metaRobots, xRobots: page.xRobots },
|
|
7673
8175
|
canonicalUrl: page.canonicalUrl,
|
|
7674
|
-
canonicalNodeKey
|
|
8176
|
+
canonicalNodeKey,
|
|
7675
8177
|
indexabilityState: page.indexability.state,
|
|
7676
8178
|
indexabilityReasons: page.indexability.reasons,
|
|
8179
|
+
healthState,
|
|
7677
8180
|
auditState: pageAuditState(page),
|
|
7678
8181
|
auditScore: audit?.overallScore ?? null,
|
|
7679
8182
|
auditFields: audit ? crawlAuditFields(audit) : {},
|
|
@@ -7696,6 +8199,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7696
8199
|
bindReferences(tx, changedUrls, now);
|
|
7697
8200
|
};
|
|
7698
8201
|
const persistEdge = (tx, edge, now) => {
|
|
8202
|
+
if (isSelfLink(edge.from, edge.to)) return;
|
|
7699
8203
|
observedEdges.set(edge.key, edge);
|
|
7700
8204
|
tx.insert(siteCrawlEdges).values({
|
|
7701
8205
|
id: crypto17.randomUUID(),
|
|
@@ -7744,20 +8248,20 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7744
8248
|
linkScoreRaw: metrics.linkScoreRaw,
|
|
7745
8249
|
linkScoreNormalized: metrics.linkScore,
|
|
7746
8250
|
updatedAt: now
|
|
7747
|
-
}).where(
|
|
7748
|
-
|
|
7749
|
-
|
|
7750
|
-
|
|
7751
|
-
|
|
8251
|
+
}).where(and12(
|
|
8252
|
+
eq15(siteCrawlPages.projectId, projectId),
|
|
8253
|
+
eq15(siteCrawlPages.runId, runId),
|
|
8254
|
+
eq15(siteCrawlPages.attemptId, attemptId),
|
|
8255
|
+
eq15(siteCrawlPages.nodeKey, key)
|
|
7752
8256
|
)).run();
|
|
7753
8257
|
};
|
|
7754
8258
|
const persistEvent = async (event) => {
|
|
7755
8259
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7756
8260
|
db.transaction((tx) => {
|
|
7757
|
-
const receipt = tx.select({ checksum: siteCrawlEventReceipts.checksum }).from(siteCrawlEventReceipts).where(
|
|
7758
|
-
|
|
7759
|
-
|
|
7760
|
-
|
|
8261
|
+
const receipt = tx.select({ checksum: siteCrawlEventReceipts.checksum }).from(siteCrawlEventReceipts).where(and12(
|
|
8262
|
+
eq15(siteCrawlEventReceipts.attemptId, attemptId),
|
|
8263
|
+
eq15(siteCrawlEventReceipts.sequence, event.sequence),
|
|
8264
|
+
eq15(siteCrawlEventReceipts.batchId, event.batchId)
|
|
7761
8265
|
)).get();
|
|
7762
8266
|
if (receipt) {
|
|
7763
8267
|
if (receipt.checksum !== event.checksum) {
|
|
@@ -7790,26 +8294,39 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7790
8294
|
pagesErrored: event.type === "summary" ? observedErrorCount(observedPages.values()) : sql6`${siteCrawlAttempts.pagesErrored}`,
|
|
7791
8295
|
edgesDiscovered: progress.edgesObserved,
|
|
7792
8296
|
updatedAt: now
|
|
7793
|
-
}).where(
|
|
8297
|
+
}).where(eq15(siteCrawlAttempts.id, attemptId)).run();
|
|
7794
8298
|
} else {
|
|
7795
8299
|
tx.update(siteCrawlAttempts).set({
|
|
7796
8300
|
lastEventSequence: event.sequence,
|
|
7797
8301
|
lastEventChecksum: event.checksum,
|
|
7798
8302
|
updatedAt: now
|
|
7799
|
-
}).where(
|
|
8303
|
+
}).where(eq15(siteCrawlAttempts.id, attemptId)).run();
|
|
7800
8304
|
}
|
|
7801
8305
|
});
|
|
7802
8306
|
};
|
|
7803
8307
|
try {
|
|
7804
|
-
const project = db.select().from(projects).where(
|
|
8308
|
+
const project = db.select().from(projects).where(eq15(projects.id, projectId)).get();
|
|
7805
8309
|
if (!project) throw new Error(`Project not found: ${projectId}`);
|
|
7806
8310
|
const homepageUrl = toHomepageUrl(project.canonicalDomain);
|
|
7807
8311
|
const maxPages = clampSiteAuditLimit(opts.maxPages ?? opts.limit);
|
|
7808
8312
|
const maxEdges = clampSiteAuditEdgeLimit(opts.maxEdges);
|
|
7809
|
-
await
|
|
8313
|
+
const resolvedRoot = await resolveSiteAuditRootUrl(homepageUrl, {
|
|
8314
|
+
resolveTarget: resolveMeasurementSitemapTarget,
|
|
8315
|
+
signal: opts.signal
|
|
8316
|
+
});
|
|
8317
|
+
const crawlRootUrl = resolvedRoot.effectiveUrl;
|
|
7810
8318
|
if (opts.sitemapUrl) await assertSiteAuditUrlAllowed(opts.sitemapUrl, "sitemapUrl");
|
|
7811
|
-
log12.info("start", {
|
|
7812
|
-
|
|
8319
|
+
log12.info("start", {
|
|
8320
|
+
runId,
|
|
8321
|
+
projectId,
|
|
8322
|
+
homepageUrl,
|
|
8323
|
+
crawlRootUrl,
|
|
8324
|
+
redirects: resolvedRoot.redirects.length,
|
|
8325
|
+
maxPages,
|
|
8326
|
+
maxEdges,
|
|
8327
|
+
maxDepth: opts.maxDepth ?? null
|
|
8328
|
+
});
|
|
8329
|
+
const report = await runSiteCrawl(crawlRootUrl, {
|
|
7813
8330
|
mode: "summary",
|
|
7814
8331
|
sitemapUrl: opts.sitemapUrl,
|
|
7815
8332
|
maxPages,
|
|
@@ -7824,6 +8341,17 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7824
8341
|
if (crawlSummary.complete && !hasAuditedPages) {
|
|
7825
8342
|
throw emptyCompleteCrawlError(crawlSummary.rootUrl, crawlSummary.finalRootUrl, crawlSummary.pagesObserved);
|
|
7826
8343
|
}
|
|
8344
|
+
const templateLinks = classifySiteCrawlTemplateLinks(
|
|
8345
|
+
db,
|
|
8346
|
+
{ projectId, runId, attemptId },
|
|
8347
|
+
crawlSummary.pagesFetched
|
|
8348
|
+
);
|
|
8349
|
+
log12.info("template-links", {
|
|
8350
|
+
runId,
|
|
8351
|
+
projectId,
|
|
8352
|
+
detection: templateLinks.detection,
|
|
8353
|
+
templateEdges: templateLinks.templateEdgeCount
|
|
8354
|
+
});
|
|
7827
8355
|
const graphLayout = await prepareSiteCrawlGraphLayout(db, {
|
|
7828
8356
|
projectId,
|
|
7829
8357
|
runId,
|
|
@@ -7851,6 +8379,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7851
8379
|
failureCode: "layout-error",
|
|
7852
8380
|
totalNodes: graphLayout.totalNodes,
|
|
7853
8381
|
totalEdges: graphLayout.totalEdges,
|
|
8382
|
+
totalTemplateEdges: graphLayout.totalTemplateEdges,
|
|
7854
8383
|
nodeCount: 0,
|
|
7855
8384
|
edgeCount: 0,
|
|
7856
8385
|
nodes: [],
|
|
@@ -7865,14 +8394,14 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7865
8394
|
}
|
|
7866
8395
|
}
|
|
7867
8396
|
const published = db.transaction((tx) => {
|
|
7868
|
-
const claim2 = tx.update(runs).set({ status: terminalStatus, finishedAt }).where(
|
|
8397
|
+
const claim2 = tx.update(runs).set({ status: terminalStatus, finishedAt }).where(and12(eq15(runs.id, runId), eq15(runs.status, "running"))).run();
|
|
7869
8398
|
if (claim2.changes === 0) return false;
|
|
7870
8399
|
if (hasAuditedPages) {
|
|
7871
8400
|
tx.insert(siteAuditSnapshots).values({
|
|
7872
8401
|
id: crypto17.randomUUID(),
|
|
7873
8402
|
projectId,
|
|
7874
8403
|
runId,
|
|
7875
|
-
sitemapUrl: opts.sitemapUrl ??
|
|
8404
|
+
sitemapUrl: opts.sitemapUrl ?? new URL("/sitemap.xml", crawlRootUrl).href,
|
|
7876
8405
|
auditedAt: crawlSummary.completedAt,
|
|
7877
8406
|
aggregateScore: crawlSummary.auditRollup.aggregateScore ?? 0,
|
|
7878
8407
|
pagesDiscovered: crawlSummary.pagesDiscovered,
|
|
@@ -7904,7 +8433,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7904
8433
|
projectId,
|
|
7905
8434
|
runId,
|
|
7906
8435
|
attemptId,
|
|
7907
|
-
requestedRootUrl:
|
|
8436
|
+
requestedRootUrl: resolvedRoot.requestedUrl,
|
|
7908
8437
|
rootUrl: crawlSummary.finalRootUrl ?? crawlSummary.rootUrl,
|
|
7909
8438
|
crawlSchemaVersion: crawlSummary.crawlSchemaVersion,
|
|
7910
8439
|
engineVersion: crawlSummary.engineVersion,
|
|
@@ -7935,6 +8464,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7935
8464
|
deadLinkState: report.deadLinks.state,
|
|
7936
8465
|
deadLinksChecked,
|
|
7937
8466
|
deadLinksFound,
|
|
8467
|
+
templateDetection: templateLinks.detection,
|
|
7938
8468
|
createdAt: finishedAt,
|
|
7939
8469
|
updatedAt: finishedAt
|
|
7940
8470
|
}).run();
|
|
@@ -7967,7 +8497,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7967
8497
|
edgesDiscovered: crawlSummary.edgesObserved,
|
|
7968
8498
|
finishedAt,
|
|
7969
8499
|
updatedAt: finishedAt
|
|
7970
|
-
}).where(
|
|
8500
|
+
}).where(eq15(siteCrawlAttempts.id, attemptId)).run();
|
|
7971
8501
|
return true;
|
|
7972
8502
|
});
|
|
7973
8503
|
if (!published) {
|
|
@@ -7980,7 +8510,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7980
8510
|
error: cleanupError instanceof Error ? cleanupError.message : String(cleanupError)
|
|
7981
8511
|
});
|
|
7982
8512
|
}
|
|
7983
|
-
const current = db.select({ status: runs.status }).from(runs).where(
|
|
8513
|
+
const current = db.select({ status: runs.status }).from(runs).where(eq15(runs.id, runId)).get();
|
|
7984
8514
|
if (current?.status === "cancelled") throw new SiteAuditCancelledError(`Site audit cancelled before publication: ${runId}`);
|
|
7985
8515
|
throw new Error(`Site audit run was no longer running before publication: ${runId}`);
|
|
7986
8516
|
}
|
|
@@ -7995,12 +8525,12 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
7995
8525
|
error: message,
|
|
7996
8526
|
finishedAt,
|
|
7997
8527
|
updatedAt: finishedAt
|
|
7998
|
-
}).where(
|
|
8528
|
+
}).where(eq15(siteCrawlAttempts.id, attemptId)).run();
|
|
7999
8529
|
tx.update(runs).set({
|
|
8000
8530
|
status: cancelled ? "cancelled" : "failed",
|
|
8001
8531
|
error: message,
|
|
8002
8532
|
finishedAt
|
|
8003
|
-
}).where(
|
|
8533
|
+
}).where(and12(eq15(runs.id, runId), eq15(runs.status, "running"))).run();
|
|
8004
8534
|
});
|
|
8005
8535
|
log12.error("failed", { runId, projectId, cancelled, error: message });
|
|
8006
8536
|
throw error;
|
|
@@ -8008,7 +8538,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
8008
8538
|
}
|
|
8009
8539
|
|
|
8010
8540
|
// src/commands/backfill.ts
|
|
8011
|
-
import { and as
|
|
8541
|
+
import { and as and13, eq as eq16, inArray as inArray5, isNull as isNull2, sql as sql7 } from "drizzle-orm";
|
|
8012
8542
|
var SNAPSHOT_BATCH_SIZE = 500;
|
|
8013
8543
|
async function backfillAnswerVisibilityCommand(opts) {
|
|
8014
8544
|
const config = loadConfig();
|
|
@@ -8016,7 +8546,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
8016
8546
|
migrate(db);
|
|
8017
8547
|
const projectFilter = opts?.project?.trim();
|
|
8018
8548
|
const isDryRun = opts?.dryRun === true;
|
|
8019
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
8549
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq16(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
8020
8550
|
let examined = 0;
|
|
8021
8551
|
let updated = 0;
|
|
8022
8552
|
let wouldUpdate = 0;
|
|
@@ -8024,10 +8554,10 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
8024
8554
|
let reparsed = 0;
|
|
8025
8555
|
let providerErrors = 0;
|
|
8026
8556
|
if (scopedProjects.length > 0) {
|
|
8027
|
-
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
8028
|
-
|
|
8029
|
-
|
|
8030
|
-
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
8557
|
+
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(and13(
|
|
8558
|
+
eq16(runs.kind, RunKinds["answer-visibility"]),
|
|
8559
|
+
inArray5(runs.projectId, scopedProjects.map((project) => project.id))
|
|
8560
|
+
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(eq16(runs.kind, RunKinds["answer-visibility"])).all();
|
|
8031
8561
|
const runIdsByProject = /* @__PURE__ */ new Map();
|
|
8032
8562
|
for (const run of runRows) {
|
|
8033
8563
|
const existing = runIdsByProject.get(run.projectId);
|
|
@@ -8035,7 +8565,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
8035
8565
|
else runIdsByProject.set(run.projectId, [run.id]);
|
|
8036
8566
|
}
|
|
8037
8567
|
for (const project of scopedProjects) {
|
|
8038
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
8568
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq16(competitors.projectId, project.id)).all().map((row) => row.domain);
|
|
8039
8569
|
const runIds = runIdsByProject.get(project.id) ?? [];
|
|
8040
8570
|
if (runIds.length === 0) continue;
|
|
8041
8571
|
const projectDomains = effectiveDomains({
|
|
@@ -8058,7 +8588,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
8058
8588
|
competitorOverlap: querySnapshots.competitorOverlap,
|
|
8059
8589
|
recommendedCompetitors: querySnapshots.recommendedCompetitors,
|
|
8060
8590
|
rawResponse: querySnapshots.rawResponse
|
|
8061
|
-
}).from(querySnapshots).where(
|
|
8591
|
+
}).from(querySnapshots).where(inArray5(querySnapshots.runId, batchRunIds)).all();
|
|
8062
8592
|
const pendingUpdates = [];
|
|
8063
8593
|
for (const snapshot of snapshotRows) {
|
|
8064
8594
|
examined++;
|
|
@@ -8128,7 +8658,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
8128
8658
|
} else {
|
|
8129
8659
|
db.transaction((tx) => {
|
|
8130
8660
|
for (const update of pendingUpdates) {
|
|
8131
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
8661
|
+
tx.update(querySnapshots).set(update.patch).where(eq16(querySnapshots.id, update.id)).run();
|
|
8132
8662
|
}
|
|
8133
8663
|
});
|
|
8134
8664
|
updated += pendingUpdates.length;
|
|
@@ -8177,13 +8707,13 @@ No DB writes performed. Re-run without --dry-run to apply.`);
|
|
|
8177
8707
|
function backfillNormalizedPaths(db, opts) {
|
|
8178
8708
|
const baseConditions = [];
|
|
8179
8709
|
if (opts?.projectId) {
|
|
8180
|
-
baseConditions.push(
|
|
8710
|
+
baseConditions.push(eq16(gaTrafficSnapshots.projectId, opts.projectId));
|
|
8181
8711
|
}
|
|
8182
8712
|
const rows = db.select({
|
|
8183
8713
|
id: gaTrafficSnapshots.id,
|
|
8184
8714
|
landingPage: gaTrafficSnapshots.landingPage,
|
|
8185
8715
|
landingPageNormalized: gaTrafficSnapshots.landingPageNormalized
|
|
8186
|
-
}).from(gaTrafficSnapshots).where(baseConditions.length > 0 ?
|
|
8716
|
+
}).from(gaTrafficSnapshots).where(baseConditions.length > 0 ? and13(...baseConditions) : void 0).all();
|
|
8187
8717
|
let updated = 0;
|
|
8188
8718
|
let unchanged = 0;
|
|
8189
8719
|
if (rows.length > 0) {
|
|
@@ -8198,7 +8728,7 @@ function backfillNormalizedPaths(db, opts) {
|
|
|
8198
8728
|
unchanged++;
|
|
8199
8729
|
continue;
|
|
8200
8730
|
}
|
|
8201
|
-
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(
|
|
8731
|
+
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(eq16(gaTrafficSnapshots.id, row.id)).run();
|
|
8202
8732
|
updated++;
|
|
8203
8733
|
}
|
|
8204
8734
|
});
|
|
@@ -8212,7 +8742,7 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
8212
8742
|
const projectFilter = opts?.project?.trim();
|
|
8213
8743
|
let projectId;
|
|
8214
8744
|
if (projectFilter) {
|
|
8215
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
8745
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq16(projects.name, projectFilter)).get();
|
|
8216
8746
|
if (!project) {
|
|
8217
8747
|
const result2 = {
|
|
8218
8748
|
project: projectFilter,
|
|
@@ -8249,13 +8779,13 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
8249
8779
|
function backfillAiReferralPaths(db, opts) {
|
|
8250
8780
|
const baseConditions = [];
|
|
8251
8781
|
if (opts?.projectId) {
|
|
8252
|
-
baseConditions.push(
|
|
8782
|
+
baseConditions.push(eq16(gaAiReferrals.projectId, opts.projectId));
|
|
8253
8783
|
}
|
|
8254
8784
|
const rows = db.select({
|
|
8255
8785
|
id: gaAiReferrals.id,
|
|
8256
8786
|
landingPage: gaAiReferrals.landingPage,
|
|
8257
8787
|
landingPageNormalized: gaAiReferrals.landingPageNormalized
|
|
8258
|
-
}).from(gaAiReferrals).where(baseConditions.length > 0 ?
|
|
8788
|
+
}).from(gaAiReferrals).where(baseConditions.length > 0 ? and13(...baseConditions) : void 0).all();
|
|
8259
8789
|
let updated = 0;
|
|
8260
8790
|
let unchanged = 0;
|
|
8261
8791
|
if (rows.length > 0) {
|
|
@@ -8270,7 +8800,7 @@ function backfillAiReferralPaths(db, opts) {
|
|
|
8270
8800
|
unchanged++;
|
|
8271
8801
|
continue;
|
|
8272
8802
|
}
|
|
8273
|
-
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(
|
|
8803
|
+
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(eq16(gaAiReferrals.id, row.id)).run();
|
|
8274
8804
|
updated++;
|
|
8275
8805
|
}
|
|
8276
8806
|
});
|
|
@@ -8284,7 +8814,7 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
8284
8814
|
const projectFilter = opts?.project?.trim();
|
|
8285
8815
|
let projectId;
|
|
8286
8816
|
if (projectFilter) {
|
|
8287
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
8817
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq16(projects.name, projectFilter)).get();
|
|
8288
8818
|
if (!project) {
|
|
8289
8819
|
const result2 = {
|
|
8290
8820
|
project: projectFilter,
|
|
@@ -8320,10 +8850,10 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
8320
8850
|
}
|
|
8321
8851
|
function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
8322
8852
|
const isDryRun = opts?.dryRun === true;
|
|
8323
|
-
const project = db.select().from(projects).where(
|
|
8853
|
+
const project = db.select().from(projects).where(eq16(projects.id, projectId)).get();
|
|
8324
8854
|
if (!project) return { examined: 0, updated: 0, mentioned: 0 };
|
|
8325
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
8326
|
-
const runRows = db.select({ id: runs.id }).from(runs).where(
|
|
8855
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq16(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
8856
|
+
const runRows = db.select({ id: runs.id }).from(runs).where(and13(eq16(runs.kind, RunKinds["answer-visibility"]), eq16(runs.projectId, projectId))).all();
|
|
8327
8857
|
const runIds = runRows.map((r) => r.id);
|
|
8328
8858
|
let examined = 0;
|
|
8329
8859
|
let updated = 0;
|
|
@@ -8351,7 +8881,7 @@ function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
|
8351
8881
|
competitorOverlap: querySnapshots.competitorOverlap,
|
|
8352
8882
|
recommendedCompetitors: querySnapshots.recommendedCompetitors,
|
|
8353
8883
|
rawResponse: querySnapshots.rawResponse
|
|
8354
|
-
}).from(querySnapshots).where(
|
|
8884
|
+
}).from(querySnapshots).where(inArray5(querySnapshots.runId, batchRunIds)).all();
|
|
8355
8885
|
const pendingUpdates = [];
|
|
8356
8886
|
for (const snapshot of snapshotRows) {
|
|
8357
8887
|
examined++;
|
|
@@ -8397,7 +8927,7 @@ function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
|
8397
8927
|
} else {
|
|
8398
8928
|
db.transaction((tx) => {
|
|
8399
8929
|
for (const update of pendingUpdates) {
|
|
8400
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
8930
|
+
tx.update(querySnapshots).set(update.patch).where(eq16(querySnapshots.id, update.id)).run();
|
|
8401
8931
|
}
|
|
8402
8932
|
});
|
|
8403
8933
|
updated += pendingUpdates.length;
|
|
@@ -8412,7 +8942,7 @@ async function backfillAnswerMentionsCommand(opts) {
|
|
|
8412
8942
|
migrate(db);
|
|
8413
8943
|
const projectFilter = opts?.project?.trim();
|
|
8414
8944
|
const isDryRun = opts?.dryRun === true;
|
|
8415
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
8945
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq16(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
8416
8946
|
let examined = 0;
|
|
8417
8947
|
let updated = 0;
|
|
8418
8948
|
let wouldUpdate = 0;
|
|
@@ -8472,7 +9002,7 @@ function readStoredGroundingSources(rawResponse) {
|
|
|
8472
9002
|
return result;
|
|
8473
9003
|
}
|
|
8474
9004
|
async function backfillInsightsCommand(project, opts) {
|
|
8475
|
-
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-
|
|
9005
|
+
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-DHY5CYWX.js");
|
|
8476
9006
|
const config = loadConfig();
|
|
8477
9007
|
const db = createClient(config.database);
|
|
8478
9008
|
migrate(db);
|
|
@@ -8631,7 +9161,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
8631
9161
|
const config = loadConfig();
|
|
8632
9162
|
const db = createClient(config.database);
|
|
8633
9163
|
migrate(db);
|
|
8634
|
-
const project = db.select().from(projects).where(
|
|
9164
|
+
const project = db.select().from(projects).where(eq16(projects.name, opts.project)).get();
|
|
8635
9165
|
if (!project) {
|
|
8636
9166
|
throw new Error(`Project "${opts.project}" not found`);
|
|
8637
9167
|
}
|
|
@@ -8642,17 +9172,17 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
8642
9172
|
process.stderr.write(`Recovering orphan snapshot attribution for "${project.name}"${mode}...
|
|
8643
9173
|
`);
|
|
8644
9174
|
}
|
|
8645
|
-
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(
|
|
8646
|
-
|
|
8647
|
-
|
|
9175
|
+
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(and13(
|
|
9176
|
+
eq16(auditLog.projectId, project.id),
|
|
9177
|
+
inArray5(auditLog.action, ["keywords.appended", "keywords.deleted", "queries.appended", "queries.deleted", "queries.replaced"])
|
|
8648
9178
|
)).orderBy(auditLog.createdAt).all();
|
|
8649
9179
|
const history = replayQueryAuditLog(events);
|
|
8650
9180
|
const orphanRuns = db.select({
|
|
8651
9181
|
runId: runs.id,
|
|
8652
9182
|
createdAt: runs.createdAt,
|
|
8653
9183
|
location: runs.location
|
|
8654
|
-
}).from(runs).innerJoin(querySnapshots,
|
|
8655
|
-
|
|
9184
|
+
}).from(runs).innerJoin(querySnapshots, eq16(querySnapshots.runId, runs.id)).where(and13(
|
|
9185
|
+
eq16(runs.projectId, project.id),
|
|
8656
9186
|
isNull2(querySnapshots.queryId),
|
|
8657
9187
|
isNull2(querySnapshots.queryText)
|
|
8658
9188
|
)).groupBy(runs.id).orderBy(runs.createdAt).all();
|
|
@@ -8674,8 +9204,8 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
8674
9204
|
provider: querySnapshots.provider,
|
|
8675
9205
|
createdAt: querySnapshots.createdAt,
|
|
8676
9206
|
answerText: querySnapshots.answerText
|
|
8677
|
-
}).from(querySnapshots).where(
|
|
8678
|
-
|
|
9207
|
+
}).from(querySnapshots).where(and13(
|
|
9208
|
+
eq16(querySnapshots.runId, run.runId),
|
|
8679
9209
|
isNull2(querySnapshots.queryId),
|
|
8680
9210
|
isNull2(querySnapshots.queryText)
|
|
8681
9211
|
)).orderBy(querySnapshots.provider, querySnapshots.createdAt).all();
|
|
@@ -8741,7 +9271,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
8741
9271
|
if (!isDryRun && updates.length > 0) {
|
|
8742
9272
|
db.transaction((tx) => {
|
|
8743
9273
|
for (const u of updates) {
|
|
8744
|
-
tx.update(querySnapshots).set({ queryText: u.queryText }).where(
|
|
9274
|
+
tx.update(querySnapshots).set({ queryText: u.queryText }).where(eq16(querySnapshots.id, u.id)).run();
|
|
8745
9275
|
}
|
|
8746
9276
|
});
|
|
8747
9277
|
}
|
|
@@ -8815,7 +9345,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
8815
9345
|
const projectFilter = opts?.project?.trim();
|
|
8816
9346
|
const isDryRun = opts?.dryRun === true;
|
|
8817
9347
|
const isJson = isMachineFormat(opts?.format);
|
|
8818
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
9348
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq16(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
8819
9349
|
if (scopedProjects.length === 0) {
|
|
8820
9350
|
if (projectFilter && !isJson) {
|
|
8821
9351
|
process.stderr.write(`No project named "${projectFilter}".
|
|
@@ -8840,9 +9370,9 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
8840
9370
|
dryRun: isDryRun,
|
|
8841
9371
|
byBot: {}
|
|
8842
9372
|
};
|
|
8843
|
-
const unknownCountRow = db.select({ n: sql7`count(*)` }).from(rawEventSamples).where(
|
|
8844
|
-
|
|
8845
|
-
|
|
9373
|
+
const unknownCountRow = db.select({ n: sql7`count(*)` }).from(rawEventSamples).where(and13(
|
|
9374
|
+
eq16(rawEventSamples.eventType, "unknown"),
|
|
9375
|
+
inArray5(rawEventSamples.projectId, projectIds)
|
|
8846
9376
|
)).get();
|
|
8847
9377
|
result.unknownBefore = Number(unknownCountRow?.n ?? 0);
|
|
8848
9378
|
const unknownSamples = db.select({
|
|
@@ -8853,9 +9383,9 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
8853
9383
|
userAgent: rawEventSamples.userAgent,
|
|
8854
9384
|
pathNormalized: rawEventSamples.pathNormalized,
|
|
8855
9385
|
status: rawEventSamples.status
|
|
8856
|
-
}).from(rawEventSamples).where(
|
|
8857
|
-
|
|
8858
|
-
|
|
9386
|
+
}).from(rawEventSamples).where(and13(
|
|
9387
|
+
eq16(rawEventSamples.eventType, "unknown"),
|
|
9388
|
+
inArray5(rawEventSamples.projectId, projectIds)
|
|
8859
9389
|
)).all();
|
|
8860
9390
|
result.examined = unknownSamples.length;
|
|
8861
9391
|
if (unknownSamples.length === 0) {
|
|
@@ -8893,7 +9423,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
8893
9423
|
result.reclassified++;
|
|
8894
9424
|
result.byBot[classified.botId] = (result.byBot[classified.botId] ?? 0) + 1;
|
|
8895
9425
|
if (isDryRun) continue;
|
|
8896
|
-
db.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(
|
|
9426
|
+
db.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(eq16(rawEventSamples.id, snap.id)).run();
|
|
8897
9427
|
const tsHour = new Date(snap.ts);
|
|
8898
9428
|
tsHour.setUTCMinutes(0, 0, 0);
|
|
8899
9429
|
if (userFetch) {
|
|
@@ -8957,9 +9487,9 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
8957
9487
|
}
|
|
8958
9488
|
}
|
|
8959
9489
|
if (!isDryRun) {
|
|
8960
|
-
const afterRow = db.select({ n: sql7`count(*)` }).from(rawEventSamples).where(
|
|
8961
|
-
|
|
8962
|
-
|
|
9490
|
+
const afterRow = db.select({ n: sql7`count(*)` }).from(rawEventSamples).where(and13(
|
|
9491
|
+
eq16(rawEventSamples.eventType, "unknown"),
|
|
9492
|
+
inArray5(rawEventSamples.projectId, projectIds)
|
|
8963
9493
|
)).get();
|
|
8964
9494
|
result.unknownAfter = Number(afterRow?.n ?? 0);
|
|
8965
9495
|
} else {
|
|
@@ -9392,7 +9922,7 @@ var ProviderRegistry = class {
|
|
|
9392
9922
|
// src/scheduler.ts
|
|
9393
9923
|
import crypto19 from "crypto";
|
|
9394
9924
|
import cron from "node-cron";
|
|
9395
|
-
import { and as
|
|
9925
|
+
import { and as and14, eq as eq17, inArray as inArray6, notExists, sql as sql8 } from "drizzle-orm";
|
|
9396
9926
|
var log13 = createLogger("Scheduler");
|
|
9397
9927
|
var DEFAULT_HEALTH_CRON = "0 */6 * * *";
|
|
9398
9928
|
function taskKey(projectId, kind) {
|
|
@@ -9424,9 +9954,9 @@ var Scheduler = class {
|
|
|
9424
9954
|
*/
|
|
9425
9955
|
ensureHealthSchedules() {
|
|
9426
9956
|
const projectsWithoutHealth = this.db.select({ id: projects.id }).from(projects).where(notExists(
|
|
9427
|
-
this.db.select({ one: sql8`1` }).from(schedules).where(
|
|
9428
|
-
|
|
9429
|
-
|
|
9957
|
+
this.db.select({ one: sql8`1` }).from(schedules).where(and14(
|
|
9958
|
+
eq17(schedules.projectId, projects.id),
|
|
9959
|
+
eq17(schedules.kind, SchedulableRunKinds.doctor)
|
|
9430
9960
|
))
|
|
9431
9961
|
)).all();
|
|
9432
9962
|
if (projectsWithoutHealth.length === 0) return;
|
|
@@ -9477,7 +10007,7 @@ var Scheduler = class {
|
|
|
9477
10007
|
start() {
|
|
9478
10008
|
this.ensureHealthSchedules();
|
|
9479
10009
|
this.ensureQueryBaskets();
|
|
9480
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
10010
|
+
const allSchedules = this.db.select().from(schedules).where(eq17(schedules.enabled, true)).all();
|
|
9481
10011
|
for (const schedule of allSchedules) {
|
|
9482
10012
|
const missedRunAt = schedule.nextRunAt;
|
|
9483
10013
|
this.registerCronTask(schedule);
|
|
@@ -9507,7 +10037,7 @@ var Scheduler = class {
|
|
|
9507
10037
|
this.stopTask(key, existing, "Stopped");
|
|
9508
10038
|
this.tasks.delete(key);
|
|
9509
10039
|
}
|
|
9510
|
-
const schedule = this.db.select().from(schedules).where(
|
|
10040
|
+
const schedule = this.db.select().from(schedules).where(and14(eq17(schedules.projectId, projectId), eq17(schedules.kind, kind))).get();
|
|
9511
10041
|
if (schedule && schedule.enabled) {
|
|
9512
10042
|
this.registerCronTask(schedule);
|
|
9513
10043
|
}
|
|
@@ -9548,21 +10078,21 @@ var Scheduler = class {
|
|
|
9548
10078
|
this.db.update(schedules).set({
|
|
9549
10079
|
nextRunAt: nextRunFromCron(cronExpr, timezone),
|
|
9550
10080
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
9551
|
-
}).where(
|
|
10081
|
+
}).where(eq17(schedules.id, scheduleId)).run();
|
|
9552
10082
|
const label = schedule.preset ?? cronExpr;
|
|
9553
10083
|
log13.info("cron.registered", { projectId, kind, schedule: label, timezone });
|
|
9554
10084
|
}
|
|
9555
10085
|
triggerRun(scheduleId, projectId, kind) {
|
|
9556
10086
|
try {
|
|
9557
10087
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9558
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
10088
|
+
const currentSchedule = this.db.select().from(schedules).where(eq17(schedules.id, scheduleId)).get();
|
|
9559
10089
|
if (!currentSchedule || !currentSchedule.enabled) {
|
|
9560
10090
|
log13.warn("schedule.stale", { scheduleId, projectId, kind, msg: "schedule no longer exists or is disabled" });
|
|
9561
10091
|
this.remove(projectId, kind);
|
|
9562
10092
|
return;
|
|
9563
10093
|
}
|
|
9564
10094
|
const nextRunAt = nextRunFromCron(currentSchedule.cronExpr, currentSchedule.timezone);
|
|
9565
|
-
const project = this.db.select().from(projects).where(
|
|
10095
|
+
const project = this.db.select().from(projects).where(eq17(projects.id, projectId)).get();
|
|
9566
10096
|
if (!project) {
|
|
9567
10097
|
log13.error("project.not-found", { projectId, kind, msg: "skipping scheduled run" });
|
|
9568
10098
|
this.remove(projectId, kind);
|
|
@@ -9582,7 +10112,7 @@ var Scheduler = class {
|
|
|
9582
10112
|
lastRunAt: now,
|
|
9583
10113
|
nextRunAt,
|
|
9584
10114
|
updatedAt: now
|
|
9585
|
-
}).where(
|
|
10115
|
+
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9586
10116
|
log13.info("traffic-sync.triggered", { projectName: project.name, sourceId });
|
|
9587
10117
|
this.callbacks.onTrafficSyncRequested(project.name, sourceId);
|
|
9588
10118
|
return;
|
|
@@ -9605,7 +10135,7 @@ var Scheduler = class {
|
|
|
9605
10135
|
lastRunAt: now,
|
|
9606
10136
|
nextRunAt,
|
|
9607
10137
|
updatedAt: now
|
|
9608
|
-
}).where(
|
|
10138
|
+
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9609
10139
|
log13.info("gbp-sync.triggered", { runId: runId2, projectName: project.name });
|
|
9610
10140
|
this.callbacks.onGbpSyncRequested(runId2, projectId);
|
|
9611
10141
|
return;
|
|
@@ -9615,14 +10145,14 @@ var Scheduler = class {
|
|
|
9615
10145
|
log13.warn("ads-sync.no-callback", { scheduleId, projectId, msg: "host did not register onAdsSyncRequested" });
|
|
9616
10146
|
return;
|
|
9617
10147
|
}
|
|
9618
|
-
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(
|
|
9619
|
-
|
|
9620
|
-
|
|
9621
|
-
|
|
10148
|
+
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(and14(
|
|
10149
|
+
eq17(runs.projectId, projectId),
|
|
10150
|
+
eq17(runs.kind, RunKinds["ads-sync"]),
|
|
10151
|
+
inArray6(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
9622
10152
|
)).get();
|
|
9623
10153
|
if (activeAdsRun) {
|
|
9624
10154
|
log13.info("ads-sync.skipped-active", { projectName: project.name, activeRunId: activeAdsRun.id });
|
|
9625
|
-
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(
|
|
10155
|
+
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9626
10156
|
return;
|
|
9627
10157
|
}
|
|
9628
10158
|
const runId2 = crypto19.randomUUID();
|
|
@@ -9638,7 +10168,7 @@ var Scheduler = class {
|
|
|
9638
10168
|
lastRunAt: now,
|
|
9639
10169
|
nextRunAt,
|
|
9640
10170
|
updatedAt: now
|
|
9641
|
-
}).where(
|
|
10171
|
+
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9642
10172
|
log13.info("ads-sync.triggered", { runId: runId2, projectName: project.name });
|
|
9643
10173
|
this.callbacks.onAdsSyncRequested(runId2, projectId);
|
|
9644
10174
|
return;
|
|
@@ -9652,7 +10182,7 @@ var Scheduler = class {
|
|
|
9652
10182
|
lastRunAt: now,
|
|
9653
10183
|
nextRunAt,
|
|
9654
10184
|
updatedAt: now
|
|
9655
|
-
}).where(
|
|
10185
|
+
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9656
10186
|
log13.info("data-refresh.triggered", { projectName: project.name });
|
|
9657
10187
|
this.callbacks.onDataRefreshRequested(project.name);
|
|
9658
10188
|
return;
|
|
@@ -9666,7 +10196,7 @@ var Scheduler = class {
|
|
|
9666
10196
|
lastRunAt: now,
|
|
9667
10197
|
nextRunAt,
|
|
9668
10198
|
updatedAt: now
|
|
9669
|
-
}).where(
|
|
10199
|
+
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9670
10200
|
log13.info("doctor.triggered", { projectName: project.name });
|
|
9671
10201
|
this.callbacks.onDoctorRequested(project.name);
|
|
9672
10202
|
return;
|
|
@@ -9680,7 +10210,7 @@ var Scheduler = class {
|
|
|
9680
10210
|
lastRunAt: now,
|
|
9681
10211
|
nextRunAt,
|
|
9682
10212
|
updatedAt: now
|
|
9683
|
-
}).where(
|
|
10213
|
+
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9684
10214
|
log13.info("backlinks-sync.triggered", { projectName: project.name });
|
|
9685
10215
|
this.callbacks.onBacklinksSyncRequested(project.name);
|
|
9686
10216
|
return;
|
|
@@ -9690,14 +10220,14 @@ var Scheduler = class {
|
|
|
9690
10220
|
log13.warn("site-audit.no-callback", { scheduleId, projectId, msg: "host did not register onSiteAuditRequested" });
|
|
9691
10221
|
return;
|
|
9692
10222
|
}
|
|
9693
|
-
const active = this.db.select({ id: runs.id }).from(runs).where(
|
|
9694
|
-
|
|
9695
|
-
|
|
9696
|
-
|
|
10223
|
+
const active = this.db.select({ id: runs.id }).from(runs).where(and14(
|
|
10224
|
+
eq17(runs.projectId, projectId),
|
|
10225
|
+
eq17(runs.kind, RunKinds["site-audit"]),
|
|
10226
|
+
inArray6(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
9697
10227
|
)).get();
|
|
9698
10228
|
if (active) {
|
|
9699
10229
|
log13.info("site-audit.skipped-active", { projectName: project.name, activeRunId: active.id });
|
|
9700
|
-
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(
|
|
10230
|
+
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9701
10231
|
return;
|
|
9702
10232
|
}
|
|
9703
10233
|
const runId2 = crypto19.randomUUID();
|
|
@@ -9723,7 +10253,7 @@ var Scheduler = class {
|
|
|
9723
10253
|
lastRunAt: now,
|
|
9724
10254
|
nextRunAt,
|
|
9725
10255
|
updatedAt: now
|
|
9726
|
-
}).where(
|
|
10256
|
+
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9727
10257
|
log13.info("site-audit.triggered", { runId: runId2, projectName: project.name });
|
|
9728
10258
|
this.callbacks.onSiteAuditRequested(runId2, projectId);
|
|
9729
10259
|
return;
|
|
@@ -9756,7 +10286,7 @@ var Scheduler = class {
|
|
|
9756
10286
|
this.db.update(schedules).set({
|
|
9757
10287
|
nextRunAt,
|
|
9758
10288
|
updatedAt: now
|
|
9759
|
-
}).where(
|
|
10289
|
+
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9760
10290
|
return;
|
|
9761
10291
|
}
|
|
9762
10292
|
const runId = queueResult.runId;
|
|
@@ -9764,7 +10294,7 @@ var Scheduler = class {
|
|
|
9764
10294
|
lastRunAt: now,
|
|
9765
10295
|
nextRunAt,
|
|
9766
10296
|
updatedAt: now
|
|
9767
|
-
}).where(
|
|
10297
|
+
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
9768
10298
|
log13.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
9769
10299
|
this.callbacks.onRunCreated(runId, projectId, providers, resolvedLocation);
|
|
9770
10300
|
} catch (err) {
|
|
@@ -9797,7 +10327,7 @@ async function refreshAllIntegrations(client, projectName) {
|
|
|
9797
10327
|
}
|
|
9798
10328
|
|
|
9799
10329
|
// src/notifier.ts
|
|
9800
|
-
import { eq as
|
|
10330
|
+
import { eq as eq18, desc as desc7, and as and15, inArray as inArray7, or as or2 } from "drizzle-orm";
|
|
9801
10331
|
import crypto20 from "crypto";
|
|
9802
10332
|
var log15 = createLogger("Notifier");
|
|
9803
10333
|
var Notifier = class {
|
|
@@ -9810,18 +10340,18 @@ var Notifier = class {
|
|
|
9810
10340
|
/** Called after a run completes (success, partial, or failed). */
|
|
9811
10341
|
async onRunCompleted(runId, projectId) {
|
|
9812
10342
|
log15.info("run.completed", { runId, projectId });
|
|
9813
|
-
const notifs = this.db.select().from(notifications).where(
|
|
10343
|
+
const notifs = this.db.select().from(notifications).where(eq18(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
9814
10344
|
if (notifs.length === 0) {
|
|
9815
10345
|
log15.info("notifications.none-enabled", { projectId });
|
|
9816
10346
|
return;
|
|
9817
10347
|
}
|
|
9818
10348
|
log15.info("notifications.found", { projectId, count: notifs.length });
|
|
9819
|
-
const run = this.db.select().from(runs).where(
|
|
10349
|
+
const run = this.db.select().from(runs).where(eq18(runs.id, runId)).get();
|
|
9820
10350
|
if (!run) {
|
|
9821
10351
|
log15.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
9822
10352
|
return;
|
|
9823
10353
|
}
|
|
9824
|
-
const project = this.db.select().from(projects).where(
|
|
10354
|
+
const project = this.db.select().from(projects).where(eq18(projects.id, projectId)).get();
|
|
9825
10355
|
if (!project) {
|
|
9826
10356
|
log15.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
9827
10357
|
return;
|
|
@@ -9874,7 +10404,7 @@ var Notifier = class {
|
|
|
9874
10404
|
* inferred from the absence of a webhook.
|
|
9875
10405
|
*/
|
|
9876
10406
|
async onHealthChecked(projectId, report) {
|
|
9877
|
-
const project = this.db.select().from(projects).where(
|
|
10407
|
+
const project = this.db.select().from(projects).where(eq18(projects.id, projectId)).get();
|
|
9878
10408
|
if (!project) {
|
|
9879
10409
|
log15.error("project.not-found", { projectId, msg: "skipping health notification" });
|
|
9880
10410
|
return null;
|
|
@@ -9905,7 +10435,7 @@ var Notifier = class {
|
|
|
9905
10435
|
const status = noSignal ? "warn" : worst ? worst.status : "ok";
|
|
9906
10436
|
const code = noSignal ? "health.no-signal" : worst?.code ?? "health.ok";
|
|
9907
10437
|
const summary = noSignal ? `No health check produced a result (${report.checks.length} skipped) \u2014 health is unknown, not confirmed.` : worst?.summary ?? `All ${graded.length} health check(s) passing.`;
|
|
9908
|
-
const previous = this.db.select().from(doctorHealthState).where(
|
|
10438
|
+
const previous = this.db.select().from(doctorHealthState).where(eq18(doctorHealthState.projectId, projectId)).get();
|
|
9909
10439
|
const previousStatus = previous?.status ?? null;
|
|
9910
10440
|
let event = null;
|
|
9911
10441
|
if (previous === void 0) {
|
|
@@ -9920,13 +10450,13 @@ var Notifier = class {
|
|
|
9920
10450
|
if (previous === void 0) {
|
|
9921
10451
|
this.db.insert(doctorHealthState).values({ ...observation, notifiedAt: null }).run();
|
|
9922
10452
|
} else {
|
|
9923
|
-
this.db.update(doctorHealthState).set(observation).where(
|
|
10453
|
+
this.db.update(doctorHealthState).set(observation).where(eq18(doctorHealthState.projectId, projectId)).run();
|
|
9924
10454
|
}
|
|
9925
10455
|
if (!event) {
|
|
9926
10456
|
log15.info("health.unchanged", { projectId, status, code });
|
|
9927
10457
|
return null;
|
|
9928
10458
|
}
|
|
9929
|
-
const notifs = this.db.select().from(notifications).where(
|
|
10459
|
+
const notifs = this.db.select().from(notifications).where(eq18(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
9930
10460
|
const payload = {
|
|
9931
10461
|
source: "canonry",
|
|
9932
10462
|
event,
|
|
@@ -9950,7 +10480,7 @@ var Notifier = class {
|
|
|
9950
10480
|
delivered += 1;
|
|
9951
10481
|
}
|
|
9952
10482
|
if (delivered > 0) {
|
|
9953
|
-
this.db.update(doctorHealthState).set({ notifiedAt: now }).where(
|
|
10483
|
+
this.db.update(doctorHealthState).set({ notifiedAt: now }).where(eq18(doctorHealthState.projectId, projectId)).run();
|
|
9954
10484
|
}
|
|
9955
10485
|
log15.info("health.notified", { projectId, event, status, code, subscribers: notifs.length, delivered });
|
|
9956
10486
|
return event;
|
|
@@ -9963,11 +10493,11 @@ var Notifier = class {
|
|
|
9963
10493
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
9964
10494
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
9965
10495
|
if (insightEvents.length === 0) return;
|
|
9966
|
-
const notifs = this.db.select().from(notifications).where(
|
|
10496
|
+
const notifs = this.db.select().from(notifications).where(eq18(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
9967
10497
|
if (notifs.length === 0) return;
|
|
9968
|
-
const run = this.db.select().from(runs).where(
|
|
10498
|
+
const run = this.db.select().from(runs).where(eq18(runs.id, runId)).get();
|
|
9969
10499
|
if (!run) return;
|
|
9970
|
-
const project = this.db.select().from(projects).where(
|
|
10500
|
+
const project = this.db.select().from(projects).where(eq18(projects.id, projectId)).get();
|
|
9971
10501
|
if (!project) return;
|
|
9972
10502
|
for (const notif of notifs) {
|
|
9973
10503
|
const config = notif.config;
|
|
@@ -9997,7 +10527,7 @@ var Notifier = class {
|
|
|
9997
10527
|
}
|
|
9998
10528
|
}
|
|
9999
10529
|
computeTransitions(runId, projectId) {
|
|
10000
|
-
const thisRun = this.db.select().from(runs).where(
|
|
10530
|
+
const thisRun = this.db.select().from(runs).where(eq18(runs.id, runId)).get();
|
|
10001
10531
|
if (!thisRun) return [];
|
|
10002
10532
|
const completeness = measurementRunCompleteness(this.db, runId);
|
|
10003
10533
|
if (completeness.planned && !completeness.complete) {
|
|
@@ -10008,10 +10538,10 @@ var Notifier = class {
|
|
|
10008
10538
|
});
|
|
10009
10539
|
return [];
|
|
10010
10540
|
}
|
|
10011
|
-
const groupSiblings = this.db.select().from(runs).where(
|
|
10012
|
-
|
|
10013
|
-
|
|
10014
|
-
|
|
10541
|
+
const groupSiblings = this.db.select().from(runs).where(and15(
|
|
10542
|
+
eq18(runs.projectId, projectId),
|
|
10543
|
+
eq18(runs.kind, thisRun.kind),
|
|
10544
|
+
eq18(runs.createdAt, thisRun.createdAt)
|
|
10015
10545
|
)).all();
|
|
10016
10546
|
const stillPending = groupSiblings.some((r) => r.status === "queued" || r.status === "running");
|
|
10017
10547
|
if (stillPending) return [];
|
|
@@ -10027,17 +10557,17 @@ var Notifier = class {
|
|
|
10027
10557
|
return candidate.id > best.id ? candidate : best;
|
|
10028
10558
|
});
|
|
10029
10559
|
if (winner.id !== runId) return [];
|
|
10030
|
-
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
10560
|
+
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(eq18(projects.id, projectId)).get();
|
|
10031
10561
|
const locationCount = Math.max(
|
|
10032
10562
|
1,
|
|
10033
10563
|
(projectLocations?.locations ?? []).length
|
|
10034
10564
|
);
|
|
10035
10565
|
const RECENT_FETCH_LIMIT = Math.max(8, locationCount * 4);
|
|
10036
10566
|
const recentRuns = this.db.select().from(runs).where(
|
|
10037
|
-
|
|
10038
|
-
|
|
10039
|
-
|
|
10040
|
-
or2(
|
|
10567
|
+
and15(
|
|
10568
|
+
eq18(runs.projectId, projectId),
|
|
10569
|
+
eq18(runs.kind, thisRun.kind),
|
|
10570
|
+
or2(eq18(runs.status, "completed"), eq18(runs.status, "partial"))
|
|
10041
10571
|
)
|
|
10042
10572
|
).orderBy(desc7(runs.createdAt), desc7(runs.id)).limit(RECENT_FETCH_LIMIT).all();
|
|
10043
10573
|
const groups = groupRunsByCreatedAt(recentRuns);
|
|
@@ -10054,13 +10584,13 @@ var Notifier = class {
|
|
|
10054
10584
|
provider: querySnapshots.provider,
|
|
10055
10585
|
location: querySnapshots.location,
|
|
10056
10586
|
citationState: querySnapshots.citationState
|
|
10057
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
10587
|
+
}).from(querySnapshots).leftJoin(queries, eq18(querySnapshots.queryId, queries.id)).where(inArray7(querySnapshots.runId, currentRunIds)).all();
|
|
10058
10588
|
const previousSnapshots = this.db.select({
|
|
10059
10589
|
queryId: querySnapshots.queryId,
|
|
10060
10590
|
provider: querySnapshots.provider,
|
|
10061
10591
|
location: querySnapshots.location,
|
|
10062
10592
|
citationState: querySnapshots.citationState
|
|
10063
|
-
}).from(querySnapshots).where(
|
|
10593
|
+
}).from(querySnapshots).where(inArray7(querySnapshots.runId, previousRunIds)).all();
|
|
10064
10594
|
const prevMap = /* @__PURE__ */ new Map();
|
|
10065
10595
|
for (const s of previousSnapshots) {
|
|
10066
10596
|
if (s.queryId == null) continue;
|
|
@@ -10137,7 +10667,7 @@ var Notifier = class {
|
|
|
10137
10667
|
};
|
|
10138
10668
|
|
|
10139
10669
|
// src/run-coordinator.ts
|
|
10140
|
-
import { eq as
|
|
10670
|
+
import { eq as eq19 } from "drizzle-orm";
|
|
10141
10671
|
var log16 = createLogger("RunCoordinator");
|
|
10142
10672
|
var RunCoordinator = class {
|
|
10143
10673
|
constructor(db, notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
@@ -10153,7 +10683,7 @@ var RunCoordinator = class {
|
|
|
10153
10683
|
onInsightsGenerated;
|
|
10154
10684
|
onAeroEvent;
|
|
10155
10685
|
async onRunCompleted(runId, projectId) {
|
|
10156
|
-
const runRow = this.db.select().from(runs).where(
|
|
10686
|
+
const runRow = this.db.select().from(runs).where(eq19(runs.id, runId)).get();
|
|
10157
10687
|
const kind = runRow?.kind ?? RunKinds["answer-visibility"];
|
|
10158
10688
|
if (runRow?.trigger === RunTriggers.probe) {
|
|
10159
10689
|
log16.info("probe.skip-side-effects", { runId, projectId, kind });
|
|
@@ -10244,7 +10774,7 @@ var RunCoordinator = class {
|
|
|
10244
10774
|
* so the Aero queue is never starved of a follow-up.
|
|
10245
10775
|
*/
|
|
10246
10776
|
buildDiscoveryAeroContext(runId, projectId, status, error) {
|
|
10247
|
-
const session = this.db.select().from(discoverySessions).where(
|
|
10777
|
+
const session = this.db.select().from(discoverySessions).where(eq19(discoverySessions.runId, runId)).get();
|
|
10248
10778
|
const competitorMap = session ? session.competitorMap : [];
|
|
10249
10779
|
return {
|
|
10250
10780
|
kind: RunKinds["aeo-discover-probe"],
|
|
@@ -10287,7 +10817,7 @@ function analysisResultFromInsights(insights2) {
|
|
|
10287
10817
|
|
|
10288
10818
|
// src/agent/session-registry.ts
|
|
10289
10819
|
import crypto24 from "crypto";
|
|
10290
|
-
import { eq as
|
|
10820
|
+
import { eq as eq21 } from "drizzle-orm";
|
|
10291
10821
|
|
|
10292
10822
|
// src/agent/session.ts
|
|
10293
10823
|
import fs7 from "fs";
|
|
@@ -11416,7 +11946,7 @@ async function loadExternalMcpTools(servers, opts = {}) {
|
|
|
11416
11946
|
|
|
11417
11947
|
// src/agent/memory-store.ts
|
|
11418
11948
|
import crypto23 from "crypto";
|
|
11419
|
-
import { and as
|
|
11949
|
+
import { and as and16, desc as desc8, eq as eq20, like, sql as sql9 } from "drizzle-orm";
|
|
11420
11950
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
11421
11951
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
11422
11952
|
function rowToDto(row) {
|
|
@@ -11430,7 +11960,7 @@ function rowToDto(row) {
|
|
|
11430
11960
|
};
|
|
11431
11961
|
}
|
|
11432
11962
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
11433
|
-
const query = db.select().from(agentMemory).where(
|
|
11963
|
+
const query = db.select().from(agentMemory).where(eq20(agentMemory.projectId, projectId)).orderBy(desc8(agentMemory.updatedAt));
|
|
11434
11964
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
11435
11965
|
return rows.map(rowToDto);
|
|
11436
11966
|
}
|
|
@@ -11461,12 +11991,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
11461
11991
|
updatedAt: now
|
|
11462
11992
|
}
|
|
11463
11993
|
}).run();
|
|
11464
|
-
const row = db.select().from(agentMemory).where(
|
|
11994
|
+
const row = db.select().from(agentMemory).where(and16(eq20(agentMemory.projectId, args.projectId), eq20(agentMemory.key, args.key))).get();
|
|
11465
11995
|
if (!row) throw new Error("memory upsert produced no row");
|
|
11466
11996
|
return rowToDto(row);
|
|
11467
11997
|
}
|
|
11468
11998
|
function deleteMemoryEntry(db, projectId, key) {
|
|
11469
|
-
const result = db.delete(agentMemory).where(
|
|
11999
|
+
const result = db.delete(agentMemory).where(and16(eq20(agentMemory.projectId, projectId), eq20(agentMemory.key, key))).run();
|
|
11470
12000
|
const changes = result.changes ?? 0;
|
|
11471
12001
|
return changes > 0;
|
|
11472
12002
|
}
|
|
@@ -11495,8 +12025,8 @@ function writeCompactionNote(db, args) {
|
|
|
11495
12025
|
}).run();
|
|
11496
12026
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
11497
12027
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
11498
|
-
|
|
11499
|
-
|
|
12028
|
+
and16(
|
|
12029
|
+
eq20(agentMemory.projectId, args.projectId),
|
|
11500
12030
|
like(agentMemory.key, `${sessionPrefix}%`)
|
|
11501
12031
|
)
|
|
11502
12032
|
).orderBy(desc8(agentMemory.updatedAt)).all();
|
|
@@ -11504,7 +12034,7 @@ function writeCompactionNote(db, args) {
|
|
|
11504
12034
|
if (stale.length > 0) {
|
|
11505
12035
|
tx.delete(agentMemory).where(sql9`${agentMemory.id} IN (${sql9.join(stale.map((s) => sql9`${s}`), sql9`, `)})`).run();
|
|
11506
12036
|
}
|
|
11507
|
-
const row = tx.select().from(agentMemory).where(
|
|
12037
|
+
const row = tx.select().from(agentMemory).where(and16(eq20(agentMemory.projectId, args.projectId), eq20(agentMemory.key, key))).get();
|
|
11508
12038
|
if (row) inserted = rowToDto(row);
|
|
11509
12039
|
});
|
|
11510
12040
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -11731,7 +12261,7 @@ var SessionRegistry = class {
|
|
|
11731
12261
|
modelProvider: effectiveProvider,
|
|
11732
12262
|
modelId: effectiveModelId,
|
|
11733
12263
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
11734
|
-
}).where(
|
|
12264
|
+
}).where(eq21(agentSessions.projectId, projectId)).run();
|
|
11735
12265
|
}
|
|
11736
12266
|
const agent2 = createAeroSession({
|
|
11737
12267
|
projectName,
|
|
@@ -11965,7 +12495,7 @@ ${lines.join("\n")}
|
|
|
11965
12495
|
modelProvider: nextProvider,
|
|
11966
12496
|
modelId: nextModelId,
|
|
11967
12497
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
11968
|
-
}).where(
|
|
12498
|
+
}).where(eq21(agentSessions.projectId, projectId)).run();
|
|
11969
12499
|
}
|
|
11970
12500
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
11971
12501
|
save(projectName) {
|
|
@@ -12129,11 +12659,11 @@ ${lines.join("\n")}
|
|
|
12129
12659
|
return id;
|
|
12130
12660
|
}
|
|
12131
12661
|
tryResolveProjectId(projectName) {
|
|
12132
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
12662
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq21(projects.name, projectName)).get();
|
|
12133
12663
|
return row?.id;
|
|
12134
12664
|
}
|
|
12135
12665
|
loadRow(projectId) {
|
|
12136
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
12666
|
+
const row = this.opts.db.select().from(agentSessions).where(eq21(agentSessions.projectId, projectId)).get();
|
|
12137
12667
|
return row ?? null;
|
|
12138
12668
|
}
|
|
12139
12669
|
insertRow(params) {
|
|
@@ -12152,14 +12682,14 @@ ${lines.join("\n")}
|
|
|
12152
12682
|
}
|
|
12153
12683
|
updateRow(projectId, patch) {
|
|
12154
12684
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
12155
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
12685
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq21(agentSessions.projectId, projectId)).run();
|
|
12156
12686
|
}
|
|
12157
12687
|
};
|
|
12158
12688
|
|
|
12159
12689
|
// src/agent/agent-routes.ts
|
|
12160
|
-
import { eq as
|
|
12690
|
+
import { eq as eq22 } from "drizzle-orm";
|
|
12161
12691
|
function resolveProject(db, name) {
|
|
12162
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
12692
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq22(projects.name, name)).get();
|
|
12163
12693
|
if (!row) throw notFound("project", name);
|
|
12164
12694
|
return row;
|
|
12165
12695
|
}
|
|
@@ -12168,7 +12698,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
12168
12698
|
"/projects/:name/agent/transcript",
|
|
12169
12699
|
async (request) => {
|
|
12170
12700
|
const project = resolveProject(opts.db, request.params.name);
|
|
12171
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
12701
|
+
const row = opts.db.select().from(agentSessions).where(eq22(agentSessions.projectId, project.id)).get();
|
|
12172
12702
|
if (!row) {
|
|
12173
12703
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
12174
12704
|
}
|
|
@@ -12192,7 +12722,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
12192
12722
|
async (request) => {
|
|
12193
12723
|
const project = resolveProject(opts.db, request.params.name);
|
|
12194
12724
|
opts.sessionRegistry.reset(project.name);
|
|
12195
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
12725
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq22(agentSessions.projectId, project.id)).run();
|
|
12196
12726
|
return { status: "reset" };
|
|
12197
12727
|
}
|
|
12198
12728
|
);
|
|
@@ -12519,16 +13049,16 @@ Your previous reply was not valid JSON. Return ONLY a JSON object with exactly t
|
|
|
12519
13049
|
import { runAeoAudit } from "@canonry/aeo-audit";
|
|
12520
13050
|
|
|
12521
13051
|
// src/site-fetch.ts
|
|
12522
|
-
import
|
|
13052
|
+
import https2 from "https";
|
|
12523
13053
|
var FETCH_TIMEOUT_MS2 = 1e4;
|
|
12524
13054
|
var MAX_TEXT_LENGTH = 4e3;
|
|
12525
13055
|
var MAX_BODY_BYTES = 512e3;
|
|
12526
|
-
var
|
|
13056
|
+
var USER_AGENT2 = "Canonry/1.0 (site-analysis)";
|
|
12527
13057
|
function fetchWithPinnedAddress(target) {
|
|
12528
13058
|
return new Promise((resolve) => {
|
|
12529
13059
|
const port = target.url.port ? Number(target.url.port) : 443;
|
|
12530
13060
|
const path10 = target.url.pathname + target.url.search;
|
|
12531
|
-
const req =
|
|
13061
|
+
const req = https2.request(
|
|
12532
13062
|
{
|
|
12533
13063
|
hostname: target.address,
|
|
12534
13064
|
family: target.family,
|
|
@@ -12540,7 +13070,7 @@ function fetchWithPinnedAddress(target) {
|
|
|
12540
13070
|
// SNI for TLS
|
|
12541
13071
|
headers: {
|
|
12542
13072
|
Host: target.url.host,
|
|
12543
|
-
"User-Agent":
|
|
13073
|
+
"User-Agent": USER_AGENT2,
|
|
12544
13074
|
Accept: "text/html"
|
|
12545
13075
|
}
|
|
12546
13076
|
},
|
|
@@ -13135,20 +13665,20 @@ function clipText(value, length) {
|
|
|
13135
13665
|
}
|
|
13136
13666
|
|
|
13137
13667
|
// src/research-runner.ts
|
|
13138
|
-
import { and as
|
|
13668
|
+
import { and as and17, eq as eq23, inArray as inArray8, sql as sql10 } from "drizzle-orm";
|
|
13139
13669
|
var unfinishedResearchQueryStatuses = [ResearchQueryStatuses.queued, ResearchQueryStatuses.running];
|
|
13140
13670
|
function finalResearchRunStatus(completed, failed) {
|
|
13141
13671
|
if (failed === 0) return ResearchRunStatuses.completed;
|
|
13142
13672
|
return completed > 0 ? ResearchRunStatuses.partial : ResearchRunStatuses.failed;
|
|
13143
13673
|
}
|
|
13144
13674
|
async function executeResearchRun(db, registry, runId, projectId) {
|
|
13145
|
-
const run = db.select().from(researchRuns).where(
|
|
13675
|
+
const run = db.select().from(researchRuns).where(and17(eq23(researchRuns.id, runId), eq23(researchRuns.projectId, projectId))).get();
|
|
13146
13676
|
if (!run || run.status !== ResearchRunStatuses.queued) return;
|
|
13147
|
-
const project = db.select().from(projects).where(
|
|
13677
|
+
const project = db.select().from(projects).where(eq23(projects.id, projectId)).get();
|
|
13148
13678
|
if (!project) return;
|
|
13149
13679
|
const provider = registry.get(run.provider);
|
|
13150
13680
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
13151
|
-
const claim = db.update(researchRuns).set({ status: ResearchRunStatuses.running, startedAt: now }).where(
|
|
13681
|
+
const claim = db.update(researchRuns).set({ status: ResearchRunStatuses.running, startedAt: now }).where(and17(eq23(researchRuns.id, runId), eq23(researchRuns.status, ResearchRunStatuses.queued))).run();
|
|
13152
13682
|
if (claim.changes !== 1) return;
|
|
13153
13683
|
let reserved = 0;
|
|
13154
13684
|
let dispatched = 0;
|
|
@@ -13172,8 +13702,8 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
13172
13702
|
}
|
|
13173
13703
|
reserved = run.totalQueries;
|
|
13174
13704
|
reservation = { scope, period };
|
|
13175
|
-
const rows = db.select().from(researchRunQueries).where(
|
|
13176
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
13705
|
+
const rows = db.select().from(researchRunQueries).where(eq23(researchRunQueries.researchRunId, runId)).orderBy(researchRunQueries.position).all();
|
|
13706
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq23(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
13177
13707
|
const domains = effectiveDomains(project);
|
|
13178
13708
|
const brands = effectiveBrandNames(project);
|
|
13179
13709
|
const config = { ...provider.config, model: run.resolvedModel };
|
|
@@ -13181,7 +13711,7 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
13181
13711
|
await mapWithConcurrency(rows, Math.max(1, provider.config.quotaPolicy.maxConcurrency), async (row) => {
|
|
13182
13712
|
try {
|
|
13183
13713
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
13184
|
-
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.running, startedAt }).where(
|
|
13714
|
+
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.running, startedAt }).where(and17(eq23(researchRunQueries.id, row.id), eq23(researchRunQueries.status, ResearchQueryStatuses.queued))).run();
|
|
13185
13715
|
const raw = await gate.run(async () => {
|
|
13186
13716
|
dispatched++;
|
|
13187
13717
|
return provider.adapter.executeTrackedQuery({ query: row.queryText, canonicalDomains: domains, competitorDomains, ...run.location ? { location: run.location } : {} }, config);
|
|
@@ -13208,7 +13738,7 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
13208
13738
|
citationState: determineCitationState(normalized, domains),
|
|
13209
13739
|
rawResponse: raw.rawResponse,
|
|
13210
13740
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
13211
|
-
}).where(
|
|
13741
|
+
}).where(and17(eq23(researchRunQueries.id, row.id), eq23(researchRunQueries.status, ResearchQueryStatuses.running))).run();
|
|
13212
13742
|
if (completed.changes === 1) incrementResearchProgress(db, runId, "completedQueries");
|
|
13213
13743
|
} catch (error) {
|
|
13214
13744
|
try {
|
|
@@ -13232,18 +13762,18 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
13232
13762
|
}
|
|
13233
13763
|
}
|
|
13234
13764
|
function incrementResearchProgress(db, runId, column) {
|
|
13235
|
-
db.update(researchRuns).set({ [column]: sql10`${researchRuns[column]} + 1` }).where(
|
|
13765
|
+
db.update(researchRuns).set({ [column]: sql10`${researchRuns[column]} + 1` }).where(eq23(researchRuns.id, runId)).run();
|
|
13236
13766
|
}
|
|
13237
13767
|
function markResearchQueryFailed(db, runId, queryId, error) {
|
|
13238
13768
|
const message = error instanceof Error ? error.message : String(error);
|
|
13239
|
-
const failed = db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error: message, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
13769
|
+
const failed = db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error: message, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and17(eq23(researchRunQueries.id, queryId), inArray8(researchRunQueries.status, unfinishedResearchQueryStatuses))).run();
|
|
13240
13770
|
if (failed.changes === 1) incrementResearchProgress(db, runId, "failedQueries");
|
|
13241
13771
|
}
|
|
13242
13772
|
function markUnfinishedResearchQueriesFailed(db, runId, error) {
|
|
13243
|
-
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
13773
|
+
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and17(eq23(researchRunQueries.researchRunId, runId), inArray8(researchRunQueries.status, unfinishedResearchQueryStatuses))).run();
|
|
13244
13774
|
}
|
|
13245
13775
|
function finalizeResearchRun(db, runId, fatalError) {
|
|
13246
|
-
const rows = db.select({ status: researchRunQueries.status }).from(researchRunQueries).where(
|
|
13776
|
+
const rows = db.select({ status: researchRunQueries.status }).from(researchRunQueries).where(eq23(researchRunQueries.researchRunId, runId)).all();
|
|
13247
13777
|
const completed = rows.filter((row) => row.status === ResearchQueryStatuses.completed).length;
|
|
13248
13778
|
const failed = rows.filter((row) => row.status === ResearchQueryStatuses.failed).length;
|
|
13249
13779
|
db.update(researchRuns).set({
|
|
@@ -13252,7 +13782,7 @@ function finalizeResearchRun(db, runId, fatalError) {
|
|
|
13252
13782
|
failedQueries: failed,
|
|
13253
13783
|
error: fatalError ?? (failed === rows.length ? "Every research query failed." : null),
|
|
13254
13784
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
13255
|
-
}).where(
|
|
13785
|
+
}).where(eq23(researchRuns.id, runId)).run();
|
|
13256
13786
|
}
|
|
13257
13787
|
|
|
13258
13788
|
// src/server.ts
|
|
@@ -13290,6 +13820,13 @@ function resolveDashboardShowResourceLinks(env, config) {
|
|
|
13290
13820
|
function resolveDashboardShowUpdateNotification(env, config) {
|
|
13291
13821
|
return parseBooleanEnv(env.CANONRY_DASHBOARD_SHOW_UPDATE_NOTIFICATION) ?? config.dashboard?.showUpdateNotification ?? true;
|
|
13292
13822
|
}
|
|
13823
|
+
function parseDashboardOnboardingMode(value) {
|
|
13824
|
+
const normalized = value?.trim().toLowerCase();
|
|
13825
|
+
return normalized === "legacy" || normalized === "platform" || normalized === "auto" ? normalized : void 0;
|
|
13826
|
+
}
|
|
13827
|
+
function resolveDashboardOnboardingMode(env, config) {
|
|
13828
|
+
return parseDashboardOnboardingMode(env.CANONRY_ONBOARDING_MODE) ?? config.dashboard?.onboardingMode;
|
|
13829
|
+
}
|
|
13293
13830
|
var API_ADAPTERS = [
|
|
13294
13831
|
geminiAdapter,
|
|
13295
13832
|
openaiAdapter,
|
|
@@ -13567,7 +14104,7 @@ async function createServer(opts) {
|
|
|
13567
14104
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
13568
14105
|
async (ctx) => {
|
|
13569
14106
|
if (!sessionRegistry) return;
|
|
13570
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
14107
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq24(projects.id, ctx.projectId)).get();
|
|
13571
14108
|
if (!project) return;
|
|
13572
14109
|
let content;
|
|
13573
14110
|
if (ctx.kind === RunKinds["aeo-discover-probe"]) {
|
|
@@ -13917,7 +14454,7 @@ async function createServer(opts) {
|
|
|
13917
14454
|
void (async () => {
|
|
13918
14455
|
try {
|
|
13919
14456
|
const report = await aeroClient.runDoctor({ project: projectName });
|
|
13920
|
-
const project = opts.db.select().from(projects).where(
|
|
14457
|
+
const project = opts.db.select().from(projects).where(eq24(projects.name, projectName)).get();
|
|
13921
14458
|
if (!project) {
|
|
13922
14459
|
app.log.warn({ projectName }, "doctor schedule fired for an unknown project");
|
|
13923
14460
|
return;
|
|
@@ -13945,9 +14482,9 @@ async function createServer(opts) {
|
|
|
13945
14482
|
});
|
|
13946
14483
|
if (!probed) return;
|
|
13947
14484
|
const alreadySynced = opts.db.select().from(ccReleaseSyncs).where(
|
|
13948
|
-
|
|
13949
|
-
|
|
13950
|
-
|
|
14485
|
+
and18(
|
|
14486
|
+
eq24(ccReleaseSyncs.release, probed.release),
|
|
14487
|
+
eq24(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)
|
|
13951
14488
|
)
|
|
13952
14489
|
).limit(1).get();
|
|
13953
14490
|
if (alreadySynced) {
|
|
@@ -14187,6 +14724,7 @@ async function createServer(opts) {
|
|
|
14187
14724
|
const dashboardRequirePassword = resolveDashboardRequirePassword(process.env, opts.config);
|
|
14188
14725
|
const dashboardShowResourceLinks = resolveDashboardShowResourceLinks(process.env, opts.config);
|
|
14189
14726
|
const dashboardShowUpdateNotification = resolveDashboardShowUpdateNotification(process.env, opts.config);
|
|
14727
|
+
const dashboardOnboardingMode = resolveDashboardOnboardingMode(process.env, opts.config);
|
|
14190
14728
|
app.log.info(
|
|
14191
14729
|
{ dashboardRequirePassword },
|
|
14192
14730
|
"Dashboard password gate resolved"
|
|
@@ -14195,7 +14733,7 @@ async function createServer(opts) {
|
|
|
14195
14733
|
const googlePublicUrl = resolveGooglePublicUrl(opts.config, basePath);
|
|
14196
14734
|
if (opts.config.apiKey) {
|
|
14197
14735
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
14198
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
14736
|
+
const existing = opts.db.select().from(apiKeys).where(eq24(apiKeys.keyHash, keyHash)).get();
|
|
14199
14737
|
if (!existing) {
|
|
14200
14738
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
14201
14739
|
opts.db.insert(apiKeys).values({
|
|
@@ -14247,7 +14785,7 @@ async function createServer(opts) {
|
|
|
14247
14785
|
};
|
|
14248
14786
|
const getDefaultApiKey = () => {
|
|
14249
14787
|
if (!opts.config.apiKey) return void 0;
|
|
14250
|
-
return opts.db.select().from(apiKeys).where(
|
|
14788
|
+
return opts.db.select().from(apiKeys).where(eq24(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
14251
14789
|
};
|
|
14252
14790
|
const createPasswordSession = (reply) => {
|
|
14253
14791
|
const key = getDefaultApiKey();
|
|
@@ -14271,7 +14809,7 @@ async function createServer(opts) {
|
|
|
14271
14809
|
if (!header) return false;
|
|
14272
14810
|
const parts = header.split(" ");
|
|
14273
14811
|
if (parts.length !== 2 || parts[0] !== "Bearer") return false;
|
|
14274
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
14812
|
+
const key = opts.db.select().from(apiKeys).where(eq24(apiKeys.keyHash, hashApiKey(parts[1]))).get();
|
|
14275
14813
|
return Boolean(key && !key.revokedAt);
|
|
14276
14814
|
};
|
|
14277
14815
|
const namedAccountsInUse = () => anyUsersExist(opts.db);
|
|
@@ -14361,12 +14899,12 @@ async function createServer(opts) {
|
|
|
14361
14899
|
return reply.send({ authenticated: true });
|
|
14362
14900
|
}
|
|
14363
14901
|
if (apiKey) {
|
|
14364
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
14902
|
+
const key = opts.db.select().from(apiKeys).where(eq24(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
14365
14903
|
if (!key || key.revokedAt) {
|
|
14366
14904
|
const err2 = authInvalid();
|
|
14367
14905
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
14368
14906
|
}
|
|
14369
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
14907
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq24(apiKeys.id, key.id)).run();
|
|
14370
14908
|
const sessionId = createSession(key.id);
|
|
14371
14909
|
reply.header(
|
|
14372
14910
|
"set-cookie",
|
|
@@ -14708,7 +15246,7 @@ async function createServer(opts) {
|
|
|
14708
15246
|
...inspectOpts,
|
|
14709
15247
|
config: opts.config
|
|
14710
15248
|
}).then(() => {
|
|
14711
|
-
const finished = opts.db.select({ status: runs.status }).from(runs).where(
|
|
15249
|
+
const finished = opts.db.select({ status: runs.status }).from(runs).where(eq24(runs.id, runId)).get();
|
|
14712
15250
|
if (finished?.status === RunStatuses.completed || finished?.status === RunStatuses.partial) {
|
|
14713
15251
|
return maybeRefreshGscCoverage(
|
|
14714
15252
|
opts.db,
|
|
@@ -14985,6 +15523,9 @@ async function createServer(opts) {
|
|
|
14985
15523
|
const clientConfig = {};
|
|
14986
15524
|
if (basePath) clientConfig.basePath = basePath;
|
|
14987
15525
|
const dashboardConfig = {};
|
|
15526
|
+
if (dashboardOnboardingMode) {
|
|
15527
|
+
dashboardConfig.onboardingMode = dashboardOnboardingMode;
|
|
15528
|
+
}
|
|
14988
15529
|
if (!dashboardShowResourceLinks) {
|
|
14989
15530
|
dashboardConfig.showResourceLinks = false;
|
|
14990
15531
|
}
|
|
@@ -15086,7 +15627,7 @@ async function createServer(opts) {
|
|
|
15086
15627
|
}
|
|
15087
15628
|
checkLatestVersionForServer();
|
|
15088
15629
|
scheduler.start();
|
|
15089
|
-
for (const run of opts.db.select({ id: researchRuns.id, projectId: researchRuns.projectId }).from(researchRuns).where(
|
|
15630
|
+
for (const run of opts.db.select({ id: researchRuns.id, projectId: researchRuns.projectId }).from(researchRuns).where(eq24(researchRuns.status, ResearchRunStatuses.queued)).all()) {
|
|
15090
15631
|
dispatchResearchRun(run.id, run.projectId);
|
|
15091
15632
|
}
|
|
15092
15633
|
app.addHook("onClose", async () => {
|