@ainyc/canonry 2.4.6 → 2.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/assets/agent-workspace/skills/canonry-setup/SKILL.md +3 -3
- package/assets/agent-workspace/skills/canonry-setup/references/canonry-cli.md +2 -0
- package/assets/agent-workspace/skills/canonry-setup/references/indexing.md +8 -0
- package/assets/assets/{index-Nrl3ecFY.js → index-agELvqT1.js} +14 -14
- package/assets/index.html +1 -1
- package/dist/{chunk-6UY2PETG.js → chunk-CFS35BKX.js} +443 -141
- package/dist/cli.js +90 -9
- package/dist/index.js +1 -1
- package/package.json +6 -6
|
@@ -346,11 +346,11 @@ function printCliError(err, format) {
|
|
|
346
346
|
|
|
347
347
|
// src/server.ts
|
|
348
348
|
import { createRequire as createRequire3 } from "module";
|
|
349
|
-
import
|
|
349
|
+
import crypto28 from "crypto";
|
|
350
350
|
import fs13 from "fs";
|
|
351
351
|
import path15 from "path";
|
|
352
352
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
353
|
-
import { eq as
|
|
353
|
+
import { eq as eq30 } from "drizzle-orm";
|
|
354
354
|
import Fastify from "fastify";
|
|
355
355
|
|
|
356
356
|
// ../contracts/src/config-schema.ts
|
|
@@ -957,6 +957,7 @@ var runKindSchema = z8.enum([
|
|
|
957
957
|
"inspect-sitemap",
|
|
958
958
|
"ga-sync",
|
|
959
959
|
"bing-inspect",
|
|
960
|
+
"bing-inspect-sitemap",
|
|
960
961
|
"backlink-extract"
|
|
961
962
|
]);
|
|
962
963
|
var RunKinds = runKindSchema.enum;
|
|
@@ -5072,6 +5073,31 @@ var routeCatalog = [
|
|
|
5072
5073
|
404: { description: "Project or connection not found." }
|
|
5073
5074
|
}
|
|
5074
5075
|
},
|
|
5076
|
+
{
|
|
5077
|
+
method: "post",
|
|
5078
|
+
path: "/api/v1/projects/{name}/bing/inspect-sitemap",
|
|
5079
|
+
summary: "Inspect every URL in a sitemap through Bing Webmaster Tools",
|
|
5080
|
+
tags: ["bing"],
|
|
5081
|
+
parameters: [nameParameter],
|
|
5082
|
+
requestBody: {
|
|
5083
|
+
required: false,
|
|
5084
|
+
content: {
|
|
5085
|
+
"application/json": {
|
|
5086
|
+
schema: {
|
|
5087
|
+
type: "object",
|
|
5088
|
+
properties: {
|
|
5089
|
+
sitemapUrl: stringSchema
|
|
5090
|
+
}
|
|
5091
|
+
}
|
|
5092
|
+
}
|
|
5093
|
+
}
|
|
5094
|
+
},
|
|
5095
|
+
responses: {
|
|
5096
|
+
200: { description: "Sitemap inspection run queued." },
|
|
5097
|
+
400: { description: "Bing is not configured for this project." },
|
|
5098
|
+
404: { description: "Project not found." }
|
|
5099
|
+
}
|
|
5100
|
+
},
|
|
5075
5101
|
{
|
|
5076
5102
|
method: "post",
|
|
5077
5103
|
path: "/api/v1/projects/{name}/bing/request-indexing",
|
|
@@ -8509,6 +8535,32 @@ async function bingRoutes(app, opts) {
|
|
|
8509
8535
|
throw e;
|
|
8510
8536
|
}
|
|
8511
8537
|
});
|
|
8538
|
+
app.post("/projects/:name/bing/inspect-sitemap", async (request) => {
|
|
8539
|
+
const store = requireConnectionStore();
|
|
8540
|
+
const project = resolveProject(app.db, request.params.name);
|
|
8541
|
+
const conn = requireConnection(store, project.canonicalDomain);
|
|
8542
|
+
if (!conn.siteUrl) {
|
|
8543
|
+
throw validationError('No Bing site configured. Run "canonry bing set-site <project> <url>" first.');
|
|
8544
|
+
}
|
|
8545
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8546
|
+
const runId = crypto15.randomUUID();
|
|
8547
|
+
app.db.insert(runs).values({
|
|
8548
|
+
id: runId,
|
|
8549
|
+
projectId: project.id,
|
|
8550
|
+
kind: RunKinds["bing-inspect-sitemap"],
|
|
8551
|
+
status: RunStatuses.queued,
|
|
8552
|
+
trigger: RunTriggers.manual,
|
|
8553
|
+
createdAt: now
|
|
8554
|
+
}).run();
|
|
8555
|
+
const { sitemapUrl } = request.body ?? {};
|
|
8556
|
+
if (opts.onInspectSitemapRequested) {
|
|
8557
|
+
opts.onInspectSitemapRequested(runId, project.id, { sitemapUrl: sitemapUrl ?? void 0 });
|
|
8558
|
+
} else {
|
|
8559
|
+
bingLog("warn", "inspect-sitemap.no-callback", { domain: project.canonicalDomain, runId });
|
|
8560
|
+
}
|
|
8561
|
+
const run = app.db.select().from(runs).where(eq15(runs.id, runId)).get();
|
|
8562
|
+
return run;
|
|
8563
|
+
});
|
|
8512
8564
|
app.post("/projects/:name/bing/request-indexing", async (request) => {
|
|
8513
8565
|
const store = requireConnectionStore();
|
|
8514
8566
|
const project = resolveProject(app.db, request.params.name);
|
|
@@ -11693,7 +11745,8 @@ async function apiRoutes(app, opts) {
|
|
|
11693
11745
|
setTelemetryEnabled: opts.setTelemetryEnabled
|
|
11694
11746
|
});
|
|
11695
11747
|
await api.register(bingRoutes, {
|
|
11696
|
-
bingConnectionStore: opts.bingConnectionStore
|
|
11748
|
+
bingConnectionStore: opts.bingConnectionStore,
|
|
11749
|
+
onInspectSitemapRequested: opts.onBingInspectSitemapRequested
|
|
11697
11750
|
});
|
|
11698
11751
|
await api.register(googleRoutes, {
|
|
11699
11752
|
getGoogleAuthConfig: opts.getGoogleAuthConfig,
|
|
@@ -14945,6 +14998,7 @@ import crypto21 from "crypto";
|
|
|
14945
14998
|
import { eq as eq21, and as and10 } from "drizzle-orm";
|
|
14946
14999
|
|
|
14947
15000
|
// src/sitemap-parser.ts
|
|
15001
|
+
var log3 = createLogger("SitemapParser");
|
|
14948
15002
|
var LOC_REGEX = /<loc>\s*([^<]+?)\s*<\/loc>/gi;
|
|
14949
15003
|
var SITEMAP_TAG_REGEX = /<sitemap>[\s\S]*?<\/sitemap>/gi;
|
|
14950
15004
|
var PRIVATE_IP_PATTERNS = [
|
|
@@ -14974,26 +15028,77 @@ function validateSitemapUrl(url) {
|
|
|
14974
15028
|
}
|
|
14975
15029
|
}
|
|
14976
15030
|
}
|
|
15031
|
+
async function readSitemapBody(res) {
|
|
15032
|
+
const buf = await res.arrayBuffer();
|
|
15033
|
+
const bytes = new Uint8Array(buf);
|
|
15034
|
+
const isGzipped = bytes.length >= 2 && bytes[0] === 31 && bytes[1] === 139;
|
|
15035
|
+
if (!isGzipped) {
|
|
15036
|
+
return new TextDecoder().decode(bytes);
|
|
15037
|
+
}
|
|
15038
|
+
const decompressed = new Blob([buf]).stream().pipeThrough(new DecompressionStream("gzip"));
|
|
15039
|
+
return new Response(decompressed).text();
|
|
15040
|
+
}
|
|
14977
15041
|
async function fetchAndParseSitemap(sitemapUrl) {
|
|
14978
15042
|
const urls = /* @__PURE__ */ new Set();
|
|
14979
|
-
|
|
15043
|
+
const visited = /* @__PURE__ */ new Set();
|
|
15044
|
+
await parseSitemapRecursive(
|
|
15045
|
+
sitemapUrl,
|
|
15046
|
+
urls,
|
|
15047
|
+
visited,
|
|
15048
|
+
0,
|
|
15049
|
+
/* isChild */
|
|
15050
|
+
false
|
|
15051
|
+
);
|
|
14980
15052
|
return [...urls];
|
|
14981
15053
|
}
|
|
14982
|
-
async function parseSitemapRecursive(url, urls, depth) {
|
|
15054
|
+
async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
14983
15055
|
if (depth > 3) return;
|
|
15056
|
+
if (visited.has(url)) return;
|
|
15057
|
+
visited.add(url);
|
|
14984
15058
|
validateSitemapUrl(url);
|
|
14985
|
-
|
|
15059
|
+
let res;
|
|
15060
|
+
try {
|
|
15061
|
+
res = await fetch(url);
|
|
15062
|
+
} catch (err) {
|
|
15063
|
+
if (!isChild) throw err;
|
|
15064
|
+
log3.warn("child-sitemap.fetch-failed", {
|
|
15065
|
+
url,
|
|
15066
|
+
error: err instanceof Error ? err.message : String(err)
|
|
15067
|
+
});
|
|
15068
|
+
return;
|
|
15069
|
+
}
|
|
14986
15070
|
if (!res.ok) {
|
|
14987
|
-
|
|
15071
|
+
if (!isChild) {
|
|
15072
|
+
throw new Error(`Failed to fetch sitemap at ${url}: ${res.status} ${res.statusText}`);
|
|
15073
|
+
}
|
|
15074
|
+
log3.warn("child-sitemap.http-error", { url, status: res.status, statusText: res.statusText });
|
|
15075
|
+
return;
|
|
15076
|
+
}
|
|
15077
|
+
let xml;
|
|
15078
|
+
try {
|
|
15079
|
+
xml = await readSitemapBody(res);
|
|
15080
|
+
} catch (err) {
|
|
15081
|
+
if (!isChild) throw err;
|
|
15082
|
+
log3.warn("child-sitemap.parse-failed", {
|
|
15083
|
+
url,
|
|
15084
|
+
error: err instanceof Error ? err.message : String(err)
|
|
15085
|
+
});
|
|
15086
|
+
return;
|
|
14988
15087
|
}
|
|
14989
|
-
const xml = await res.text();
|
|
14990
15088
|
const sitemapEntries = xml.match(SITEMAP_TAG_REGEX);
|
|
14991
15089
|
if (sitemapEntries) {
|
|
14992
15090
|
for (const entry of sitemapEntries) {
|
|
14993
15091
|
const locMatch = LOC_REGEX.exec(entry);
|
|
14994
15092
|
LOC_REGEX.lastIndex = 0;
|
|
14995
15093
|
if (locMatch?.[1]) {
|
|
14996
|
-
await parseSitemapRecursive(
|
|
15094
|
+
await parseSitemapRecursive(
|
|
15095
|
+
locMatch[1],
|
|
15096
|
+
urls,
|
|
15097
|
+
visited,
|
|
15098
|
+
depth + 1,
|
|
15099
|
+
/* isChild */
|
|
15100
|
+
true
|
|
15101
|
+
);
|
|
14997
15102
|
}
|
|
14998
15103
|
}
|
|
14999
15104
|
return;
|
|
@@ -15008,7 +15113,7 @@ async function parseSitemapRecursive(url, urls, depth) {
|
|
|
15008
15113
|
}
|
|
15009
15114
|
|
|
15010
15115
|
// src/gsc-inspect-sitemap.ts
|
|
15011
|
-
var
|
|
15116
|
+
var log4 = createLogger("InspectSitemap");
|
|
15012
15117
|
async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
15013
15118
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
15014
15119
|
db.update(runs).set({ status: "running", startedAt: now }).where(eq21(runs.id, runId)).run();
|
|
@@ -15041,9 +15146,9 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
15041
15146
|
saveConfigPatch(opts.config);
|
|
15042
15147
|
}
|
|
15043
15148
|
const sitemapUrl = opts.sitemapUrl || conn.sitemapUrl || `https://${project.canonicalDomain}/sitemap.xml`;
|
|
15044
|
-
|
|
15149
|
+
log4.info("sitemap.fetch", { runId, projectId, sitemapUrl });
|
|
15045
15150
|
const urls = await fetchAndParseSitemap(sitemapUrl);
|
|
15046
|
-
|
|
15151
|
+
log4.info("sitemap.parsed", { runId, projectId, urlCount: urls.length, sitemapUrl });
|
|
15047
15152
|
if (urls.length === 0) {
|
|
15048
15153
|
throw new Error("No URLs found in sitemap");
|
|
15049
15154
|
}
|
|
@@ -15076,10 +15181,10 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
15076
15181
|
createdAt: inspectedAt
|
|
15077
15182
|
}).run();
|
|
15078
15183
|
inspected++;
|
|
15079
|
-
|
|
15184
|
+
log4.info("inspect.url-done", { runId, projectId, url: pageUrl, progress: `${inspected}/${urls.length}` });
|
|
15080
15185
|
} catch (err) {
|
|
15081
15186
|
errors++;
|
|
15082
|
-
|
|
15187
|
+
log4.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
|
|
15083
15188
|
}
|
|
15084
15189
|
if (inspected + errors < urls.length) {
|
|
15085
15190
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
@@ -15119,20 +15224,206 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
15119
15224
|
}).run();
|
|
15120
15225
|
const status = errors > 0 && inspected > 0 ? "partial" : errors === urls.length ? "failed" : "completed";
|
|
15121
15226
|
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq21(runs.id, runId)).run();
|
|
15122
|
-
|
|
15227
|
+
log4.info("inspect.completed", { runId, projectId, inspected, errors, total: urls.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
15123
15228
|
} catch (err) {
|
|
15124
15229
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
15125
15230
|
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq21(runs.id, runId)).run();
|
|
15126
|
-
|
|
15231
|
+
log4.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
15127
15232
|
throw err;
|
|
15128
15233
|
}
|
|
15129
15234
|
}
|
|
15130
15235
|
|
|
15131
|
-
// src/
|
|
15236
|
+
// src/bing-inspect-sitemap.ts
|
|
15132
15237
|
import crypto22 from "crypto";
|
|
15238
|
+
import { eq as eq22, desc as desc9 } from "drizzle-orm";
|
|
15239
|
+
var log5 = createLogger("BingInspectSitemap");
|
|
15240
|
+
function parseBingDate2(value) {
|
|
15241
|
+
if (!value) return null;
|
|
15242
|
+
const match = /\/Date\((-?\d+)[^)]*\)\//.exec(value);
|
|
15243
|
+
if (!match) return null;
|
|
15244
|
+
const ms = parseInt(match[1], 10);
|
|
15245
|
+
if (ms <= 0) return null;
|
|
15246
|
+
return new Date(ms).toISOString();
|
|
15247
|
+
}
|
|
15248
|
+
function isBlockingIssueType2(issueType) {
|
|
15249
|
+
if (!issueType) return true;
|
|
15250
|
+
const trimmed = issueType.trim();
|
|
15251
|
+
if (!trimmed) return true;
|
|
15252
|
+
return trimmed.split(/\s+/).some((flag) => !/^(None|Seo(Issues|Concerns))$/i.test(flag));
|
|
15253
|
+
}
|
|
15254
|
+
async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
15255
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
15256
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq22(runs.id, runId)).run();
|
|
15257
|
+
try {
|
|
15258
|
+
const project = db.select().from(projects).where(eq22(projects.id, projectId)).get();
|
|
15259
|
+
if (!project) {
|
|
15260
|
+
throw new Error(`Project not found: ${projectId}`);
|
|
15261
|
+
}
|
|
15262
|
+
const conn = opts.config.bing?.connections?.find((c) => c.domain === project.canonicalDomain);
|
|
15263
|
+
if (!conn) {
|
|
15264
|
+
throw new Error('No Bing connection found for this project. Run "canonry bing connect <project>" first.');
|
|
15265
|
+
}
|
|
15266
|
+
if (!conn.siteUrl) {
|
|
15267
|
+
throw new Error('No Bing site configured. Run "canonry bing set-site <project> <url>" first.');
|
|
15268
|
+
}
|
|
15269
|
+
const sitemapUrl = opts.sitemapUrl ?? `https://${project.canonicalDomain}/sitemap.xml`;
|
|
15270
|
+
log5.info("sitemap.fetch", { runId, projectId, sitemapUrl });
|
|
15271
|
+
const sitemapUrls = await fetchAndParseSitemap(sitemapUrl);
|
|
15272
|
+
log5.info("sitemap.parsed", { runId, projectId, urlCount: sitemapUrls.length, sitemapUrl });
|
|
15273
|
+
if (sitemapUrls.length === 0) {
|
|
15274
|
+
throw new Error("No URLs found in sitemap");
|
|
15275
|
+
}
|
|
15276
|
+
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(eq22(bingUrlInspections.projectId, projectId)).all();
|
|
15277
|
+
const trackedUrls = new Set(trackedRows.map((r) => r.url));
|
|
15278
|
+
const discovered = sitemapUrls.filter((u) => !trackedUrls.has(u));
|
|
15279
|
+
log5.info("sitemap.diff", {
|
|
15280
|
+
runId,
|
|
15281
|
+
projectId,
|
|
15282
|
+
sitemapTotal: sitemapUrls.length,
|
|
15283
|
+
alreadyTracked: sitemapUrls.length - discovered.length,
|
|
15284
|
+
newlyDiscovered: discovered.length
|
|
15285
|
+
});
|
|
15286
|
+
let blockedUrls = /* @__PURE__ */ new Set();
|
|
15287
|
+
try {
|
|
15288
|
+
const issues = await getCrawlIssues(conn.apiKey, conn.siteUrl);
|
|
15289
|
+
for (const issue of issues) {
|
|
15290
|
+
if (issue.Url && isBlockingIssueType2(issue.IssueType ?? null)) {
|
|
15291
|
+
blockedUrls.add(issue.Url);
|
|
15292
|
+
}
|
|
15293
|
+
}
|
|
15294
|
+
log5.info("crawl-issues.loaded", { runId, projectId, blockedCount: blockedUrls.size });
|
|
15295
|
+
} catch (err) {
|
|
15296
|
+
log5.warn("crawl-issues.lookup-failed", {
|
|
15297
|
+
runId,
|
|
15298
|
+
projectId,
|
|
15299
|
+
error: err instanceof Error ? err.message : String(err)
|
|
15300
|
+
});
|
|
15301
|
+
blockedUrls = /* @__PURE__ */ new Set();
|
|
15302
|
+
}
|
|
15303
|
+
let inspected = 0;
|
|
15304
|
+
let errors = 0;
|
|
15305
|
+
for (const pageUrl of sitemapUrls) {
|
|
15306
|
+
try {
|
|
15307
|
+
const result = await getUrlInfo(conn.apiKey, conn.siteUrl, pageUrl);
|
|
15308
|
+
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
15309
|
+
const httpCode = result.HttpStatus ?? result.HttpCode ?? null;
|
|
15310
|
+
const lastCrawledDate = parseBingDate2(result.LastCrawledDate);
|
|
15311
|
+
const inIndexDate = parseBingDate2(result.InIndexDate);
|
|
15312
|
+
const discoveryDate = parseBingDate2(result.DiscoveryDate);
|
|
15313
|
+
let derivedInIndex = null;
|
|
15314
|
+
if (result.DocumentSize != null && result.DocumentSize > 0) {
|
|
15315
|
+
derivedInIndex = true;
|
|
15316
|
+
} else if (lastCrawledDate != null) {
|
|
15317
|
+
derivedInIndex = httpCode != null && httpCode >= 400 ? false : true;
|
|
15318
|
+
} else if (discoveryDate != null) {
|
|
15319
|
+
derivedInIndex = false;
|
|
15320
|
+
}
|
|
15321
|
+
if (derivedInIndex === true && blockedUrls.has(pageUrl)) {
|
|
15322
|
+
derivedInIndex = false;
|
|
15323
|
+
}
|
|
15324
|
+
db.insert(bingUrlInspections).values({
|
|
15325
|
+
id: crypto22.randomUUID(),
|
|
15326
|
+
projectId,
|
|
15327
|
+
url: pageUrl,
|
|
15328
|
+
httpCode,
|
|
15329
|
+
inIndex: derivedInIndex === true ? 1 : derivedInIndex === false ? 0 : null,
|
|
15330
|
+
lastCrawledDate,
|
|
15331
|
+
inIndexDate,
|
|
15332
|
+
inspectedAt,
|
|
15333
|
+
syncRunId: runId,
|
|
15334
|
+
createdAt: inspectedAt,
|
|
15335
|
+
documentSize: result.DocumentSize ?? null,
|
|
15336
|
+
anchorCount: result.AnchorCount ?? null,
|
|
15337
|
+
discoveryDate
|
|
15338
|
+
}).run();
|
|
15339
|
+
inspected++;
|
|
15340
|
+
log5.info("inspect.url-done", {
|
|
15341
|
+
runId,
|
|
15342
|
+
projectId,
|
|
15343
|
+
url: pageUrl,
|
|
15344
|
+
progress: `${inspected}/${sitemapUrls.length}`
|
|
15345
|
+
});
|
|
15346
|
+
} catch (err) {
|
|
15347
|
+
errors++;
|
|
15348
|
+
log5.error("inspect.url-failed", {
|
|
15349
|
+
runId,
|
|
15350
|
+
projectId,
|
|
15351
|
+
url: pageUrl,
|
|
15352
|
+
error: err instanceof Error ? err.message : String(err)
|
|
15353
|
+
});
|
|
15354
|
+
}
|
|
15355
|
+
if (inspected + errors < sitemapUrls.length) {
|
|
15356
|
+
await new Promise((r) => setTimeout(r, 1e3));
|
|
15357
|
+
}
|
|
15358
|
+
}
|
|
15359
|
+
const allInspections = db.select().from(bingUrlInspections).where(eq22(bingUrlInspections.projectId, projectId)).orderBy(desc9(bingUrlInspections.inspectedAt)).all();
|
|
15360
|
+
const latestByUrl = /* @__PURE__ */ new Map();
|
|
15361
|
+
const definitiveByUrl = /* @__PURE__ */ new Map();
|
|
15362
|
+
for (const row of allInspections) {
|
|
15363
|
+
if (!latestByUrl.has(row.url)) latestByUrl.set(row.url, row);
|
|
15364
|
+
if (!definitiveByUrl.has(row.url) && row.inIndex != null) definitiveByUrl.set(row.url, row);
|
|
15365
|
+
}
|
|
15366
|
+
for (const [url, latest] of latestByUrl) {
|
|
15367
|
+
if (latest.inIndex == null) {
|
|
15368
|
+
const def = definitiveByUrl.get(url);
|
|
15369
|
+
if (def) latestByUrl.set(url, def);
|
|
15370
|
+
}
|
|
15371
|
+
}
|
|
15372
|
+
let snapIndexed = 0;
|
|
15373
|
+
let snapNotIndexed = 0;
|
|
15374
|
+
let snapUnknown = 0;
|
|
15375
|
+
for (const [, row] of latestByUrl) {
|
|
15376
|
+
if (row.inIndex === 1) snapIndexed++;
|
|
15377
|
+
else if (row.inIndex === 0) snapNotIndexed++;
|
|
15378
|
+
else snapUnknown++;
|
|
15379
|
+
}
|
|
15380
|
+
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
15381
|
+
const snapNow = (/* @__PURE__ */ new Date()).toISOString();
|
|
15382
|
+
db.insert(bingCoverageSnapshots).values({
|
|
15383
|
+
id: crypto22.randomUUID(),
|
|
15384
|
+
projectId,
|
|
15385
|
+
syncRunId: runId,
|
|
15386
|
+
date: snapshotDate,
|
|
15387
|
+
indexed: snapIndexed,
|
|
15388
|
+
notIndexed: snapNotIndexed,
|
|
15389
|
+
unknown: snapUnknown,
|
|
15390
|
+
createdAt: snapNow
|
|
15391
|
+
}).onConflictDoUpdate({
|
|
15392
|
+
target: [bingCoverageSnapshots.projectId, bingCoverageSnapshots.date],
|
|
15393
|
+
set: {
|
|
15394
|
+
indexed: snapIndexed,
|
|
15395
|
+
notIndexed: snapNotIndexed,
|
|
15396
|
+
unknown: snapUnknown,
|
|
15397
|
+
createdAt: snapNow,
|
|
15398
|
+
syncRunId: runId
|
|
15399
|
+
}
|
|
15400
|
+
}).run();
|
|
15401
|
+
const status = errors === sitemapUrls.length ? RunStatuses.failed : errors > 0 ? RunStatuses.partial : RunStatuses.completed;
|
|
15402
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq22(runs.id, runId)).run();
|
|
15403
|
+
log5.info("inspect.completed", {
|
|
15404
|
+
runId,
|
|
15405
|
+
projectId,
|
|
15406
|
+
inspected,
|
|
15407
|
+
errors,
|
|
15408
|
+
total: sitemapUrls.length,
|
|
15409
|
+
newlyDiscovered: discovered.length,
|
|
15410
|
+
indexed: snapIndexed,
|
|
15411
|
+
notIndexed: snapNotIndexed,
|
|
15412
|
+
unknown: snapUnknown
|
|
15413
|
+
});
|
|
15414
|
+
} catch (err) {
|
|
15415
|
+
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
15416
|
+
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq22(runs.id, runId)).run();
|
|
15417
|
+
log5.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
15418
|
+
throw err;
|
|
15419
|
+
}
|
|
15420
|
+
}
|
|
15421
|
+
|
|
15422
|
+
// src/commoncrawl-sync.ts
|
|
15423
|
+
import crypto23 from "crypto";
|
|
15133
15424
|
import path11 from "path";
|
|
15134
|
-
import { and as and11, eq as
|
|
15135
|
-
var
|
|
15425
|
+
import { and as and11, eq as eq23, sql as sql8 } from "drizzle-orm";
|
|
15426
|
+
var log6 = createLogger("CommonCrawlSync");
|
|
15136
15427
|
var INSERT_CHUNK_SIZE = 1e4;
|
|
15137
15428
|
function defaultDeps() {
|
|
15138
15429
|
return {
|
|
@@ -15157,7 +15448,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
15157
15448
|
phaseDetail: "downloading vertices + edges",
|
|
15158
15449
|
updatedAt: downloadStartedAt,
|
|
15159
15450
|
error: null
|
|
15160
|
-
}).where(
|
|
15451
|
+
}).where(eq23(ccReleaseSyncs.id, syncId)).run();
|
|
15161
15452
|
const paths = ccReleasePaths(release);
|
|
15162
15453
|
const releaseCacheDir = path11.join(deps.cacheDir, release);
|
|
15163
15454
|
const vertexPath = path11.join(releaseCacheDir, paths.vertexFilename);
|
|
@@ -15180,7 +15471,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
15180
15471
|
vertexSha256: vertex.sha256,
|
|
15181
15472
|
edgesSha256: edges.sha256,
|
|
15182
15473
|
updatedAt: downloadFinishedAt
|
|
15183
|
-
}).where(
|
|
15474
|
+
}).where(eq23(ccReleaseSyncs.id, syncId)).run();
|
|
15184
15475
|
const allProjects = db.select().from(projects).all();
|
|
15185
15476
|
const targets = Array.from(new Set(allProjects.map((p) => p.canonicalDomain)));
|
|
15186
15477
|
let rows = [];
|
|
@@ -15196,15 +15487,15 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
15196
15487
|
}
|
|
15197
15488
|
const queriedAt = deps.now().toISOString();
|
|
15198
15489
|
db.transaction((tx) => {
|
|
15199
|
-
tx.delete(backlinkDomains).where(
|
|
15200
|
-
tx.delete(backlinkSummaries).where(
|
|
15490
|
+
tx.delete(backlinkDomains).where(eq23(backlinkDomains.releaseSyncId, syncId)).run();
|
|
15491
|
+
tx.delete(backlinkSummaries).where(eq23(backlinkSummaries.releaseSyncId, syncId)).run();
|
|
15201
15492
|
const expanded = [];
|
|
15202
15493
|
for (const r of rows) {
|
|
15203
15494
|
const projectIds = projectsByDomain.get(r.targetDomain);
|
|
15204
15495
|
if (!projectIds) continue;
|
|
15205
15496
|
for (const projectId of projectIds) {
|
|
15206
15497
|
expanded.push({
|
|
15207
|
-
id:
|
|
15498
|
+
id: crypto23.randomUUID(),
|
|
15208
15499
|
projectId,
|
|
15209
15500
|
releaseSyncId: syncId,
|
|
15210
15501
|
release,
|
|
@@ -15224,7 +15515,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
15224
15515
|
const projectRows = rowsByProject.get(p.id) ?? [];
|
|
15225
15516
|
const summary = computeSummary(projectRows);
|
|
15226
15517
|
tx.insert(backlinkSummaries).values({
|
|
15227
|
-
id:
|
|
15518
|
+
id: crypto23.randomUUID(),
|
|
15228
15519
|
projectId: p.id,
|
|
15229
15520
|
releaseSyncId: syncId,
|
|
15230
15521
|
release,
|
|
@@ -15256,8 +15547,8 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
15256
15547
|
domainsDiscovered: rows.length,
|
|
15257
15548
|
updatedAt: finishedAt,
|
|
15258
15549
|
error: null
|
|
15259
|
-
}).where(
|
|
15260
|
-
|
|
15550
|
+
}).where(eq23(ccReleaseSyncs.id, syncId)).run();
|
|
15551
|
+
log6.info("sync.completed", {
|
|
15261
15552
|
syncId,
|
|
15262
15553
|
release,
|
|
15263
15554
|
projectsProcessed: allProjects.length,
|
|
@@ -15269,7 +15560,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
15269
15560
|
try {
|
|
15270
15561
|
deps.enqueueAutoExtract({ projectId: p.id, release });
|
|
15271
15562
|
} catch (err) {
|
|
15272
|
-
|
|
15563
|
+
log6.error("auto-extract.enqueue-failed", {
|
|
15273
15564
|
syncId,
|
|
15274
15565
|
release,
|
|
15275
15566
|
projectId: p.id,
|
|
@@ -15286,8 +15577,8 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
15286
15577
|
error: errorMsg,
|
|
15287
15578
|
phaseDetail: null,
|
|
15288
15579
|
updatedAt: finishedAt
|
|
15289
|
-
}).where(
|
|
15290
|
-
|
|
15580
|
+
}).where(eq23(ccReleaseSyncs.id, syncId)).run();
|
|
15581
|
+
log6.error("sync.failed", { syncId, release, error: errorMsg });
|
|
15291
15582
|
throw err;
|
|
15292
15583
|
}
|
|
15293
15584
|
}
|
|
@@ -15320,10 +15611,10 @@ function computeSummary(rows) {
|
|
|
15320
15611
|
}
|
|
15321
15612
|
|
|
15322
15613
|
// src/backlink-extract.ts
|
|
15323
|
-
import
|
|
15614
|
+
import crypto24 from "crypto";
|
|
15324
15615
|
import fs9 from "fs";
|
|
15325
|
-
import { and as and12, desc as
|
|
15326
|
-
var
|
|
15616
|
+
import { and as and12, desc as desc10, eq as eq24 } from "drizzle-orm";
|
|
15617
|
+
var log7 = createLogger("BacklinkExtract");
|
|
15327
15618
|
function defaultDeps2() {
|
|
15328
15619
|
return {
|
|
15329
15620
|
queryBacklinks,
|
|
@@ -15334,13 +15625,13 @@ function defaultDeps2() {
|
|
|
15334
15625
|
async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
15335
15626
|
const deps = { ...defaultDeps2(), ...opts.deps };
|
|
15336
15627
|
const startedAt = deps.now().toISOString();
|
|
15337
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
15628
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq24(runs.id, runId)).run();
|
|
15338
15629
|
try {
|
|
15339
|
-
const project = db.select().from(projects).where(
|
|
15630
|
+
const project = db.select().from(projects).where(eq24(projects.id, projectId)).get();
|
|
15340
15631
|
if (!project) {
|
|
15341
15632
|
throw new Error(`Project not found: ${projectId}`);
|
|
15342
15633
|
}
|
|
15343
|
-
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(
|
|
15634
|
+
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(eq24(ccReleaseSyncs.release, opts.release)).get() : db.select().from(ccReleaseSyncs).where(eq24(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).orderBy(desc10(ccReleaseSyncs.createdAt)).limit(1).get();
|
|
15344
15635
|
if (!sync) {
|
|
15345
15636
|
throw new Error("No ready release sync available \u2014 run `canonry backlinks sync` first");
|
|
15346
15637
|
}
|
|
@@ -15368,11 +15659,11 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
15368
15659
|
const targetDomain = project.canonicalDomain;
|
|
15369
15660
|
db.transaction((tx) => {
|
|
15370
15661
|
tx.delete(backlinkDomains).where(
|
|
15371
|
-
and12(
|
|
15662
|
+
and12(eq24(backlinkDomains.projectId, projectId), eq24(backlinkDomains.release, release))
|
|
15372
15663
|
).run();
|
|
15373
15664
|
if (rows.length > 0) {
|
|
15374
15665
|
const values = rows.map((r) => ({
|
|
15375
|
-
id:
|
|
15666
|
+
id: crypto24.randomUUID(),
|
|
15376
15667
|
projectId,
|
|
15377
15668
|
releaseSyncId: syncId,
|
|
15378
15669
|
release,
|
|
@@ -15385,7 +15676,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
15385
15676
|
}
|
|
15386
15677
|
const summary = computeSummary2(rows);
|
|
15387
15678
|
tx.insert(backlinkSummaries).values({
|
|
15388
|
-
id:
|
|
15679
|
+
id: crypto24.randomUUID(),
|
|
15389
15680
|
projectId,
|
|
15390
15681
|
releaseSyncId: syncId,
|
|
15391
15682
|
release,
|
|
@@ -15408,8 +15699,8 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
15408
15699
|
}).run();
|
|
15409
15700
|
});
|
|
15410
15701
|
const finishedAt = deps.now().toISOString();
|
|
15411
|
-
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
15412
|
-
|
|
15702
|
+
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq24(runs.id, runId)).run();
|
|
15703
|
+
log7.info("extract.completed", { runId, projectId, release, rows: rows.length });
|
|
15413
15704
|
} catch (err) {
|
|
15414
15705
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
15415
15706
|
const finishedAt = deps.now().toISOString();
|
|
@@ -15417,8 +15708,8 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
15417
15708
|
status: RunStatuses.failed,
|
|
15418
15709
|
error: errorMsg,
|
|
15419
15710
|
finishedAt
|
|
15420
|
-
}).where(
|
|
15421
|
-
|
|
15711
|
+
}).where(eq24(runs.id, runId)).run();
|
|
15712
|
+
log7.error("extract.failed", { runId, projectId, error: errorMsg });
|
|
15422
15713
|
throw err;
|
|
15423
15714
|
}
|
|
15424
15715
|
}
|
|
@@ -15490,8 +15781,8 @@ var ProviderRegistry = class {
|
|
|
15490
15781
|
|
|
15491
15782
|
// src/scheduler.ts
|
|
15492
15783
|
import cron from "node-cron";
|
|
15493
|
-
import { eq as
|
|
15494
|
-
var
|
|
15784
|
+
import { eq as eq25 } from "drizzle-orm";
|
|
15785
|
+
var log8 = createLogger("Scheduler");
|
|
15495
15786
|
var Scheduler = class {
|
|
15496
15787
|
db;
|
|
15497
15788
|
callbacks;
|
|
@@ -15502,16 +15793,16 @@ var Scheduler = class {
|
|
|
15502
15793
|
}
|
|
15503
15794
|
/** Load all enabled schedules from DB and register cron jobs. */
|
|
15504
15795
|
start() {
|
|
15505
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
15796
|
+
const allSchedules = this.db.select().from(schedules).where(eq25(schedules.enabled, 1)).all();
|
|
15506
15797
|
for (const schedule of allSchedules) {
|
|
15507
15798
|
const missedRunAt = schedule.nextRunAt;
|
|
15508
15799
|
this.registerCronTask(schedule);
|
|
15509
15800
|
if (missedRunAt && new Date(missedRunAt) < /* @__PURE__ */ new Date()) {
|
|
15510
|
-
|
|
15801
|
+
log8.info("run.catch-up", { projectId: schedule.projectId, missedRunAt });
|
|
15511
15802
|
this.triggerRun(schedule.id, schedule.projectId);
|
|
15512
15803
|
}
|
|
15513
15804
|
}
|
|
15514
|
-
|
|
15805
|
+
log8.info("started", { scheduleCount: allSchedules.length });
|
|
15515
15806
|
}
|
|
15516
15807
|
/** Stop all cron tasks for graceful shutdown. */
|
|
15517
15808
|
stop() {
|
|
@@ -15527,7 +15818,7 @@ var Scheduler = class {
|
|
|
15527
15818
|
this.stopTask(projectId, existing, "Stopped");
|
|
15528
15819
|
this.tasks.delete(projectId);
|
|
15529
15820
|
}
|
|
15530
|
-
const schedule = this.db.select().from(schedules).where(
|
|
15821
|
+
const schedule = this.db.select().from(schedules).where(eq25(schedules.projectId, projectId)).get();
|
|
15531
15822
|
if (schedule && schedule.enabled === 1) {
|
|
15532
15823
|
this.registerCronTask(schedule);
|
|
15533
15824
|
}
|
|
@@ -15543,12 +15834,12 @@ var Scheduler = class {
|
|
|
15543
15834
|
stopTask(projectId, task, verb) {
|
|
15544
15835
|
task.stop();
|
|
15545
15836
|
task.destroy();
|
|
15546
|
-
|
|
15837
|
+
log8.info(`task.${verb.toLowerCase()}`, { projectId });
|
|
15547
15838
|
}
|
|
15548
15839
|
registerCronTask(schedule) {
|
|
15549
15840
|
const { id: scheduleId, projectId, cronExpr, timezone } = schedule;
|
|
15550
15841
|
if (!cron.validate(cronExpr)) {
|
|
15551
|
-
|
|
15842
|
+
log8.error("cron.invalid", { projectId, cronExpr });
|
|
15552
15843
|
return;
|
|
15553
15844
|
}
|
|
15554
15845
|
const task = cron.schedule(cronExpr, () => {
|
|
@@ -15560,24 +15851,24 @@ var Scheduler = class {
|
|
|
15560
15851
|
this.db.update(schedules).set({
|
|
15561
15852
|
nextRunAt: task.getNextRun()?.toISOString() ?? null,
|
|
15562
15853
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
15563
|
-
}).where(
|
|
15854
|
+
}).where(eq25(schedules.id, scheduleId)).run();
|
|
15564
15855
|
const label = schedule.preset ?? cronExpr;
|
|
15565
|
-
|
|
15856
|
+
log8.info("cron.registered", { projectId, schedule: label, timezone });
|
|
15566
15857
|
}
|
|
15567
15858
|
triggerRun(scheduleId, projectId) {
|
|
15568
15859
|
try {
|
|
15569
15860
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
15570
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
15861
|
+
const currentSchedule = this.db.select().from(schedules).where(eq25(schedules.id, scheduleId)).get();
|
|
15571
15862
|
if (!currentSchedule || currentSchedule.enabled !== 1) {
|
|
15572
|
-
|
|
15863
|
+
log8.warn("schedule.stale", { scheduleId, projectId, msg: "schedule no longer exists or is disabled" });
|
|
15573
15864
|
this.remove(projectId);
|
|
15574
15865
|
return;
|
|
15575
15866
|
}
|
|
15576
15867
|
const task = this.tasks.get(projectId);
|
|
15577
15868
|
const nextRunAt = task?.getNextRun()?.toISOString() ?? null;
|
|
15578
|
-
const project = this.db.select().from(projects).where(
|
|
15869
|
+
const project = this.db.select().from(projects).where(eq25(projects.id, projectId)).get();
|
|
15579
15870
|
if (!project) {
|
|
15580
|
-
|
|
15871
|
+
log8.error("project.not-found", { projectId, msg: "skipping scheduled run" });
|
|
15581
15872
|
this.remove(projectId);
|
|
15582
15873
|
return;
|
|
15583
15874
|
}
|
|
@@ -15586,7 +15877,7 @@ var Scheduler = class {
|
|
|
15586
15877
|
if (project.defaultLocation) {
|
|
15587
15878
|
const loc = projectLocations.find((l) => l.label === project.defaultLocation);
|
|
15588
15879
|
if (!loc) {
|
|
15589
|
-
|
|
15880
|
+
log8.warn("default-location.stale", { scheduleId, projectId, label: project.defaultLocation });
|
|
15590
15881
|
return;
|
|
15591
15882
|
}
|
|
15592
15883
|
resolvedLocation = loc;
|
|
@@ -15600,11 +15891,11 @@ var Scheduler = class {
|
|
|
15600
15891
|
location: locationLabel
|
|
15601
15892
|
});
|
|
15602
15893
|
if (queueResult.conflict) {
|
|
15603
|
-
|
|
15894
|
+
log8.info("run.skipped-active", { projectName: project.name, activeRunId: queueResult.activeRunId });
|
|
15604
15895
|
this.db.update(schedules).set({
|
|
15605
15896
|
nextRunAt,
|
|
15606
15897
|
updatedAt: now
|
|
15607
|
-
}).where(
|
|
15898
|
+
}).where(eq25(schedules.id, currentSchedule.id)).run();
|
|
15608
15899
|
return;
|
|
15609
15900
|
}
|
|
15610
15901
|
const runId = queueResult.runId;
|
|
@@ -15612,21 +15903,21 @@ var Scheduler = class {
|
|
|
15612
15903
|
lastRunAt: now,
|
|
15613
15904
|
nextRunAt,
|
|
15614
15905
|
updatedAt: now
|
|
15615
|
-
}).where(
|
|
15906
|
+
}).where(eq25(schedules.id, currentSchedule.id)).run();
|
|
15616
15907
|
const scheduleProviders = parseJsonColumn(currentSchedule.providers, []);
|
|
15617
15908
|
const providers = scheduleProviders.length > 0 ? scheduleProviders : void 0;
|
|
15618
|
-
|
|
15909
|
+
log8.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
15619
15910
|
this.callbacks.onRunCreated(runId, projectId, providers, resolvedLocation);
|
|
15620
15911
|
} catch (err) {
|
|
15621
|
-
|
|
15912
|
+
log8.error("trigger.error", { scheduleId, projectId, error: err instanceof Error ? err.message : String(err) });
|
|
15622
15913
|
}
|
|
15623
15914
|
}
|
|
15624
15915
|
};
|
|
15625
15916
|
|
|
15626
15917
|
// src/notifier.ts
|
|
15627
|
-
import { eq as
|
|
15628
|
-
import
|
|
15629
|
-
var
|
|
15918
|
+
import { eq as eq26, desc as desc11, and as and13, or as or2 } from "drizzle-orm";
|
|
15919
|
+
import crypto25 from "crypto";
|
|
15920
|
+
var log9 = createLogger("Notifier");
|
|
15630
15921
|
var Notifier = class {
|
|
15631
15922
|
db;
|
|
15632
15923
|
serverUrl;
|
|
@@ -15636,26 +15927,26 @@ var Notifier = class {
|
|
|
15636
15927
|
}
|
|
15637
15928
|
/** Called after a run completes (success, partial, or failed). */
|
|
15638
15929
|
async onRunCompleted(runId, projectId) {
|
|
15639
|
-
|
|
15640
|
-
const notifs = this.db.select().from(notifications).where(
|
|
15930
|
+
log9.info("run.completed", { runId, projectId });
|
|
15931
|
+
const notifs = this.db.select().from(notifications).where(eq26(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
15641
15932
|
if (notifs.length === 0) {
|
|
15642
|
-
|
|
15933
|
+
log9.info("notifications.none-enabled", { projectId });
|
|
15643
15934
|
return;
|
|
15644
15935
|
}
|
|
15645
|
-
|
|
15646
|
-
const run = this.db.select().from(runs).where(
|
|
15936
|
+
log9.info("notifications.found", { projectId, count: notifs.length });
|
|
15937
|
+
const run = this.db.select().from(runs).where(eq26(runs.id, runId)).get();
|
|
15647
15938
|
if (!run) {
|
|
15648
|
-
|
|
15939
|
+
log9.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
15649
15940
|
return;
|
|
15650
15941
|
}
|
|
15651
|
-
const project = this.db.select().from(projects).where(
|
|
15942
|
+
const project = this.db.select().from(projects).where(eq26(projects.id, projectId)).get();
|
|
15652
15943
|
if (!project) {
|
|
15653
|
-
|
|
15944
|
+
log9.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
15654
15945
|
return;
|
|
15655
15946
|
}
|
|
15656
15947
|
const transitions = this.computeTransitions(runId, projectId);
|
|
15657
15948
|
const events = [];
|
|
15658
|
-
|
|
15949
|
+
log9.info("run.status", { runId: run.id, status: run.status, projectId });
|
|
15659
15950
|
if (run.status === "completed" || run.status === "partial") {
|
|
15660
15951
|
events.push("run.completed");
|
|
15661
15952
|
}
|
|
@@ -15671,7 +15962,7 @@ var Notifier = class {
|
|
|
15671
15962
|
if (!config.url) continue;
|
|
15672
15963
|
const subscribedEvents = config.events;
|
|
15673
15964
|
const matchingEvents = events.filter((e) => subscribedEvents.includes(e));
|
|
15674
|
-
|
|
15965
|
+
log9.info("notification.match", { notificationId: notif.id, subscribedEvents, matchedEvents: matchingEvents });
|
|
15675
15966
|
if (matchingEvents.length === 0) continue;
|
|
15676
15967
|
for (const event of matchingEvents) {
|
|
15677
15968
|
const relevantTransitions = event === "citation.lost" ? lostTransitions : event === "citation.gained" ? gainedTransitions : transitions;
|
|
@@ -15695,11 +15986,11 @@ var Notifier = class {
|
|
|
15695
15986
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
15696
15987
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
15697
15988
|
if (insightEvents.length === 0) return;
|
|
15698
|
-
const notifs = this.db.select().from(notifications).where(
|
|
15989
|
+
const notifs = this.db.select().from(notifications).where(eq26(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
15699
15990
|
if (notifs.length === 0) return;
|
|
15700
|
-
const run = this.db.select().from(runs).where(
|
|
15991
|
+
const run = this.db.select().from(runs).where(eq26(runs.id, runId)).get();
|
|
15701
15992
|
if (!run) return;
|
|
15702
|
-
const project = this.db.select().from(projects).where(
|
|
15993
|
+
const project = this.db.select().from(projects).where(eq26(projects.id, projectId)).get();
|
|
15703
15994
|
if (!project) return;
|
|
15704
15995
|
for (const notif of notifs) {
|
|
15705
15996
|
const config = parseJsonColumn(notif.config, { url: "", events: [] });
|
|
@@ -15731,10 +16022,10 @@ var Notifier = class {
|
|
|
15731
16022
|
computeTransitions(runId, projectId) {
|
|
15732
16023
|
const recentRuns = this.db.select().from(runs).where(
|
|
15733
16024
|
and13(
|
|
15734
|
-
|
|
15735
|
-
or2(
|
|
16025
|
+
eq26(runs.projectId, projectId),
|
|
16026
|
+
or2(eq26(runs.status, "completed"), eq26(runs.status, "partial"))
|
|
15736
16027
|
)
|
|
15737
|
-
).orderBy(
|
|
16028
|
+
).orderBy(desc11(runs.createdAt)).limit(2).all();
|
|
15738
16029
|
if (recentRuns.length < 2) return [];
|
|
15739
16030
|
const currentRunId = recentRuns[0].id;
|
|
15740
16031
|
const previousRunId = recentRuns[1].id;
|
|
@@ -15744,12 +16035,12 @@ var Notifier = class {
|
|
|
15744
16035
|
keyword: keywords.keyword,
|
|
15745
16036
|
provider: querySnapshots.provider,
|
|
15746
16037
|
citationState: querySnapshots.citationState
|
|
15747
|
-
}).from(querySnapshots).leftJoin(keywords,
|
|
16038
|
+
}).from(querySnapshots).leftJoin(keywords, eq26(querySnapshots.keywordId, keywords.id)).where(eq26(querySnapshots.runId, currentRunId)).all();
|
|
15748
16039
|
const previousSnapshots = this.db.select({
|
|
15749
16040
|
keywordId: querySnapshots.keywordId,
|
|
15750
16041
|
provider: querySnapshots.provider,
|
|
15751
16042
|
citationState: querySnapshots.citationState
|
|
15752
|
-
}).from(querySnapshots).where(
|
|
16043
|
+
}).from(querySnapshots).where(eq26(querySnapshots.runId, previousRunId)).all();
|
|
15753
16044
|
const prevMap = /* @__PURE__ */ new Map();
|
|
15754
16045
|
for (const s of previousSnapshots) {
|
|
15755
16046
|
prevMap.set(`${s.keywordId}:${s.provider}`, s.citationState);
|
|
@@ -15773,23 +16064,23 @@ var Notifier = class {
|
|
|
15773
16064
|
const targetLabel = redactNotificationUrl(url).urlDisplay;
|
|
15774
16065
|
const targetCheck = await resolveWebhookTarget(url);
|
|
15775
16066
|
if (!targetCheck.ok) {
|
|
15776
|
-
|
|
16067
|
+
log9.error("webhook.ssrf-blocked", { url: targetLabel, reason: targetCheck.message });
|
|
15777
16068
|
this.logDelivery(projectId, notificationId, payload.event, "failed", `SSRF: ${targetCheck.message}`);
|
|
15778
16069
|
return;
|
|
15779
16070
|
}
|
|
15780
|
-
|
|
16071
|
+
log9.info("webhook.send", { event: payload.event, url: targetLabel });
|
|
15781
16072
|
const maxRetries = 3;
|
|
15782
16073
|
const delays = [1e3, 4e3, 16e3];
|
|
15783
16074
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
15784
16075
|
try {
|
|
15785
16076
|
const response = await deliverWebhook(targetCheck.target, payload, webhookSecret);
|
|
15786
16077
|
if (response.status >= 200 && response.status < 300) {
|
|
15787
|
-
|
|
16078
|
+
log9.info("webhook.delivered", { event: payload.event, url: targetLabel, httpStatus: response.status });
|
|
15788
16079
|
this.logDelivery(projectId, notificationId, payload.event, "sent", null);
|
|
15789
16080
|
return;
|
|
15790
16081
|
}
|
|
15791
16082
|
const errorDetail = response.error ?? `HTTP ${response.status}`;
|
|
15792
|
-
|
|
16083
|
+
log9.warn("webhook.attempt-failed", { event: payload.event, url: targetLabel, attempt: attempt + 1, maxRetries, httpStatus: response.status, error: errorDetail });
|
|
15793
16084
|
if (attempt === maxRetries - 1) {
|
|
15794
16085
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
15795
16086
|
}
|
|
@@ -15797,7 +16088,7 @@ var Notifier = class {
|
|
|
15797
16088
|
const errorDetail = err instanceof Error ? err.message : String(err);
|
|
15798
16089
|
if (attempt === maxRetries - 1) {
|
|
15799
16090
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
15800
|
-
|
|
16091
|
+
log9.error("webhook.exhausted", { event: payload.event, url: targetLabel, maxRetries, error: errorDetail });
|
|
15801
16092
|
}
|
|
15802
16093
|
}
|
|
15803
16094
|
if (attempt < maxRetries - 1) {
|
|
@@ -15807,7 +16098,7 @@ var Notifier = class {
|
|
|
15807
16098
|
}
|
|
15808
16099
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
15809
16100
|
this.db.insert(auditLog).values({
|
|
15810
|
-
id:
|
|
16101
|
+
id: crypto25.randomUUID(),
|
|
15811
16102
|
projectId,
|
|
15812
16103
|
actor: "scheduler",
|
|
15813
16104
|
action: `notification.${status}`,
|
|
@@ -15820,7 +16111,7 @@ var Notifier = class {
|
|
|
15820
16111
|
};
|
|
15821
16112
|
|
|
15822
16113
|
// src/run-coordinator.ts
|
|
15823
|
-
var
|
|
16114
|
+
var log10 = createLogger("RunCoordinator");
|
|
15824
16115
|
var RunCoordinator = class {
|
|
15825
16116
|
constructor(notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
15826
16117
|
this.notifier = notifier;
|
|
@@ -15842,31 +16133,31 @@ var RunCoordinator = class {
|
|
|
15842
16133
|
try {
|
|
15843
16134
|
await this.onInsightsGenerated(runId, projectId, result);
|
|
15844
16135
|
} catch (err) {
|
|
15845
|
-
|
|
16136
|
+
log10.error("insight-webhook.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
15846
16137
|
}
|
|
15847
16138
|
}
|
|
15848
16139
|
}
|
|
15849
16140
|
} catch (err) {
|
|
15850
|
-
|
|
16141
|
+
log10.error("intelligence.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
15851
16142
|
}
|
|
15852
16143
|
try {
|
|
15853
16144
|
await this.notifier.onRunCompleted(runId, projectId);
|
|
15854
16145
|
} catch (err) {
|
|
15855
|
-
|
|
16146
|
+
log10.error("notifier.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
15856
16147
|
}
|
|
15857
16148
|
if (this.onAeroEvent) {
|
|
15858
16149
|
try {
|
|
15859
16150
|
await this.onAeroEvent({ runId, projectId, insightCount, criticalOrHigh });
|
|
15860
16151
|
} catch (err) {
|
|
15861
|
-
|
|
16152
|
+
log10.error("aero.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
15862
16153
|
}
|
|
15863
16154
|
}
|
|
15864
16155
|
}
|
|
15865
16156
|
};
|
|
15866
16157
|
|
|
15867
16158
|
// src/agent/session-registry.ts
|
|
15868
|
-
import
|
|
15869
|
-
import { eq as
|
|
16159
|
+
import crypto27 from "crypto";
|
|
16160
|
+
import { eq as eq28 } from "drizzle-orm";
|
|
15870
16161
|
|
|
15871
16162
|
// src/agent/session.ts
|
|
15872
16163
|
import fs12 from "fs";
|
|
@@ -16085,8 +16376,8 @@ function buildSkillDocTools() {
|
|
|
16085
16376
|
import { Type as Type2 } from "@sinclair/typebox";
|
|
16086
16377
|
|
|
16087
16378
|
// src/agent/memory-store.ts
|
|
16088
|
-
import
|
|
16089
|
-
import { and as and14, desc as
|
|
16379
|
+
import crypto26 from "crypto";
|
|
16380
|
+
import { and as and14, desc as desc12, eq as eq27, like, sql as sql9 } from "drizzle-orm";
|
|
16090
16381
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
16091
16382
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
16092
16383
|
function rowToDto(row) {
|
|
@@ -16100,7 +16391,7 @@ function rowToDto(row) {
|
|
|
16100
16391
|
};
|
|
16101
16392
|
}
|
|
16102
16393
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
16103
|
-
const query = db.select().from(agentMemory).where(
|
|
16394
|
+
const query = db.select().from(agentMemory).where(eq27(agentMemory.projectId, projectId)).orderBy(desc12(agentMemory.updatedAt));
|
|
16104
16395
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
16105
16396
|
return rows.map(rowToDto);
|
|
16106
16397
|
}
|
|
@@ -16114,7 +16405,7 @@ function upsertMemoryEntry(db, args) {
|
|
|
16114
16405
|
throw new Error(`memory key prefix "${COMPACTION_KEY_PREFIX}" is reserved for compaction notes`);
|
|
16115
16406
|
}
|
|
16116
16407
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
16117
|
-
const id =
|
|
16408
|
+
const id = crypto26.randomUUID();
|
|
16118
16409
|
db.insert(agentMemory).values({
|
|
16119
16410
|
id,
|
|
16120
16411
|
projectId: args.projectId,
|
|
@@ -16131,12 +16422,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
16131
16422
|
updatedAt: now
|
|
16132
16423
|
}
|
|
16133
16424
|
}).run();
|
|
16134
|
-
const row = db.select().from(agentMemory).where(and14(
|
|
16425
|
+
const row = db.select().from(agentMemory).where(and14(eq27(agentMemory.projectId, args.projectId), eq27(agentMemory.key, args.key))).get();
|
|
16135
16426
|
if (!row) throw new Error("memory upsert produced no row");
|
|
16136
16427
|
return rowToDto(row);
|
|
16137
16428
|
}
|
|
16138
16429
|
function deleteMemoryEntry(db, projectId, key) {
|
|
16139
|
-
const result = db.delete(agentMemory).where(and14(
|
|
16430
|
+
const result = db.delete(agentMemory).where(and14(eq27(agentMemory.projectId, projectId), eq27(agentMemory.key, key))).run();
|
|
16140
16431
|
const changes = result.changes ?? 0;
|
|
16141
16432
|
return changes > 0;
|
|
16142
16433
|
}
|
|
@@ -16151,7 +16442,7 @@ function writeCompactionNote(db, args) {
|
|
|
16151
16442
|
}
|
|
16152
16443
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
16153
16444
|
const key = `${COMPACTION_KEY_PREFIX}${args.sessionId}:${now}`;
|
|
16154
|
-
const id =
|
|
16445
|
+
const id = crypto26.randomUUID();
|
|
16155
16446
|
let inserted;
|
|
16156
16447
|
db.transaction((tx) => {
|
|
16157
16448
|
tx.insert(agentMemory).values({
|
|
@@ -16166,15 +16457,15 @@ function writeCompactionNote(db, args) {
|
|
|
16166
16457
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
16167
16458
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
16168
16459
|
and14(
|
|
16169
|
-
|
|
16460
|
+
eq27(agentMemory.projectId, args.projectId),
|
|
16170
16461
|
like(agentMemory.key, `${sessionPrefix}%`)
|
|
16171
16462
|
)
|
|
16172
|
-
).orderBy(
|
|
16463
|
+
).orderBy(desc12(agentMemory.updatedAt)).all();
|
|
16173
16464
|
const stale = existing.slice(COMPACTION_NOTES_PER_SESSION).map((r) => r.id);
|
|
16174
16465
|
if (stale.length > 0) {
|
|
16175
16466
|
tx.delete(agentMemory).where(sql9`${agentMemory.id} IN (${sql9.join(stale.map((s) => sql9`${s}`), sql9`, `)})`).run();
|
|
16176
16467
|
}
|
|
16177
|
-
const row = tx.select().from(agentMemory).where(and14(
|
|
16468
|
+
const row = tx.select().from(agentMemory).where(and14(eq27(agentMemory.projectId, args.projectId), eq27(agentMemory.key, key))).get();
|
|
16178
16469
|
if (row) inserted = rowToDto(row);
|
|
16179
16470
|
});
|
|
16180
16471
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -16802,7 +17093,7 @@ async function compactMessages(args) {
|
|
|
16802
17093
|
}
|
|
16803
17094
|
|
|
16804
17095
|
// src/agent/session-registry.ts
|
|
16805
|
-
var
|
|
17096
|
+
var log11 = createLogger("SessionRegistry");
|
|
16806
17097
|
var MAX_HYDRATE_NOTES = 20;
|
|
16807
17098
|
var MAX_HYDRATE_BYTES = 32 * 1024;
|
|
16808
17099
|
function escapeMemoryFragment(value) {
|
|
@@ -16851,7 +17142,7 @@ var SessionRegistry = class {
|
|
|
16851
17142
|
modelProvider: effectiveProvider,
|
|
16852
17143
|
modelId: effectiveModelId,
|
|
16853
17144
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
16854
|
-
}).where(
|
|
17145
|
+
}).where(eq28(agentSessions.projectId, projectId)).run();
|
|
16855
17146
|
}
|
|
16856
17147
|
const agent2 = createAeroSession({
|
|
16857
17148
|
projectName,
|
|
@@ -17033,13 +17324,13 @@ ${lines.join("\n")}
|
|
|
17033
17324
|
agent.state.messages = result.messages;
|
|
17034
17325
|
agent.state.systemPrompt = this.buildHydratedSystemPrompt(projectId, row.systemPrompt);
|
|
17035
17326
|
this.save(projectName);
|
|
17036
|
-
|
|
17327
|
+
log11.info("compaction.completed", {
|
|
17037
17328
|
projectName,
|
|
17038
17329
|
removedCount: result.removedCount,
|
|
17039
17330
|
summaryBytes: Buffer.byteLength(result.summary, "utf8")
|
|
17040
17331
|
});
|
|
17041
17332
|
} catch (err) {
|
|
17042
|
-
|
|
17333
|
+
log11.error("compaction.failed", {
|
|
17043
17334
|
projectName,
|
|
17044
17335
|
error: err instanceof Error ? err.message : String(err)
|
|
17045
17336
|
});
|
|
@@ -17069,7 +17360,7 @@ ${lines.join("\n")}
|
|
|
17069
17360
|
modelProvider: nextProvider,
|
|
17070
17361
|
modelId: nextModelId,
|
|
17071
17362
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
17072
|
-
}).where(
|
|
17363
|
+
}).where(eq28(agentSessions.projectId, projectId)).run();
|
|
17073
17364
|
}
|
|
17074
17365
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
17075
17366
|
save(projectName) {
|
|
@@ -17136,7 +17427,7 @@ ${lines.join("\n")}
|
|
|
17136
17427
|
await agent.prompt(msgs);
|
|
17137
17428
|
this.save(projectName);
|
|
17138
17429
|
} catch (err) {
|
|
17139
|
-
|
|
17430
|
+
log11.error("drain.failed", {
|
|
17140
17431
|
projectName,
|
|
17141
17432
|
error: err instanceof Error ? err.message : String(err)
|
|
17142
17433
|
});
|
|
@@ -17231,17 +17522,17 @@ ${lines.join("\n")}
|
|
|
17231
17522
|
return id;
|
|
17232
17523
|
}
|
|
17233
17524
|
tryResolveProjectId(projectName) {
|
|
17234
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
17525
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq28(projects.name, projectName)).get();
|
|
17235
17526
|
return row?.id;
|
|
17236
17527
|
}
|
|
17237
17528
|
loadRow(projectId) {
|
|
17238
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
17529
|
+
const row = this.opts.db.select().from(agentSessions).where(eq28(agentSessions.projectId, projectId)).get();
|
|
17239
17530
|
return row ?? null;
|
|
17240
17531
|
}
|
|
17241
17532
|
insertRow(params) {
|
|
17242
17533
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
17243
17534
|
this.opts.db.insert(agentSessions).values({
|
|
17244
|
-
id:
|
|
17535
|
+
id: crypto27.randomUUID(),
|
|
17245
17536
|
projectId: params.projectId,
|
|
17246
17537
|
systemPrompt: params.systemPrompt,
|
|
17247
17538
|
modelProvider: params.provider ?? params.modelProvider ?? AgentProviderIds.claude,
|
|
@@ -17254,14 +17545,14 @@ ${lines.join("\n")}
|
|
|
17254
17545
|
}
|
|
17255
17546
|
updateRow(projectId, patch) {
|
|
17256
17547
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
17257
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
17548
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq28(agentSessions.projectId, projectId)).run();
|
|
17258
17549
|
}
|
|
17259
17550
|
};
|
|
17260
17551
|
|
|
17261
17552
|
// src/agent/agent-routes.ts
|
|
17262
|
-
import { eq as
|
|
17553
|
+
import { eq as eq29 } from "drizzle-orm";
|
|
17263
17554
|
function resolveProject2(db, name) {
|
|
17264
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
17555
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq29(projects.name, name)).get();
|
|
17265
17556
|
if (!row) throw notFound("project", name);
|
|
17266
17557
|
return row;
|
|
17267
17558
|
}
|
|
@@ -17270,7 +17561,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
17270
17561
|
"/projects/:name/agent/transcript",
|
|
17271
17562
|
async (request) => {
|
|
17272
17563
|
const project = resolveProject2(opts.db, request.params.name);
|
|
17273
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
17564
|
+
const row = opts.db.select().from(agentSessions).where(eq29(agentSessions.projectId, project.id)).get();
|
|
17274
17565
|
if (!row) {
|
|
17275
17566
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
17276
17567
|
}
|
|
@@ -17294,7 +17585,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
17294
17585
|
async (request) => {
|
|
17295
17586
|
const project = resolveProject2(opts.db, request.params.name);
|
|
17296
17587
|
opts.sessionRegistry.reset(project.name);
|
|
17297
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
17588
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq29(agentSessions.projectId, project.id)).run();
|
|
17298
17589
|
return { status: "reset" };
|
|
17299
17590
|
}
|
|
17300
17591
|
);
|
|
@@ -17805,6 +18096,9 @@ var ApiClient = class {
|
|
|
17805
18096
|
async bingInspectUrl(project, url) {
|
|
17806
18097
|
return this.request("POST", `/projects/${encodeURIComponent(project)}/bing/inspect-url`, { url });
|
|
17807
18098
|
}
|
|
18099
|
+
async bingInspectSitemap(project, body) {
|
|
18100
|
+
return this.request("POST", `/projects/${encodeURIComponent(project)}/bing/inspect-sitemap`, body ?? {});
|
|
18101
|
+
}
|
|
17808
18102
|
async bingRequestIndexing(project, body) {
|
|
17809
18103
|
return this.request("POST", `/projects/${encodeURIComponent(project)}/bing/request-indexing`, body);
|
|
17810
18104
|
}
|
|
@@ -18111,7 +18405,7 @@ function formatAuditFactorScore(factor) {
|
|
|
18111
18405
|
}
|
|
18112
18406
|
|
|
18113
18407
|
// src/snapshot-service.ts
|
|
18114
|
-
var
|
|
18408
|
+
var log12 = createLogger("Snapshot");
|
|
18115
18409
|
var ANALYSIS_PROVIDER_PRIORITY = ["openai", "claude", "gemini", "perplexity", "local"];
|
|
18116
18410
|
var SNAPSHOT_QUERY_COUNT = 6;
|
|
18117
18411
|
var ProviderExecutionGate2 = class {
|
|
@@ -18254,7 +18548,7 @@ var SnapshotService = class {
|
|
|
18254
18548
|
return mapAuditReport(report);
|
|
18255
18549
|
} catch (err) {
|
|
18256
18550
|
const message = err instanceof Error ? err.message : String(err);
|
|
18257
|
-
|
|
18551
|
+
log12.warn("audit.failed", { homepageUrl, error: message });
|
|
18258
18552
|
return {
|
|
18259
18553
|
url: homepageUrl,
|
|
18260
18554
|
finalUrl: homepageUrl,
|
|
@@ -18284,7 +18578,7 @@ var SnapshotService = class {
|
|
|
18284
18578
|
phrases: parsedPhrases
|
|
18285
18579
|
};
|
|
18286
18580
|
} catch (err) {
|
|
18287
|
-
|
|
18581
|
+
log12.warn("profile.generation-failed", {
|
|
18288
18582
|
domain: ctx.domain,
|
|
18289
18583
|
provider: ctx.analysisProvider.adapter.name,
|
|
18290
18584
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -18426,7 +18720,7 @@ var SnapshotService = class {
|
|
|
18426
18720
|
recommendedActions: uniqueStrings(parsed.recommendedActions ?? []).slice(0, 4)
|
|
18427
18721
|
};
|
|
18428
18722
|
} catch (err) {
|
|
18429
|
-
|
|
18723
|
+
log12.warn("response.analysis-failed", {
|
|
18430
18724
|
provider: ctx.analysisProvider.adapter.name,
|
|
18431
18725
|
error: err instanceof Error ? err.message : String(err)
|
|
18432
18726
|
});
|
|
@@ -18711,7 +19005,7 @@ function clipText(value, length) {
|
|
|
18711
19005
|
// src/server.ts
|
|
18712
19006
|
var _require2 = createRequire3(import.meta.url);
|
|
18713
19007
|
var { version: PKG_VERSION } = _require2("../package.json");
|
|
18714
|
-
var
|
|
19008
|
+
var log13 = createLogger("Server");
|
|
18715
19009
|
var DEFAULT_QUOTA = {
|
|
18716
19010
|
maxConcurrency: 2,
|
|
18717
19011
|
maxRequestsPerMinute: 10,
|
|
@@ -18742,7 +19036,7 @@ function summarizeProviderConfig(provider, config) {
|
|
|
18742
19036
|
};
|
|
18743
19037
|
}
|
|
18744
19038
|
function hashApiKey(key) {
|
|
18745
|
-
return
|
|
19039
|
+
return crypto28.createHash("sha256").update(key).digest("hex");
|
|
18746
19040
|
}
|
|
18747
19041
|
function parseCookies2(header) {
|
|
18748
19042
|
if (!header) return {};
|
|
@@ -18798,7 +19092,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
18798
19092
|
}
|
|
18799
19093
|
if (migratedGoogle > 0) {
|
|
18800
19094
|
saveConfigPatch({ google: config.google });
|
|
18801
|
-
|
|
19095
|
+
log13.info("credentials.migrated", { type: "google", count: migratedGoogle });
|
|
18802
19096
|
}
|
|
18803
19097
|
let migratedGa4 = 0;
|
|
18804
19098
|
for (const row of rows.ga4) {
|
|
@@ -18816,7 +19110,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
18816
19110
|
}
|
|
18817
19111
|
if (migratedGa4 > 0) {
|
|
18818
19112
|
saveConfigPatch({ ga4: config.ga4 });
|
|
18819
|
-
|
|
19113
|
+
log13.info("credentials.migrated", { type: "ga4", count: migratedGa4 });
|
|
18820
19114
|
}
|
|
18821
19115
|
}
|
|
18822
19116
|
async function createServer(opts) {
|
|
@@ -18848,11 +19142,11 @@ async function createServer(opts) {
|
|
|
18848
19142
|
applyLegacyCredentials(legacyRows, opts.config);
|
|
18849
19143
|
dropLegacyCredentialColumns(opts.db);
|
|
18850
19144
|
} catch (err) {
|
|
18851
|
-
|
|
19145
|
+
log13.warn("credentials.migration.failed", {
|
|
18852
19146
|
error: err instanceof Error ? err.message : String(err)
|
|
18853
19147
|
});
|
|
18854
19148
|
}
|
|
18855
|
-
|
|
19149
|
+
log13.info("providers.configured", { providers: Object.keys(providers).filter((k) => {
|
|
18856
19150
|
const p = providers[k];
|
|
18857
19151
|
return p?.apiKey || p?.baseUrl || p?.vertexProject;
|
|
18858
19152
|
}) });
|
|
@@ -18900,7 +19194,7 @@ async function createServer(opts) {
|
|
|
18900
19194
|
intelligenceService,
|
|
18901
19195
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
18902
19196
|
async ({ runId, projectId, insightCount, criticalOrHigh }) => {
|
|
18903
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
19197
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq30(projects.id, projectId)).get();
|
|
18904
19198
|
if (!project) return;
|
|
18905
19199
|
sessionRegistry.queueFollowUp(project.name, {
|
|
18906
19200
|
role: "user",
|
|
@@ -18994,7 +19288,7 @@ async function createServer(opts) {
|
|
|
18994
19288
|
return removed;
|
|
18995
19289
|
}
|
|
18996
19290
|
};
|
|
18997
|
-
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ??
|
|
19291
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto28.randomBytes(32).toString("hex");
|
|
18998
19292
|
const googleConnectionStore = {
|
|
18999
19293
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
19000
19294
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -19040,11 +19334,11 @@ async function createServer(opts) {
|
|
|
19040
19334
|
const apiPrefix = basePath ? `${basePath}api/v1` : "/api/v1";
|
|
19041
19335
|
if (opts.config.apiKey) {
|
|
19042
19336
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
19043
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
19337
|
+
const existing = opts.db.select().from(apiKeys).where(eq30(apiKeys.keyHash, keyHash)).get();
|
|
19044
19338
|
if (!existing) {
|
|
19045
19339
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
19046
19340
|
opts.db.insert(apiKeys).values({
|
|
19047
|
-
id: `key_${
|
|
19341
|
+
id: `key_${crypto28.randomBytes(8).toString("hex")}`,
|
|
19048
19342
|
name: "default",
|
|
19049
19343
|
keyHash,
|
|
19050
19344
|
keyPrefix: prefix,
|
|
@@ -19068,7 +19362,7 @@ async function createServer(opts) {
|
|
|
19068
19362
|
};
|
|
19069
19363
|
const createSession = (apiKeyId) => {
|
|
19070
19364
|
pruneExpiredSessions();
|
|
19071
|
-
const sessionId =
|
|
19365
|
+
const sessionId = crypto28.randomBytes(32).toString("hex");
|
|
19072
19366
|
sessions.set(sessionId, {
|
|
19073
19367
|
apiKeyId,
|
|
19074
19368
|
expiresAt: Date.now() + SESSION_TTL_MS
|
|
@@ -19092,7 +19386,7 @@ async function createServer(opts) {
|
|
|
19092
19386
|
};
|
|
19093
19387
|
const getDefaultApiKey = () => {
|
|
19094
19388
|
if (!opts.config.apiKey) return void 0;
|
|
19095
|
-
return opts.db.select().from(apiKeys).where(
|
|
19389
|
+
return opts.db.select().from(apiKeys).where(eq30(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
19096
19390
|
};
|
|
19097
19391
|
const createPasswordSession = (reply) => {
|
|
19098
19392
|
const key = getDefaultApiKey();
|
|
@@ -19149,12 +19443,12 @@ async function createServer(opts) {
|
|
|
19149
19443
|
return reply.send({ authenticated: true });
|
|
19150
19444
|
}
|
|
19151
19445
|
if (apiKey) {
|
|
19152
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
19446
|
+
const key = opts.db.select().from(apiKeys).where(eq30(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
19153
19447
|
if (!key || key.revokedAt) {
|
|
19154
19448
|
const err2 = authInvalid();
|
|
19155
19449
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
19156
19450
|
}
|
|
19157
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
19451
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq30(apiKeys.id, key.id)).run();
|
|
19158
19452
|
const sessionId = createSession(key.id);
|
|
19159
19453
|
reply.header("set-cookie", serializeSessionCookie({
|
|
19160
19454
|
name: SESSION_COOKIE_NAME,
|
|
@@ -19242,7 +19536,7 @@ async function createServer(opts) {
|
|
|
19242
19536
|
deps: {
|
|
19243
19537
|
enqueueAutoExtract: ({ projectId, release: r }) => {
|
|
19244
19538
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
19245
|
-
const runId =
|
|
19539
|
+
const runId = crypto28.randomUUID();
|
|
19246
19540
|
opts.db.insert(runs).values({
|
|
19247
19541
|
id: runId,
|
|
19248
19542
|
projectId,
|
|
@@ -19304,6 +19598,14 @@ async function createServer(opts) {
|
|
|
19304
19598
|
googleSettingsSummary,
|
|
19305
19599
|
bingSettingsSummary,
|
|
19306
19600
|
bingConnectionStore,
|
|
19601
|
+
onBingInspectSitemapRequested: (runId, projectId, inspectOpts) => {
|
|
19602
|
+
executeBingInspectSitemap(opts.db, runId, projectId, {
|
|
19603
|
+
...inspectOpts,
|
|
19604
|
+
config: opts.config
|
|
19605
|
+
}).catch((err) => {
|
|
19606
|
+
app.log.error({ runId, err }, "Bing inspect sitemap failed");
|
|
19607
|
+
});
|
|
19608
|
+
},
|
|
19307
19609
|
wordpressConnectionStore,
|
|
19308
19610
|
ga4CredentialStore,
|
|
19309
19611
|
onRunCreated: (runId, projectId, providers2, location) => {
|
|
@@ -19368,7 +19670,7 @@ async function createServer(opts) {
|
|
|
19368
19670
|
const targetProjectIds = affectedProjectIds.length > 0 ? affectedProjectIds : [null];
|
|
19369
19671
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
19370
19672
|
opts.db.insert(auditLog).values(targetProjectIds.map((projectId) => ({
|
|
19371
|
-
id:
|
|
19673
|
+
id: crypto28.randomUUID(),
|
|
19372
19674
|
projectId,
|
|
19373
19675
|
actor: "api",
|
|
19374
19676
|
action: existing ? "provider.updated" : "provider.created",
|