@canonry/canonry 4.148.7 → 4.148.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-4VNOYFCG.js → chunk-U2ETSHDL.js} +424 -374
- package/dist/cli.js +1 -1
- package/dist/index.js +1 -1
- package/package.json +10 -10
|
@@ -549,11 +549,11 @@ function checkLatestVersionForServer(opts) {
|
|
|
549
549
|
|
|
550
550
|
// src/server.ts
|
|
551
551
|
import { createRequire as createRequire3 } from "module";
|
|
552
|
-
import
|
|
552
|
+
import crypto23 from "crypto";
|
|
553
553
|
import fs8 from "fs";
|
|
554
554
|
import path9 from "path";
|
|
555
555
|
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
556
|
-
import { and as and15, eq as
|
|
556
|
+
import { and as and15, eq as eq22 } from "drizzle-orm";
|
|
557
557
|
import Fastify from "fastify";
|
|
558
558
|
import os5 from "os";
|
|
559
559
|
|
|
@@ -4225,8 +4225,83 @@ function maybeShowActivationNotice(io = {}) {
|
|
|
4225
4225
|
}
|
|
4226
4226
|
|
|
4227
4227
|
// src/gsc-sync.ts
|
|
4228
|
+
import crypto6 from "crypto";
|
|
4229
|
+
import { eq as eq4, and as and4, sql as sql3 } from "drizzle-orm";
|
|
4230
|
+
|
|
4231
|
+
// src/gsc-coverage-snapshot.ts
|
|
4228
4232
|
import crypto5 from "crypto";
|
|
4229
|
-
import {
|
|
4233
|
+
import { and as and3, desc, eq as eq3, gte } from "drizzle-orm";
|
|
4234
|
+
var COVERAGE_IMPRESSION_WINDOW_DAYS = 30;
|
|
4235
|
+
function pageKey(raw) {
|
|
4236
|
+
if (!raw) return null;
|
|
4237
|
+
try {
|
|
4238
|
+
return normalizeUrlPath(new URL(raw).pathname);
|
|
4239
|
+
} catch {
|
|
4240
|
+
return normalizeUrlPath(raw);
|
|
4241
|
+
}
|
|
4242
|
+
}
|
|
4243
|
+
function isoDaysAgo(days) {
|
|
4244
|
+
const d = /* @__PURE__ */ new Date();
|
|
4245
|
+
d.setUTCDate(d.getUTCDate() - days);
|
|
4246
|
+
return d.toISOString().split("T")[0];
|
|
4247
|
+
}
|
|
4248
|
+
function writeCoverageSnapshot(db, projectId, runId, opts = {}) {
|
|
4249
|
+
const windowDays = opts.windowDays ?? COVERAGE_IMPRESSION_WINDOW_DAYS;
|
|
4250
|
+
const since = isoDaysAgo(windowDays);
|
|
4251
|
+
const pageRows = db.select({ page: gscSearchData.page, impressions: gscSearchData.impressions }).from(gscSearchData).where(and3(eq3(gscSearchData.projectId, projectId), gte(gscSearchData.date, since))).all();
|
|
4252
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq3(gscUrlInspections.projectId, projectId)).all();
|
|
4253
|
+
const latestAudit = db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).where(eq3(siteAuditSnapshots.projectId, projectId)).orderBy(desc(siteAuditSnapshots.createdAt)).limit(1).get();
|
|
4254
|
+
const sitemapUrls = latestAudit ? db.select({ url: siteAuditPages.url }).from(siteAuditPages).where(eq3(siteAuditPages.runId, latestAudit.runId)).all() : [];
|
|
4255
|
+
const impressionsByKey = /* @__PURE__ */ new Map();
|
|
4256
|
+
for (const row of pageRows) {
|
|
4257
|
+
const key = pageKey(row.page);
|
|
4258
|
+
if (key) impressionsByKey.set(key, (impressionsByKey.get(key) ?? 0) + row.impressions);
|
|
4259
|
+
}
|
|
4260
|
+
for (const row of sitemapUrls) {
|
|
4261
|
+
const key = pageKey(row.url);
|
|
4262
|
+
if (key && !impressionsByKey.has(key)) impressionsByKey.set(key, 0);
|
|
4263
|
+
}
|
|
4264
|
+
const latestByKey = /* @__PURE__ */ new Map();
|
|
4265
|
+
for (const row of allInspections) {
|
|
4266
|
+
const key = pageKey(row.url);
|
|
4267
|
+
if (!key) continue;
|
|
4268
|
+
const existing = latestByKey.get(key);
|
|
4269
|
+
if (!existing || row.inspectedAt > existing.inspectedAt) latestByKey.set(key, row);
|
|
4270
|
+
}
|
|
4271
|
+
const coverage = deriveIndexCoverage({
|
|
4272
|
+
pages: [...impressionsByKey].map(([page, impressions]) => ({ page, impressions })),
|
|
4273
|
+
inspections: [...latestByKey].map(([key, r]) => ({
|
|
4274
|
+
url: key,
|
|
4275
|
+
indexingState: r.indexingState,
|
|
4276
|
+
coverageState: r.coverageState
|
|
4277
|
+
}))
|
|
4278
|
+
});
|
|
4279
|
+
const now = opts.now ?? /* @__PURE__ */ new Date();
|
|
4280
|
+
const snapshotDate = now.toISOString().split("T")[0];
|
|
4281
|
+
db.delete(gscCoverageSnapshots).where(and3(eq3(gscCoverageSnapshots.projectId, projectId), eq3(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
4282
|
+
db.insert(gscCoverageSnapshots).values({
|
|
4283
|
+
id: crypto5.randomUUID(),
|
|
4284
|
+
projectId,
|
|
4285
|
+
syncRunId: runId,
|
|
4286
|
+
date: snapshotDate,
|
|
4287
|
+
indexed: coverage.indexed,
|
|
4288
|
+
notIndexed: coverage.notIndexed,
|
|
4289
|
+
unknownPages: coverage.unknown,
|
|
4290
|
+
verifiedByInspection: coverage.verifiedByInspection,
|
|
4291
|
+
derivedFromImpressions: coverage.derivedFromImpressions,
|
|
4292
|
+
reasonBreakdown: coverage.reasonBreakdown,
|
|
4293
|
+
createdAt: now.toISOString()
|
|
4294
|
+
}).run();
|
|
4295
|
+
return {
|
|
4296
|
+
indexed: coverage.indexed,
|
|
4297
|
+
notIndexed: coverage.notIndexed,
|
|
4298
|
+
unknown: coverage.unknown,
|
|
4299
|
+
verifiedByInspection: coverage.verifiedByInspection,
|
|
4300
|
+
derivedFromImpressions: coverage.derivedFromImpressions
|
|
4301
|
+
};
|
|
4302
|
+
}
|
|
4303
|
+
|
|
4304
|
+
// src/gsc-sync.ts
|
|
4230
4305
|
var log2 = createLogger("GscSync");
|
|
4231
4306
|
function formatDate(d) {
|
|
4232
4307
|
return d.toISOString().split("T")[0];
|
|
@@ -4238,13 +4313,13 @@ function daysAgo(n) {
|
|
|
4238
4313
|
}
|
|
4239
4314
|
async function executeGscSync(db, runId, projectId, opts) {
|
|
4240
4315
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4241
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
4316
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq4(runs.id, runId)).run();
|
|
4242
4317
|
try {
|
|
4243
4318
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
4244
4319
|
if (!googleClientId || !googleClientSecret) {
|
|
4245
4320
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
4246
4321
|
}
|
|
4247
|
-
const project = db.select().from(projects).where(
|
|
4322
|
+
const project = db.select().from(projects).where(eq4(projects.id, projectId)).get();
|
|
4248
4323
|
if (!project) {
|
|
4249
4324
|
throw new Error(`Project not found: ${projectId}`);
|
|
4250
4325
|
}
|
|
@@ -4279,8 +4354,8 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
4279
4354
|
});
|
|
4280
4355
|
log2.info("fetch.complete", { runId, projectId, rowCount: rows.length });
|
|
4281
4356
|
db.delete(gscSearchData).where(
|
|
4282
|
-
|
|
4283
|
-
|
|
4357
|
+
and4(
|
|
4358
|
+
eq4(gscSearchData.projectId, projectId),
|
|
4284
4359
|
sql3`${gscSearchData.date} >= ${startDate}`,
|
|
4285
4360
|
sql3`${gscSearchData.date} <= ${endDate}`
|
|
4286
4361
|
)
|
|
@@ -4292,7 +4367,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
4292
4367
|
for (const row of batch) {
|
|
4293
4368
|
const [query, page, country, device, date] = row.keys;
|
|
4294
4369
|
db.insert(gscSearchData).values({
|
|
4295
|
-
id:
|
|
4370
|
+
id: crypto6.randomUUID(),
|
|
4296
4371
|
projectId,
|
|
4297
4372
|
syncRunId: runId,
|
|
4298
4373
|
date: date ?? "",
|
|
@@ -4314,8 +4389,8 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
4314
4389
|
dimensions: ["date"]
|
|
4315
4390
|
});
|
|
4316
4391
|
db.delete(gscDailyTotals).where(
|
|
4317
|
-
|
|
4318
|
-
|
|
4392
|
+
and4(
|
|
4393
|
+
eq4(gscDailyTotals.projectId, projectId),
|
|
4319
4394
|
sql3`${gscDailyTotals.date} >= ${startDate}`,
|
|
4320
4395
|
sql3`${gscDailyTotals.date} <= ${endDate}`
|
|
4321
4396
|
)
|
|
@@ -4324,7 +4399,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
4324
4399
|
for (const row of totalRows) {
|
|
4325
4400
|
const [date] = row.keys;
|
|
4326
4401
|
db.insert(gscDailyTotals).values({
|
|
4327
|
-
id:
|
|
4402
|
+
id: crypto6.randomUUID(),
|
|
4328
4403
|
projectId,
|
|
4329
4404
|
date: date ?? "",
|
|
4330
4405
|
clicks: row.clicks,
|
|
@@ -4340,8 +4415,8 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
4340
4415
|
dimensions: ["date", "query"]
|
|
4341
4416
|
});
|
|
4342
4417
|
db.delete(gscQueryDailyTotals).where(
|
|
4343
|
-
|
|
4344
|
-
|
|
4418
|
+
and4(
|
|
4419
|
+
eq4(gscQueryDailyTotals.projectId, projectId),
|
|
4345
4420
|
sql3`${gscQueryDailyTotals.date} >= ${startDate}`,
|
|
4346
4421
|
sql3`${gscQueryDailyTotals.date} <= ${endDate}`
|
|
4347
4422
|
)
|
|
@@ -4351,7 +4426,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
4351
4426
|
const [date, query] = row.keys;
|
|
4352
4427
|
if (!date || !query) continue;
|
|
4353
4428
|
db.insert(gscQueryDailyTotals).values({
|
|
4354
|
-
id:
|
|
4429
|
+
id: crypto6.randomUUID(),
|
|
4355
4430
|
projectId,
|
|
4356
4431
|
date,
|
|
4357
4432
|
query,
|
|
@@ -4364,50 +4439,20 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
4364
4439
|
}).run();
|
|
4365
4440
|
}
|
|
4366
4441
|
log2.info("query-totals.complete", { runId, projectId, rowCount: queryTotalRows.length });
|
|
4367
|
-
const
|
|
4368
|
-
|
|
4369
|
-
for (const row of allInspections) {
|
|
4370
|
-
const existing = latestByUrl.get(row.url);
|
|
4371
|
-
if (!existing || row.inspectedAt > existing.inspectedAt) {
|
|
4372
|
-
latestByUrl.set(row.url, row);
|
|
4373
|
-
}
|
|
4374
|
-
}
|
|
4375
|
-
const coverage = deriveIndexCoverage({
|
|
4376
|
-
pages: rows.map((row) => ({ page: row.keys[1] ?? "", impressions: row.impressions })),
|
|
4377
|
-
inspections: [...latestByUrl.values()].map((row) => ({
|
|
4378
|
-
url: row.url,
|
|
4379
|
-
indexingState: row.indexingState,
|
|
4380
|
-
coverageState: row.coverageState
|
|
4381
|
-
}))
|
|
4382
|
-
});
|
|
4383
|
-
const snapshotDate = formatDate(/* @__PURE__ */ new Date());
|
|
4384
|
-
db.delete(gscCoverageSnapshots).where(and3(eq3(gscCoverageSnapshots.projectId, projectId), eq3(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
4385
|
-
db.insert(gscCoverageSnapshots).values({
|
|
4386
|
-
id: crypto5.randomUUID(),
|
|
4387
|
-
projectId,
|
|
4388
|
-
syncRunId: runId,
|
|
4389
|
-
date: snapshotDate,
|
|
4390
|
-
indexed: coverage.indexed,
|
|
4391
|
-
notIndexed: coverage.notIndexed,
|
|
4392
|
-
unknownPages: coverage.unknown,
|
|
4393
|
-
verifiedByInspection: coverage.verifiedByInspection,
|
|
4394
|
-
derivedFromImpressions: coverage.derivedFromImpressions,
|
|
4395
|
-
reasonBreakdown: coverage.reasonBreakdown,
|
|
4396
|
-
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
4397
|
-
}).run();
|
|
4398
|
-
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq3(runs.id, runId)).run();
|
|
4442
|
+
const coverage = writeCoverageSnapshot(db, projectId, runId);
|
|
4443
|
+
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq4(runs.id, runId)).run();
|
|
4399
4444
|
log2.info("sync.completed", { runId, projectId, searchDataRows: rows.length, indexed: coverage.indexed, notIndexed: coverage.notIndexed, unknown: coverage.unknown, verifiedByInspection: coverage.verifiedByInspection });
|
|
4400
4445
|
} catch (err) {
|
|
4401
4446
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
4402
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
4447
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq4(runs.id, runId)).run();
|
|
4403
4448
|
log2.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
4404
4449
|
throw err;
|
|
4405
4450
|
}
|
|
4406
4451
|
}
|
|
4407
4452
|
|
|
4408
4453
|
// src/gbp-sync.ts
|
|
4409
|
-
import
|
|
4410
|
-
import { eq as
|
|
4454
|
+
import crypto7 from "crypto";
|
|
4455
|
+
import { eq as eq5, and as and5, desc as desc2, inArray as inArray2, lt } from "drizzle-orm";
|
|
4411
4456
|
var MS_PER_DAY = 864e5;
|
|
4412
4457
|
var log3 = createLogger("GbpSync");
|
|
4413
4458
|
var LOCATION_CONCURRENCY = 4;
|
|
@@ -4428,13 +4473,13 @@ function monthMinus(n) {
|
|
|
4428
4473
|
}
|
|
4429
4474
|
async function executeGbpSync(db, runId, projectId, opts) {
|
|
4430
4475
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4431
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
4476
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq5(runs.id, runId)).run();
|
|
4432
4477
|
try {
|
|
4433
4478
|
const { clientId, clientSecret } = getGoogleAuthConfig(opts.config);
|
|
4434
4479
|
if (!clientId || !clientSecret) {
|
|
4435
4480
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
4436
4481
|
}
|
|
4437
|
-
const project = db.select().from(projects).where(
|
|
4482
|
+
const project = db.select().from(projects).where(eq5(projects.id, projectId)).get();
|
|
4438
4483
|
if (!project) throw new Error(`Project not found: ${projectId}`);
|
|
4439
4484
|
const conn = getGoogleConnection(opts.config, project.canonicalDomain, "gbp");
|
|
4440
4485
|
if (!conn || !conn.refreshToken) {
|
|
@@ -4452,7 +4497,7 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4452
4497
|
});
|
|
4453
4498
|
saveConfigPatch(opts.config);
|
|
4454
4499
|
}
|
|
4455
|
-
let locationRows = db.select().from(gbpLocations).where(
|
|
4500
|
+
let locationRows = db.select().from(gbpLocations).where(and5(eq5(gbpLocations.projectId, projectId), eq5(gbpLocations.selected, true))).all();
|
|
4456
4501
|
if (opts.locationNames?.length) {
|
|
4457
4502
|
const wanted = new Set(opts.locationNames);
|
|
4458
4503
|
locationRows = locationRows.filter((l) => wanted.has(l.locationName));
|
|
@@ -4502,15 +4547,15 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4502
4547
|
})))
|
|
4503
4548
|
]);
|
|
4504
4549
|
const lodgingHash = lodging ? hashLodging(lodging) : null;
|
|
4505
|
-
const latestLodging = lodging ? db.select().from(gbpLodgingSnapshots).where(
|
|
4550
|
+
const latestLodging = lodging ? db.select().from(gbpLodgingSnapshots).where(and5(eq5(gbpLodgingSnapshots.projectId, projectId), eq5(gbpLodgingSnapshots.locationName, loc.locationName))).orderBy(desc2(gbpLodgingSnapshots.syncedAt)).limit(1).get() : void 0;
|
|
4506
4551
|
const lodgingChanged = lodging !== null && latestLodging?.contentHash !== lodgingHash;
|
|
4507
4552
|
const attributesHash = hashAttributes(attributes);
|
|
4508
|
-
const latestAttributes = db.select().from(gbpAttributesSnapshots).where(
|
|
4553
|
+
const latestAttributes = db.select().from(gbpAttributesSnapshots).where(and5(eq5(gbpAttributesSnapshots.projectId, projectId), eq5(gbpAttributesSnapshots.locationName, loc.locationName))).orderBy(desc2(gbpAttributesSnapshots.syncedAt)).limit(1).get();
|
|
4509
4554
|
const attributesChanged = latestAttributes?.contentHash !== attributesHash;
|
|
4510
4555
|
let placeToWrite = null;
|
|
4511
4556
|
let placeToTouch = null;
|
|
4512
4557
|
if (placesTier !== "off" && placesApiKey && lodging !== null && loc.placeId) {
|
|
4513
|
-
const latestPlace = db.select().from(gbpPlaceDetails).where(
|
|
4558
|
+
const latestPlace = db.select().from(gbpPlaceDetails).where(and5(eq5(gbpPlaceDetails.projectId, projectId), eq5(gbpPlaceDetails.locationName, loc.locationName))).orderBy(desc2(gbpPlaceDetails.syncedAt)).limit(1).get();
|
|
4514
4559
|
const ageDays = latestPlace ? (Date.now() - new Date(latestPlace.syncedAt).getTime()) / MS_PER_DAY : Infinity;
|
|
4515
4560
|
if (ageDays >= placesRefreshDays) {
|
|
4516
4561
|
try {
|
|
@@ -4528,10 +4573,10 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4528
4573
|
}
|
|
4529
4574
|
const insertNow = (/* @__PURE__ */ new Date()).toISOString();
|
|
4530
4575
|
db.transaction((tx) => {
|
|
4531
|
-
tx.delete(gbpDailyMetrics).where(
|
|
4576
|
+
tx.delete(gbpDailyMetrics).where(and5(eq5(gbpDailyMetrics.projectId, projectId), eq5(gbpDailyMetrics.locationName, loc.locationName))).run();
|
|
4532
4577
|
for (const row of metricRows) {
|
|
4533
4578
|
tx.insert(gbpDailyMetrics).values({
|
|
4534
|
-
id:
|
|
4579
|
+
id: crypto7.randomUUID(),
|
|
4535
4580
|
projectId,
|
|
4536
4581
|
locationName: loc.locationName,
|
|
4537
4582
|
date: row.date,
|
|
@@ -4540,10 +4585,10 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4540
4585
|
syncRunId: runId
|
|
4541
4586
|
}).run();
|
|
4542
4587
|
}
|
|
4543
|
-
tx.delete(gbpKeywordImpressions).where(
|
|
4588
|
+
tx.delete(gbpKeywordImpressions).where(and5(eq5(gbpKeywordImpressions.projectId, projectId), eq5(gbpKeywordImpressions.locationName, loc.locationName))).run();
|
|
4544
4589
|
for (const row of keywordRows) {
|
|
4545
4590
|
tx.insert(gbpKeywordImpressions).values({
|
|
4546
|
-
id:
|
|
4591
|
+
id: crypto7.randomUUID(),
|
|
4547
4592
|
projectId,
|
|
4548
4593
|
locationName: loc.locationName,
|
|
4549
4594
|
periodStart: monthKey(keywordsStart),
|
|
@@ -4556,15 +4601,15 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4556
4601
|
}
|
|
4557
4602
|
const fetchedMonths = monthlyKeywordResults.map((r) => r.month);
|
|
4558
4603
|
if (fetchedMonths.length > 0) {
|
|
4559
|
-
tx.delete(gbpKeywordMonthly).where(
|
|
4560
|
-
|
|
4561
|
-
|
|
4604
|
+
tx.delete(gbpKeywordMonthly).where(and5(
|
|
4605
|
+
eq5(gbpKeywordMonthly.projectId, projectId),
|
|
4606
|
+
eq5(gbpKeywordMonthly.locationName, loc.locationName),
|
|
4562
4607
|
inArray2(gbpKeywordMonthly.month, fetchedMonths)
|
|
4563
4608
|
)).run();
|
|
4564
4609
|
for (const { month, rows } of monthlyKeywordResults) {
|
|
4565
4610
|
for (const row of rows) {
|
|
4566
4611
|
tx.insert(gbpKeywordMonthly).values({
|
|
4567
|
-
id:
|
|
4612
|
+
id: crypto7.randomUUID(),
|
|
4568
4613
|
projectId,
|
|
4569
4614
|
locationName: loc.locationName,
|
|
4570
4615
|
month,
|
|
@@ -4576,16 +4621,16 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4576
4621
|
}).run();
|
|
4577
4622
|
}
|
|
4578
4623
|
}
|
|
4579
|
-
tx.delete(gbpKeywordMonthly).where(
|
|
4580
|
-
|
|
4581
|
-
|
|
4624
|
+
tx.delete(gbpKeywordMonthly).where(and5(
|
|
4625
|
+
eq5(gbpKeywordMonthly.projectId, projectId),
|
|
4626
|
+
eq5(gbpKeywordMonthly.locationName, loc.locationName),
|
|
4582
4627
|
lt(gbpKeywordMonthly.month, keywordRetentionCutoff)
|
|
4583
4628
|
)).run();
|
|
4584
4629
|
}
|
|
4585
|
-
tx.delete(gbpPlaceActions).where(
|
|
4630
|
+
tx.delete(gbpPlaceActions).where(and5(eq5(gbpPlaceActions.projectId, projectId), eq5(gbpPlaceActions.locationName, loc.locationName))).run();
|
|
4586
4631
|
for (const row of placeActionRows) {
|
|
4587
4632
|
tx.insert(gbpPlaceActions).values({
|
|
4588
|
-
id:
|
|
4633
|
+
id: crypto7.randomUUID(),
|
|
4589
4634
|
projectId,
|
|
4590
4635
|
locationName: loc.locationName,
|
|
4591
4636
|
placeActionLinkName: row.placeActionLinkName,
|
|
@@ -4598,7 +4643,7 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4598
4643
|
}
|
|
4599
4644
|
if (lodging !== null && lodgingChanged) {
|
|
4600
4645
|
tx.insert(gbpLodgingSnapshots).values({
|
|
4601
|
-
id:
|
|
4646
|
+
id: crypto7.randomUUID(),
|
|
4602
4647
|
projectId,
|
|
4603
4648
|
locationName: loc.locationName,
|
|
4604
4649
|
contentHash: lodgingHash,
|
|
@@ -4608,11 +4653,11 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4608
4653
|
syncRunId: runId
|
|
4609
4654
|
}).run();
|
|
4610
4655
|
} else if (lodging !== null && latestLodging) {
|
|
4611
|
-
tx.update(gbpLodgingSnapshots).set({ syncedAt: insertNow, syncRunId: runId }).where(
|
|
4656
|
+
tx.update(gbpLodgingSnapshots).set({ syncedAt: insertNow, syncRunId: runId }).where(eq5(gbpLodgingSnapshots.id, latestLodging.id)).run();
|
|
4612
4657
|
}
|
|
4613
4658
|
if (attributesChanged) {
|
|
4614
4659
|
tx.insert(gbpAttributesSnapshots).values({
|
|
4615
|
-
id:
|
|
4660
|
+
id: crypto7.randomUUID(),
|
|
4616
4661
|
projectId,
|
|
4617
4662
|
locationName: loc.locationName,
|
|
4618
4663
|
contentHash: attributesHash,
|
|
@@ -4622,11 +4667,11 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4622
4667
|
syncRunId: runId
|
|
4623
4668
|
}).run();
|
|
4624
4669
|
} else if (latestAttributes) {
|
|
4625
|
-
tx.update(gbpAttributesSnapshots).set({ syncedAt: insertNow, syncRunId: runId }).where(
|
|
4670
|
+
tx.update(gbpAttributesSnapshots).set({ syncedAt: insertNow, syncRunId: runId }).where(eq5(gbpAttributesSnapshots.id, latestAttributes.id)).run();
|
|
4626
4671
|
}
|
|
4627
4672
|
if (placeToWrite) {
|
|
4628
4673
|
tx.insert(gbpPlaceDetails).values({
|
|
4629
|
-
id:
|
|
4674
|
+
id: crypto7.randomUUID(),
|
|
4630
4675
|
projectId,
|
|
4631
4676
|
locationName: loc.locationName,
|
|
4632
4677
|
placeId: placeToWrite.placeId,
|
|
@@ -4637,9 +4682,9 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4637
4682
|
syncRunId: runId
|
|
4638
4683
|
}).run();
|
|
4639
4684
|
} else if (placeToTouch) {
|
|
4640
|
-
tx.update(gbpPlaceDetails).set({ syncedAt: insertNow, syncRunId: runId }).where(
|
|
4685
|
+
tx.update(gbpPlaceDetails).set({ syncedAt: insertNow, syncRunId: runId }).where(eq5(gbpPlaceDetails.id, placeToTouch)).run();
|
|
4641
4686
|
}
|
|
4642
|
-
tx.update(gbpLocations).set({ syncedAt: insertNow, updatedAt: insertNow }).where(
|
|
4687
|
+
tx.update(gbpLocations).set({ syncedAt: insertNow, updatedAt: insertNow }).where(eq5(gbpLocations.id, loc.id)).run();
|
|
4643
4688
|
});
|
|
4644
4689
|
okCount++;
|
|
4645
4690
|
} catch (err) {
|
|
@@ -4650,24 +4695,24 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4650
4695
|
}
|
|
4651
4696
|
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
4652
4697
|
if (errors.size === 0) {
|
|
4653
|
-
db.update(runs).set({ status: "completed", finishedAt }).where(
|
|
4698
|
+
db.update(runs).set({ status: "completed", finishedAt }).where(eq5(runs.id, runId)).run();
|
|
4654
4699
|
} else if (okCount > 0) {
|
|
4655
4700
|
db.update(runs).set({
|
|
4656
4701
|
status: "partial",
|
|
4657
4702
|
error: serializeRunError(buildRunErrorFromMessages(errors)),
|
|
4658
4703
|
finishedAt
|
|
4659
|
-
}).where(
|
|
4704
|
+
}).where(eq5(runs.id, runId)).run();
|
|
4660
4705
|
} else {
|
|
4661
4706
|
db.update(runs).set({
|
|
4662
4707
|
status: "failed",
|
|
4663
4708
|
error: serializeRunError(buildRunErrorFromMessages(errors)),
|
|
4664
4709
|
finishedAt
|
|
4665
|
-
}).where(
|
|
4710
|
+
}).where(eq5(runs.id, runId)).run();
|
|
4666
4711
|
}
|
|
4667
4712
|
log3.info("sync.done", { runId, projectId, ok: okCount, failed: errors.size });
|
|
4668
4713
|
} catch (err) {
|
|
4669
4714
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
4670
|
-
db.update(runs).set({ status: "failed", error: serializeRunError({ message: errorMsg }), finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
4715
|
+
db.update(runs).set({ status: "failed", error: serializeRunError({ message: errorMsg }), finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq5(runs.id, runId)).run();
|
|
4671
4716
|
log3.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
4672
4717
|
throw err;
|
|
4673
4718
|
}
|
|
@@ -4705,7 +4750,7 @@ async function refreshSelectedLocationProfiles(db, projectId, accessToken, locat
|
|
|
4705
4750
|
...profile,
|
|
4706
4751
|
updatedAt
|
|
4707
4752
|
};
|
|
4708
|
-
db.update(gbpLocations).set(update).where(
|
|
4753
|
+
db.update(gbpLocations).set(update).where(and5(eq5(gbpLocations.projectId, projectId), eq5(gbpLocations.id, row.id))).run();
|
|
4709
4754
|
refreshedRows.set(row.id, { ...row, ...update });
|
|
4710
4755
|
}
|
|
4711
4756
|
}
|
|
@@ -4713,8 +4758,8 @@ async function refreshSelectedLocationProfiles(db, projectId, accessToken, locat
|
|
|
4713
4758
|
}
|
|
4714
4759
|
|
|
4715
4760
|
// src/ads-sync.ts
|
|
4716
|
-
import
|
|
4717
|
-
import { eq as
|
|
4761
|
+
import crypto8 from "crypto";
|
|
4762
|
+
import { eq as eq6 } from "drizzle-orm";
|
|
4718
4763
|
|
|
4719
4764
|
// ../integration-openai-ads/src/constants.ts
|
|
4720
4765
|
var OPENAI_ADS_API_BASE = "https://api.ads.openai.com/v1";
|
|
@@ -5322,11 +5367,11 @@ function toDailyUpserts(level, entityId, read) {
|
|
|
5322
5367
|
}
|
|
5323
5368
|
async function executeAdsSync(db, runId, projectId, opts) {
|
|
5324
5369
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
5325
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
5370
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq6(runs.id, runId)).run();
|
|
5326
5371
|
try {
|
|
5327
|
-
const project = db.select().from(projects).where(
|
|
5372
|
+
const project = db.select().from(projects).where(eq6(projects.id, projectId)).get();
|
|
5328
5373
|
if (!project) throw new Error(`Project not found: ${projectId}`);
|
|
5329
|
-
const connRow = db.select().from(adsConnections).where(
|
|
5374
|
+
const connRow = db.select().from(adsConnections).where(eq6(adsConnections.projectId, projectId)).get();
|
|
5330
5375
|
if (!connRow) {
|
|
5331
5376
|
throw new Error('No ads connection found for this project. Run "canonry ads connect" first.');
|
|
5332
5377
|
}
|
|
@@ -5386,7 +5431,7 @@ async function executeAdsSync(db, runId, projectId, opts) {
|
|
|
5386
5431
|
);
|
|
5387
5432
|
const insertNow = (/* @__PURE__ */ new Date()).toISOString();
|
|
5388
5433
|
db.transaction((tx) => {
|
|
5389
|
-
tx.delete(adsCampaigns).where(
|
|
5434
|
+
tx.delete(adsCampaigns).where(eq6(adsCampaigns.projectId, projectId)).run();
|
|
5390
5435
|
for (const campaign of syncedCampaigns) {
|
|
5391
5436
|
tx.insert(adsCampaigns).values({
|
|
5392
5437
|
id: campaign.id,
|
|
@@ -5441,7 +5486,7 @@ async function executeAdsSync(db, runId, projectId, opts) {
|
|
|
5441
5486
|
}
|
|
5442
5487
|
for (const upsert of insightUpserts) {
|
|
5443
5488
|
tx.insert(adsInsightsDaily).values({
|
|
5444
|
-
id:
|
|
5489
|
+
id: crypto8.randomUUID(),
|
|
5445
5490
|
projectId,
|
|
5446
5491
|
level: upsert.level,
|
|
5447
5492
|
entityId: upsert.entityId,
|
|
@@ -5480,36 +5525,36 @@ async function executeAdsSync(db, runId, projectId, opts) {
|
|
|
5480
5525
|
conversionTrackingConfigured,
|
|
5481
5526
|
lastSyncedAt: insertNow,
|
|
5482
5527
|
updatedAt: insertNow
|
|
5483
|
-
}).where(
|
|
5528
|
+
}).where(eq6(adsConnections.projectId, projectId)).run();
|
|
5484
5529
|
});
|
|
5485
5530
|
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
5486
5531
|
if (errors.size === 0) {
|
|
5487
|
-
db.update(runs).set({ status: "completed", finishedAt }).where(
|
|
5532
|
+
db.update(runs).set({ status: "completed", finishedAt }).where(eq6(runs.id, runId)).run();
|
|
5488
5533
|
} else if (syncedCampaigns.length > 0) {
|
|
5489
5534
|
db.update(runs).set({
|
|
5490
5535
|
status: "partial",
|
|
5491
5536
|
error: serializeRunError(buildRunErrorFromMessages(errors)),
|
|
5492
5537
|
finishedAt
|
|
5493
|
-
}).where(
|
|
5538
|
+
}).where(eq6(runs.id, runId)).run();
|
|
5494
5539
|
} else {
|
|
5495
5540
|
db.update(runs).set({
|
|
5496
5541
|
status: "failed",
|
|
5497
5542
|
error: serializeRunError(buildRunErrorFromMessages(errors)),
|
|
5498
5543
|
finishedAt
|
|
5499
|
-
}).where(
|
|
5544
|
+
}).where(eq6(runs.id, runId)).run();
|
|
5500
5545
|
}
|
|
5501
5546
|
log4.info("sync.done", { runId, projectId, campaigns: syncedCampaigns.length, insightRows: insightUpserts.length, failed: errors.size });
|
|
5502
5547
|
} catch (err) {
|
|
5503
5548
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
5504
|
-
db.update(runs).set({ status: "failed", error: serializeRunError({ message: errorMsg }), finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
5549
|
+
db.update(runs).set({ status: "failed", error: serializeRunError({ message: errorMsg }), finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq6(runs.id, runId)).run();
|
|
5505
5550
|
log4.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
5506
5551
|
throw err;
|
|
5507
5552
|
}
|
|
5508
5553
|
}
|
|
5509
5554
|
|
|
5510
5555
|
// src/gsc-inspect-sitemap.ts
|
|
5511
|
-
import
|
|
5512
|
-
import { eq as
|
|
5556
|
+
import crypto9 from "crypto";
|
|
5557
|
+
import { eq as eq7 } from "drizzle-orm";
|
|
5513
5558
|
|
|
5514
5559
|
// src/sitemap-parser.ts
|
|
5515
5560
|
var log5 = createLogger("SitemapParser");
|
|
@@ -5609,6 +5654,9 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
5609
5654
|
|
|
5610
5655
|
// src/gsc-inspect-paced.ts
|
|
5611
5656
|
var INSPECT_BASE_DELAY_MS = 1e3;
|
|
5657
|
+
var INSPECT_MAX_CONCURRENCY = 5;
|
|
5658
|
+
var INSPECT_DAILY_QUOTA = 2e3;
|
|
5659
|
+
var INSPECT_SWEEP_MAX_URLS = 1500;
|
|
5612
5660
|
var INSPECT_PACING_JITTER_MS = 250;
|
|
5613
5661
|
var INSPECT_MAX_RETRIES = 3;
|
|
5614
5662
|
var INSPECT_MAX_BACKOFF_MS = 3e4;
|
|
@@ -5625,62 +5673,79 @@ function defaultSleep(ms) {
|
|
|
5625
5673
|
async function inspectUrlsPaced(urls, cb, deps = {}) {
|
|
5626
5674
|
const sleep3 = deps.sleep ?? defaultSleep;
|
|
5627
5675
|
const jitter = deps.jitter ?? Math.random;
|
|
5676
|
+
const concurrency = Math.max(1, Math.min(deps.concurrency ?? INSPECT_MAX_CONCURRENCY, urls.length || 1));
|
|
5628
5677
|
let inspected = 0;
|
|
5629
5678
|
let errors = 0;
|
|
5630
5679
|
let consecutiveRetryableFailures = 0;
|
|
5631
|
-
|
|
5632
|
-
|
|
5633
|
-
|
|
5634
|
-
|
|
5635
|
-
|
|
5636
|
-
|
|
5637
|
-
|
|
5638
|
-
|
|
5639
|
-
|
|
5640
|
-
|
|
5641
|
-
|
|
5642
|
-
|
|
5643
|
-
|
|
5644
|
-
|
|
5645
|
-
|
|
5646
|
-
|
|
5647
|
-
|
|
5648
|
-
|
|
5649
|
-
|
|
5650
|
-
|
|
5651
|
-
|
|
5652
|
-
|
|
5653
|
-
|
|
5654
|
-
|
|
5655
|
-
|
|
5656
|
-
|
|
5657
|
-
|
|
5658
|
-
|
|
5659
|
-
|
|
5660
|
-
|
|
5661
|
-
|
|
5662
|
-
|
|
5680
|
+
let nextIndex = 0;
|
|
5681
|
+
let aborted = false;
|
|
5682
|
+
let abortError;
|
|
5683
|
+
let ratePromise = Promise.resolve();
|
|
5684
|
+
const takeRateSlot = () => {
|
|
5685
|
+
const wait = ratePromise.then(() => sleep3(INSPECT_BASE_DELAY_MS + jitter() * INSPECT_PACING_JITTER_MS));
|
|
5686
|
+
ratePromise = wait;
|
|
5687
|
+
return wait;
|
|
5688
|
+
};
|
|
5689
|
+
async function worker() {
|
|
5690
|
+
for (; ; ) {
|
|
5691
|
+
if (aborted) return;
|
|
5692
|
+
const index = nextIndex++;
|
|
5693
|
+
if (index >= urls.length) return;
|
|
5694
|
+
const url = urls[index];
|
|
5695
|
+
if (index > 0) await takeRateSlot();
|
|
5696
|
+
if (aborted) return;
|
|
5697
|
+
try {
|
|
5698
|
+
const result = await withRetry(() => cb.inspectOne(url), {
|
|
5699
|
+
maxRetries: INSPECT_MAX_RETRIES,
|
|
5700
|
+
baseDelayMs: INSPECT_BASE_DELAY_MS,
|
|
5701
|
+
maxDelayMs: INSPECT_MAX_BACKOFF_MS,
|
|
5702
|
+
isRetryable: isRetryableGscInspectError,
|
|
5703
|
+
sleep: sleep3,
|
|
5704
|
+
onRetry: ({ attempt, delayMs, err }) => deps.log?.info("inspect.retry", {
|
|
5705
|
+
url,
|
|
5706
|
+
attempt,
|
|
5707
|
+
delayMs: Math.round(delayMs),
|
|
5708
|
+
error: err instanceof Error ? err.message : String(err)
|
|
5709
|
+
})
|
|
5710
|
+
});
|
|
5711
|
+
cb.onResult(url, result, index);
|
|
5712
|
+
inspected++;
|
|
5713
|
+
consecutiveRetryableFailures = 0;
|
|
5714
|
+
} catch (err) {
|
|
5715
|
+
errors++;
|
|
5716
|
+
cb.onError(url, err, index);
|
|
5717
|
+
if (isRetryableGscInspectError(err)) {
|
|
5718
|
+
consecutiveRetryableFailures++;
|
|
5719
|
+
if (consecutiveRetryableFailures >= INSPECT_FAILFAST_THRESHOLD) {
|
|
5720
|
+
deps.log?.error("inspect.circuit-break", {
|
|
5721
|
+
consecutiveFailures: consecutiveRetryableFailures,
|
|
5722
|
+
inspected,
|
|
5723
|
+
errors,
|
|
5724
|
+
remaining: urls.length - nextIndex
|
|
5725
|
+
});
|
|
5726
|
+
aborted = true;
|
|
5727
|
+
abortError = err;
|
|
5728
|
+
return;
|
|
5729
|
+
}
|
|
5663
5730
|
}
|
|
5664
5731
|
}
|
|
5665
5732
|
}
|
|
5666
|
-
if (index < urls.length - 1) {
|
|
5667
|
-
await sleep3(INSPECT_BASE_DELAY_MS + jitter() * INSPECT_PACING_JITTER_MS);
|
|
5668
|
-
}
|
|
5669
5733
|
}
|
|
5670
|
-
|
|
5734
|
+
await Promise.all(Array.from({ length: concurrency }, () => worker()));
|
|
5735
|
+
return aborted ? { inspected, errors, aborted: true, abortError } : { inspected, errors, aborted: false };
|
|
5671
5736
|
}
|
|
5672
5737
|
|
|
5673
5738
|
// src/gsc-inspect-sitemap.ts
|
|
5674
5739
|
var log6 = createLogger("InspectSitemap");
|
|
5675
5740
|
async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
5676
5741
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
5677
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
5742
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq7(runs.id, runId)).run();
|
|
5678
5743
|
try {
|
|
5679
5744
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
5680
5745
|
if (!googleClientId || !googleClientSecret) {
|
|
5681
5746
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
5682
5747
|
}
|
|
5683
|
-
const project = db.select().from(projects).where(
|
|
5748
|
+
const project = db.select().from(projects).where(eq7(projects.id, projectId)).get();
|
|
5684
5749
|
if (!project) {
|
|
5685
5750
|
throw new Error(`Project not found: ${projectId}`);
|
|
5686
5751
|
}
|
|
@@ -5711,8 +5776,21 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
5711
5776
|
if (urls.length === 0) {
|
|
5712
5777
|
throw new Error("No URLs found in sitemap");
|
|
5713
5778
|
}
|
|
5779
|
+
const skipped = Math.max(0, urls.length - INSPECT_SWEEP_MAX_URLS);
|
|
5780
|
+
const targetUrls = skipped > 0 ? urls.slice(0, INSPECT_SWEEP_MAX_URLS) : urls;
|
|
5781
|
+
if (skipped > 0) {
|
|
5782
|
+
log6.warn("sitemap.over-budget", {
|
|
5783
|
+
runId,
|
|
5784
|
+
projectId,
|
|
5785
|
+
sitemapUrls: urls.length,
|
|
5786
|
+
inspecting: targetUrls.length,
|
|
5787
|
+
skipped,
|
|
5788
|
+
dailyQuota: INSPECT_DAILY_QUOTA,
|
|
5789
|
+
note: `Sitemap has ${urls.length} pages; Google allows ${INSPECT_DAILY_QUOTA} URL inspections per property per day. Inspecting the first ${targetUrls.length}; ${skipped} pages will not have a verdict from this run.`
|
|
5790
|
+
});
|
|
5791
|
+
}
|
|
5714
5792
|
const { inspected, errors, aborted, abortError } = await inspectUrlsPaced(
|
|
5715
|
-
|
|
5793
|
+
targetUrls,
|
|
5716
5794
|
{
|
|
5717
5795
|
inspectOne: (pageUrl) => inspectUrl(accessToken, pageUrl, propertyId),
|
|
5718
5796
|
onResult: (pageUrl, result, index) => {
|
|
@@ -5722,7 +5800,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
5722
5800
|
const rich = ir.richResultsResult;
|
|
5723
5801
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
5724
5802
|
db.insert(gscUrlInspections).values({
|
|
5725
|
-
id:
|
|
5803
|
+
id: crypto9.randomUUID(),
|
|
5726
5804
|
projectId,
|
|
5727
5805
|
syncRunId: runId,
|
|
5728
5806
|
url: pageUrl,
|
|
@@ -5758,52 +5836,24 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
5758
5836
|
`URL inspection aborted after ${INSPECT_FAILFAST_THRESHOLD} consecutive rate/access failures (likely GSC URL Inspection quota exhaustion or property access loss). Last error: ${detail}`
|
|
5759
5837
|
);
|
|
5760
5838
|
}
|
|
5761
|
-
const
|
|
5762
|
-
const
|
|
5763
|
-
|
|
5764
|
-
|
|
5765
|
-
|
|
5766
|
-
|
|
5767
|
-
}
|
|
5768
|
-
}
|
|
5769
|
-
let snapIndexed = 0;
|
|
5770
|
-
let snapNotIndexed = 0;
|
|
5771
|
-
const reasonCounts = {};
|
|
5772
|
-
for (const [, row] of latestByUrl) {
|
|
5773
|
-
if (row.indexingState === "INDEXING_ALLOWED") {
|
|
5774
|
-
snapIndexed++;
|
|
5775
|
-
} else {
|
|
5776
|
-
snapNotIndexed++;
|
|
5777
|
-
const reason = row.coverageState ?? "Unknown";
|
|
5778
|
-
reasonCounts[reason] = (reasonCounts[reason] ?? 0) + 1;
|
|
5779
|
-
}
|
|
5780
|
-
}
|
|
5781
|
-
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
5782
|
-
db.delete(gscCoverageSnapshots).where(and5(eq6(gscCoverageSnapshots.projectId, projectId), eq6(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
5783
|
-
db.insert(gscCoverageSnapshots).values({
|
|
5784
|
-
id: crypto8.randomUUID(),
|
|
5785
|
-
projectId,
|
|
5786
|
-
syncRunId: runId,
|
|
5787
|
-
date: snapshotDate,
|
|
5788
|
-
indexed: snapIndexed,
|
|
5789
|
-
notIndexed: snapNotIndexed,
|
|
5790
|
-
reasonBreakdown: reasonCounts,
|
|
5791
|
-
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
5792
|
-
}).run();
|
|
5793
|
-
const status = errors > 0 && inspected > 0 ? "partial" : errors === urls.length ? "failed" : "completed";
|
|
5794
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq6(runs.id, runId)).run();
|
|
5839
|
+
const coverage = writeCoverageSnapshot(db, projectId, runId);
|
|
5840
|
+
const snapIndexed = coverage.indexed;
|
|
5841
|
+
const snapNotIndexed = coverage.notIndexed;
|
|
5842
|
+
const attempted = targetUrls.length;
|
|
5843
|
+
const status = skipped > 0 || errors > 0 && inspected > 0 ? "partial" : errors === attempted ? "failed" : "completed";
|
|
5844
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq7(runs.id, runId)).run();
|
|
5795
5845
|
log6.info("inspect.completed", { runId, projectId, inspected, errors, total: urls.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
5796
5846
|
} catch (err) {
|
|
5797
5847
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
5798
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
5848
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq7(runs.id, runId)).run();
|
|
5799
5849
|
log6.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
5800
5850
|
throw err;
|
|
5801
5851
|
}
|
|
5802
5852
|
}
|
|
5803
5853
|
|
|
5804
5854
|
// src/bing-inspect-sitemap.ts
|
|
5805
|
-
import
|
|
5806
|
-
import { eq as
|
|
5855
|
+
import crypto10 from "crypto";
|
|
5856
|
+
import { eq as eq8, desc as desc3 } from "drizzle-orm";
|
|
5807
5857
|
var log7 = createLogger("BingInspectSitemap");
|
|
5808
5858
|
function parseBingDate(value) {
|
|
5809
5859
|
if (!value) return null;
|
|
@@ -5821,9 +5871,9 @@ function isBlockingIssueType(issueType) {
|
|
|
5821
5871
|
}
|
|
5822
5872
|
async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
5823
5873
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
5824
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
5874
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq8(runs.id, runId)).run();
|
|
5825
5875
|
try {
|
|
5826
|
-
const project = db.select().from(projects).where(
|
|
5876
|
+
const project = db.select().from(projects).where(eq8(projects.id, projectId)).get();
|
|
5827
5877
|
if (!project) {
|
|
5828
5878
|
throw new Error(`Project not found: ${projectId}`);
|
|
5829
5879
|
}
|
|
@@ -5841,7 +5891,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
5841
5891
|
if (sitemapUrls.length === 0) {
|
|
5842
5892
|
throw new Error("No URLs found in sitemap");
|
|
5843
5893
|
}
|
|
5844
|
-
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(
|
|
5894
|
+
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(eq8(bingUrlInspections.projectId, projectId)).all();
|
|
5845
5895
|
const trackedUrls = new Set(trackedRows.map((r) => r.url));
|
|
5846
5896
|
const discovered = sitemapUrls.filter((u) => !trackedUrls.has(u));
|
|
5847
5897
|
log7.info("sitemap.diff", {
|
|
@@ -5890,7 +5940,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
5890
5940
|
derivedInIndex = false;
|
|
5891
5941
|
}
|
|
5892
5942
|
db.insert(bingUrlInspections).values({
|
|
5893
|
-
id:
|
|
5943
|
+
id: crypto10.randomUUID(),
|
|
5894
5944
|
projectId,
|
|
5895
5945
|
url: pageUrl,
|
|
5896
5946
|
httpCode,
|
|
@@ -5924,7 +5974,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
5924
5974
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
5925
5975
|
}
|
|
5926
5976
|
}
|
|
5927
|
-
const allInspections = db.select().from(bingUrlInspections).where(
|
|
5977
|
+
const allInspections = db.select().from(bingUrlInspections).where(eq8(bingUrlInspections.projectId, projectId)).orderBy(desc3(bingUrlInspections.inspectedAt)).all();
|
|
5928
5978
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
5929
5979
|
const definitiveByUrl = /* @__PURE__ */ new Map();
|
|
5930
5980
|
for (const row of allInspections) {
|
|
@@ -5948,7 +5998,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
5948
5998
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
5949
5999
|
const snapNow = (/* @__PURE__ */ new Date()).toISOString();
|
|
5950
6000
|
db.insert(bingCoverageSnapshots).values({
|
|
5951
|
-
id:
|
|
6001
|
+
id: crypto10.randomUUID(),
|
|
5952
6002
|
projectId,
|
|
5953
6003
|
syncRunId: runId,
|
|
5954
6004
|
date: snapshotDate,
|
|
@@ -5967,7 +6017,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
5967
6017
|
}
|
|
5968
6018
|
}).run();
|
|
5969
6019
|
const status = errors === sitemapUrls.length ? RunStatuses.failed : errors > 0 ? RunStatuses.partial : RunStatuses.completed;
|
|
5970
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
6020
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq8(runs.id, runId)).run();
|
|
5971
6021
|
log7.info("inspect.completed", {
|
|
5972
6022
|
runId,
|
|
5973
6023
|
projectId,
|
|
@@ -5981,15 +6031,15 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
5981
6031
|
});
|
|
5982
6032
|
} catch (err) {
|
|
5983
6033
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
5984
|
-
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
6034
|
+
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq8(runs.id, runId)).run();
|
|
5985
6035
|
log7.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
5986
6036
|
throw err;
|
|
5987
6037
|
}
|
|
5988
6038
|
}
|
|
5989
6039
|
|
|
5990
6040
|
// src/coverage-refresh.ts
|
|
5991
|
-
import
|
|
5992
|
-
import { and as and6, desc as
|
|
6041
|
+
import crypto11 from "crypto";
|
|
6042
|
+
import { and as and6, desc as desc4, eq as eq9, inArray as inArray3 } from "drizzle-orm";
|
|
5993
6043
|
var log8 = createLogger("CoverageRefresh");
|
|
5994
6044
|
var COVERAGE_REFRESH_MIN_INTERVAL_MS = 60 * 60 * 1e3;
|
|
5995
6045
|
var ACTIVE_OR_DONE_STATUSES = [
|
|
@@ -6000,7 +6050,7 @@ var ACTIVE_OR_DONE_STATUSES = [
|
|
|
6000
6050
|
];
|
|
6001
6051
|
var defaultDeps = { executeInspectSitemap };
|
|
6002
6052
|
async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps, nowMs = Date.now()) {
|
|
6003
|
-
const project = db.select({ canonicalDomain: projects.canonicalDomain }).from(projects).where(
|
|
6053
|
+
const project = db.select({ canonicalDomain: projects.canonicalDomain }).from(projects).where(eq9(projects.id, projectId)).get();
|
|
6004
6054
|
if (!project) return null;
|
|
6005
6055
|
const { clientId, clientSecret } = getGoogleAuthConfig(config);
|
|
6006
6056
|
if (!clientId || !clientSecret) return null;
|
|
@@ -6008,11 +6058,11 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
6008
6058
|
if (!conn?.refreshToken || !conn.propertyId) return null;
|
|
6009
6059
|
const recent = db.select({ createdAt: runs.createdAt }).from(runs).where(
|
|
6010
6060
|
and6(
|
|
6011
|
-
|
|
6012
|
-
|
|
6061
|
+
eq9(runs.projectId, projectId),
|
|
6062
|
+
eq9(runs.kind, RunKinds["inspect-sitemap"]),
|
|
6013
6063
|
inArray3(runs.status, ACTIVE_OR_DONE_STATUSES)
|
|
6014
6064
|
)
|
|
6015
|
-
).orderBy(
|
|
6065
|
+
).orderBy(desc4(runs.createdAt)).limit(1).get();
|
|
6016
6066
|
if (recent) {
|
|
6017
6067
|
const ageMs = nowMs - Date.parse(recent.createdAt);
|
|
6018
6068
|
if (Number.isFinite(ageMs) && ageMs < COVERAGE_REFRESH_MIN_INTERVAL_MS) {
|
|
@@ -6020,7 +6070,7 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
6020
6070
|
return null;
|
|
6021
6071
|
}
|
|
6022
6072
|
}
|
|
6023
|
-
const runId =
|
|
6073
|
+
const runId = crypto11.randomUUID();
|
|
6024
6074
|
db.insert(runs).values({
|
|
6025
6075
|
id: runId,
|
|
6026
6076
|
projectId,
|
|
@@ -6043,9 +6093,9 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
6043
6093
|
}
|
|
6044
6094
|
|
|
6045
6095
|
// src/commoncrawl-sync.ts
|
|
6046
|
-
import
|
|
6096
|
+
import crypto12 from "crypto";
|
|
6047
6097
|
import path4 from "path";
|
|
6048
|
-
import { and as and7, eq as
|
|
6098
|
+
import { and as and7, eq as eq10, sql as sql4 } from "drizzle-orm";
|
|
6049
6099
|
var log9 = createLogger("CommonCrawlSync");
|
|
6050
6100
|
var INSERT_CHUNK_SIZE = 1e4;
|
|
6051
6101
|
function defaultDeps2() {
|
|
@@ -6071,7 +6121,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
6071
6121
|
phaseDetail: "downloading vertices + edges",
|
|
6072
6122
|
updatedAt: downloadStartedAt,
|
|
6073
6123
|
error: null
|
|
6074
|
-
}).where(
|
|
6124
|
+
}).where(eq10(ccReleaseSyncs.id, syncId)).run();
|
|
6075
6125
|
const paths = ccReleasePaths(release);
|
|
6076
6126
|
const releaseCacheDir = path4.join(deps.cacheDir, release);
|
|
6077
6127
|
const vertexPath = path4.join(releaseCacheDir, paths.vertexFilename);
|
|
@@ -6094,7 +6144,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
6094
6144
|
vertexSha256: vertex.sha256,
|
|
6095
6145
|
edgesSha256: edges.sha256,
|
|
6096
6146
|
updatedAt: downloadFinishedAt
|
|
6097
|
-
}).where(
|
|
6147
|
+
}).where(eq10(ccReleaseSyncs.id, syncId)).run();
|
|
6098
6148
|
const allProjects = db.select().from(projects).all();
|
|
6099
6149
|
const targets = Array.from(new Set(allProjects.map((p) => p.canonicalDomain)));
|
|
6100
6150
|
let rows = [];
|
|
@@ -6110,15 +6160,15 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
6110
6160
|
}
|
|
6111
6161
|
const queriedAt = deps.now().toISOString();
|
|
6112
6162
|
db.transaction((tx) => {
|
|
6113
|
-
tx.delete(backlinkDomains).where(
|
|
6114
|
-
tx.delete(backlinkSummaries).where(
|
|
6163
|
+
tx.delete(backlinkDomains).where(eq10(backlinkDomains.releaseSyncId, syncId)).run();
|
|
6164
|
+
tx.delete(backlinkSummaries).where(eq10(backlinkSummaries.releaseSyncId, syncId)).run();
|
|
6115
6165
|
const expanded = [];
|
|
6116
6166
|
for (const r of rows) {
|
|
6117
6167
|
const projectIds = projectsByDomain.get(r.targetDomain);
|
|
6118
6168
|
if (!projectIds) continue;
|
|
6119
6169
|
for (const projectId of projectIds) {
|
|
6120
6170
|
expanded.push({
|
|
6121
|
-
id:
|
|
6171
|
+
id: crypto12.randomUUID(),
|
|
6122
6172
|
projectId,
|
|
6123
6173
|
releaseSyncId: syncId,
|
|
6124
6174
|
release,
|
|
@@ -6138,7 +6188,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
6138
6188
|
const projectRows = rowsByProject.get(p.id) ?? [];
|
|
6139
6189
|
const summary = computeSummary(projectRows);
|
|
6140
6190
|
tx.insert(backlinkSummaries).values({
|
|
6141
|
-
id:
|
|
6191
|
+
id: crypto12.randomUUID(),
|
|
6142
6192
|
projectId: p.id,
|
|
6143
6193
|
releaseSyncId: syncId,
|
|
6144
6194
|
source: BacklinkSources.commoncrawl,
|
|
@@ -6171,7 +6221,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
6171
6221
|
domainsDiscovered: rows.length,
|
|
6172
6222
|
updatedAt: finishedAt,
|
|
6173
6223
|
error: null
|
|
6174
|
-
}).where(
|
|
6224
|
+
}).where(eq10(ccReleaseSyncs.id, syncId)).run();
|
|
6175
6225
|
log9.info("sync.completed", {
|
|
6176
6226
|
syncId,
|
|
6177
6227
|
release,
|
|
@@ -6201,7 +6251,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
6201
6251
|
error: errorMsg,
|
|
6202
6252
|
phaseDetail: null,
|
|
6203
6253
|
updatedAt: finishedAt
|
|
6204
|
-
}).where(
|
|
6254
|
+
}).where(eq10(ccReleaseSyncs.id, syncId)).run();
|
|
6205
6255
|
log9.error("sync.failed", { syncId, release, error: errorMsg });
|
|
6206
6256
|
throw err;
|
|
6207
6257
|
}
|
|
@@ -6235,9 +6285,9 @@ function computeSummary(rows) {
|
|
|
6235
6285
|
}
|
|
6236
6286
|
|
|
6237
6287
|
// src/backlink-extract.ts
|
|
6238
|
-
import
|
|
6288
|
+
import crypto13 from "crypto";
|
|
6239
6289
|
import fs3 from "fs";
|
|
6240
|
-
import { and as and8, desc as
|
|
6290
|
+
import { and as and8, desc as desc5, eq as eq11 } from "drizzle-orm";
|
|
6241
6291
|
var log10 = createLogger("BacklinkExtract");
|
|
6242
6292
|
function defaultDeps3() {
|
|
6243
6293
|
return {
|
|
@@ -6249,13 +6299,13 @@ function defaultDeps3() {
|
|
|
6249
6299
|
async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
6250
6300
|
const deps = { ...defaultDeps3(), ...opts.deps };
|
|
6251
6301
|
const startedAt = deps.now().toISOString();
|
|
6252
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
6302
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq11(runs.id, runId)).run();
|
|
6253
6303
|
try {
|
|
6254
|
-
const project = db.select().from(projects).where(
|
|
6304
|
+
const project = db.select().from(projects).where(eq11(projects.id, projectId)).get();
|
|
6255
6305
|
if (!project) {
|
|
6256
6306
|
throw new Error(`Project not found: ${projectId}`);
|
|
6257
6307
|
}
|
|
6258
|
-
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(
|
|
6308
|
+
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(eq11(ccReleaseSyncs.release, opts.release)).get() : db.select().from(ccReleaseSyncs).where(eq11(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).orderBy(desc5(ccReleaseSyncs.createdAt)).limit(1).get();
|
|
6259
6309
|
if (!sync) {
|
|
6260
6310
|
throw new Error("No ready release sync available \u2014 run `canonry backlinks sync` first");
|
|
6261
6311
|
}
|
|
@@ -6284,14 +6334,14 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
6284
6334
|
db.transaction((tx) => {
|
|
6285
6335
|
tx.delete(backlinkDomains).where(
|
|
6286
6336
|
and8(
|
|
6287
|
-
|
|
6288
|
-
|
|
6289
|
-
|
|
6337
|
+
eq11(backlinkDomains.projectId, projectId),
|
|
6338
|
+
eq11(backlinkDomains.source, BacklinkSources.commoncrawl),
|
|
6339
|
+
eq11(backlinkDomains.release, release)
|
|
6290
6340
|
)
|
|
6291
6341
|
).run();
|
|
6292
6342
|
if (rows.length > 0) {
|
|
6293
6343
|
const values = rows.map((r) => ({
|
|
6294
|
-
id:
|
|
6344
|
+
id: crypto13.randomUUID(),
|
|
6295
6345
|
projectId,
|
|
6296
6346
|
releaseSyncId: syncId,
|
|
6297
6347
|
source: BacklinkSources.commoncrawl,
|
|
@@ -6305,7 +6355,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
6305
6355
|
}
|
|
6306
6356
|
const summary = computeSummary2(rows);
|
|
6307
6357
|
tx.insert(backlinkSummaries).values({
|
|
6308
|
-
id:
|
|
6358
|
+
id: crypto13.randomUUID(),
|
|
6309
6359
|
projectId,
|
|
6310
6360
|
releaseSyncId: syncId,
|
|
6311
6361
|
source: BacklinkSources.commoncrawl,
|
|
@@ -6329,7 +6379,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
6329
6379
|
}).run();
|
|
6330
6380
|
});
|
|
6331
6381
|
const finishedAt = deps.now().toISOString();
|
|
6332
|
-
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
6382
|
+
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq11(runs.id, runId)).run();
|
|
6333
6383
|
log10.info("extract.completed", { runId, projectId, release, rows: rows.length });
|
|
6334
6384
|
} catch (err) {
|
|
6335
6385
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
@@ -6338,7 +6388,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
6338
6388
|
status: RunStatuses.failed,
|
|
6339
6389
|
error: errorMsg,
|
|
6340
6390
|
finishedAt
|
|
6341
|
-
}).where(
|
|
6391
|
+
}).where(eq11(runs.id, runId)).run();
|
|
6342
6392
|
log10.error("extract.failed", { runId, projectId, error: errorMsg });
|
|
6343
6393
|
throw err;
|
|
6344
6394
|
}
|
|
@@ -6348,8 +6398,8 @@ function computeSummary2(rows) {
|
|
|
6348
6398
|
}
|
|
6349
6399
|
|
|
6350
6400
|
// src/discovery-run.ts
|
|
6351
|
-
import
|
|
6352
|
-
import { and as and9, eq as
|
|
6401
|
+
import crypto14 from "crypto";
|
|
6402
|
+
import { and as and9, eq as eq12 } from "drizzle-orm";
|
|
6353
6403
|
var log11 = createLogger("DiscoveryRun");
|
|
6354
6404
|
var DEFAULT_SEED_COUNT = 30;
|
|
6355
6405
|
var EMBED_RETRY_MAX_RETRIES = 3;
|
|
@@ -6391,11 +6441,11 @@ async function embedWithRetry(fn, opts = {}) {
|
|
|
6391
6441
|
var QUERIES_PER_INTENT_BUCKET = 6;
|
|
6392
6442
|
async function executeDiscoveryRun(opts) {
|
|
6393
6443
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
6394
|
-
opts.db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
6444
|
+
opts.db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq12(runs.id, opts.runId)).run();
|
|
6395
6445
|
try {
|
|
6396
|
-
const projectRow = opts.db.select().from(projects).where(
|
|
6446
|
+
const projectRow = opts.db.select().from(projects).where(eq12(projects.id, opts.projectId)).get();
|
|
6397
6447
|
if (!projectRow) throw new Error(`Project ${opts.projectId} not found`);
|
|
6398
|
-
const projectCompetitors = opts.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
6448
|
+
const projectCompetitors = opts.db.select({ domain: competitors.domain }).from(competitors).where(eq12(competitors.projectId, opts.projectId)).all().map((r) => r.domain.toLowerCase());
|
|
6399
6449
|
const canonicalDomains = effectiveDomains({
|
|
6400
6450
|
canonicalDomain: projectRow.canonicalDomain,
|
|
6401
6451
|
ownedDomains: projectRow.ownedDomains
|
|
@@ -6433,7 +6483,7 @@ async function executeDiscoveryRun(opts) {
|
|
|
6433
6483
|
seedProvider: result.seedProvider,
|
|
6434
6484
|
result
|
|
6435
6485
|
});
|
|
6436
|
-
opts.db.update(runs).set({ status: RunStatuses.completed, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
6486
|
+
opts.db.update(runs).set({ status: RunStatuses.completed, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq12(runs.id, opts.runId)).run();
|
|
6437
6487
|
log11.info("discovery.completed", {
|
|
6438
6488
|
runId: opts.runId,
|
|
6439
6489
|
sessionId: opts.sessionId,
|
|
@@ -6448,7 +6498,7 @@ async function executeDiscoveryRun(opts) {
|
|
|
6448
6498
|
status: RunStatuses.failed,
|
|
6449
6499
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
6450
6500
|
error: errorMsg
|
|
6451
|
-
}).where(
|
|
6501
|
+
}).where(eq12(runs.id, opts.runId)).run();
|
|
6452
6502
|
}
|
|
6453
6503
|
}
|
|
6454
6504
|
function buildDefaultDeps(registry) {
|
|
@@ -6698,12 +6748,12 @@ function writeDiscoveryInsight(db, input) {
|
|
|
6698
6748
|
});
|
|
6699
6749
|
db.transaction((tx) => {
|
|
6700
6750
|
tx.update(insights).set({ dismissed: true }).where(and9(
|
|
6701
|
-
|
|
6702
|
-
|
|
6703
|
-
|
|
6751
|
+
eq12(insights.projectId, input.projectId),
|
|
6752
|
+
eq12(insights.type, "discovery.basket-divergence"),
|
|
6753
|
+
eq12(insights.dismissed, false)
|
|
6704
6754
|
)).run();
|
|
6705
6755
|
tx.insert(insights).values({
|
|
6706
|
-
id:
|
|
6756
|
+
id: crypto14.randomUUID(),
|
|
6707
6757
|
projectId: input.projectId,
|
|
6708
6758
|
runId: input.runId,
|
|
6709
6759
|
type: "discovery.basket-divergence",
|
|
@@ -6739,8 +6789,8 @@ function buildDiscoveryInsightTitle(input) {
|
|
|
6739
6789
|
}
|
|
6740
6790
|
|
|
6741
6791
|
// src/execute-site-audit.ts
|
|
6742
|
-
import
|
|
6743
|
-
import { eq as
|
|
6792
|
+
import crypto15 from "crypto";
|
|
6793
|
+
import { eq as eq13 } from "drizzle-orm";
|
|
6744
6794
|
import { runSitemapAudit } from "@ainyc/aeo-audit";
|
|
6745
6795
|
var log12 = createLogger("SiteAudit");
|
|
6746
6796
|
var SITE_AUDIT_DEFAULT_PAGE_LIMIT = 500;
|
|
@@ -6803,9 +6853,9 @@ function computeFactorAverages(pages) {
|
|
|
6803
6853
|
}
|
|
6804
6854
|
async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
6805
6855
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
6806
|
-
db.update(runs).set({ status: "running", startedAt }).where(
|
|
6856
|
+
db.update(runs).set({ status: "running", startedAt }).where(eq13(runs.id, runId)).run();
|
|
6807
6857
|
try {
|
|
6808
|
-
const project = db.select().from(projects).where(
|
|
6858
|
+
const project = db.select().from(projects).where(eq13(projects.id, projectId)).get();
|
|
6809
6859
|
if (!project) {
|
|
6810
6860
|
throw new Error(`Project not found: ${projectId}`);
|
|
6811
6861
|
}
|
|
@@ -6838,7 +6888,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
6838
6888
|
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
6839
6889
|
db.transaction((tx) => {
|
|
6840
6890
|
tx.insert(siteAuditSnapshots).values({
|
|
6841
|
-
id:
|
|
6891
|
+
id: crypto15.randomUUID(),
|
|
6842
6892
|
projectId,
|
|
6843
6893
|
runId,
|
|
6844
6894
|
sitemapUrl: report.sitemapUrl,
|
|
@@ -6867,7 +6917,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
6867
6917
|
}).run();
|
|
6868
6918
|
for (const page of report.pages) {
|
|
6869
6919
|
tx.insert(siteAuditPages).values({
|
|
6870
|
-
id:
|
|
6920
|
+
id: crypto15.randomUUID(),
|
|
6871
6921
|
projectId,
|
|
6872
6922
|
runId,
|
|
6873
6923
|
url: page.url,
|
|
@@ -6878,7 +6928,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
6878
6928
|
createdAt: finishedAt
|
|
6879
6929
|
}).run();
|
|
6880
6930
|
}
|
|
6881
|
-
tx.update(runs).set({ status, finishedAt }).where(
|
|
6931
|
+
tx.update(runs).set({ status, finishedAt }).where(eq13(runs.id, runId)).run();
|
|
6882
6932
|
});
|
|
6883
6933
|
log12.info("completed", {
|
|
6884
6934
|
runId,
|
|
@@ -6890,14 +6940,14 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
6890
6940
|
});
|
|
6891
6941
|
} catch (err) {
|
|
6892
6942
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
6893
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
6943
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq13(runs.id, runId)).run();
|
|
6894
6944
|
log12.error("failed", { runId, projectId, error: errorMsg });
|
|
6895
6945
|
throw err;
|
|
6896
6946
|
}
|
|
6897
6947
|
}
|
|
6898
6948
|
|
|
6899
6949
|
// src/commands/backfill.ts
|
|
6900
|
-
import { and as and10, eq as
|
|
6950
|
+
import { and as and10, eq as eq14, inArray as inArray4, isNull, sql as sql5 } from "drizzle-orm";
|
|
6901
6951
|
var SNAPSHOT_BATCH_SIZE = 500;
|
|
6902
6952
|
async function backfillAnswerVisibilityCommand(opts) {
|
|
6903
6953
|
const config = loadConfig();
|
|
@@ -6905,7 +6955,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
6905
6955
|
migrate(db);
|
|
6906
6956
|
const projectFilter = opts?.project?.trim();
|
|
6907
6957
|
const isDryRun = opts?.dryRun === true;
|
|
6908
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
6958
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq14(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
6909
6959
|
let examined = 0;
|
|
6910
6960
|
let updated = 0;
|
|
6911
6961
|
let wouldUpdate = 0;
|
|
@@ -6914,9 +6964,9 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
6914
6964
|
let providerErrors = 0;
|
|
6915
6965
|
if (scopedProjects.length > 0) {
|
|
6916
6966
|
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(and10(
|
|
6917
|
-
|
|
6967
|
+
eq14(runs.kind, RunKinds["answer-visibility"]),
|
|
6918
6968
|
inArray4(runs.projectId, scopedProjects.map((project) => project.id))
|
|
6919
|
-
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
6969
|
+
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(eq14(runs.kind, RunKinds["answer-visibility"])).all();
|
|
6920
6970
|
const runIdsByProject = /* @__PURE__ */ new Map();
|
|
6921
6971
|
for (const run of runRows) {
|
|
6922
6972
|
const existing = runIdsByProject.get(run.projectId);
|
|
@@ -6924,7 +6974,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
6924
6974
|
else runIdsByProject.set(run.projectId, [run.id]);
|
|
6925
6975
|
}
|
|
6926
6976
|
for (const project of scopedProjects) {
|
|
6927
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
6977
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq14(competitors.projectId, project.id)).all().map((row) => row.domain);
|
|
6928
6978
|
const runIds = runIdsByProject.get(project.id) ?? [];
|
|
6929
6979
|
if (runIds.length === 0) continue;
|
|
6930
6980
|
const projectDomains = effectiveDomains({
|
|
@@ -7017,7 +7067,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
7017
7067
|
} else {
|
|
7018
7068
|
db.transaction((tx) => {
|
|
7019
7069
|
for (const update of pendingUpdates) {
|
|
7020
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
7070
|
+
tx.update(querySnapshots).set(update.patch).where(eq14(querySnapshots.id, update.id)).run();
|
|
7021
7071
|
}
|
|
7022
7072
|
});
|
|
7023
7073
|
updated += pendingUpdates.length;
|
|
@@ -7066,7 +7116,7 @@ No DB writes performed. Re-run without --dry-run to apply.`);
|
|
|
7066
7116
|
function backfillNormalizedPaths(db, opts) {
|
|
7067
7117
|
const baseConditions = [];
|
|
7068
7118
|
if (opts?.projectId) {
|
|
7069
|
-
baseConditions.push(
|
|
7119
|
+
baseConditions.push(eq14(gaTrafficSnapshots.projectId, opts.projectId));
|
|
7070
7120
|
}
|
|
7071
7121
|
const rows = db.select({
|
|
7072
7122
|
id: gaTrafficSnapshots.id,
|
|
@@ -7087,7 +7137,7 @@ function backfillNormalizedPaths(db, opts) {
|
|
|
7087
7137
|
unchanged++;
|
|
7088
7138
|
continue;
|
|
7089
7139
|
}
|
|
7090
|
-
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(
|
|
7140
|
+
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(eq14(gaTrafficSnapshots.id, row.id)).run();
|
|
7091
7141
|
updated++;
|
|
7092
7142
|
}
|
|
7093
7143
|
});
|
|
@@ -7101,7 +7151,7 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
7101
7151
|
const projectFilter = opts?.project?.trim();
|
|
7102
7152
|
let projectId;
|
|
7103
7153
|
if (projectFilter) {
|
|
7104
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
7154
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq14(projects.name, projectFilter)).get();
|
|
7105
7155
|
if (!project) {
|
|
7106
7156
|
const result2 = {
|
|
7107
7157
|
project: projectFilter,
|
|
@@ -7138,7 +7188,7 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
7138
7188
|
function backfillAiReferralPaths(db, opts) {
|
|
7139
7189
|
const baseConditions = [];
|
|
7140
7190
|
if (opts?.projectId) {
|
|
7141
|
-
baseConditions.push(
|
|
7191
|
+
baseConditions.push(eq14(gaAiReferrals.projectId, opts.projectId));
|
|
7142
7192
|
}
|
|
7143
7193
|
const rows = db.select({
|
|
7144
7194
|
id: gaAiReferrals.id,
|
|
@@ -7159,7 +7209,7 @@ function backfillAiReferralPaths(db, opts) {
|
|
|
7159
7209
|
unchanged++;
|
|
7160
7210
|
continue;
|
|
7161
7211
|
}
|
|
7162
|
-
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(
|
|
7212
|
+
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(eq14(gaAiReferrals.id, row.id)).run();
|
|
7163
7213
|
updated++;
|
|
7164
7214
|
}
|
|
7165
7215
|
});
|
|
@@ -7173,7 +7223,7 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
7173
7223
|
const projectFilter = opts?.project?.trim();
|
|
7174
7224
|
let projectId;
|
|
7175
7225
|
if (projectFilter) {
|
|
7176
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
7226
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq14(projects.name, projectFilter)).get();
|
|
7177
7227
|
if (!project) {
|
|
7178
7228
|
const result2 = {
|
|
7179
7229
|
project: projectFilter,
|
|
@@ -7209,10 +7259,10 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
7209
7259
|
}
|
|
7210
7260
|
function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
7211
7261
|
const isDryRun = opts?.dryRun === true;
|
|
7212
|
-
const project = db.select().from(projects).where(
|
|
7262
|
+
const project = db.select().from(projects).where(eq14(projects.id, projectId)).get();
|
|
7213
7263
|
if (!project) return { examined: 0, updated: 0, mentioned: 0 };
|
|
7214
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
7215
|
-
const runRows = db.select({ id: runs.id }).from(runs).where(and10(
|
|
7264
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq14(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
7265
|
+
const runRows = db.select({ id: runs.id }).from(runs).where(and10(eq14(runs.kind, RunKinds["answer-visibility"]), eq14(runs.projectId, projectId))).all();
|
|
7216
7266
|
const runIds = runRows.map((r) => r.id);
|
|
7217
7267
|
let examined = 0;
|
|
7218
7268
|
let updated = 0;
|
|
@@ -7286,7 +7336,7 @@ function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
|
7286
7336
|
} else {
|
|
7287
7337
|
db.transaction((tx) => {
|
|
7288
7338
|
for (const update of pendingUpdates) {
|
|
7289
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
7339
|
+
tx.update(querySnapshots).set(update.patch).where(eq14(querySnapshots.id, update.id)).run();
|
|
7290
7340
|
}
|
|
7291
7341
|
});
|
|
7292
7342
|
updated += pendingUpdates.length;
|
|
@@ -7301,7 +7351,7 @@ async function backfillAnswerMentionsCommand(opts) {
|
|
|
7301
7351
|
migrate(db);
|
|
7302
7352
|
const projectFilter = opts?.project?.trim();
|
|
7303
7353
|
const isDryRun = opts?.dryRun === true;
|
|
7304
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
7354
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq14(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
7305
7355
|
let examined = 0;
|
|
7306
7356
|
let updated = 0;
|
|
7307
7357
|
let wouldUpdate = 0;
|
|
@@ -7520,7 +7570,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
7520
7570
|
const config = loadConfig();
|
|
7521
7571
|
const db = createClient(config.database);
|
|
7522
7572
|
migrate(db);
|
|
7523
|
-
const project = db.select().from(projects).where(
|
|
7573
|
+
const project = db.select().from(projects).where(eq14(projects.name, opts.project)).get();
|
|
7524
7574
|
if (!project) {
|
|
7525
7575
|
throw new Error(`Project "${opts.project}" not found`);
|
|
7526
7576
|
}
|
|
@@ -7532,7 +7582,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
7532
7582
|
`);
|
|
7533
7583
|
}
|
|
7534
7584
|
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(and10(
|
|
7535
|
-
|
|
7585
|
+
eq14(auditLog.projectId, project.id),
|
|
7536
7586
|
inArray4(auditLog.action, ["keywords.appended", "keywords.deleted", "queries.appended", "queries.deleted", "queries.replaced"])
|
|
7537
7587
|
)).orderBy(auditLog.createdAt).all();
|
|
7538
7588
|
const history = replayQueryAuditLog(events);
|
|
@@ -7540,8 +7590,8 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
7540
7590
|
runId: runs.id,
|
|
7541
7591
|
createdAt: runs.createdAt,
|
|
7542
7592
|
location: runs.location
|
|
7543
|
-
}).from(runs).innerJoin(querySnapshots,
|
|
7544
|
-
|
|
7593
|
+
}).from(runs).innerJoin(querySnapshots, eq14(querySnapshots.runId, runs.id)).where(and10(
|
|
7594
|
+
eq14(runs.projectId, project.id),
|
|
7545
7595
|
isNull(querySnapshots.queryId),
|
|
7546
7596
|
isNull(querySnapshots.queryText)
|
|
7547
7597
|
)).groupBy(runs.id).orderBy(runs.createdAt).all();
|
|
@@ -7564,7 +7614,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
7564
7614
|
createdAt: querySnapshots.createdAt,
|
|
7565
7615
|
answerText: querySnapshots.answerText
|
|
7566
7616
|
}).from(querySnapshots).where(and10(
|
|
7567
|
-
|
|
7617
|
+
eq14(querySnapshots.runId, run.runId),
|
|
7568
7618
|
isNull(querySnapshots.queryId),
|
|
7569
7619
|
isNull(querySnapshots.queryText)
|
|
7570
7620
|
)).orderBy(querySnapshots.provider, querySnapshots.createdAt).all();
|
|
@@ -7630,7 +7680,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
7630
7680
|
if (!isDryRun && updates.length > 0) {
|
|
7631
7681
|
db.transaction((tx) => {
|
|
7632
7682
|
for (const u of updates) {
|
|
7633
|
-
tx.update(querySnapshots).set({ queryText: u.queryText }).where(
|
|
7683
|
+
tx.update(querySnapshots).set({ queryText: u.queryText }).where(eq14(querySnapshots.id, u.id)).run();
|
|
7634
7684
|
}
|
|
7635
7685
|
});
|
|
7636
7686
|
}
|
|
@@ -7704,7 +7754,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7704
7754
|
const projectFilter = opts?.project?.trim();
|
|
7705
7755
|
const isDryRun = opts?.dryRun === true;
|
|
7706
7756
|
const isJson = isMachineFormat(opts?.format);
|
|
7707
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
7757
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq14(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
7708
7758
|
if (scopedProjects.length === 0) {
|
|
7709
7759
|
if (projectFilter && !isJson) {
|
|
7710
7760
|
process.stderr.write(`No project named "${projectFilter}".
|
|
@@ -7730,7 +7780,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7730
7780
|
byBot: {}
|
|
7731
7781
|
};
|
|
7732
7782
|
const unknownCountRow = db.select({ n: sql5`count(*)` }).from(rawEventSamples).where(and10(
|
|
7733
|
-
|
|
7783
|
+
eq14(rawEventSamples.eventType, "unknown"),
|
|
7734
7784
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
7735
7785
|
)).get();
|
|
7736
7786
|
result.unknownBefore = Number(unknownCountRow?.n ?? 0);
|
|
@@ -7743,7 +7793,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7743
7793
|
pathNormalized: rawEventSamples.pathNormalized,
|
|
7744
7794
|
status: rawEventSamples.status
|
|
7745
7795
|
}).from(rawEventSamples).where(and10(
|
|
7746
|
-
|
|
7796
|
+
eq14(rawEventSamples.eventType, "unknown"),
|
|
7747
7797
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
7748
7798
|
)).all();
|
|
7749
7799
|
result.examined = unknownSamples.length;
|
|
@@ -7782,7 +7832,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7782
7832
|
result.reclassified++;
|
|
7783
7833
|
result.byBot[classified.botId] = (result.byBot[classified.botId] ?? 0) + 1;
|
|
7784
7834
|
if (isDryRun) continue;
|
|
7785
|
-
db.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(
|
|
7835
|
+
db.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(eq14(rawEventSamples.id, snap.id)).run();
|
|
7786
7836
|
const tsHour = new Date(snap.ts);
|
|
7787
7837
|
tsHour.setUTCMinutes(0, 0, 0);
|
|
7788
7838
|
if (userFetch) {
|
|
@@ -7847,7 +7897,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7847
7897
|
}
|
|
7848
7898
|
if (!isDryRun) {
|
|
7849
7899
|
const afterRow = db.select({ n: sql5`count(*)` }).from(rawEventSamples).where(and10(
|
|
7850
|
-
|
|
7900
|
+
eq14(rawEventSamples.eventType, "unknown"),
|
|
7851
7901
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
7852
7902
|
)).get();
|
|
7853
7903
|
result.unknownAfter = Number(afterRow?.n ?? 0);
|
|
@@ -7882,7 +7932,7 @@ No DB writes performed. Re-run without --dry-run to apply.`);
|
|
|
7882
7932
|
}
|
|
7883
7933
|
|
|
7884
7934
|
// src/commands/skills.ts
|
|
7885
|
-
import
|
|
7935
|
+
import crypto16 from "crypto";
|
|
7886
7936
|
import fs4 from "fs";
|
|
7887
7937
|
import os4 from "os";
|
|
7888
7938
|
import path5 from "path";
|
|
@@ -7937,7 +7987,7 @@ function walkRelative(dir, prefix = "") {
|
|
|
7937
7987
|
return out.sort();
|
|
7938
7988
|
}
|
|
7939
7989
|
function sha256File(filePath) {
|
|
7940
|
-
return
|
|
7990
|
+
return crypto16.createHash("sha256").update(fs4.readFileSync(filePath)).digest("hex");
|
|
7941
7991
|
}
|
|
7942
7992
|
function readSkillManifest(skillDir) {
|
|
7943
7993
|
try {
|
|
@@ -8279,9 +8329,9 @@ var ProviderRegistry = class {
|
|
|
8279
8329
|
};
|
|
8280
8330
|
|
|
8281
8331
|
// src/scheduler.ts
|
|
8282
|
-
import
|
|
8332
|
+
import crypto17 from "crypto";
|
|
8283
8333
|
import cron from "node-cron";
|
|
8284
|
-
import { and as and11, eq as
|
|
8334
|
+
import { and as and11, eq as eq15, inArray as inArray5, notExists, sql as sql6 } from "drizzle-orm";
|
|
8285
8335
|
var log13 = createLogger("Scheduler");
|
|
8286
8336
|
var DEFAULT_HEALTH_CRON = "0 */6 * * *";
|
|
8287
8337
|
function taskKey(projectId, kind) {
|
|
@@ -8314,15 +8364,15 @@ var Scheduler = class {
|
|
|
8314
8364
|
ensureHealthSchedules() {
|
|
8315
8365
|
const projectsWithoutHealth = this.db.select({ id: projects.id }).from(projects).where(notExists(
|
|
8316
8366
|
this.db.select({ one: sql6`1` }).from(schedules).where(and11(
|
|
8317
|
-
|
|
8318
|
-
|
|
8367
|
+
eq15(schedules.projectId, projects.id),
|
|
8368
|
+
eq15(schedules.kind, SchedulableRunKinds.doctor)
|
|
8319
8369
|
))
|
|
8320
8370
|
)).all();
|
|
8321
8371
|
if (projectsWithoutHealth.length === 0) return;
|
|
8322
8372
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8323
8373
|
for (const project of projectsWithoutHealth) {
|
|
8324
8374
|
this.db.insert(schedules).values({
|
|
8325
|
-
id:
|
|
8375
|
+
id: crypto17.randomUUID(),
|
|
8326
8376
|
projectId: project.id,
|
|
8327
8377
|
kind: SchedulableRunKinds.doctor,
|
|
8328
8378
|
cronExpr: DEFAULT_HEALTH_CRON,
|
|
@@ -8366,7 +8416,7 @@ var Scheduler = class {
|
|
|
8366
8416
|
start() {
|
|
8367
8417
|
this.ensureHealthSchedules();
|
|
8368
8418
|
this.ensureQueryBaskets();
|
|
8369
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
8419
|
+
const allSchedules = this.db.select().from(schedules).where(eq15(schedules.enabled, true)).all();
|
|
8370
8420
|
for (const schedule of allSchedules) {
|
|
8371
8421
|
const missedRunAt = schedule.nextRunAt;
|
|
8372
8422
|
this.registerCronTask(schedule);
|
|
@@ -8396,7 +8446,7 @@ var Scheduler = class {
|
|
|
8396
8446
|
this.stopTask(key, existing, "Stopped");
|
|
8397
8447
|
this.tasks.delete(key);
|
|
8398
8448
|
}
|
|
8399
|
-
const schedule = this.db.select().from(schedules).where(and11(
|
|
8449
|
+
const schedule = this.db.select().from(schedules).where(and11(eq15(schedules.projectId, projectId), eq15(schedules.kind, kind))).get();
|
|
8400
8450
|
if (schedule && schedule.enabled) {
|
|
8401
8451
|
this.registerCronTask(schedule);
|
|
8402
8452
|
}
|
|
@@ -8437,21 +8487,21 @@ var Scheduler = class {
|
|
|
8437
8487
|
this.db.update(schedules).set({
|
|
8438
8488
|
nextRunAt: nextRunFromCron(cronExpr, timezone),
|
|
8439
8489
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
8440
|
-
}).where(
|
|
8490
|
+
}).where(eq15(schedules.id, scheduleId)).run();
|
|
8441
8491
|
const label = schedule.preset ?? cronExpr;
|
|
8442
8492
|
log13.info("cron.registered", { projectId, kind, schedule: label, timezone });
|
|
8443
8493
|
}
|
|
8444
8494
|
triggerRun(scheduleId, projectId, kind) {
|
|
8445
8495
|
try {
|
|
8446
8496
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8447
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
8497
|
+
const currentSchedule = this.db.select().from(schedules).where(eq15(schedules.id, scheduleId)).get();
|
|
8448
8498
|
if (!currentSchedule || !currentSchedule.enabled) {
|
|
8449
8499
|
log13.warn("schedule.stale", { scheduleId, projectId, kind, msg: "schedule no longer exists or is disabled" });
|
|
8450
8500
|
this.remove(projectId, kind);
|
|
8451
8501
|
return;
|
|
8452
8502
|
}
|
|
8453
8503
|
const nextRunAt = nextRunFromCron(currentSchedule.cronExpr, currentSchedule.timezone);
|
|
8454
|
-
const project = this.db.select().from(projects).where(
|
|
8504
|
+
const project = this.db.select().from(projects).where(eq15(projects.id, projectId)).get();
|
|
8455
8505
|
if (!project) {
|
|
8456
8506
|
log13.error("project.not-found", { projectId, kind, msg: "skipping scheduled run" });
|
|
8457
8507
|
this.remove(projectId, kind);
|
|
@@ -8471,7 +8521,7 @@ var Scheduler = class {
|
|
|
8471
8521
|
lastRunAt: now,
|
|
8472
8522
|
nextRunAt,
|
|
8473
8523
|
updatedAt: now
|
|
8474
|
-
}).where(
|
|
8524
|
+
}).where(eq15(schedules.id, currentSchedule.id)).run();
|
|
8475
8525
|
log13.info("traffic-sync.triggered", { projectName: project.name, sourceId });
|
|
8476
8526
|
this.callbacks.onTrafficSyncRequested(project.name, sourceId);
|
|
8477
8527
|
return;
|
|
@@ -8481,7 +8531,7 @@ var Scheduler = class {
|
|
|
8481
8531
|
log13.warn("gbp-sync.no-callback", { scheduleId, projectId, msg: "host did not register onGbpSyncRequested" });
|
|
8482
8532
|
return;
|
|
8483
8533
|
}
|
|
8484
|
-
const runId2 =
|
|
8534
|
+
const runId2 = crypto17.randomUUID();
|
|
8485
8535
|
this.db.insert(runs).values({
|
|
8486
8536
|
id: runId2,
|
|
8487
8537
|
projectId,
|
|
@@ -8494,7 +8544,7 @@ var Scheduler = class {
|
|
|
8494
8544
|
lastRunAt: now,
|
|
8495
8545
|
nextRunAt,
|
|
8496
8546
|
updatedAt: now
|
|
8497
|
-
}).where(
|
|
8547
|
+
}).where(eq15(schedules.id, currentSchedule.id)).run();
|
|
8498
8548
|
log13.info("gbp-sync.triggered", { runId: runId2, projectName: project.name });
|
|
8499
8549
|
this.callbacks.onGbpSyncRequested(runId2, projectId);
|
|
8500
8550
|
return;
|
|
@@ -8505,16 +8555,16 @@ var Scheduler = class {
|
|
|
8505
8555
|
return;
|
|
8506
8556
|
}
|
|
8507
8557
|
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(and11(
|
|
8508
|
-
|
|
8509
|
-
|
|
8558
|
+
eq15(runs.projectId, projectId),
|
|
8559
|
+
eq15(runs.kind, RunKinds["ads-sync"]),
|
|
8510
8560
|
inArray5(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
8511
8561
|
)).get();
|
|
8512
8562
|
if (activeAdsRun) {
|
|
8513
8563
|
log13.info("ads-sync.skipped-active", { projectName: project.name, activeRunId: activeAdsRun.id });
|
|
8514
|
-
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(
|
|
8564
|
+
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq15(schedules.id, currentSchedule.id)).run();
|
|
8515
8565
|
return;
|
|
8516
8566
|
}
|
|
8517
|
-
const runId2 =
|
|
8567
|
+
const runId2 = crypto17.randomUUID();
|
|
8518
8568
|
this.db.insert(runs).values({
|
|
8519
8569
|
id: runId2,
|
|
8520
8570
|
projectId,
|
|
@@ -8527,7 +8577,7 @@ var Scheduler = class {
|
|
|
8527
8577
|
lastRunAt: now,
|
|
8528
8578
|
nextRunAt,
|
|
8529
8579
|
updatedAt: now
|
|
8530
|
-
}).where(
|
|
8580
|
+
}).where(eq15(schedules.id, currentSchedule.id)).run();
|
|
8531
8581
|
log13.info("ads-sync.triggered", { runId: runId2, projectName: project.name });
|
|
8532
8582
|
this.callbacks.onAdsSyncRequested(runId2, projectId);
|
|
8533
8583
|
return;
|
|
@@ -8541,7 +8591,7 @@ var Scheduler = class {
|
|
|
8541
8591
|
lastRunAt: now,
|
|
8542
8592
|
nextRunAt,
|
|
8543
8593
|
updatedAt: now
|
|
8544
|
-
}).where(
|
|
8594
|
+
}).where(eq15(schedules.id, currentSchedule.id)).run();
|
|
8545
8595
|
log13.info("data-refresh.triggered", { projectName: project.name });
|
|
8546
8596
|
this.callbacks.onDataRefreshRequested(project.name);
|
|
8547
8597
|
return;
|
|
@@ -8555,7 +8605,7 @@ var Scheduler = class {
|
|
|
8555
8605
|
lastRunAt: now,
|
|
8556
8606
|
nextRunAt,
|
|
8557
8607
|
updatedAt: now
|
|
8558
|
-
}).where(
|
|
8608
|
+
}).where(eq15(schedules.id, currentSchedule.id)).run();
|
|
8559
8609
|
log13.info("doctor.triggered", { projectName: project.name });
|
|
8560
8610
|
this.callbacks.onDoctorRequested(project.name);
|
|
8561
8611
|
return;
|
|
@@ -8569,7 +8619,7 @@ var Scheduler = class {
|
|
|
8569
8619
|
lastRunAt: now,
|
|
8570
8620
|
nextRunAt,
|
|
8571
8621
|
updatedAt: now
|
|
8572
|
-
}).where(
|
|
8622
|
+
}).where(eq15(schedules.id, currentSchedule.id)).run();
|
|
8573
8623
|
log13.info("backlinks-sync.triggered", { projectName: project.name });
|
|
8574
8624
|
this.callbacks.onBacklinksSyncRequested(project.name);
|
|
8575
8625
|
return;
|
|
@@ -8580,16 +8630,16 @@ var Scheduler = class {
|
|
|
8580
8630
|
return;
|
|
8581
8631
|
}
|
|
8582
8632
|
const active = this.db.select({ id: runs.id }).from(runs).where(and11(
|
|
8583
|
-
|
|
8584
|
-
|
|
8633
|
+
eq15(runs.projectId, projectId),
|
|
8634
|
+
eq15(runs.kind, RunKinds["site-audit"]),
|
|
8585
8635
|
inArray5(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
8586
8636
|
)).get();
|
|
8587
8637
|
if (active) {
|
|
8588
8638
|
log13.info("site-audit.skipped-active", { projectName: project.name, activeRunId: active.id });
|
|
8589
|
-
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(
|
|
8639
|
+
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq15(schedules.id, currentSchedule.id)).run();
|
|
8590
8640
|
return;
|
|
8591
8641
|
}
|
|
8592
|
-
const runId2 =
|
|
8642
|
+
const runId2 = crypto17.randomUUID();
|
|
8593
8643
|
this.db.insert(runs).values({
|
|
8594
8644
|
id: runId2,
|
|
8595
8645
|
projectId,
|
|
@@ -8602,7 +8652,7 @@ var Scheduler = class {
|
|
|
8602
8652
|
lastRunAt: now,
|
|
8603
8653
|
nextRunAt,
|
|
8604
8654
|
updatedAt: now
|
|
8605
|
-
}).where(
|
|
8655
|
+
}).where(eq15(schedules.id, currentSchedule.id)).run();
|
|
8606
8656
|
log13.info("site-audit.triggered", { runId: runId2, projectName: project.name });
|
|
8607
8657
|
this.callbacks.onSiteAuditRequested(runId2, projectId);
|
|
8608
8658
|
return;
|
|
@@ -8635,7 +8685,7 @@ var Scheduler = class {
|
|
|
8635
8685
|
this.db.update(schedules).set({
|
|
8636
8686
|
nextRunAt,
|
|
8637
8687
|
updatedAt: now
|
|
8638
|
-
}).where(
|
|
8688
|
+
}).where(eq15(schedules.id, currentSchedule.id)).run();
|
|
8639
8689
|
return;
|
|
8640
8690
|
}
|
|
8641
8691
|
const runId = queueResult.runId;
|
|
@@ -8643,7 +8693,7 @@ var Scheduler = class {
|
|
|
8643
8693
|
lastRunAt: now,
|
|
8644
8694
|
nextRunAt,
|
|
8645
8695
|
updatedAt: now
|
|
8646
|
-
}).where(
|
|
8696
|
+
}).where(eq15(schedules.id, currentSchedule.id)).run();
|
|
8647
8697
|
log13.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
8648
8698
|
this.callbacks.onRunCreated(runId, projectId, providers, resolvedLocation);
|
|
8649
8699
|
} catch (err) {
|
|
@@ -8676,8 +8726,8 @@ async function refreshAllIntegrations(client, projectName) {
|
|
|
8676
8726
|
}
|
|
8677
8727
|
|
|
8678
8728
|
// src/notifier.ts
|
|
8679
|
-
import { eq as
|
|
8680
|
-
import
|
|
8729
|
+
import { eq as eq16, desc as desc6, and as and12, inArray as inArray6, or } from "drizzle-orm";
|
|
8730
|
+
import crypto18 from "crypto";
|
|
8681
8731
|
var log15 = createLogger("Notifier");
|
|
8682
8732
|
var Notifier = class {
|
|
8683
8733
|
db;
|
|
@@ -8689,18 +8739,18 @@ var Notifier = class {
|
|
|
8689
8739
|
/** Called after a run completes (success, partial, or failed). */
|
|
8690
8740
|
async onRunCompleted(runId, projectId) {
|
|
8691
8741
|
log15.info("run.completed", { runId, projectId });
|
|
8692
|
-
const notifs = this.db.select().from(notifications).where(
|
|
8742
|
+
const notifs = this.db.select().from(notifications).where(eq16(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
8693
8743
|
if (notifs.length === 0) {
|
|
8694
8744
|
log15.info("notifications.none-enabled", { projectId });
|
|
8695
8745
|
return;
|
|
8696
8746
|
}
|
|
8697
8747
|
log15.info("notifications.found", { projectId, count: notifs.length });
|
|
8698
|
-
const run = this.db.select().from(runs).where(
|
|
8748
|
+
const run = this.db.select().from(runs).where(eq16(runs.id, runId)).get();
|
|
8699
8749
|
if (!run) {
|
|
8700
8750
|
log15.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
8701
8751
|
return;
|
|
8702
8752
|
}
|
|
8703
|
-
const project = this.db.select().from(projects).where(
|
|
8753
|
+
const project = this.db.select().from(projects).where(eq16(projects.id, projectId)).get();
|
|
8704
8754
|
if (!project) {
|
|
8705
8755
|
log15.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
8706
8756
|
return;
|
|
@@ -8753,7 +8803,7 @@ var Notifier = class {
|
|
|
8753
8803
|
* inferred from the absence of a webhook.
|
|
8754
8804
|
*/
|
|
8755
8805
|
async onHealthChecked(projectId, report) {
|
|
8756
|
-
const project = this.db.select().from(projects).where(
|
|
8806
|
+
const project = this.db.select().from(projects).where(eq16(projects.id, projectId)).get();
|
|
8757
8807
|
if (!project) {
|
|
8758
8808
|
log15.error("project.not-found", { projectId, msg: "skipping health notification" });
|
|
8759
8809
|
return null;
|
|
@@ -8784,7 +8834,7 @@ var Notifier = class {
|
|
|
8784
8834
|
const status = noSignal ? "warn" : worst ? worst.status : "ok";
|
|
8785
8835
|
const code = noSignal ? "health.no-signal" : worst?.code ?? "health.ok";
|
|
8786
8836
|
const summary = noSignal ? `No health check produced a result (${report.checks.length} skipped) \u2014 health is unknown, not confirmed.` : worst?.summary ?? `All ${graded.length} health check(s) passing.`;
|
|
8787
|
-
const previous = this.db.select().from(doctorHealthState).where(
|
|
8837
|
+
const previous = this.db.select().from(doctorHealthState).where(eq16(doctorHealthState.projectId, projectId)).get();
|
|
8788
8838
|
const previousStatus = previous?.status ?? null;
|
|
8789
8839
|
let event = null;
|
|
8790
8840
|
if (previous === void 0) {
|
|
@@ -8799,13 +8849,13 @@ var Notifier = class {
|
|
|
8799
8849
|
if (previous === void 0) {
|
|
8800
8850
|
this.db.insert(doctorHealthState).values({ ...observation, notifiedAt: null }).run();
|
|
8801
8851
|
} else {
|
|
8802
|
-
this.db.update(doctorHealthState).set(observation).where(
|
|
8852
|
+
this.db.update(doctorHealthState).set(observation).where(eq16(doctorHealthState.projectId, projectId)).run();
|
|
8803
8853
|
}
|
|
8804
8854
|
if (!event) {
|
|
8805
8855
|
log15.info("health.unchanged", { projectId, status, code });
|
|
8806
8856
|
return null;
|
|
8807
8857
|
}
|
|
8808
|
-
const notifs = this.db.select().from(notifications).where(
|
|
8858
|
+
const notifs = this.db.select().from(notifications).where(eq16(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
8809
8859
|
const payload = {
|
|
8810
8860
|
source: "canonry",
|
|
8811
8861
|
event,
|
|
@@ -8829,7 +8879,7 @@ var Notifier = class {
|
|
|
8829
8879
|
delivered += 1;
|
|
8830
8880
|
}
|
|
8831
8881
|
if (delivered > 0) {
|
|
8832
|
-
this.db.update(doctorHealthState).set({ notifiedAt: now }).where(
|
|
8882
|
+
this.db.update(doctorHealthState).set({ notifiedAt: now }).where(eq16(doctorHealthState.projectId, projectId)).run();
|
|
8833
8883
|
}
|
|
8834
8884
|
log15.info("health.notified", { projectId, event, status, code, subscribers: notifs.length, delivered });
|
|
8835
8885
|
return event;
|
|
@@ -8842,11 +8892,11 @@ var Notifier = class {
|
|
|
8842
8892
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
8843
8893
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
8844
8894
|
if (insightEvents.length === 0) return;
|
|
8845
|
-
const notifs = this.db.select().from(notifications).where(
|
|
8895
|
+
const notifs = this.db.select().from(notifications).where(eq16(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
8846
8896
|
if (notifs.length === 0) return;
|
|
8847
|
-
const run = this.db.select().from(runs).where(
|
|
8897
|
+
const run = this.db.select().from(runs).where(eq16(runs.id, runId)).get();
|
|
8848
8898
|
if (!run) return;
|
|
8849
|
-
const project = this.db.select().from(projects).where(
|
|
8899
|
+
const project = this.db.select().from(projects).where(eq16(projects.id, projectId)).get();
|
|
8850
8900
|
if (!project) return;
|
|
8851
8901
|
for (const notif of notifs) {
|
|
8852
8902
|
const config = notif.config;
|
|
@@ -8876,7 +8926,7 @@ var Notifier = class {
|
|
|
8876
8926
|
}
|
|
8877
8927
|
}
|
|
8878
8928
|
computeTransitions(runId, projectId) {
|
|
8879
|
-
const thisRun = this.db.select().from(runs).where(
|
|
8929
|
+
const thisRun = this.db.select().from(runs).where(eq16(runs.id, runId)).get();
|
|
8880
8930
|
if (!thisRun) return [];
|
|
8881
8931
|
const completeness = measurementRunCompleteness(this.db, runId);
|
|
8882
8932
|
if (completeness.planned && !completeness.complete) {
|
|
@@ -8888,9 +8938,9 @@ var Notifier = class {
|
|
|
8888
8938
|
return [];
|
|
8889
8939
|
}
|
|
8890
8940
|
const groupSiblings = this.db.select().from(runs).where(and12(
|
|
8891
|
-
|
|
8892
|
-
|
|
8893
|
-
|
|
8941
|
+
eq16(runs.projectId, projectId),
|
|
8942
|
+
eq16(runs.kind, thisRun.kind),
|
|
8943
|
+
eq16(runs.createdAt, thisRun.createdAt)
|
|
8894
8944
|
)).all();
|
|
8895
8945
|
const stillPending = groupSiblings.some((r) => r.status === "queued" || r.status === "running");
|
|
8896
8946
|
if (stillPending) return [];
|
|
@@ -8906,7 +8956,7 @@ var Notifier = class {
|
|
|
8906
8956
|
return candidate.id > best.id ? candidate : best;
|
|
8907
8957
|
});
|
|
8908
8958
|
if (winner.id !== runId) return [];
|
|
8909
|
-
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
8959
|
+
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(eq16(projects.id, projectId)).get();
|
|
8910
8960
|
const locationCount = Math.max(
|
|
8911
8961
|
1,
|
|
8912
8962
|
(projectLocations?.locations ?? []).length
|
|
@@ -8914,11 +8964,11 @@ var Notifier = class {
|
|
|
8914
8964
|
const RECENT_FETCH_LIMIT = Math.max(8, locationCount * 4);
|
|
8915
8965
|
const recentRuns = this.db.select().from(runs).where(
|
|
8916
8966
|
and12(
|
|
8917
|
-
|
|
8918
|
-
|
|
8919
|
-
or(
|
|
8967
|
+
eq16(runs.projectId, projectId),
|
|
8968
|
+
eq16(runs.kind, thisRun.kind),
|
|
8969
|
+
or(eq16(runs.status, "completed"), eq16(runs.status, "partial"))
|
|
8920
8970
|
)
|
|
8921
|
-
).orderBy(
|
|
8971
|
+
).orderBy(desc6(runs.createdAt), desc6(runs.id)).limit(RECENT_FETCH_LIMIT).all();
|
|
8922
8972
|
const groups = groupRunsByCreatedAt(recentRuns);
|
|
8923
8973
|
const currentGroupIdx = groups.findIndex((g) => g[0]?.createdAt === thisRun.createdAt);
|
|
8924
8974
|
if (currentGroupIdx < 0) return [];
|
|
@@ -8933,7 +8983,7 @@ var Notifier = class {
|
|
|
8933
8983
|
provider: querySnapshots.provider,
|
|
8934
8984
|
location: querySnapshots.location,
|
|
8935
8985
|
citationState: querySnapshots.citationState
|
|
8936
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
8986
|
+
}).from(querySnapshots).leftJoin(queries, eq16(querySnapshots.queryId, queries.id)).where(inArray6(querySnapshots.runId, currentRunIds)).all();
|
|
8937
8987
|
const previousSnapshots = this.db.select({
|
|
8938
8988
|
queryId: querySnapshots.queryId,
|
|
8939
8989
|
provider: querySnapshots.provider,
|
|
@@ -9003,7 +9053,7 @@ var Notifier = class {
|
|
|
9003
9053
|
}
|
|
9004
9054
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
9005
9055
|
this.db.insert(auditLog).values({
|
|
9006
|
-
id:
|
|
9056
|
+
id: crypto18.randomUUID(),
|
|
9007
9057
|
projectId,
|
|
9008
9058
|
actor: "scheduler",
|
|
9009
9059
|
action: `notification.${status}`,
|
|
@@ -9016,7 +9066,7 @@ var Notifier = class {
|
|
|
9016
9066
|
};
|
|
9017
9067
|
|
|
9018
9068
|
// src/run-coordinator.ts
|
|
9019
|
-
import { eq as
|
|
9069
|
+
import { eq as eq17 } from "drizzle-orm";
|
|
9020
9070
|
var log16 = createLogger("RunCoordinator");
|
|
9021
9071
|
var RunCoordinator = class {
|
|
9022
9072
|
constructor(db, notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
@@ -9032,7 +9082,7 @@ var RunCoordinator = class {
|
|
|
9032
9082
|
onInsightsGenerated;
|
|
9033
9083
|
onAeroEvent;
|
|
9034
9084
|
async onRunCompleted(runId, projectId) {
|
|
9035
|
-
const runRow = this.db.select().from(runs).where(
|
|
9085
|
+
const runRow = this.db.select().from(runs).where(eq17(runs.id, runId)).get();
|
|
9036
9086
|
const kind = runRow?.kind ?? RunKinds["answer-visibility"];
|
|
9037
9087
|
if (runRow?.trigger === RunTriggers.probe) {
|
|
9038
9088
|
log16.info("probe.skip-side-effects", { runId, projectId, kind });
|
|
@@ -9123,7 +9173,7 @@ var RunCoordinator = class {
|
|
|
9123
9173
|
* so the Aero queue is never starved of a follow-up.
|
|
9124
9174
|
*/
|
|
9125
9175
|
buildDiscoveryAeroContext(runId, projectId, status, error) {
|
|
9126
|
-
const session = this.db.select().from(discoverySessions).where(
|
|
9176
|
+
const session = this.db.select().from(discoverySessions).where(eq17(discoverySessions.runId, runId)).get();
|
|
9127
9177
|
const competitorMap = session ? session.competitorMap : [];
|
|
9128
9178
|
return {
|
|
9129
9179
|
kind: RunKinds["aeo-discover-probe"],
|
|
@@ -9165,8 +9215,8 @@ function analysisResultFromInsights(insights2) {
|
|
|
9165
9215
|
}
|
|
9166
9216
|
|
|
9167
9217
|
// src/agent/session-registry.ts
|
|
9168
|
-
import
|
|
9169
|
-
import { eq as
|
|
9218
|
+
import crypto22 from "crypto";
|
|
9219
|
+
import { eq as eq19 } from "drizzle-orm";
|
|
9170
9220
|
|
|
9171
9221
|
// src/agent/session.ts
|
|
9172
9222
|
import fs7 from "fs";
|
|
@@ -9909,7 +9959,7 @@ function buildAeroStateTools(ctx, opts = {}) {
|
|
|
9909
9959
|
}
|
|
9910
9960
|
|
|
9911
9961
|
// src/agent/llm-usage.ts
|
|
9912
|
-
import
|
|
9962
|
+
import crypto19 from "crypto";
|
|
9913
9963
|
var AeroLlmUsageFeatures = {
|
|
9914
9964
|
turn: "aero.turn"
|
|
9915
9965
|
};
|
|
@@ -9928,7 +9978,7 @@ function recordLlmUsageEvent(args) {
|
|
|
9928
9978
|
const usage = args.message.usage;
|
|
9929
9979
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9930
9980
|
args.db.insert(llmUsageEvents).values({
|
|
9931
|
-
id:
|
|
9981
|
+
id: crypto19.randomUUID(),
|
|
9932
9982
|
projectId: args.projectId,
|
|
9933
9983
|
runId: args.runId,
|
|
9934
9984
|
agentSessionId: args.agentSessionId,
|
|
@@ -10004,7 +10054,7 @@ function splitAeroAnthropicSystemCachePayload(payload, model) {
|
|
|
10004
10054
|
}
|
|
10005
10055
|
|
|
10006
10056
|
// src/agent/tool-usage.ts
|
|
10007
|
-
import
|
|
10057
|
+
import crypto20 from "crypto";
|
|
10008
10058
|
var AeroToolEventStatuses = {
|
|
10009
10059
|
success: "success",
|
|
10010
10060
|
error: "error"
|
|
@@ -10067,7 +10117,7 @@ function createAeroToolUsageHooks(args) {
|
|
|
10067
10117
|
function recordAgentToolEvent(args) {
|
|
10068
10118
|
try {
|
|
10069
10119
|
args.db.insert(agentToolEvents).values({
|
|
10070
|
-
id:
|
|
10120
|
+
id: crypto20.randomUUID(),
|
|
10071
10121
|
projectId: args.projectId,
|
|
10072
10122
|
agentSessionId: args.agentSessionId,
|
|
10073
10123
|
toolCallId: args.toolCallId,
|
|
@@ -10294,8 +10344,8 @@ async function loadExternalMcpTools(servers, opts = {}) {
|
|
|
10294
10344
|
}
|
|
10295
10345
|
|
|
10296
10346
|
// src/agent/memory-store.ts
|
|
10297
|
-
import
|
|
10298
|
-
import { and as and13, desc as
|
|
10347
|
+
import crypto21 from "crypto";
|
|
10348
|
+
import { and as and13, desc as desc7, eq as eq18, like, sql as sql7 } from "drizzle-orm";
|
|
10299
10349
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
10300
10350
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
10301
10351
|
function rowToDto(row) {
|
|
@@ -10309,7 +10359,7 @@ function rowToDto(row) {
|
|
|
10309
10359
|
};
|
|
10310
10360
|
}
|
|
10311
10361
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
10312
|
-
const query = db.select().from(agentMemory).where(
|
|
10362
|
+
const query = db.select().from(agentMemory).where(eq18(agentMemory.projectId, projectId)).orderBy(desc7(agentMemory.updatedAt));
|
|
10313
10363
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
10314
10364
|
return rows.map(rowToDto);
|
|
10315
10365
|
}
|
|
@@ -10323,7 +10373,7 @@ function upsertMemoryEntry(db, args) {
|
|
|
10323
10373
|
throw new Error(`memory key prefix "${COMPACTION_KEY_PREFIX}" is reserved for compaction notes`);
|
|
10324
10374
|
}
|
|
10325
10375
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
10326
|
-
const id =
|
|
10376
|
+
const id = crypto21.randomUUID();
|
|
10327
10377
|
db.insert(agentMemory).values({
|
|
10328
10378
|
id,
|
|
10329
10379
|
projectId: args.projectId,
|
|
@@ -10340,12 +10390,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
10340
10390
|
updatedAt: now
|
|
10341
10391
|
}
|
|
10342
10392
|
}).run();
|
|
10343
|
-
const row = db.select().from(agentMemory).where(and13(
|
|
10393
|
+
const row = db.select().from(agentMemory).where(and13(eq18(agentMemory.projectId, args.projectId), eq18(agentMemory.key, args.key))).get();
|
|
10344
10394
|
if (!row) throw new Error("memory upsert produced no row");
|
|
10345
10395
|
return rowToDto(row);
|
|
10346
10396
|
}
|
|
10347
10397
|
function deleteMemoryEntry(db, projectId, key) {
|
|
10348
|
-
const result = db.delete(agentMemory).where(and13(
|
|
10398
|
+
const result = db.delete(agentMemory).where(and13(eq18(agentMemory.projectId, projectId), eq18(agentMemory.key, key))).run();
|
|
10349
10399
|
const changes = result.changes ?? 0;
|
|
10350
10400
|
return changes > 0;
|
|
10351
10401
|
}
|
|
@@ -10360,7 +10410,7 @@ function writeCompactionNote(db, args) {
|
|
|
10360
10410
|
}
|
|
10361
10411
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
10362
10412
|
const key = `${COMPACTION_KEY_PREFIX}${args.sessionId}:${now}`;
|
|
10363
|
-
const id =
|
|
10413
|
+
const id = crypto21.randomUUID();
|
|
10364
10414
|
let inserted;
|
|
10365
10415
|
db.transaction((tx) => {
|
|
10366
10416
|
tx.insert(agentMemory).values({
|
|
@@ -10375,15 +10425,15 @@ function writeCompactionNote(db, args) {
|
|
|
10375
10425
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
10376
10426
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
10377
10427
|
and13(
|
|
10378
|
-
|
|
10428
|
+
eq18(agentMemory.projectId, args.projectId),
|
|
10379
10429
|
like(agentMemory.key, `${sessionPrefix}%`)
|
|
10380
10430
|
)
|
|
10381
|
-
).orderBy(
|
|
10431
|
+
).orderBy(desc7(agentMemory.updatedAt)).all();
|
|
10382
10432
|
const stale = existing.slice(COMPACTION_NOTES_PER_SESSION).map((r) => r.id);
|
|
10383
10433
|
if (stale.length > 0) {
|
|
10384
10434
|
tx.delete(agentMemory).where(sql7`${agentMemory.id} IN (${sql7.join(stale.map((s) => sql7`${s}`), sql7`, `)})`).run();
|
|
10385
10435
|
}
|
|
10386
|
-
const row = tx.select().from(agentMemory).where(and13(
|
|
10436
|
+
const row = tx.select().from(agentMemory).where(and13(eq18(agentMemory.projectId, args.projectId), eq18(agentMemory.key, key))).get();
|
|
10387
10437
|
if (row) inserted = rowToDto(row);
|
|
10388
10438
|
});
|
|
10389
10439
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -10610,7 +10660,7 @@ var SessionRegistry = class {
|
|
|
10610
10660
|
modelProvider: effectiveProvider,
|
|
10611
10661
|
modelId: effectiveModelId,
|
|
10612
10662
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
10613
|
-
}).where(
|
|
10663
|
+
}).where(eq19(agentSessions.projectId, projectId)).run();
|
|
10614
10664
|
}
|
|
10615
10665
|
const agent2 = createAeroSession({
|
|
10616
10666
|
projectName,
|
|
@@ -10639,7 +10689,7 @@ var SessionRegistry = class {
|
|
|
10639
10689
|
}
|
|
10640
10690
|
const { provider, modelId } = resolveSessionProviderAndModel(this.opts.config, preferences);
|
|
10641
10691
|
const systemPrompt = loadAeroSystemPrompt();
|
|
10642
|
-
const sessionId =
|
|
10692
|
+
const sessionId = crypto22.randomUUID();
|
|
10643
10693
|
const agent = createAeroSession({
|
|
10644
10694
|
projectName,
|
|
10645
10695
|
client: this.opts.client,
|
|
@@ -10844,7 +10894,7 @@ ${lines.join("\n")}
|
|
|
10844
10894
|
modelProvider: nextProvider,
|
|
10845
10895
|
modelId: nextModelId,
|
|
10846
10896
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
10847
|
-
}).where(
|
|
10897
|
+
}).where(eq19(agentSessions.projectId, projectId)).run();
|
|
10848
10898
|
}
|
|
10849
10899
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
10850
10900
|
save(projectName) {
|
|
@@ -11008,17 +11058,17 @@ ${lines.join("\n")}
|
|
|
11008
11058
|
return id;
|
|
11009
11059
|
}
|
|
11010
11060
|
tryResolveProjectId(projectName) {
|
|
11011
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
11061
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq19(projects.name, projectName)).get();
|
|
11012
11062
|
return row?.id;
|
|
11013
11063
|
}
|
|
11014
11064
|
loadRow(projectId) {
|
|
11015
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
11065
|
+
const row = this.opts.db.select().from(agentSessions).where(eq19(agentSessions.projectId, projectId)).get();
|
|
11016
11066
|
return row ?? null;
|
|
11017
11067
|
}
|
|
11018
11068
|
insertRow(params) {
|
|
11019
11069
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
11020
11070
|
this.opts.db.insert(agentSessions).values({
|
|
11021
|
-
id: params.id ??
|
|
11071
|
+
id: params.id ?? crypto22.randomUUID(),
|
|
11022
11072
|
projectId: params.projectId,
|
|
11023
11073
|
systemPrompt: params.systemPrompt,
|
|
11024
11074
|
modelProvider: params.provider ?? params.modelProvider ?? AgentProviderIds.claude,
|
|
@@ -11031,14 +11081,14 @@ ${lines.join("\n")}
|
|
|
11031
11081
|
}
|
|
11032
11082
|
updateRow(projectId, patch) {
|
|
11033
11083
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
11034
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
11084
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq19(agentSessions.projectId, projectId)).run();
|
|
11035
11085
|
}
|
|
11036
11086
|
};
|
|
11037
11087
|
|
|
11038
11088
|
// src/agent/agent-routes.ts
|
|
11039
|
-
import { eq as
|
|
11089
|
+
import { eq as eq20 } from "drizzle-orm";
|
|
11040
11090
|
function resolveProject(db, name) {
|
|
11041
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
11091
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq20(projects.name, name)).get();
|
|
11042
11092
|
if (!row) throw notFound("project", name);
|
|
11043
11093
|
return row;
|
|
11044
11094
|
}
|
|
@@ -11047,7 +11097,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
11047
11097
|
"/projects/:name/agent/transcript",
|
|
11048
11098
|
async (request) => {
|
|
11049
11099
|
const project = resolveProject(opts.db, request.params.name);
|
|
11050
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
11100
|
+
const row = opts.db.select().from(agentSessions).where(eq20(agentSessions.projectId, project.id)).get();
|
|
11051
11101
|
if (!row) {
|
|
11052
11102
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
11053
11103
|
}
|
|
@@ -11071,7 +11121,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
11071
11121
|
async (request) => {
|
|
11072
11122
|
const project = resolveProject(opts.db, request.params.name);
|
|
11073
11123
|
opts.sessionRegistry.reset(project.name);
|
|
11074
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
11124
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq20(agentSessions.projectId, project.id)).run();
|
|
11075
11125
|
return { status: "reset" };
|
|
11076
11126
|
}
|
|
11077
11127
|
);
|
|
@@ -12014,20 +12064,20 @@ function clipText(value, length) {
|
|
|
12014
12064
|
}
|
|
12015
12065
|
|
|
12016
12066
|
// src/research-runner.ts
|
|
12017
|
-
import { and as and14, eq as
|
|
12067
|
+
import { and as and14, eq as eq21, inArray as inArray7, sql as sql8 } from "drizzle-orm";
|
|
12018
12068
|
var unfinishedResearchQueryStatuses = [ResearchQueryStatuses.queued, ResearchQueryStatuses.running];
|
|
12019
12069
|
function finalResearchRunStatus(completed, failed) {
|
|
12020
12070
|
if (failed === 0) return ResearchRunStatuses.completed;
|
|
12021
12071
|
return completed > 0 ? ResearchRunStatuses.partial : ResearchRunStatuses.failed;
|
|
12022
12072
|
}
|
|
12023
12073
|
async function executeResearchRun(db, registry, runId, projectId) {
|
|
12024
|
-
const run = db.select().from(researchRuns).where(and14(
|
|
12074
|
+
const run = db.select().from(researchRuns).where(and14(eq21(researchRuns.id, runId), eq21(researchRuns.projectId, projectId))).get();
|
|
12025
12075
|
if (!run || run.status !== ResearchRunStatuses.queued) return;
|
|
12026
|
-
const project = db.select().from(projects).where(
|
|
12076
|
+
const project = db.select().from(projects).where(eq21(projects.id, projectId)).get();
|
|
12027
12077
|
if (!project) return;
|
|
12028
12078
|
const provider = registry.get(run.provider);
|
|
12029
12079
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
12030
|
-
const claim = db.update(researchRuns).set({ status: ResearchRunStatuses.running, startedAt: now }).where(and14(
|
|
12080
|
+
const claim = db.update(researchRuns).set({ status: ResearchRunStatuses.running, startedAt: now }).where(and14(eq21(researchRuns.id, runId), eq21(researchRuns.status, ResearchRunStatuses.queued))).run();
|
|
12031
12081
|
if (claim.changes !== 1) return;
|
|
12032
12082
|
let reserved = 0;
|
|
12033
12083
|
let dispatched = 0;
|
|
@@ -12051,8 +12101,8 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
12051
12101
|
}
|
|
12052
12102
|
reserved = run.totalQueries;
|
|
12053
12103
|
reservation = { scope, period };
|
|
12054
|
-
const rows = db.select().from(researchRunQueries).where(
|
|
12055
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
12104
|
+
const rows = db.select().from(researchRunQueries).where(eq21(researchRunQueries.researchRunId, runId)).orderBy(researchRunQueries.position).all();
|
|
12105
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq21(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
12056
12106
|
const domains = effectiveDomains(project);
|
|
12057
12107
|
const brands = effectiveBrandNames(project);
|
|
12058
12108
|
const config = { ...provider.config, model: run.resolvedModel };
|
|
@@ -12060,7 +12110,7 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
12060
12110
|
await mapWithConcurrency(rows, Math.max(1, provider.config.quotaPolicy.maxConcurrency), async (row) => {
|
|
12061
12111
|
try {
|
|
12062
12112
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
12063
|
-
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.running, startedAt }).where(and14(
|
|
12113
|
+
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.running, startedAt }).where(and14(eq21(researchRunQueries.id, row.id), eq21(researchRunQueries.status, ResearchQueryStatuses.queued))).run();
|
|
12064
12114
|
const raw = await gate.run(async () => {
|
|
12065
12115
|
dispatched++;
|
|
12066
12116
|
return provider.adapter.executeTrackedQuery({ query: row.queryText, canonicalDomains: domains, competitorDomains, ...run.location ? { location: run.location } : {} }, config);
|
|
@@ -12087,7 +12137,7 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
12087
12137
|
citationState: determineCitationState(normalized, domains),
|
|
12088
12138
|
rawResponse: raw.rawResponse,
|
|
12089
12139
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
12090
|
-
}).where(and14(
|
|
12140
|
+
}).where(and14(eq21(researchRunQueries.id, row.id), eq21(researchRunQueries.status, ResearchQueryStatuses.running))).run();
|
|
12091
12141
|
if (completed.changes === 1) incrementResearchProgress(db, runId, "completedQueries");
|
|
12092
12142
|
} catch (error) {
|
|
12093
12143
|
try {
|
|
@@ -12111,18 +12161,18 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
12111
12161
|
}
|
|
12112
12162
|
}
|
|
12113
12163
|
function incrementResearchProgress(db, runId, column) {
|
|
12114
|
-
db.update(researchRuns).set({ [column]: sql8`${researchRuns[column]} + 1` }).where(
|
|
12164
|
+
db.update(researchRuns).set({ [column]: sql8`${researchRuns[column]} + 1` }).where(eq21(researchRuns.id, runId)).run();
|
|
12115
12165
|
}
|
|
12116
12166
|
function markResearchQueryFailed(db, runId, queryId, error) {
|
|
12117
12167
|
const message = error instanceof Error ? error.message : String(error);
|
|
12118
|
-
const failed = db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error: message, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and14(
|
|
12168
|
+
const failed = db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error: message, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and14(eq21(researchRunQueries.id, queryId), inArray7(researchRunQueries.status, unfinishedResearchQueryStatuses))).run();
|
|
12119
12169
|
if (failed.changes === 1) incrementResearchProgress(db, runId, "failedQueries");
|
|
12120
12170
|
}
|
|
12121
12171
|
function markUnfinishedResearchQueriesFailed(db, runId, error) {
|
|
12122
|
-
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and14(
|
|
12172
|
+
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and14(eq21(researchRunQueries.researchRunId, runId), inArray7(researchRunQueries.status, unfinishedResearchQueryStatuses))).run();
|
|
12123
12173
|
}
|
|
12124
12174
|
function finalizeResearchRun(db, runId, fatalError) {
|
|
12125
|
-
const rows = db.select({ status: researchRunQueries.status }).from(researchRunQueries).where(
|
|
12175
|
+
const rows = db.select({ status: researchRunQueries.status }).from(researchRunQueries).where(eq21(researchRunQueries.researchRunId, runId)).all();
|
|
12126
12176
|
const completed = rows.filter((row) => row.status === ResearchQueryStatuses.completed).length;
|
|
12127
12177
|
const failed = rows.filter((row) => row.status === ResearchQueryStatuses.failed).length;
|
|
12128
12178
|
db.update(researchRuns).set({
|
|
@@ -12131,7 +12181,7 @@ function finalizeResearchRun(db, runId, fatalError) {
|
|
|
12131
12181
|
failedQueries: failed,
|
|
12132
12182
|
error: fatalError ?? (failed === rows.length ? "Every research query failed." : null),
|
|
12133
12183
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
12134
|
-
}).where(
|
|
12184
|
+
}).where(eq21(researchRuns.id, runId)).run();
|
|
12135
12185
|
}
|
|
12136
12186
|
|
|
12137
12187
|
// src/server.ts
|
|
@@ -12194,14 +12244,14 @@ function summarizeProviderConfig(config) {
|
|
|
12194
12244
|
};
|
|
12195
12245
|
}
|
|
12196
12246
|
function hashApiKey(key) {
|
|
12197
|
-
return
|
|
12247
|
+
return crypto23.createHash("sha256").update(key).digest("hex");
|
|
12198
12248
|
}
|
|
12199
12249
|
var DASHBOARD_SCRYPT_KEYLEN = 64;
|
|
12200
12250
|
var DASHBOARD_SCRYPT_COST = 1 << 15;
|
|
12201
12251
|
var DASHBOARD_SCRYPT_MAXMEM = 64 * 1024 * 1024;
|
|
12202
12252
|
function hashDashboardPassword(password) {
|
|
12203
|
-
const salt =
|
|
12204
|
-
const derived =
|
|
12253
|
+
const salt = crypto23.randomBytes(16);
|
|
12254
|
+
const derived = crypto23.scryptSync(password, salt, DASHBOARD_SCRYPT_KEYLEN, {
|
|
12205
12255
|
N: DASHBOARD_SCRYPT_COST,
|
|
12206
12256
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
12207
12257
|
});
|
|
@@ -12222,14 +12272,14 @@ function verifyDashboardPassword(password, storedHash) {
|
|
|
12222
12272
|
} catch {
|
|
12223
12273
|
return { ok: false, needsRehash: false };
|
|
12224
12274
|
}
|
|
12225
|
-
const derived =
|
|
12275
|
+
const derived = crypto23.scryptSync(password, salt, expected.length, {
|
|
12226
12276
|
N: DASHBOARD_SCRYPT_COST,
|
|
12227
12277
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
12228
12278
|
});
|
|
12229
12279
|
if (derived.length !== expected.length)
|
|
12230
12280
|
return { ok: false, needsRehash: false };
|
|
12231
12281
|
return {
|
|
12232
|
-
ok:
|
|
12282
|
+
ok: crypto23.timingSafeEqual(derived, expected),
|
|
12233
12283
|
needsRehash: false
|
|
12234
12284
|
};
|
|
12235
12285
|
}
|
|
@@ -12238,7 +12288,7 @@ function verifyDashboardPassword(password, storedHash) {
|
|
|
12238
12288
|
const expected = Buffer.from(storedHash, "hex");
|
|
12239
12289
|
if (candidate.length !== expected.length)
|
|
12240
12290
|
return { ok: false, needsRehash: false };
|
|
12241
|
-
const ok =
|
|
12291
|
+
const ok = crypto23.timingSafeEqual(candidate, expected);
|
|
12242
12292
|
return { ok, needsRehash: ok };
|
|
12243
12293
|
}
|
|
12244
12294
|
return { ok: false, needsRehash: false };
|
|
@@ -12446,7 +12496,7 @@ async function createServer(opts) {
|
|
|
12446
12496
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
12447
12497
|
async (ctx) => {
|
|
12448
12498
|
if (!sessionRegistry) return;
|
|
12449
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
12499
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq22(projects.id, ctx.projectId)).get();
|
|
12450
12500
|
if (!project) return;
|
|
12451
12501
|
let content;
|
|
12452
12502
|
if (ctx.kind === RunKinds["aeo-discover-probe"]) {
|
|
@@ -12789,7 +12839,7 @@ async function createServer(opts) {
|
|
|
12789
12839
|
void (async () => {
|
|
12790
12840
|
try {
|
|
12791
12841
|
const report = await aeroClient.runDoctor({ project: projectName });
|
|
12792
|
-
const project = opts.db.select().from(projects).where(
|
|
12842
|
+
const project = opts.db.select().from(projects).where(eq22(projects.name, projectName)).get();
|
|
12793
12843
|
if (!project) {
|
|
12794
12844
|
app.log.warn({ projectName }, "doctor schedule fired for an unknown project");
|
|
12795
12845
|
return;
|
|
@@ -12818,8 +12868,8 @@ async function createServer(opts) {
|
|
|
12818
12868
|
if (!probed) return;
|
|
12819
12869
|
const alreadySynced = opts.db.select().from(ccReleaseSyncs).where(
|
|
12820
12870
|
and15(
|
|
12821
|
-
|
|
12822
|
-
|
|
12871
|
+
eq22(ccReleaseSyncs.release, probed.release),
|
|
12872
|
+
eq22(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)
|
|
12823
12873
|
)
|
|
12824
12874
|
).limit(1).get();
|
|
12825
12875
|
if (alreadySynced) {
|
|
@@ -12974,7 +13024,7 @@ async function createServer(opts) {
|
|
|
12974
13024
|
return removed;
|
|
12975
13025
|
}
|
|
12976
13026
|
};
|
|
12977
|
-
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ??
|
|
13027
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto23.randomBytes(32).toString("hex");
|
|
12978
13028
|
const googleConnectionStore = {
|
|
12979
13029
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
12980
13030
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -13049,11 +13099,11 @@ async function createServer(opts) {
|
|
|
13049
13099
|
const googlePublicUrl = resolveGooglePublicUrl(opts.config, basePath);
|
|
13050
13100
|
if (opts.config.apiKey) {
|
|
13051
13101
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
13052
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
13102
|
+
const existing = opts.db.select().from(apiKeys).where(eq22(apiKeys.keyHash, keyHash)).get();
|
|
13053
13103
|
if (!existing) {
|
|
13054
13104
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
13055
13105
|
opts.db.insert(apiKeys).values({
|
|
13056
|
-
id: `key_${
|
|
13106
|
+
id: `key_${crypto23.randomBytes(8).toString("hex")}`,
|
|
13057
13107
|
name: "default",
|
|
13058
13108
|
keyHash,
|
|
13059
13109
|
keyPrefix: prefix,
|
|
@@ -13077,7 +13127,7 @@ async function createServer(opts) {
|
|
|
13077
13127
|
};
|
|
13078
13128
|
const createSession = (apiKeyId) => {
|
|
13079
13129
|
pruneExpiredSessions();
|
|
13080
|
-
const sessionId =
|
|
13130
|
+
const sessionId = crypto23.randomBytes(32).toString("hex");
|
|
13081
13131
|
sessions.set(sessionId, {
|
|
13082
13132
|
apiKeyId,
|
|
13083
13133
|
expiresAt: Date.now() + SESSION_TTL_MS
|
|
@@ -13101,7 +13151,7 @@ async function createServer(opts) {
|
|
|
13101
13151
|
};
|
|
13102
13152
|
const getDefaultApiKey = () => {
|
|
13103
13153
|
if (!opts.config.apiKey) return void 0;
|
|
13104
|
-
return opts.db.select().from(apiKeys).where(
|
|
13154
|
+
return opts.db.select().from(apiKeys).where(eq22(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
13105
13155
|
};
|
|
13106
13156
|
const createPasswordSession = (reply) => {
|
|
13107
13157
|
const key = getDefaultApiKey();
|
|
@@ -13125,7 +13175,7 @@ async function createServer(opts) {
|
|
|
13125
13175
|
if (!header) return false;
|
|
13126
13176
|
const parts = header.split(" ");
|
|
13127
13177
|
if (parts.length !== 2 || parts[0] !== "Bearer") return false;
|
|
13128
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
13178
|
+
const key = opts.db.select().from(apiKeys).where(eq22(apiKeys.keyHash, hashApiKey(parts[1]))).get();
|
|
13129
13179
|
return Boolean(key && !key.revokedAt);
|
|
13130
13180
|
};
|
|
13131
13181
|
const namedAccountsInUse = () => anyUsersExist(opts.db);
|
|
@@ -13215,12 +13265,12 @@ async function createServer(opts) {
|
|
|
13215
13265
|
return reply.send({ authenticated: true });
|
|
13216
13266
|
}
|
|
13217
13267
|
if (apiKey) {
|
|
13218
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
13268
|
+
const key = opts.db.select().from(apiKeys).where(eq22(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
13219
13269
|
if (!key || key.revokedAt) {
|
|
13220
13270
|
const err2 = authInvalid();
|
|
13221
13271
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
13222
13272
|
}
|
|
13223
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
13273
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq22(apiKeys.id, key.id)).run();
|
|
13224
13274
|
const sessionId = createSession(key.id);
|
|
13225
13275
|
reply.header(
|
|
13226
13276
|
"set-cookie",
|
|
@@ -13433,7 +13483,7 @@ async function createServer(opts) {
|
|
|
13433
13483
|
deps: {
|
|
13434
13484
|
enqueueAutoExtract: ({ projectId, release: r }) => {
|
|
13435
13485
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
13436
|
-
const runId =
|
|
13486
|
+
const runId = crypto23.randomUUID();
|
|
13437
13487
|
opts.db.insert(runs).values({
|
|
13438
13488
|
id: runId,
|
|
13439
13489
|
projectId,
|
|
@@ -13558,7 +13608,7 @@ async function createServer(opts) {
|
|
|
13558
13608
|
...inspectOpts,
|
|
13559
13609
|
config: opts.config
|
|
13560
13610
|
}).then(() => {
|
|
13561
|
-
const finished = opts.db.select({ status: runs.status }).from(runs).where(
|
|
13611
|
+
const finished = opts.db.select({ status: runs.status }).from(runs).where(eq22(runs.id, runId)).get();
|
|
13562
13612
|
if (finished?.status === RunStatuses.completed || finished?.status === RunStatuses.partial) {
|
|
13563
13613
|
return maybeRefreshGscCoverage(opts.db, opts.config, projectId);
|
|
13564
13614
|
}
|
|
@@ -13653,7 +13703,7 @@ async function createServer(opts) {
|
|
|
13653
13703
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
13654
13704
|
opts.db.insert(auditLog).values(
|
|
13655
13705
|
targetProjectIds.map((projectId) => ({
|
|
13656
|
-
id:
|
|
13706
|
+
id: crypto23.randomUUID(),
|
|
13657
13707
|
projectId,
|
|
13658
13708
|
actor: "api",
|
|
13659
13709
|
action: existing ? "provider.updated" : "provider.created",
|
|
@@ -13921,7 +13971,7 @@ async function createServer(opts) {
|
|
|
13921
13971
|
}
|
|
13922
13972
|
checkLatestVersionForServer();
|
|
13923
13973
|
scheduler.start();
|
|
13924
|
-
for (const run of opts.db.select({ id: researchRuns.id, projectId: researchRuns.projectId }).from(researchRuns).where(
|
|
13974
|
+
for (const run of opts.db.select({ id: researchRuns.id, projectId: researchRuns.projectId }).from(researchRuns).where(eq22(researchRuns.status, ResearchRunStatuses.queued)).all()) {
|
|
13925
13975
|
dispatchResearchRun(run.id, run.projectId);
|
|
13926
13976
|
}
|
|
13927
13977
|
app.addHook("onClose", async () => {
|