@ainyc/canonry 4.34.0 → 4.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/assets/assets/index-Cfv0_lwq.css +1 -0
- package/assets/assets/index-u7ZXZ5mA.js +302 -0
- package/assets/index.html +2 -2
- package/dist/{chunk-5EBN7736.js → chunk-B3FBOECD.js} +1 -1
- package/dist/{chunk-XW3F5EEW.js → chunk-EM5GVF3C.js} +73 -20
- package/dist/{chunk-7256SFYT.js → chunk-MLS5KJWK.js} +155 -38
- package/dist/{chunk-7AF6B3L6.js → chunk-NLV4MZZF.js} +1084 -279
- package/dist/cli.js +100 -563
- package/dist/index.d.ts +3 -0
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-3P2DMYRR.js → intelligence-service-WAJOEOJV.js} +2 -2
- package/dist/mcp.js +2 -2
- package/package.json +5 -5
- package/assets/assets/index-47V0U52s.js +0 -302
- package/assets/assets/index-CNKAwZMB.css +0 -1
|
@@ -5,7 +5,7 @@ import {
|
|
|
5
5
|
loadConfig,
|
|
6
6
|
loadConfigRaw,
|
|
7
7
|
saveConfigPatch
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-B3FBOECD.js";
|
|
9
9
|
import {
|
|
10
10
|
DEFAULT_RUN_HISTORY_LIMIT,
|
|
11
11
|
IntelligenceService,
|
|
@@ -39,6 +39,7 @@ import {
|
|
|
39
39
|
ccReleaseSyncs,
|
|
40
40
|
competitors,
|
|
41
41
|
crawlerEventsHourly,
|
|
42
|
+
createClient,
|
|
42
43
|
createLogger,
|
|
43
44
|
discoveryProbes,
|
|
44
45
|
discoverySessions,
|
|
@@ -60,6 +61,7 @@ import {
|
|
|
60
61
|
isBlogShapedQuery,
|
|
61
62
|
isTrendBaseline,
|
|
62
63
|
mapOpportunitiesToNextSteps,
|
|
64
|
+
migrate,
|
|
63
65
|
notifications,
|
|
64
66
|
parseJsonColumn,
|
|
65
67
|
pickGroupRepresentative,
|
|
@@ -71,7 +73,7 @@ import {
|
|
|
71
73
|
schedules,
|
|
72
74
|
trafficSources,
|
|
73
75
|
usageCounters
|
|
74
|
-
} from "./chunk-
|
|
76
|
+
} from "./chunk-MLS5KJWK.js";
|
|
75
77
|
import {
|
|
76
78
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
77
79
|
AGENT_PROVIDER_IDS,
|
|
@@ -90,6 +92,7 @@ import {
|
|
|
90
92
|
DiscoveryCompetitorTypes,
|
|
91
93
|
DiscoverySessionStatuses,
|
|
92
94
|
MemorySources,
|
|
95
|
+
ProviderNames,
|
|
93
96
|
RunKinds,
|
|
94
97
|
RunStatuses,
|
|
95
98
|
RunTriggers,
|
|
@@ -127,6 +130,7 @@ import {
|
|
|
127
130
|
discoveryBucketSchema,
|
|
128
131
|
discoveryPromoteRequestSchema,
|
|
129
132
|
discoveryRunRequestSchema,
|
|
133
|
+
effectiveBrandNames,
|
|
130
134
|
effectiveDomains,
|
|
131
135
|
emptyCitationVisibility,
|
|
132
136
|
extractAnswerMentions,
|
|
@@ -144,7 +148,9 @@ import {
|
|
|
144
148
|
isBrowserProvider,
|
|
145
149
|
keywordGenerateRequestSchema,
|
|
146
150
|
locationContextSchema,
|
|
151
|
+
mentionStateFromAnswerMentioned,
|
|
147
152
|
missingDependency,
|
|
153
|
+
normalizeProjectAliases,
|
|
148
154
|
normalizeProjectDomain,
|
|
149
155
|
normalizeUrlPath,
|
|
150
156
|
notFound,
|
|
@@ -180,7 +186,7 @@ import {
|
|
|
180
186
|
visibilityStateFromAnswerMentioned,
|
|
181
187
|
windowCutoff,
|
|
182
188
|
wordpressEnvSchema
|
|
183
|
-
} from "./chunk-
|
|
189
|
+
} from "./chunk-EM5GVF3C.js";
|
|
184
190
|
|
|
185
191
|
// src/telemetry.ts
|
|
186
192
|
import crypto from "crypto";
|
|
@@ -333,13 +339,146 @@ function trackEvent(event, properties, options) {
|
|
|
333
339
|
}).finally(() => clearTimeout(timeout));
|
|
334
340
|
}
|
|
335
341
|
|
|
342
|
+
// src/update-check.ts
|
|
343
|
+
import { createRequire as createRequire2 } from "module";
|
|
344
|
+
var _require2 = createRequire2(import.meta.url);
|
|
345
|
+
var { version: PKG_VERSION } = _require2("../package.json");
|
|
346
|
+
var PKG_NAME = "@ainyc/canonry";
|
|
347
|
+
var NPM_DIST_TAGS_URL = `https://registry.npmjs.org/-/package/${PKG_NAME}/dist-tags`;
|
|
348
|
+
var NPM_PACKAGE_URL = `https://www.npmjs.com/package/${PKG_NAME}`;
|
|
349
|
+
var FETCH_TIMEOUT_MS = 1500;
|
|
350
|
+
function isUpdateCheckEnabled() {
|
|
351
|
+
if (process.env.CANONRY_DISABLE_UPDATE_CHECK === "1") return false;
|
|
352
|
+
if (process.env.DO_NOT_TRACK === "1") return false;
|
|
353
|
+
if (process.env.CI) return false;
|
|
354
|
+
if (!configExists()) return true;
|
|
355
|
+
try {
|
|
356
|
+
const raw = loadConfigRaw();
|
|
357
|
+
return raw?.updateCheck !== false;
|
|
358
|
+
} catch {
|
|
359
|
+
return true;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
function compareSemver(a, b) {
|
|
363
|
+
const parse = (v) => {
|
|
364
|
+
const core = v.split(/[-+]/)[0];
|
|
365
|
+
if (!core) return null;
|
|
366
|
+
const parts = core.split(".");
|
|
367
|
+
if (parts.length < 3) return null;
|
|
368
|
+
const nums = [];
|
|
369
|
+
for (let i = 0; i < 3; i++) {
|
|
370
|
+
const n = Number(parts[i]);
|
|
371
|
+
if (!Number.isInteger(n) || n < 0) return null;
|
|
372
|
+
nums.push(n);
|
|
373
|
+
}
|
|
374
|
+
return [nums[0], nums[1], nums[2]];
|
|
375
|
+
};
|
|
376
|
+
const pa = parse(a);
|
|
377
|
+
const pb = parse(b);
|
|
378
|
+
if (!pa || !pb) return 0;
|
|
379
|
+
for (let i = 0; i < 3; i++) {
|
|
380
|
+
if (pa[i] > pb[i]) return 1;
|
|
381
|
+
if (pa[i] < pb[i]) return -1;
|
|
382
|
+
}
|
|
383
|
+
return 0;
|
|
384
|
+
}
|
|
385
|
+
async function fetchLatestVersion(opts) {
|
|
386
|
+
const controller = new AbortController();
|
|
387
|
+
const timeout = setTimeout(() => controller.abort(), opts?.timeoutMs ?? FETCH_TIMEOUT_MS);
|
|
388
|
+
timeout.unref();
|
|
389
|
+
try {
|
|
390
|
+
const res = await fetch(NPM_DIST_TAGS_URL, {
|
|
391
|
+
signal: controller.signal,
|
|
392
|
+
headers: { accept: "application/json" }
|
|
393
|
+
});
|
|
394
|
+
if (!res.ok) return null;
|
|
395
|
+
const data = await res.json();
|
|
396
|
+
if (typeof data.latest !== "string") return null;
|
|
397
|
+
return data.latest;
|
|
398
|
+
} catch {
|
|
399
|
+
return null;
|
|
400
|
+
} finally {
|
|
401
|
+
clearTimeout(timeout);
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
function buildUpdateAvailable(current, latest) {
|
|
405
|
+
if (compareSemver(latest, current) <= 0) return null;
|
|
406
|
+
return {
|
|
407
|
+
current,
|
|
408
|
+
latest,
|
|
409
|
+
url: NPM_PACKAGE_URL,
|
|
410
|
+
upgradeCommand: `npm install -g ${PKG_NAME}`
|
|
411
|
+
};
|
|
412
|
+
}
|
|
413
|
+
async function checkLatestVersionForCli(opts) {
|
|
414
|
+
if (!isUpdateCheckEnabled()) return null;
|
|
415
|
+
if (!configExists()) return null;
|
|
416
|
+
const now = opts?.now ? opts.now() : /* @__PURE__ */ new Date();
|
|
417
|
+
const ttlMs = (opts?.ttlHours ?? 24) * 60 * 60 * 1e3;
|
|
418
|
+
let raw;
|
|
419
|
+
try {
|
|
420
|
+
raw = loadConfigRaw();
|
|
421
|
+
} catch {
|
|
422
|
+
return null;
|
|
423
|
+
}
|
|
424
|
+
if (!raw) return null;
|
|
425
|
+
const lastCheckedAt = raw.lastUpdateCheckAt ? Date.parse(raw.lastUpdateCheckAt) : NaN;
|
|
426
|
+
const cachedLatest = typeof raw.lastKnownLatestVersion === "string" ? raw.lastKnownLatestVersion : void 0;
|
|
427
|
+
if (Number.isFinite(lastCheckedAt) && now.getTime() - lastCheckedAt < ttlMs) {
|
|
428
|
+
if (!cachedLatest) return null;
|
|
429
|
+
return buildUpdateAvailable(PKG_VERSION, cachedLatest);
|
|
430
|
+
}
|
|
431
|
+
const latest = await fetchLatestVersion();
|
|
432
|
+
if (!latest) {
|
|
433
|
+
try {
|
|
434
|
+
saveConfigPatch({ lastUpdateCheckAt: now.toISOString() });
|
|
435
|
+
} catch {
|
|
436
|
+
}
|
|
437
|
+
return null;
|
|
438
|
+
}
|
|
439
|
+
try {
|
|
440
|
+
saveConfigPatch({
|
|
441
|
+
lastUpdateCheckAt: now.toISOString(),
|
|
442
|
+
lastKnownLatestVersion: latest
|
|
443
|
+
});
|
|
444
|
+
} catch {
|
|
445
|
+
}
|
|
446
|
+
return buildUpdateAvailable(PKG_VERSION, latest);
|
|
447
|
+
}
|
|
448
|
+
var memoryCache = null;
|
|
449
|
+
var inFlight = null;
|
|
450
|
+
function startBackgroundRefresh(getNow) {
|
|
451
|
+
if (inFlight) return;
|
|
452
|
+
inFlight = (async () => {
|
|
453
|
+
try {
|
|
454
|
+
const latest = await fetchLatestVersion();
|
|
455
|
+
memoryCache = { fetchedAt: getNow(), latest };
|
|
456
|
+
} catch {
|
|
457
|
+
memoryCache = { fetchedAt: getNow(), latest: null };
|
|
458
|
+
} finally {
|
|
459
|
+
inFlight = null;
|
|
460
|
+
}
|
|
461
|
+
})();
|
|
462
|
+
}
|
|
463
|
+
function checkLatestVersionForServer(opts) {
|
|
464
|
+
if (!isUpdateCheckEnabled()) return null;
|
|
465
|
+
const getNow = opts?.now ?? Date.now;
|
|
466
|
+
const ttl = opts?.ttlMs ?? 60 * 60 * 1e3;
|
|
467
|
+
const now = getNow();
|
|
468
|
+
if (!memoryCache || now - memoryCache.fetchedAt >= ttl) {
|
|
469
|
+
startBackgroundRefresh(getNow);
|
|
470
|
+
}
|
|
471
|
+
if (!memoryCache || !memoryCache.latest) return null;
|
|
472
|
+
return buildUpdateAvailable(PKG_VERSION, memoryCache.latest);
|
|
473
|
+
}
|
|
474
|
+
|
|
336
475
|
// src/server.ts
|
|
337
|
-
import { createRequire as
|
|
476
|
+
import { createRequire as createRequire4 } from "module";
|
|
338
477
|
import crypto34 from "crypto";
|
|
339
478
|
import fs12 from "fs";
|
|
340
479
|
import path14 from "path";
|
|
341
480
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
342
|
-
import { eq as
|
|
481
|
+
import { eq as eq41 } from "drizzle-orm";
|
|
343
482
|
import Fastify from "fastify";
|
|
344
483
|
|
|
345
484
|
// ../api-routes/src/auth.ts
|
|
@@ -440,7 +579,11 @@ function resolveSnapshotMentionResult(snapshot, project) {
|
|
|
440
579
|
canonicalDomain: project.canonicalDomain,
|
|
441
580
|
ownedDomains: normalizeOwnedDomains(project.ownedDomains)
|
|
442
581
|
});
|
|
443
|
-
|
|
582
|
+
const brandNames = effectiveBrandNames({
|
|
583
|
+
displayName: project.displayName,
|
|
584
|
+
aliases: normalizeOwnedDomains(project.aliases)
|
|
585
|
+
});
|
|
586
|
+
return extractAnswerMentions(snapshot.answerText, brandNames, domains);
|
|
444
587
|
}
|
|
445
588
|
if (typeof snapshot.answerMentioned === "boolean") {
|
|
446
589
|
return { mentioned: snapshot.answerMentioned, matchedTerms: [] };
|
|
@@ -453,6 +596,9 @@ function resolveSnapshotAnswerMentioned(snapshot, project) {
|
|
|
453
596
|
function resolveSnapshotVisibilityState(snapshot, project) {
|
|
454
597
|
return visibilityStateFromAnswerMentioned(resolveSnapshotMentionResult(snapshot, project).mentioned);
|
|
455
598
|
}
|
|
599
|
+
function resolveSnapshotMentionState(snapshot, project) {
|
|
600
|
+
return mentionStateFromAnswerMentioned(resolveSnapshotMentionResult(snapshot, project).mentioned);
|
|
601
|
+
}
|
|
456
602
|
function resolveSnapshotMatchedTerms(snapshot, project) {
|
|
457
603
|
return resolveSnapshotMentionResult(snapshot, project).matchedTerms;
|
|
458
604
|
}
|
|
@@ -503,12 +649,16 @@ async function projectRoutes(app, opts) {
|
|
|
503
649
|
});
|
|
504
650
|
}
|
|
505
651
|
const nextAutoExtractBacklinks = body.autoExtractBacklinks !== void 0 ? body.autoExtractBacklinks ? 1 : 0 : existing?.autoExtractBacklinks ?? 0;
|
|
652
|
+
const nextAliases = normalizeProjectAliases(body.displayName, body.aliases ?? []);
|
|
506
653
|
if (existing) {
|
|
654
|
+
const prevAliases = parseJsonColumn(existing.aliases, []);
|
|
655
|
+
const aliasesChanged = !aliasArraysEqual(prevAliases, nextAliases);
|
|
507
656
|
app.db.transaction((tx) => {
|
|
508
657
|
tx.update(projects).set({
|
|
509
658
|
displayName: body.displayName,
|
|
510
659
|
canonicalDomain: body.canonicalDomain,
|
|
511
660
|
ownedDomains: JSON.stringify(body.ownedDomains ?? []),
|
|
661
|
+
aliases: JSON.stringify(nextAliases),
|
|
512
662
|
country: body.country,
|
|
513
663
|
language: body.language,
|
|
514
664
|
tags: JSON.stringify(body.tags ?? []),
|
|
@@ -530,6 +680,7 @@ async function projectRoutes(app, opts) {
|
|
|
530
680
|
});
|
|
531
681
|
});
|
|
532
682
|
opts.onProjectUpserted?.(existing.id, name);
|
|
683
|
+
if (aliasesChanged) opts.onAliasesChanged?.(existing.id, name);
|
|
533
684
|
const updated = app.db.select().from(projects).where(eq3(projects.id, existing.id)).get();
|
|
534
685
|
return reply.status(200).send(formatProject(updated));
|
|
535
686
|
}
|
|
@@ -541,6 +692,7 @@ async function projectRoutes(app, opts) {
|
|
|
541
692
|
displayName: body.displayName,
|
|
542
693
|
canonicalDomain: body.canonicalDomain,
|
|
543
694
|
ownedDomains: JSON.stringify(body.ownedDomains ?? []),
|
|
695
|
+
aliases: JSON.stringify(nextAliases),
|
|
544
696
|
country: body.country,
|
|
545
697
|
language: body.language,
|
|
546
698
|
tags: JSON.stringify(body.tags ?? []),
|
|
@@ -688,6 +840,7 @@ async function projectRoutes(app, opts) {
|
|
|
688
840
|
displayName: project.displayName,
|
|
689
841
|
canonicalDomain: project.canonicalDomain,
|
|
690
842
|
ownedDomains: parseJsonColumn(project.ownedDomains, []),
|
|
843
|
+
aliases: parseJsonColumn(project.aliases, []),
|
|
691
844
|
country: project.country,
|
|
692
845
|
language: project.language,
|
|
693
846
|
queries: qs.map((q) => q.query),
|
|
@@ -723,6 +876,7 @@ function formatProject(row) {
|
|
|
723
876
|
displayName: row.displayName,
|
|
724
877
|
canonicalDomain: row.canonicalDomain,
|
|
725
878
|
ownedDomains: parseJsonColumn(row.ownedDomains, []),
|
|
879
|
+
aliases: parseJsonColumn(row.aliases, []),
|
|
726
880
|
country: row.country,
|
|
727
881
|
language: row.language,
|
|
728
882
|
tags: parseJsonColumn(row.tags, []),
|
|
@@ -737,6 +891,13 @@ function formatProject(row) {
|
|
|
737
891
|
updatedAt: row.updatedAt
|
|
738
892
|
};
|
|
739
893
|
}
|
|
894
|
+
function aliasArraysEqual(a, b) {
|
|
895
|
+
if (a.length !== b.length) return false;
|
|
896
|
+
for (let i = 0; i < a.length; i++) {
|
|
897
|
+
if (a[i].toLowerCase() !== b[i].toLowerCase()) return false;
|
|
898
|
+
}
|
|
899
|
+
return true;
|
|
900
|
+
}
|
|
740
901
|
|
|
741
902
|
// ../api-routes/src/queries.ts
|
|
742
903
|
import crypto5 from "crypto";
|
|
@@ -1139,7 +1300,7 @@ function parseCompetitorBatch(value) {
|
|
|
1139
1300
|
|
|
1140
1301
|
// ../api-routes/src/runs.ts
|
|
1141
1302
|
import crypto8 from "crypto";
|
|
1142
|
-
import { eq as eq7, asc, desc, sql as sql2 } from "drizzle-orm";
|
|
1303
|
+
import { and as and2, eq as eq7, asc, desc, or as or2, sql as sql2 } from "drizzle-orm";
|
|
1143
1304
|
|
|
1144
1305
|
// ../api-routes/src/run-queue.ts
|
|
1145
1306
|
import crypto7 from "crypto";
|
|
@@ -1233,23 +1394,36 @@ async function runRoutes(app, opts) {
|
|
|
1233
1394
|
if (projectLocations.length === 0) {
|
|
1234
1395
|
throw validationError("No locations configured for this project");
|
|
1235
1396
|
}
|
|
1236
|
-
const
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1397
|
+
const result = app.db.transaction((tx) => {
|
|
1398
|
+
const activeRun = tx.select({ id: runs.id }).from(runs).where(and2(
|
|
1399
|
+
eq7(runs.projectId, project.id),
|
|
1400
|
+
or2(eq7(runs.status, "queued"), eq7(runs.status, "running"))
|
|
1401
|
+
)).get();
|
|
1402
|
+
if (activeRun) {
|
|
1403
|
+
return { conflict: true };
|
|
1404
|
+
}
|
|
1405
|
+
const inserted = [];
|
|
1406
|
+
for (const loc of projectLocations) {
|
|
1407
|
+
const runId2 = crypto8.randomUUID();
|
|
1408
|
+
tx.insert(runs).values({
|
|
1409
|
+
id: runId2,
|
|
1410
|
+
projectId: project.id,
|
|
1411
|
+
kind,
|
|
1412
|
+
status: "queued",
|
|
1413
|
+
trigger,
|
|
1414
|
+
location: loc.label,
|
|
1415
|
+
queries: queriesColumn,
|
|
1416
|
+
createdAt: now
|
|
1417
|
+
}).run();
|
|
1418
|
+
inserted.push({ runId: runId2, loc });
|
|
1419
|
+
}
|
|
1420
|
+
return { conflict: false, inserted };
|
|
1421
|
+
});
|
|
1422
|
+
if (result.conflict) {
|
|
1423
|
+
throw runInProgress(project.name);
|
|
1250
1424
|
}
|
|
1251
1425
|
const results = [];
|
|
1252
|
-
for (const { runId: runId2, loc } of
|
|
1426
|
+
for (const { runId: runId2, loc } of result.inserted) {
|
|
1253
1427
|
writeAuditLog(app.db, {
|
|
1254
1428
|
projectId: project.id,
|
|
1255
1429
|
actor: "api",
|
|
@@ -1437,7 +1611,8 @@ function loadRunDetail(app, run) {
|
|
|
1437
1611
|
const project = app.db.select({
|
|
1438
1612
|
displayName: projects.displayName,
|
|
1439
1613
|
canonicalDomain: projects.canonicalDomain,
|
|
1440
|
-
ownedDomains: projects.ownedDomains
|
|
1614
|
+
ownedDomains: projects.ownedDomains,
|
|
1615
|
+
aliases: projects.aliases
|
|
1441
1616
|
}).from(projects).where(eq7(projects.id, run.projectId)).get();
|
|
1442
1617
|
const snapshots = app.db.select({
|
|
1443
1618
|
id: querySnapshots.id,
|
|
@@ -1469,7 +1644,10 @@ function loadRunDetail(app, run) {
|
|
|
1469
1644
|
provider: s.provider,
|
|
1470
1645
|
citationState: s.citationState,
|
|
1471
1646
|
answerMentioned,
|
|
1647
|
+
// Legacy alias of `mentionState`, retained for backwards compatibility.
|
|
1472
1648
|
visibilityState: project ? resolveSnapshotVisibilityState(s, project) : answerMentioned ? "visible" : "not-visible",
|
|
1649
|
+
// Canonical vocabulary for answer-text presence; new consumers prefer this.
|
|
1650
|
+
mentionState: project ? resolveSnapshotMentionState(s, project) : answerMentioned ? "mentioned" : "not-mentioned",
|
|
1473
1651
|
answerText: s.answerText,
|
|
1474
1652
|
citedDomains: parseJsonColumn(s.citedDomains, []),
|
|
1475
1653
|
competitorOverlap: parseJsonColumn(s.competitorOverlap, []),
|
|
@@ -1487,7 +1665,7 @@ function loadRunDetail(app, run) {
|
|
|
1487
1665
|
|
|
1488
1666
|
// ../api-routes/src/apply.ts
|
|
1489
1667
|
import crypto10 from "crypto";
|
|
1490
|
-
import { and as
|
|
1668
|
+
import { and as and3, eq as eq8 } from "drizzle-orm";
|
|
1491
1669
|
|
|
1492
1670
|
// ../api-routes/src/schedule-utils.ts
|
|
1493
1671
|
var DAY_MAP = {
|
|
@@ -1584,7 +1762,7 @@ import http from "http";
|
|
|
1584
1762
|
import https from "https";
|
|
1585
1763
|
import net from "net";
|
|
1586
1764
|
var REQUEST_TIMEOUT_MS = 1e4;
|
|
1587
|
-
async function resolveWebhookTarget(raw) {
|
|
1765
|
+
async function resolveWebhookTarget(raw, options = {}) {
|
|
1588
1766
|
let parsed;
|
|
1589
1767
|
try {
|
|
1590
1768
|
parsed = new URL(raw);
|
|
@@ -1605,7 +1783,7 @@ async function resolveWebhookTarget(raw) {
|
|
|
1605
1783
|
if (addresses.length === 0) {
|
|
1606
1784
|
return { ok: false, message: '"url" hostname could not be resolved' };
|
|
1607
1785
|
}
|
|
1608
|
-
const blocked = addresses.find((entry) => isBlockedAddress(entry.address));
|
|
1786
|
+
const blocked = addresses.find((entry) => isBlockedAddress(entry.address, options));
|
|
1609
1787
|
if (blocked) {
|
|
1610
1788
|
return { ok: false, message: '"url" must not resolve to a private or loopback address' };
|
|
1611
1789
|
}
|
|
@@ -1683,34 +1861,40 @@ async function resolveHostAddresses(hostname) {
|
|
|
1683
1861
|
return [];
|
|
1684
1862
|
}
|
|
1685
1863
|
}
|
|
1686
|
-
function isBlockedAddress(address) {
|
|
1864
|
+
function isBlockedAddress(address, options) {
|
|
1687
1865
|
const normalized = stripIpv6Brackets(address).toLowerCase();
|
|
1688
1866
|
const family = net.isIP(normalized);
|
|
1689
1867
|
if (family === 4) {
|
|
1690
|
-
return isBlockedIpv4(normalized);
|
|
1868
|
+
return isBlockedIpv4(normalized, options);
|
|
1691
1869
|
}
|
|
1692
1870
|
if (family === 6) {
|
|
1693
1871
|
const mappedIpv4 = extractMappedIpv4(normalized);
|
|
1694
1872
|
if (mappedIpv4) {
|
|
1695
|
-
return isBlockedIpv4(mappedIpv4);
|
|
1873
|
+
return isBlockedIpv4(mappedIpv4, options);
|
|
1696
1874
|
}
|
|
1697
|
-
return isBlockedIpv6(normalized);
|
|
1875
|
+
return isBlockedIpv6(normalized, options);
|
|
1698
1876
|
}
|
|
1699
1877
|
return true;
|
|
1700
1878
|
}
|
|
1701
|
-
function isBlockedIpv4(address) {
|
|
1879
|
+
function isBlockedIpv4(address, options) {
|
|
1702
1880
|
const octets = address.split(".").map((part) => Number.parseInt(part, 10));
|
|
1703
1881
|
if (octets.length !== 4 || octets.some(Number.isNaN)) {
|
|
1704
1882
|
return true;
|
|
1705
1883
|
}
|
|
1706
1884
|
const [first, second] = octets;
|
|
1885
|
+
if (first === 127 && !options.allowLoopback) {
|
|
1886
|
+
return true;
|
|
1887
|
+
}
|
|
1707
1888
|
return first === 0 || first === 10 || first === 100 && second >= 64 && second <= 127 || first === 169 && second === 254 || first === 172 && second >= 16 && second <= 31 || first === 192 && second === 168 || first === 198 && (second === 18 || second === 19);
|
|
1708
1889
|
}
|
|
1709
|
-
function isBlockedIpv6(address) {
|
|
1890
|
+
function isBlockedIpv6(address, options) {
|
|
1710
1891
|
const normalized = address.split("%")[0].toLowerCase();
|
|
1711
1892
|
if (normalized === "::") {
|
|
1712
1893
|
return true;
|
|
1713
1894
|
}
|
|
1895
|
+
if (normalized === "::1") {
|
|
1896
|
+
return !options.allowLoopback;
|
|
1897
|
+
}
|
|
1714
1898
|
const firstHextetText = normalized.split(":")[0] ?? "";
|
|
1715
1899
|
const firstHextet = firstHextetText === "" ? 0 : Number.parseInt(firstHextetText, 16);
|
|
1716
1900
|
if (Number.isNaN(firstHextet)) {
|
|
@@ -1741,6 +1925,7 @@ function stripIpv6Brackets(value) {
|
|
|
1741
1925
|
|
|
1742
1926
|
// ../api-routes/src/apply.ts
|
|
1743
1927
|
async function applyRoutes(app, opts) {
|
|
1928
|
+
const allowLoopback = opts?.allowLoopbackWebhooks === true;
|
|
1744
1929
|
app.post("/apply", async (request, reply) => {
|
|
1745
1930
|
const parsed = projectConfigSchema.safeParse(request.body);
|
|
1746
1931
|
if (!parsed.success) {
|
|
@@ -1795,7 +1980,7 @@ async function applyRoutes(app, opts) {
|
|
|
1795
1980
|
const hasNotifications = "notifications" in rawSpec;
|
|
1796
1981
|
if (hasNotifications) {
|
|
1797
1982
|
for (const notif of config.spec.notifications) {
|
|
1798
|
-
const urlCheck = await resolveWebhookTarget(notif.url ?? "");
|
|
1983
|
+
const urlCheck = await resolveWebhookTarget(notif.url ?? "", { allowLoopback });
|
|
1799
1984
|
if (!urlCheck.ok) throw validationError(`Notification URL invalid: ${urlCheck.message}`);
|
|
1800
1985
|
}
|
|
1801
1986
|
}
|
|
@@ -1804,14 +1989,21 @@ async function applyRoutes(app, opts) {
|
|
|
1804
1989
|
const configQueries = resolveConfigSpecQueries(config.spec);
|
|
1805
1990
|
let projectId;
|
|
1806
1991
|
let scheduleAction = null;
|
|
1992
|
+
let aliasesChanged = false;
|
|
1807
1993
|
app.db.transaction((tx) => {
|
|
1808
1994
|
const existing = tx.select().from(projects).where(eq8(projects.name, name)).get();
|
|
1995
|
+
const nextAliases = normalizeProjectAliases(config.spec.displayName, config.spec.aliases ?? []);
|
|
1996
|
+
if (existing) {
|
|
1997
|
+
const prevAliases = parseJsonColumn(existing.aliases, []);
|
|
1998
|
+
aliasesChanged = !aliasArraysEqual2(prevAliases, nextAliases);
|
|
1999
|
+
}
|
|
1809
2000
|
if (existing) {
|
|
1810
2001
|
projectId = existing.id;
|
|
1811
2002
|
tx.update(projects).set({
|
|
1812
2003
|
displayName: config.spec.displayName,
|
|
1813
2004
|
canonicalDomain: config.spec.canonicalDomain,
|
|
1814
2005
|
ownedDomains: JSON.stringify(config.spec.ownedDomains ?? []),
|
|
2006
|
+
aliases: JSON.stringify(nextAliases),
|
|
1815
2007
|
country: config.spec.country,
|
|
1816
2008
|
language: config.spec.language,
|
|
1817
2009
|
labels: JSON.stringify(config.metadata.labels),
|
|
@@ -1838,6 +2030,7 @@ async function applyRoutes(app, opts) {
|
|
|
1838
2030
|
displayName: config.spec.displayName,
|
|
1839
2031
|
canonicalDomain: config.spec.canonicalDomain,
|
|
1840
2032
|
ownedDomains: JSON.stringify(config.spec.ownedDomains ?? []),
|
|
2033
|
+
aliases: JSON.stringify(nextAliases),
|
|
1841
2034
|
country: config.spec.country,
|
|
1842
2035
|
language: config.spec.language,
|
|
1843
2036
|
tags: "[]",
|
|
@@ -1896,7 +2089,7 @@ async function applyRoutes(app, opts) {
|
|
|
1896
2089
|
});
|
|
1897
2090
|
const AV_KIND = SchedulableRunKinds["answer-visibility"];
|
|
1898
2091
|
if (resolvedSchedule) {
|
|
1899
|
-
const existingSched = tx.select().from(schedules).where(
|
|
2092
|
+
const existingSched = tx.select().from(schedules).where(and3(eq8(schedules.projectId, projectId), eq8(schedules.kind, AV_KIND))).get();
|
|
1900
2093
|
if (existingSched) {
|
|
1901
2094
|
tx.update(schedules).set({
|
|
1902
2095
|
cronExpr: resolvedSchedule.cronExpr,
|
|
@@ -1922,9 +2115,9 @@ async function applyRoutes(app, opts) {
|
|
|
1922
2115
|
}
|
|
1923
2116
|
scheduleAction = "upsert";
|
|
1924
2117
|
} else if (deleteSchedule) {
|
|
1925
|
-
const existingSched = tx.select().from(schedules).where(
|
|
2118
|
+
const existingSched = tx.select().from(schedules).where(and3(eq8(schedules.projectId, projectId), eq8(schedules.kind, AV_KIND))).get();
|
|
1926
2119
|
if (existingSched) {
|
|
1927
|
-
tx.delete(schedules).where(
|
|
2120
|
+
tx.delete(schedules).where(and3(eq8(schedules.projectId, projectId), eq8(schedules.kind, AV_KIND))).run();
|
|
1928
2121
|
scheduleAction = "delete";
|
|
1929
2122
|
}
|
|
1930
2123
|
}
|
|
@@ -1957,6 +2150,9 @@ async function applyRoutes(app, opts) {
|
|
|
1957
2150
|
if (!hasNotifications) {
|
|
1958
2151
|
opts?.onProjectUpserted?.(projectId, config.metadata.name);
|
|
1959
2152
|
}
|
|
2153
|
+
if (aliasesChanged) {
|
|
2154
|
+
opts?.onAliasesChanged?.(projectId, config.metadata.name);
|
|
2155
|
+
}
|
|
1960
2156
|
if ("google" in rawSpec && config.spec.google?.gsc?.propertyUrl) {
|
|
1961
2157
|
opts?.onGoogleConnectionPropertyUpdated?.(config.spec.canonicalDomain, "gsc", config.spec.google.gsc.propertyUrl);
|
|
1962
2158
|
}
|
|
@@ -1967,6 +2163,7 @@ async function applyRoutes(app, opts) {
|
|
|
1967
2163
|
displayName: project.displayName,
|
|
1968
2164
|
canonicalDomain: project.canonicalDomain,
|
|
1969
2165
|
ownedDomains: parseJsonColumn(project.ownedDomains, []),
|
|
2166
|
+
aliases: parseJsonColumn(project.aliases, []),
|
|
1970
2167
|
country: project.country,
|
|
1971
2168
|
language: project.language,
|
|
1972
2169
|
tags: parseJsonColumn(project.tags, []),
|
|
@@ -1982,6 +2179,13 @@ async function applyRoutes(app, opts) {
|
|
|
1982
2179
|
});
|
|
1983
2180
|
});
|
|
1984
2181
|
}
|
|
2182
|
+
function aliasArraysEqual2(a, b) {
|
|
2183
|
+
if (a.length !== b.length) return false;
|
|
2184
|
+
for (let i = 0; i < a.length; i++) {
|
|
2185
|
+
if (a[i].toLowerCase() !== b[i].toLowerCase()) return false;
|
|
2186
|
+
}
|
|
2187
|
+
return true;
|
|
2188
|
+
}
|
|
1985
2189
|
function normalizeCompetitorList2(domains) {
|
|
1986
2190
|
const seen = /* @__PURE__ */ new Set();
|
|
1987
2191
|
const result = [];
|
|
@@ -2089,6 +2293,7 @@ async function historyRoutes(app) {
|
|
|
2089
2293
|
citationState: s.citationState,
|
|
2090
2294
|
answerMentioned: resolveSnapshotAnswerMentioned(s, project),
|
|
2091
2295
|
visibilityState: resolveSnapshotVisibilityState(s, project),
|
|
2296
|
+
mentionState: resolveSnapshotMentionState(s, project),
|
|
2092
2297
|
answerText: s.answerText,
|
|
2093
2298
|
citedDomains: parseJsonColumn(s.citedDomains, []),
|
|
2094
2299
|
competitorOverlap: parseJsonColumn(s.competitorOverlap, []),
|
|
@@ -2142,9 +2347,11 @@ async function historyRoutes(app) {
|
|
|
2142
2347
|
const run = projectRuns.find((r) => r.id === snap.runId);
|
|
2143
2348
|
let transition = snap.citationState === CitationStates.cited ? "cited" : "not-cited";
|
|
2144
2349
|
let visibilityTransition = snap.answerMentioned ? "visible" : "not-visible";
|
|
2350
|
+
let mentionTransition = snap.answerMentioned ? "mentioned" : "not-mentioned";
|
|
2145
2351
|
if (idx === 0) {
|
|
2146
2352
|
transition = "new";
|
|
2147
2353
|
visibilityTransition = "new";
|
|
2354
|
+
mentionTransition = "new";
|
|
2148
2355
|
} else {
|
|
2149
2356
|
const prev = snaps[idx - 1];
|
|
2150
2357
|
if (prev.citationState === CitationStates["not-cited"] && snap.citationState === CitationStates.cited) {
|
|
@@ -2154,8 +2361,10 @@ async function historyRoutes(app) {
|
|
|
2154
2361
|
}
|
|
2155
2362
|
if (!prev.answerMentioned && snap.answerMentioned) {
|
|
2156
2363
|
visibilityTransition = "emerging";
|
|
2364
|
+
mentionTransition = "emerging";
|
|
2157
2365
|
} else if (prev.answerMentioned && !snap.answerMentioned) {
|
|
2158
2366
|
visibilityTransition = "lost";
|
|
2367
|
+
mentionTransition = "lost";
|
|
2159
2368
|
}
|
|
2160
2369
|
}
|
|
2161
2370
|
return {
|
|
@@ -2164,8 +2373,12 @@ async function historyRoutes(app) {
|
|
|
2164
2373
|
citationState: snap.citationState,
|
|
2165
2374
|
transition,
|
|
2166
2375
|
answerMentioned: snap.answerMentioned,
|
|
2376
|
+
// Legacy aliases of `mentionState` / `mentionTransition`.
|
|
2167
2377
|
visibilityState: snap.answerMentioned ? "visible" : "not-visible",
|
|
2168
2378
|
visibilityTransition,
|
|
2379
|
+
// Canonical-vocabulary fields — new consumers prefer these.
|
|
2380
|
+
mentionState: snap.answerMentioned ? "mentioned" : "not-mentioned",
|
|
2381
|
+
mentionTransition,
|
|
2169
2382
|
location: snap.location
|
|
2170
2383
|
};
|
|
2171
2384
|
});
|
|
@@ -2221,7 +2434,8 @@ async function historyRoutes(app) {
|
|
|
2221
2434
|
const resolved = {
|
|
2222
2435
|
...s,
|
|
2223
2436
|
resolvedAnswerMentioned: resolveSnapshotAnswerMentioned(s, project),
|
|
2224
|
-
resolvedVisibilityState: resolveSnapshotVisibilityState(s, project)
|
|
2437
|
+
resolvedVisibilityState: resolveSnapshotVisibilityState(s, project),
|
|
2438
|
+
resolvedMentionState: resolveSnapshotMentionState(s, project)
|
|
2225
2439
|
};
|
|
2226
2440
|
const existing = map1.get(s.queryId);
|
|
2227
2441
|
if (!existing || !existing.resolvedAnswerMentioned && resolved.resolvedAnswerMentioned || existing.resolvedAnswerMentioned === resolved.resolvedAnswerMentioned && resolved.citationState === CitationStates.cited) {
|
|
@@ -2233,7 +2447,8 @@ async function historyRoutes(app) {
|
|
|
2233
2447
|
const resolved = {
|
|
2234
2448
|
...s,
|
|
2235
2449
|
resolvedAnswerMentioned: resolveSnapshotAnswerMentioned(s, project),
|
|
2236
|
-
resolvedVisibilityState: resolveSnapshotVisibilityState(s, project)
|
|
2450
|
+
resolvedVisibilityState: resolveSnapshotVisibilityState(s, project),
|
|
2451
|
+
resolvedMentionState: resolveSnapshotMentionState(s, project)
|
|
2237
2452
|
};
|
|
2238
2453
|
const existing = map2.get(s.queryId);
|
|
2239
2454
|
if (!existing || !existing.resolvedAnswerMentioned && resolved.resolvedAnswerMentioned || existing.resolvedAnswerMentioned === resolved.resolvedAnswerMentioned && resolved.citationState === CitationStates.cited) {
|
|
@@ -2251,8 +2466,11 @@ async function historyRoutes(app) {
|
|
|
2251
2466
|
run2State: s2?.citationState ?? null,
|
|
2252
2467
|
run1AnswerMentioned: s1?.resolvedAnswerMentioned ?? null,
|
|
2253
2468
|
run2AnswerMentioned: s2?.resolvedAnswerMentioned ?? null,
|
|
2469
|
+
// Legacy aliases — same data as run{1,2}MentionState below.
|
|
2254
2470
|
run1VisibilityState: s1?.resolvedVisibilityState ?? null,
|
|
2255
2471
|
run2VisibilityState: s2?.resolvedVisibilityState ?? null,
|
|
2472
|
+
run1MentionState: s1?.resolvedMentionState ?? null,
|
|
2473
|
+
run2MentionState: s2?.resolvedMentionState ?? null,
|
|
2256
2474
|
changed: (s1?.citationState ?? null) !== (s2?.citationState ?? null),
|
|
2257
2475
|
visibilityChanged: (s1?.resolvedAnswerMentioned ?? null) !== (s2?.resolvedAnswerMentioned ?? null)
|
|
2258
2476
|
};
|
|
@@ -2621,7 +2839,7 @@ function buildCategoryCounts(counts) {
|
|
|
2621
2839
|
}
|
|
2622
2840
|
|
|
2623
2841
|
// ../api-routes/src/intelligence.ts
|
|
2624
|
-
import { eq as eq11, desc as desc4, and as
|
|
2842
|
+
import { eq as eq11, desc as desc4, and as and4, inArray as inArray3 } from "drizzle-orm";
|
|
2625
2843
|
function emptyHealthSnapshot(projectId) {
|
|
2626
2844
|
return {
|
|
2627
2845
|
id: `no-data:${projectId}`,
|
|
@@ -2710,7 +2928,7 @@ async function intelligenceRoutes(app) {
|
|
|
2710
2928
|
if (request.query.runId) {
|
|
2711
2929
|
conditions.push(eq11(insights.runId, request.query.runId));
|
|
2712
2930
|
}
|
|
2713
|
-
const rows = app.db.select().from(insights).where(conditions.length === 1 ? conditions[0] :
|
|
2931
|
+
const rows = app.db.select().from(insights).where(conditions.length === 1 ? conditions[0] : and4(...conditions)).orderBy(desc4(insights.createdAt)).all();
|
|
2714
2932
|
const showDismissed = request.query.dismissed === "true";
|
|
2715
2933
|
const result = rows.filter((r) => showDismissed || !r.dismissed).map(mapInsightRow);
|
|
2716
2934
|
return reply.send(result);
|
|
@@ -2734,7 +2952,7 @@ async function intelligenceRoutes(app) {
|
|
|
2734
2952
|
});
|
|
2735
2953
|
app.get("/projects/:name/health/latest", async (request, reply) => {
|
|
2736
2954
|
const project = resolveProject(app.db, request.params.name);
|
|
2737
|
-
const projectVisRuns = app.db.select({ id: runs.id, createdAt: runs.createdAt }).from(runs).where(
|
|
2955
|
+
const projectVisRuns = app.db.select({ id: runs.id, createdAt: runs.createdAt }).from(runs).where(and4(
|
|
2738
2956
|
eq11(runs.projectId, project.id),
|
|
2739
2957
|
eq11(runs.kind, RunKinds["answer-visibility"]),
|
|
2740
2958
|
inArray3(runs.status, [RunStatuses.completed, RunStatuses.partial])
|
|
@@ -2742,7 +2960,7 @@ async function intelligenceRoutes(app) {
|
|
|
2742
2960
|
const latestGroup = groupRunsByCreatedAt(projectVisRuns)[0] ?? [];
|
|
2743
2961
|
const latestGroupRunIds = latestGroup.map((r) => r.id);
|
|
2744
2962
|
if (latestGroupRunIds.length > 0) {
|
|
2745
|
-
const groupRows = app.db.select().from(healthSnapshots).where(
|
|
2963
|
+
const groupRows = app.db.select().from(healthSnapshots).where(and4(
|
|
2746
2964
|
eq11(healthSnapshots.projectId, project.id),
|
|
2747
2965
|
inArray3(healthSnapshots.runId, latestGroupRunIds)
|
|
2748
2966
|
)).all();
|
|
@@ -2766,7 +2984,7 @@ async function intelligenceRoutes(app) {
|
|
|
2766
2984
|
}
|
|
2767
2985
|
|
|
2768
2986
|
// ../api-routes/src/report.ts
|
|
2769
|
-
import { and as
|
|
2987
|
+
import { and as and6, desc as desc6, eq as eq13, gte, inArray as inArray5, lt, lte, ne, or as or3, sql as sql3 } from "drizzle-orm";
|
|
2770
2988
|
|
|
2771
2989
|
// ../api-routes/src/report-renderer.ts
|
|
2772
2990
|
var COLORS = {
|
|
@@ -5002,7 +5220,7 @@ function renderReportHtml(report, opts = {}) {
|
|
|
5002
5220
|
}
|
|
5003
5221
|
|
|
5004
5222
|
// ../api-routes/src/content-data.ts
|
|
5005
|
-
import { and as
|
|
5223
|
+
import { and as and5, eq as eq12, desc as desc5, inArray as inArray4 } from "drizzle-orm";
|
|
5006
5224
|
var RECENT_RUNS_WINDOW = 5;
|
|
5007
5225
|
function loadOrchestratorInput(db, project, locationFilter = void 0) {
|
|
5008
5226
|
const projectId = project.id;
|
|
@@ -5126,7 +5344,7 @@ function listCompetitorDomains(db, projectId) {
|
|
|
5126
5344
|
}
|
|
5127
5345
|
function listRecentAnswerVisibilityRunIds(db, projectId, limit, locationFilter) {
|
|
5128
5346
|
const rows = db.select({ id: runs.id, location: runs.location }).from(runs).where(
|
|
5129
|
-
|
|
5347
|
+
and5(
|
|
5130
5348
|
eq12(runs.projectId, projectId),
|
|
5131
5349
|
eq12(runs.kind, RunKinds["answer-visibility"]),
|
|
5132
5350
|
// Queued/running/failed/cancelled runs may have partial or no
|
|
@@ -5384,8 +5602,8 @@ function safeNum(value) {
|
|
|
5384
5602
|
}
|
|
5385
5603
|
return 0;
|
|
5386
5604
|
}
|
|
5387
|
-
function categorizeQuery(query,
|
|
5388
|
-
return categorizeQueryByIntent(query, buildBrandTokens(canonicalDomain,
|
|
5605
|
+
function categorizeQuery(query, projectBrandNames, canonicalDomain) {
|
|
5606
|
+
return categorizeQueryByIntent(query, buildBrandTokens(canonicalDomain, projectBrandNames));
|
|
5389
5607
|
}
|
|
5390
5608
|
function loadSnapshotsForRun(db, runId) {
|
|
5391
5609
|
return loadSnapshotsForRunIds(db, [runId]);
|
|
@@ -5414,7 +5632,7 @@ function loadQueryLookup(db, projectId) {
|
|
|
5414
5632
|
for (const row of rows) byId.set(row.id, row.query);
|
|
5415
5633
|
return { byId };
|
|
5416
5634
|
}
|
|
5417
|
-
function buildGscSection(db, projectId,
|
|
5635
|
+
function buildGscSection(db, projectId, projectBrandNames, canonicalDomain, trackedQueries) {
|
|
5418
5636
|
const allRows = db.select().from(gscSearchData).where(eq13(gscSearchData.projectId, projectId)).all();
|
|
5419
5637
|
if (allRows.length === 0) return null;
|
|
5420
5638
|
let maxDate = "";
|
|
@@ -5449,11 +5667,11 @@ function buildGscSection(db, projectId, projectDisplayName, canonicalDomain, tra
|
|
|
5449
5667
|
impressions: agg.impressions,
|
|
5450
5668
|
ctr: agg.impressions > 0 ? agg.clicks / agg.impressions : 0,
|
|
5451
5669
|
avgPosition: agg.impressions > 0 ? agg.weightedPositionSum / agg.impressions : 0,
|
|
5452
|
-
category: categorizeQuery(query,
|
|
5670
|
+
category: categorizeQuery(query, projectBrandNames, canonicalDomain)
|
|
5453
5671
|
})).sort((a, b) => b.clicks - a.clicks).slice(0, TOP_QUERIES_LIMIT);
|
|
5454
5672
|
const categoryAgg = /* @__PURE__ */ new Map();
|
|
5455
5673
|
for (const [query, agg] of queryAgg) {
|
|
5456
|
-
const cat = categorizeQuery(query,
|
|
5674
|
+
const cat = categorizeQuery(query, projectBrandNames, canonicalDomain);
|
|
5457
5675
|
const bucket = categoryAgg.get(cat) ?? { clicks: 0, impressions: 0 };
|
|
5458
5676
|
bucket.clicks += agg.clicks;
|
|
5459
5677
|
bucket.impressions += agg.impressions;
|
|
@@ -5471,7 +5689,7 @@ function buildGscSection(db, projectId, projectDisplayName, canonicalDomain, tra
|
|
|
5471
5689
|
const trackedSet = new Set(trackedQueries.map((q) => q.toLowerCase()));
|
|
5472
5690
|
const gscQuerySet = new Set([...queryAgg.keys()].map((q) => q.toLowerCase()));
|
|
5473
5691
|
const trackedButNoGsc = trackedQueries.filter((q) => !gscQuerySet.has(q.toLowerCase())).sort();
|
|
5474
|
-
const gscButNotTracked = [...queryAgg.entries()].filter(([q]) => !trackedSet.has(q.toLowerCase())).filter(([q]) => categorizeQuery(q,
|
|
5692
|
+
const gscButNotTracked = [...queryAgg.entries()].filter(([q]) => !trackedSet.has(q.toLowerCase())).filter(([q]) => categorizeQuery(q, projectBrandNames, canonicalDomain) !== "brand").sort((a, b) => b[1].impressions - a[1].impressions).map(([q]) => q).slice(0, TOP_QUERIES_LIMIT);
|
|
5475
5693
|
return {
|
|
5476
5694
|
periodStart,
|
|
5477
5695
|
periodEnd,
|
|
@@ -5488,7 +5706,7 @@ function buildGscSection(db, projectId, projectDisplayName, canonicalDomain, tra
|
|
|
5488
5706
|
}
|
|
5489
5707
|
function buildGaSection(db, projectId) {
|
|
5490
5708
|
const windowSummary = db.select().from(gaTrafficWindowSummaries).where(
|
|
5491
|
-
|
|
5709
|
+
and6(
|
|
5492
5710
|
eq13(gaTrafficWindowSummaries.projectId, projectId),
|
|
5493
5711
|
eq13(gaTrafficWindowSummaries.windowKey, "30d")
|
|
5494
5712
|
)
|
|
@@ -5666,7 +5884,7 @@ function nonSubresourceReferralPathCondition() {
|
|
|
5666
5884
|
}
|
|
5667
5885
|
function buildServerActivity(db, projectId) {
|
|
5668
5886
|
const sourceRows = db.select({ id: trafficSources.id }).from(trafficSources).where(
|
|
5669
|
-
|
|
5887
|
+
and6(
|
|
5670
5888
|
eq13(trafficSources.projectId, projectId),
|
|
5671
5889
|
ne(trafficSources.status, TrafficSourceStatuses.archived)
|
|
5672
5890
|
)
|
|
@@ -5682,7 +5900,7 @@ function buildServerActivity(db, projectId) {
|
|
|
5682
5900
|
const trendStart = new Date(trendStartMs).toISOString();
|
|
5683
5901
|
const sumVerifiedCrawlers = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
5684
5902
|
db.select({ total: sql3`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
5685
|
-
|
|
5903
|
+
and6(
|
|
5686
5904
|
eq13(crawlerEventsHourly.projectId, projectId),
|
|
5687
5905
|
eq13(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
5688
5906
|
gte(crawlerEventsHourly.tsHour, windowStartIso),
|
|
@@ -5692,7 +5910,7 @@ function buildServerActivity(db, projectId) {
|
|
|
5692
5910
|
);
|
|
5693
5911
|
const sumUnverifiedCrawlers = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
5694
5912
|
db.select({ total: sql3`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
5695
|
-
|
|
5913
|
+
and6(
|
|
5696
5914
|
eq13(crawlerEventsHourly.projectId, projectId),
|
|
5697
5915
|
ne(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
5698
5916
|
gte(crawlerEventsHourly.tsHour, windowStartIso),
|
|
@@ -5702,7 +5920,7 @@ function buildServerActivity(db, projectId) {
|
|
|
5702
5920
|
);
|
|
5703
5921
|
const sumReferrals = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
5704
5922
|
db.select({ total: sql3`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
5705
|
-
|
|
5923
|
+
and6(
|
|
5706
5924
|
eq13(aiReferralEventsHourly.projectId, projectId),
|
|
5707
5925
|
nonSubresourceReferralPathCondition(),
|
|
5708
5926
|
gte(aiReferralEventsHourly.tsHour, windowStartIso),
|
|
@@ -5721,7 +5939,7 @@ function buildServerActivity(db, projectId) {
|
|
|
5721
5939
|
verificationStatus: crawlerEventsHourly.verificationStatus,
|
|
5722
5940
|
hits: sql3`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
5723
5941
|
}).from(crawlerEventsHourly).where(
|
|
5724
|
-
|
|
5942
|
+
and6(
|
|
5725
5943
|
eq13(crawlerEventsHourly.projectId, projectId),
|
|
5726
5944
|
gte(crawlerEventsHourly.tsHour, headlineStart),
|
|
5727
5945
|
lte(crawlerEventsHourly.tsHour, headlineEnd)
|
|
@@ -5731,7 +5949,7 @@ function buildServerActivity(db, projectId) {
|
|
|
5731
5949
|
operator: crawlerEventsHourly.operator,
|
|
5732
5950
|
hits: sql3`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
5733
5951
|
}).from(crawlerEventsHourly).where(
|
|
5734
|
-
|
|
5952
|
+
and6(
|
|
5735
5953
|
eq13(crawlerEventsHourly.projectId, projectId),
|
|
5736
5954
|
eq13(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
5737
5955
|
gte(crawlerEventsHourly.tsHour, priorStart),
|
|
@@ -5742,7 +5960,7 @@ function buildServerActivity(db, projectId) {
|
|
|
5742
5960
|
operator: aiReferralEventsHourly.operator,
|
|
5743
5961
|
hits: sql3`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`
|
|
5744
5962
|
}).from(aiReferralEventsHourly).where(
|
|
5745
|
-
|
|
5963
|
+
and6(
|
|
5746
5964
|
eq13(aiReferralEventsHourly.projectId, projectId),
|
|
5747
5965
|
nonSubresourceReferralPathCondition(),
|
|
5748
5966
|
gte(aiReferralEventsHourly.tsHour, headlineStart),
|
|
@@ -5783,7 +6001,7 @@ function buildServerActivity(db, projectId) {
|
|
|
5783
6001
|
hits: sql3`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
5784
6002
|
operators: sql3`COUNT(DISTINCT ${crawlerEventsHourly.operator})`
|
|
5785
6003
|
}).from(crawlerEventsHourly).where(
|
|
5786
|
-
|
|
6004
|
+
and6(
|
|
5787
6005
|
eq13(crawlerEventsHourly.projectId, projectId),
|
|
5788
6006
|
eq13(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
5789
6007
|
gte(crawlerEventsHourly.tsHour, headlineStart),
|
|
@@ -5800,7 +6018,7 @@ function buildServerActivity(db, projectId) {
|
|
|
5800
6018
|
arrivals: sql3`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
5801
6019
|
landingPaths: sql3`COUNT(DISTINCT ${aiReferralEventsHourly.landingPathNormalized})`
|
|
5802
6020
|
}).from(aiReferralEventsHourly).where(
|
|
5803
|
-
|
|
6021
|
+
and6(
|
|
5804
6022
|
eq13(aiReferralEventsHourly.projectId, projectId),
|
|
5805
6023
|
nonSubresourceReferralPathCondition(),
|
|
5806
6024
|
gte(aiReferralEventsHourly.tsHour, headlineStart),
|
|
@@ -5817,7 +6035,7 @@ function buildServerActivity(db, projectId) {
|
|
|
5817
6035
|
arrivals: sql3`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
5818
6036
|
products: sql3`COUNT(DISTINCT ${aiReferralEventsHourly.product})`
|
|
5819
6037
|
}).from(aiReferralEventsHourly).where(
|
|
5820
|
-
|
|
6038
|
+
and6(
|
|
5821
6039
|
eq13(aiReferralEventsHourly.projectId, projectId),
|
|
5822
6040
|
nonSubresourceReferralPathCondition(),
|
|
5823
6041
|
gte(aiReferralEventsHourly.tsHour, headlineStart),
|
|
@@ -5833,7 +6051,7 @@ function buildServerActivity(db, projectId) {
|
|
|
5833
6051
|
date: sql3`SUBSTR(${crawlerEventsHourly.tsHour}, 1, 10)`,
|
|
5834
6052
|
hits: sql3`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
5835
6053
|
}).from(crawlerEventsHourly).where(
|
|
5836
|
-
|
|
6054
|
+
and6(
|
|
5837
6055
|
eq13(crawlerEventsHourly.projectId, projectId),
|
|
5838
6056
|
eq13(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
5839
6057
|
gte(crawlerEventsHourly.tsHour, trendStart),
|
|
@@ -5844,7 +6062,7 @@ function buildServerActivity(db, projectId) {
|
|
|
5844
6062
|
date: sql3`SUBSTR(${aiReferralEventsHourly.tsHour}, 1, 10)`,
|
|
5845
6063
|
hits: sql3`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`
|
|
5846
6064
|
}).from(aiReferralEventsHourly).where(
|
|
5847
|
-
|
|
6065
|
+
and6(
|
|
5848
6066
|
eq13(aiReferralEventsHourly.projectId, projectId),
|
|
5849
6067
|
nonSubresourceReferralPathCondition(),
|
|
5850
6068
|
gte(aiReferralEventsHourly.tsHour, trendStart),
|
|
@@ -5919,7 +6137,7 @@ function buildIndexingHealth(db, projectId) {
|
|
|
5919
6137
|
return null;
|
|
5920
6138
|
}
|
|
5921
6139
|
function buildCitationsTrend(db, projectId, queryLookup, locationFilter) {
|
|
5922
|
-
const visibilityRuns = db.select().from(runs).where(
|
|
6140
|
+
const visibilityRuns = db.select().from(runs).where(and6(eq13(runs.projectId, projectId), eq13(runs.kind, RunKinds["answer-visibility"]))).all().filter((r) => locationFilter === void 0 || (r.location ?? null) === locationFilter);
|
|
5923
6141
|
const totalQueries = queryLookup.byId.size;
|
|
5924
6142
|
const points = [];
|
|
5925
6143
|
for (const run of visibilityRuns) {
|
|
@@ -5965,14 +6183,14 @@ function buildCitationsTrend(db, projectId, queryLookup, locationFilter) {
|
|
|
5965
6183
|
}
|
|
5966
6184
|
function buildInsightList(db, projectId, locationFilter) {
|
|
5967
6185
|
const recentRunIds = db.select({ id: runs.id, location: runs.location }).from(runs).where(
|
|
5968
|
-
|
|
6186
|
+
and6(
|
|
5969
6187
|
eq13(runs.projectId, projectId),
|
|
5970
6188
|
eq13(runs.kind, RunKinds["answer-visibility"]),
|
|
5971
|
-
|
|
6189
|
+
or3(eq13(runs.status, RunStatuses.completed), eq13(runs.status, RunStatuses.partial))
|
|
5972
6190
|
)
|
|
5973
6191
|
).orderBy(desc6(runs.createdAt)).all().filter((r) => locationFilter === void 0 || (r.location ?? null) === locationFilter).slice(0, INSIGHT_LOOKBACK_RUNS).map((r) => r.id);
|
|
5974
6192
|
if (recentRunIds.length === 0) return [];
|
|
5975
|
-
const rows = db.select().from(insights).where(
|
|
6193
|
+
const rows = db.select().from(insights).where(and6(eq13(insights.projectId, projectId), inArray5(insights.runId, recentRunIds))).orderBy(desc6(insights.createdAt)).all();
|
|
5976
6194
|
const severityRank = { critical: 0, high: 1, medium: 2, low: 3 };
|
|
5977
6195
|
const flat = rows.filter((r) => !r.dismissed).map((r) => {
|
|
5978
6196
|
const recommendation = parseJsonColumn(r.recommendation, null);
|
|
@@ -6160,7 +6378,8 @@ function buildReportActionPlan(input) {
|
|
|
6160
6378
|
}
|
|
6161
6379
|
for (const [index, opportunity] of input.contentOpportunities.slice(0, 2).entries()) {
|
|
6162
6380
|
const verb = contentActionVerb(opportunity.action);
|
|
6163
|
-
const
|
|
6381
|
+
const bestPageUrl = opportunity.ourBestPage?.url;
|
|
6382
|
+
const hasUsablePage = !!bestPageUrl && bestPageUrl !== "/" && bestPageUrl.trim() !== "";
|
|
6164
6383
|
const evidence = [
|
|
6165
6384
|
`Opportunity score ${Math.round(opportunity.score)} with ${opportunity.actionConfidence} confidence`,
|
|
6166
6385
|
`Demand source: ${opportunity.demandSource}`
|
|
@@ -6169,17 +6388,22 @@ function buildReportActionPlan(input) {
|
|
|
6169
6388
|
evidence.push(`${opportunity.winningCompetitor.domain} is the current winning cited source`);
|
|
6170
6389
|
}
|
|
6171
6390
|
if (opportunity.ourBestPage) {
|
|
6172
|
-
|
|
6391
|
+
if (bestPageUrl === "/") {
|
|
6392
|
+
evidence.push("No topical page yet \u2014 homepage is the closest slug match");
|
|
6393
|
+
} else {
|
|
6394
|
+
evidence.push(`Best matching owned page: ${bestPageUrl}`);
|
|
6395
|
+
}
|
|
6173
6396
|
} else {
|
|
6174
6397
|
evidence.push("No matching owned page was found");
|
|
6175
6398
|
}
|
|
6399
|
+
const targetIsExistingPage = opportunity.action !== "create" && hasUsablePage;
|
|
6176
6400
|
actions.push({
|
|
6177
6401
|
audience: "both",
|
|
6178
6402
|
priority: 20 + index,
|
|
6179
6403
|
horizon: opportunity.actionConfidence === "high" ? "short-term" : "medium-term",
|
|
6180
6404
|
category: "content",
|
|
6181
6405
|
title: `${verb} content for "${opportunity.query}"`,
|
|
6182
|
-
action:
|
|
6406
|
+
action: targetIsExistingPage ? `${verb} the existing page so it directly answers the tracked query and cites the strongest supporting evidence.` : `${verb} a new page for "${opportunity.query}" that directly answers the query and earns citations from AI answer engines.`,
|
|
6183
6407
|
why: opportunity.drivers.length > 0 ? opportunity.drivers : ["Canonry ranked this as a content opportunity from search-demand and citation evidence."],
|
|
6184
6408
|
evidence,
|
|
6185
6409
|
successMetric: `A future check cites ${input.canonicalDomain} for "${opportunity.query}" and the matching GSC query/page improves.`,
|
|
@@ -6524,6 +6748,11 @@ function buildProjectReport(db, projectName) {
|
|
|
6524
6748
|
const competitorDomains = competitorRows.map((c) => c.domain);
|
|
6525
6749
|
const ownedDomains = parseJsonColumn(project.ownedDomains, []);
|
|
6526
6750
|
const projectDomains = [project.canonicalDomain, ...ownedDomains];
|
|
6751
|
+
const projectAliases = parseJsonColumn(project.aliases, []);
|
|
6752
|
+
const projectBrandNames = effectiveBrandNames({
|
|
6753
|
+
displayName: project.displayName,
|
|
6754
|
+
aliases: projectAliases
|
|
6755
|
+
});
|
|
6527
6756
|
const citationScorecard = buildCitationScorecard(latestSnapshots, queryLookup);
|
|
6528
6757
|
const competitorLandscape = buildCompetitorLandscape(
|
|
6529
6758
|
latestSnapshots,
|
|
@@ -6534,7 +6763,7 @@ function buildProjectReport(db, projectName) {
|
|
|
6534
6763
|
const mentionLandscape = buildMentionLandscape(
|
|
6535
6764
|
latestSnapshots,
|
|
6536
6765
|
competitorDomains,
|
|
6537
|
-
|
|
6766
|
+
projectBrandNames,
|
|
6538
6767
|
projectDomains,
|
|
6539
6768
|
queryLookup
|
|
6540
6769
|
);
|
|
@@ -6543,7 +6772,7 @@ function buildProjectReport(db, projectName) {
|
|
|
6543
6772
|
const gscSection = buildGscSection(
|
|
6544
6773
|
db,
|
|
6545
6774
|
project.id,
|
|
6546
|
-
|
|
6775
|
+
projectBrandNames,
|
|
6547
6776
|
project.canonicalDomain,
|
|
6548
6777
|
trackedQueries
|
|
6549
6778
|
);
|
|
@@ -6899,7 +7128,7 @@ function normalizeDomain2(domain) {
|
|
|
6899
7128
|
}
|
|
6900
7129
|
|
|
6901
7130
|
// ../api-routes/src/composites.ts
|
|
6902
|
-
import { eq as eq15, and as
|
|
7131
|
+
import { eq as eq15, and as and7, desc as desc7, sql as sql4, like, or as or4, inArray as inArray7 } from "drizzle-orm";
|
|
6903
7132
|
var TOP_INSIGHT_LIMIT = 5;
|
|
6904
7133
|
var SEARCH_HIT_HARD_LIMIT = 50;
|
|
6905
7134
|
var SEARCH_SNIPPET_RADIUS = 80;
|
|
@@ -7013,9 +7242,9 @@ async function compositeRoutes(app) {
|
|
|
7013
7242
|
rawResponse: querySnapshots.rawResponse,
|
|
7014
7243
|
createdAt: querySnapshots.createdAt
|
|
7015
7244
|
}).from(querySnapshots).innerJoin(queries, eq15(querySnapshots.queryId, queries.id)).where(
|
|
7016
|
-
|
|
7245
|
+
and7(
|
|
7017
7246
|
eq15(queries.projectId, project.id),
|
|
7018
|
-
|
|
7247
|
+
or4(
|
|
7019
7248
|
sql4`${querySnapshots.answerText} LIKE ${pattern} ESCAPE '\\'`,
|
|
7020
7249
|
sql4`${querySnapshots.citedDomains} LIKE ${pattern} ESCAPE '\\'`,
|
|
7021
7250
|
sql4`${querySnapshots.rawResponse} LIKE ${pattern} ESCAPE '\\'`,
|
|
@@ -7024,9 +7253,9 @@ async function compositeRoutes(app) {
|
|
|
7024
7253
|
)
|
|
7025
7254
|
).orderBy(desc7(querySnapshots.createdAt)).limit(limit + 1).all());
|
|
7026
7255
|
const insightMatches = app.db.select().from(insights).where(
|
|
7027
|
-
|
|
7256
|
+
and7(
|
|
7028
7257
|
eq15(insights.projectId, project.id),
|
|
7029
|
-
|
|
7258
|
+
or4(
|
|
7030
7259
|
like(insights.title, pattern),
|
|
7031
7260
|
like(insights.query, pattern),
|
|
7032
7261
|
sql4`${insights.recommendation} LIKE ${pattern} ESCAPE '\\'`,
|
|
@@ -7354,6 +7583,7 @@ function formatProject2(row) {
|
|
|
7354
7583
|
displayName: row.displayName,
|
|
7355
7584
|
canonicalDomain: row.canonicalDomain,
|
|
7356
7585
|
ownedDomains: parseJsonColumn(row.ownedDomains, []),
|
|
7586
|
+
aliases: parseJsonColumn(row.aliases, []),
|
|
7357
7587
|
country: row.country,
|
|
7358
7588
|
language: row.language,
|
|
7359
7589
|
tags: parseJsonColumn(row.tags, []),
|
|
@@ -7656,6 +7886,7 @@ var routeCatalog = [
|
|
|
7656
7886
|
displayName: stringSchema,
|
|
7657
7887
|
canonicalDomain: stringSchema,
|
|
7658
7888
|
ownedDomains: stringArraySchema,
|
|
7889
|
+
aliases: stringArraySchema,
|
|
7659
7890
|
country: stringSchema,
|
|
7660
7891
|
language: stringSchema,
|
|
7661
7892
|
tags: stringArraySchema,
|
|
@@ -10875,7 +11106,7 @@ async function telemetryRoutes(app, opts) {
|
|
|
10875
11106
|
|
|
10876
11107
|
// ../api-routes/src/schedules.ts
|
|
10877
11108
|
import crypto11 from "crypto";
|
|
10878
|
-
import { and as
|
|
11109
|
+
import { and as and8, eq as eq16 } from "drizzle-orm";
|
|
10879
11110
|
function parseKindParam(raw) {
|
|
10880
11111
|
if (raw === void 0 || raw === null || raw === "") return SchedulableRunKinds["answer-visibility"];
|
|
10881
11112
|
const parsed = schedulableRunKindSchema.safeParse(raw);
|
|
@@ -10941,7 +11172,7 @@ async function scheduleRoutes(app, opts) {
|
|
|
10941
11172
|
}
|
|
10942
11173
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
10943
11174
|
const enabledInt = enabled === false ? 0 : 1;
|
|
10944
|
-
const existing = app.db.select().from(schedules).where(
|
|
11175
|
+
const existing = app.db.select().from(schedules).where(and8(eq16(schedules.projectId, project.id), eq16(schedules.kind, kind))).get();
|
|
10945
11176
|
if (existing) {
|
|
10946
11177
|
app.db.update(schedules).set({
|
|
10947
11178
|
cronExpr,
|
|
@@ -10975,13 +11206,13 @@ async function scheduleRoutes(app, opts) {
|
|
|
10975
11206
|
diff: { kind, cronExpr, preset, timezone, providers, sourceId }
|
|
10976
11207
|
});
|
|
10977
11208
|
opts.onScheduleUpdated?.("upsert", project.id, kind);
|
|
10978
|
-
const schedule = app.db.select().from(schedules).where(
|
|
11209
|
+
const schedule = app.db.select().from(schedules).where(and8(eq16(schedules.projectId, project.id), eq16(schedules.kind, kind))).get();
|
|
10979
11210
|
return reply.status(existing ? 200 : 201).send(formatSchedule(schedule));
|
|
10980
11211
|
});
|
|
10981
11212
|
app.get("/projects/:name/schedule", async (request, reply) => {
|
|
10982
11213
|
const project = resolveProject(app.db, request.params.name);
|
|
10983
11214
|
const kind = parseKindParam(request.query?.kind);
|
|
10984
|
-
const schedule = app.db.select().from(schedules).where(
|
|
11215
|
+
const schedule = app.db.select().from(schedules).where(and8(eq16(schedules.projectId, project.id), eq16(schedules.kind, kind))).get();
|
|
10985
11216
|
if (!schedule) {
|
|
10986
11217
|
throw notFound("Schedule", `${request.params.name} (kind=${kind})`);
|
|
10987
11218
|
}
|
|
@@ -10990,7 +11221,7 @@ async function scheduleRoutes(app, opts) {
|
|
|
10990
11221
|
app.delete("/projects/:name/schedule", async (request, reply) => {
|
|
10991
11222
|
const project = resolveProject(app.db, request.params.name);
|
|
10992
11223
|
const kind = parseKindParam(request.query?.kind);
|
|
10993
|
-
const schedule = app.db.select().from(schedules).where(
|
|
11224
|
+
const schedule = app.db.select().from(schedules).where(and8(eq16(schedules.projectId, project.id), eq16(schedules.kind, kind))).get();
|
|
10994
11225
|
if (!schedule) {
|
|
10995
11226
|
throw notFound("Schedule", `${request.params.name} (kind=${kind})`);
|
|
10996
11227
|
}
|
|
@@ -11029,7 +11260,8 @@ function formatSchedule(row) {
|
|
|
11029
11260
|
import crypto12 from "crypto";
|
|
11030
11261
|
import { eq as eq17 } from "drizzle-orm";
|
|
11031
11262
|
var VALID_EVENTS = ["citation.lost", "citation.gained", "run.completed", "run.failed", "insight.critical", "insight.high"];
|
|
11032
|
-
async function notificationRoutes(app) {
|
|
11263
|
+
async function notificationRoutes(app, opts = {}) {
|
|
11264
|
+
const allowLoopback = opts.allowLoopbackWebhooks === true;
|
|
11033
11265
|
app.get("/notifications/events", async (_request, reply) => {
|
|
11034
11266
|
return reply.send(VALID_EVENTS);
|
|
11035
11267
|
});
|
|
@@ -11037,7 +11269,7 @@ async function notificationRoutes(app) {
|
|
|
11037
11269
|
const project = resolveProject(app.db, request.params.name);
|
|
11038
11270
|
const { channel, url, events, source } = request.body ?? {};
|
|
11039
11271
|
if (channel !== "webhook") throw validationError('Only "webhook" channel is supported');
|
|
11040
|
-
const urlCheck = await resolveWebhookTarget(url ?? "");
|
|
11272
|
+
const urlCheck = await resolveWebhookTarget(url ?? "", { allowLoopback });
|
|
11041
11273
|
if (!urlCheck.ok) throw validationError(urlCheck.message);
|
|
11042
11274
|
if (!events?.length) throw validationError('"events" must be a non-empty array');
|
|
11043
11275
|
const invalid = events.filter((e) => !VALID_EVENTS.includes(e));
|
|
@@ -11098,7 +11330,7 @@ async function notificationRoutes(app) {
|
|
|
11098
11330
|
throw notFound("Notification", request.params.id);
|
|
11099
11331
|
}
|
|
11100
11332
|
const config = parseJsonColumn(notification.config, { url: "", events: [] });
|
|
11101
|
-
const urlCheck = await resolveWebhookTarget(config.url);
|
|
11333
|
+
const urlCheck = await resolveWebhookTarget(config.url, { allowLoopback });
|
|
11102
11334
|
if (!urlCheck.ok) throw validationError(`Stored webhook URL is invalid: ${urlCheck.message}`);
|
|
11103
11335
|
const payload = {
|
|
11104
11336
|
source: "canonry",
|
|
@@ -11146,7 +11378,7 @@ function formatNotification(row) {
|
|
|
11146
11378
|
|
|
11147
11379
|
// ../api-routes/src/google.ts
|
|
11148
11380
|
import crypto14 from "crypto";
|
|
11149
|
-
import { eq as eq18, and as
|
|
11381
|
+
import { eq as eq18, and as and9, desc as desc8, sql as sql5 } from "drizzle-orm";
|
|
11150
11382
|
|
|
11151
11383
|
// ../integration-google/src/constants.ts
|
|
11152
11384
|
var GOOGLE_AUTH_URL = "https://accounts.google.com/o/oauth2/v2/auth";
|
|
@@ -12156,7 +12388,23 @@ async function getValidToken(store, domain, connectionType, clientId, clientSecr
|
|
|
12156
12388
|
};
|
|
12157
12389
|
}
|
|
12158
12390
|
async function googleRoutes(app, opts) {
|
|
12159
|
-
|
|
12391
|
+
if (opts.googleStateSecret === void 0) {
|
|
12392
|
+
app.log.warn(
|
|
12393
|
+
"googleStateSecret is not configured \u2014 Google OAuth routes will not be registered. Set GOOGLE_STATE_SECRET to enable Google integrations."
|
|
12394
|
+
);
|
|
12395
|
+
return;
|
|
12396
|
+
}
|
|
12397
|
+
if (opts.googleStateSecret === "") {
|
|
12398
|
+
throw new Error(
|
|
12399
|
+
"googleStateSecret is empty. Set a non-empty secret (e.g. `openssl rand -hex 32`) via the GOOGLE_STATE_SECRET environment variable."
|
|
12400
|
+
);
|
|
12401
|
+
}
|
|
12402
|
+
if (opts.googleStateSecret === "insecure-default-secret") {
|
|
12403
|
+
throw new Error(
|
|
12404
|
+
"googleStateSecret is set to the legacy insecure default. Generate a real secret (e.g. `openssl rand -hex 32`) and set GOOGLE_STATE_SECRET."
|
|
12405
|
+
);
|
|
12406
|
+
}
|
|
12407
|
+
const stateSecret = opts.googleStateSecret;
|
|
12160
12408
|
function getAuthConfig() {
|
|
12161
12409
|
return opts.getGoogleAuthConfig?.() ?? {};
|
|
12162
12410
|
}
|
|
@@ -12363,7 +12611,7 @@ async function googleRoutes(app, opts) {
|
|
|
12363
12611
|
if (page) conditions.push(sql5`${gscSearchData.page} LIKE ${"%" + page + "%"}`);
|
|
12364
12612
|
const limitVal = Math.max(parseInt(limit ?? "500", 10) || 0, 1);
|
|
12365
12613
|
const offsetVal = Math.max(parseInt(offset ?? "0", 10) || 0, 0);
|
|
12366
|
-
const rows = app.db.select().from(gscSearchData).where(
|
|
12614
|
+
const rows = app.db.select().from(gscSearchData).where(and9(...conditions)).orderBy(desc8(gscSearchData.date)).limit(limitVal).offset(offsetVal).all();
|
|
12367
12615
|
return rows.map((r) => ({
|
|
12368
12616
|
date: r.date,
|
|
12369
12617
|
query: r.query,
|
|
@@ -12437,7 +12685,7 @@ async function googleRoutes(app, opts) {
|
|
|
12437
12685
|
const { url, limit } = request.query;
|
|
12438
12686
|
const conditions = [eq18(gscUrlInspections.projectId, project.id)];
|
|
12439
12687
|
if (url) conditions.push(eq18(gscUrlInspections.url, url));
|
|
12440
|
-
const rows = app.db.select().from(gscUrlInspections).where(
|
|
12688
|
+
const rows = app.db.select().from(gscUrlInspections).where(and9(...conditions)).orderBy(desc8(gscUrlInspections.inspectedAt)).limit(parseInt(limit ?? "100", 10)).all();
|
|
12441
12689
|
return rows.map((r) => ({
|
|
12442
12690
|
id: r.id,
|
|
12443
12691
|
url: r.url,
|
|
@@ -12789,7 +13037,7 @@ async function googleRoutes(app, opts) {
|
|
|
12789
13037
|
|
|
12790
13038
|
// ../api-routes/src/bing.ts
|
|
12791
13039
|
import crypto15 from "crypto";
|
|
12792
|
-
import { eq as eq19, and as
|
|
13040
|
+
import { eq as eq19, and as and10, desc as desc9 } from "drizzle-orm";
|
|
12793
13041
|
|
|
12794
13042
|
// ../integration-bing/src/constants.ts
|
|
12795
13043
|
var BING_WMT_API_BASE = "https://ssl.bing.com/webmaster/api.svc/json";
|
|
@@ -13203,7 +13451,7 @@ async function bingRoutes(app, opts) {
|
|
|
13203
13451
|
requireConnectionStore();
|
|
13204
13452
|
const project = resolveProject(app.db, request.params.name);
|
|
13205
13453
|
const { url, limit } = request.query;
|
|
13206
|
-
const whereClause = url ?
|
|
13454
|
+
const whereClause = url ? and10(eq19(bingUrlInspections.projectId, project.id), eq19(bingUrlInspections.url, url)) : eq19(bingUrlInspections.projectId, project.id);
|
|
13207
13455
|
const filtered = app.db.select().from(bingUrlInspections).where(whereClause).orderBy(desc9(bingUrlInspections.inspectedAt)).limit(Math.max(1, Math.min(parseInt(limit ?? "100", 10) || 100, 1e3))).all();
|
|
13208
13456
|
return filtered.map((r) => ({
|
|
13209
13457
|
id: r.id,
|
|
@@ -13435,7 +13683,7 @@ async function bingRoutes(app, opts) {
|
|
|
13435
13683
|
import fs from "fs";
|
|
13436
13684
|
import path from "path";
|
|
13437
13685
|
import os2 from "os";
|
|
13438
|
-
import { eq as eq20, and as
|
|
13686
|
+
import { eq as eq20, and as and11 } from "drizzle-orm";
|
|
13439
13687
|
function getScreenshotDir() {
|
|
13440
13688
|
return path.join(os2.homedir(), ".canonry", "screenshots");
|
|
13441
13689
|
}
|
|
@@ -13508,7 +13756,7 @@ async function cdpRoutes(app, opts) {
|
|
|
13508
13756
|
async (request, reply) => {
|
|
13509
13757
|
const project = resolveProject(app.db, request.params.name);
|
|
13510
13758
|
const { runId } = request.params;
|
|
13511
|
-
const run = app.db.select().from(runs).where(
|
|
13759
|
+
const run = app.db.select().from(runs).where(and11(eq20(runs.id, runId), eq20(runs.projectId, project.id))).get();
|
|
13512
13760
|
if (!run) {
|
|
13513
13761
|
const err = notFound("Run", runId);
|
|
13514
13762
|
return reply.code(err.statusCode).send(err.toJSON());
|
|
@@ -13605,7 +13853,7 @@ async function cdpRoutes(app, opts) {
|
|
|
13605
13853
|
|
|
13606
13854
|
// ../api-routes/src/ga.ts
|
|
13607
13855
|
import crypto16 from "crypto";
|
|
13608
|
-
import { eq as eq21, desc as desc10, and as
|
|
13856
|
+
import { eq as eq21, desc as desc10, and as and12, sql as sql6 } from "drizzle-orm";
|
|
13609
13857
|
function gaLog(level, action, ctx) {
|
|
13610
13858
|
const entry = { ts: (/* @__PURE__ */ new Date()).toISOString(), level, module: "GA4Routes", action, ...ctx };
|
|
13611
13859
|
const stream = level === "error" ? process.stderr : process.stdout;
|
|
@@ -13900,7 +14148,7 @@ async function ga4Routes(app, opts) {
|
|
|
13900
14148
|
app.db.transaction((tx) => {
|
|
13901
14149
|
if (syncTraffic) {
|
|
13902
14150
|
tx.delete(gaTrafficSnapshots).where(
|
|
13903
|
-
|
|
14151
|
+
and12(
|
|
13904
14152
|
eq21(gaTrafficSnapshots.projectId, project.id),
|
|
13905
14153
|
sql6`${gaTrafficSnapshots.date} >= ${summary.periodStart}`,
|
|
13906
14154
|
sql6`${gaTrafficSnapshots.date} <= ${summary.periodEnd}`
|
|
@@ -13924,7 +14172,7 @@ async function ga4Routes(app, opts) {
|
|
|
13924
14172
|
}
|
|
13925
14173
|
if (syncAi) {
|
|
13926
14174
|
tx.delete(gaAiReferrals).where(
|
|
13927
|
-
|
|
14175
|
+
and12(
|
|
13928
14176
|
eq21(gaAiReferrals.projectId, project.id),
|
|
13929
14177
|
sql6`${gaAiReferrals.date} >= ${summary.periodStart}`,
|
|
13930
14178
|
sql6`${gaAiReferrals.date} <= ${summary.periodEnd}`
|
|
@@ -13950,7 +14198,7 @@ async function ga4Routes(app, opts) {
|
|
|
13950
14198
|
}
|
|
13951
14199
|
if (syncSocial) {
|
|
13952
14200
|
tx.delete(gaSocialReferrals).where(
|
|
13953
|
-
|
|
14201
|
+
and12(
|
|
13954
14202
|
eq21(gaSocialReferrals.projectId, project.id),
|
|
13955
14203
|
sql6`${gaSocialReferrals.date} >= ${summary.periodStart}`,
|
|
13956
14204
|
sql6`${gaSocialReferrals.date} <= ${summary.periodEnd}`
|
|
@@ -14054,7 +14302,7 @@ async function ga4Routes(app, opts) {
|
|
|
14054
14302
|
totalDirectSessions: gaTrafficWindowSummaries.totalDirectSessions,
|
|
14055
14303
|
totalUsers: gaTrafficWindowSummaries.totalUsers
|
|
14056
14304
|
}).from(gaTrafficWindowSummaries).where(
|
|
14057
|
-
|
|
14305
|
+
and12(
|
|
14058
14306
|
eq21(gaTrafficWindowSummaries.projectId, project.id),
|
|
14059
14307
|
eq21(gaTrafficWindowSummaries.windowKey, window)
|
|
14060
14308
|
)
|
|
@@ -14063,7 +14311,7 @@ async function ga4Routes(app, opts) {
|
|
|
14063
14311
|
totalSessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)`,
|
|
14064
14312
|
totalOrganicSessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)`,
|
|
14065
14313
|
totalUsers: sql6`COALESCE(SUM(${gaTrafficSnapshots.users}), 0)`
|
|
14066
|
-
}).from(gaTrafficSnapshots).where(
|
|
14314
|
+
}).from(gaTrafficSnapshots).where(and12(...snapshotConditions)).get() : null;
|
|
14067
14315
|
const summaryRow = cutoffDate ? windowSummaryRow ?? snapshotTotalsRow : app.db.select({
|
|
14068
14316
|
totalSessions: gaTrafficSummaries.totalSessions,
|
|
14069
14317
|
totalOrganicSessions: gaTrafficSummaries.totalOrganicSessions,
|
|
@@ -14071,7 +14319,7 @@ async function ga4Routes(app, opts) {
|
|
|
14071
14319
|
}).from(gaTrafficSummaries).where(eq21(gaTrafficSummaries.projectId, project.id)).get();
|
|
14072
14320
|
const directTotalRow = windowSummaryRow ? { totalDirectSessions: windowSummaryRow.totalDirectSessions } : app.db.select({
|
|
14073
14321
|
totalDirectSessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)`
|
|
14074
|
-
}).from(gaTrafficSnapshots).where(
|
|
14322
|
+
}).from(gaTrafficSnapshots).where(and12(...snapshotConditions)).get();
|
|
14075
14323
|
const summaryMeta = app.db.select({
|
|
14076
14324
|
periodStart: gaTrafficSummaries.periodStart,
|
|
14077
14325
|
periodEnd: gaTrafficSummaries.periodEnd
|
|
@@ -14082,14 +14330,14 @@ async function ga4Routes(app, opts) {
|
|
|
14082
14330
|
organicSessions: sql6`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
14083
14331
|
directSessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)`,
|
|
14084
14332
|
users: sql6`SUM(${gaTrafficSnapshots.users})`
|
|
14085
|
-
}).from(gaTrafficSnapshots).where(
|
|
14333
|
+
}).from(gaTrafficSnapshots).where(and12(...snapshotConditions)).groupBy(sql6`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`).orderBy(sql6`SUM(${gaTrafficSnapshots.sessions}) DESC`).limit(limit).all();
|
|
14086
14334
|
const aiReferralRows = app.db.select({
|
|
14087
14335
|
source: gaAiReferrals.source,
|
|
14088
14336
|
medium: gaAiReferrals.medium,
|
|
14089
14337
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
14090
14338
|
sessions: sql6`SUM(${gaAiReferrals.sessions})`,
|
|
14091
14339
|
users: sql6`SUM(${gaAiReferrals.users})`
|
|
14092
|
-
}).from(gaAiReferrals).where(
|
|
14340
|
+
}).from(gaAiReferrals).where(and12(...aiConditions)).groupBy(gaAiReferrals.source, gaAiReferrals.medium, gaAiReferrals.sourceDimension).all();
|
|
14093
14341
|
const aiReferralLandingPageRows = app.db.select({
|
|
14094
14342
|
source: gaAiReferrals.source,
|
|
14095
14343
|
medium: gaAiReferrals.medium,
|
|
@@ -14097,7 +14345,7 @@ async function ga4Routes(app, opts) {
|
|
|
14097
14345
|
landingPage: sql6`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`,
|
|
14098
14346
|
sessions: sql6`SUM(${gaAiReferrals.sessions})`,
|
|
14099
14347
|
users: sql6`SUM(${gaAiReferrals.users})`
|
|
14100
|
-
}).from(gaAiReferrals).where(
|
|
14348
|
+
}).from(gaAiReferrals).where(and12(...aiConditions)).groupBy(
|
|
14101
14349
|
gaAiReferrals.source,
|
|
14102
14350
|
gaAiReferrals.medium,
|
|
14103
14351
|
gaAiReferrals.sourceDimension,
|
|
@@ -14134,7 +14382,7 @@ async function ga4Routes(app, opts) {
|
|
|
14134
14382
|
channelGroup: gaAiReferrals.channelGroup,
|
|
14135
14383
|
sessions: sql6`COALESCE(SUM(${gaAiReferrals.sessions}), 0)`,
|
|
14136
14384
|
users: sql6`COALESCE(SUM(${gaAiReferrals.users}), 0)`
|
|
14137
|
-
}).from(gaAiReferrals).where(
|
|
14385
|
+
}).from(gaAiReferrals).where(and12(...aiConditions, eq21(gaAiReferrals.sourceDimension, "session"))).groupBy(gaAiReferrals.channelGroup).all();
|
|
14138
14386
|
const aiSessionsByChannelGroup = /* @__PURE__ */ new Map();
|
|
14139
14387
|
let aiBySessionUsers = 0;
|
|
14140
14388
|
for (const row of aiBySessionRows) {
|
|
@@ -14148,11 +14396,11 @@ async function ga4Routes(app, opts) {
|
|
|
14148
14396
|
channelGroup: gaSocialReferrals.channelGroup,
|
|
14149
14397
|
sessions: sql6`SUM(${gaSocialReferrals.sessions})`,
|
|
14150
14398
|
users: sql6`SUM(${gaSocialReferrals.users})`
|
|
14151
|
-
}).from(gaSocialReferrals).where(
|
|
14399
|
+
}).from(gaSocialReferrals).where(and12(...socialConditions)).groupBy(gaSocialReferrals.source, gaSocialReferrals.medium, gaSocialReferrals.channelGroup).orderBy(sql6`SUM(${gaSocialReferrals.sessions}) DESC`).all();
|
|
14152
14400
|
const socialTotals = app.db.select({
|
|
14153
14401
|
sessions: sql6`SUM(${gaSocialReferrals.sessions})`,
|
|
14154
14402
|
users: sql6`SUM(${gaSocialReferrals.users})`
|
|
14155
|
-
}).from(gaSocialReferrals).where(
|
|
14403
|
+
}).from(gaSocialReferrals).where(and12(...socialConditions)).get();
|
|
14156
14404
|
const latestSync = app.db.select({ syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(eq21(gaTrafficSummaries.projectId, project.id)).orderBy(desc10(gaTrafficSummaries.syncedAt)).limit(1).get();
|
|
14157
14405
|
const total = summaryRow?.totalSessions ?? 0;
|
|
14158
14406
|
const totalDirectSessions = directTotalRow?.totalDirectSessions ?? 0;
|
|
@@ -14243,7 +14491,7 @@ async function ga4Routes(app, opts) {
|
|
|
14243
14491
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
14244
14492
|
sessions: sql6`SUM(${gaAiReferrals.sessions})`,
|
|
14245
14493
|
users: sql6`SUM(${gaAiReferrals.users})`
|
|
14246
|
-
}).from(gaAiReferrals).where(
|
|
14494
|
+
}).from(gaAiReferrals).where(and12(...conditions)).groupBy(
|
|
14247
14495
|
gaAiReferrals.date,
|
|
14248
14496
|
gaAiReferrals.source,
|
|
14249
14497
|
gaAiReferrals.medium,
|
|
@@ -14265,7 +14513,7 @@ async function ga4Routes(app, opts) {
|
|
|
14265
14513
|
channelGroup: gaSocialReferrals.channelGroup,
|
|
14266
14514
|
sessions: gaSocialReferrals.sessions,
|
|
14267
14515
|
users: gaSocialReferrals.users
|
|
14268
|
-
}).from(gaSocialReferrals).where(
|
|
14516
|
+
}).from(gaSocialReferrals).where(and12(...conditions)).orderBy(gaSocialReferrals.date).all();
|
|
14269
14517
|
return rows;
|
|
14270
14518
|
});
|
|
14271
14519
|
app.get("/projects/:name/ga/social-referral-trend", async (request, _reply) => {
|
|
@@ -14278,7 +14526,7 @@ async function ga4Routes(app, opts) {
|
|
|
14278
14526
|
d.setDate(d.getDate() - n);
|
|
14279
14527
|
return fmt(d);
|
|
14280
14528
|
};
|
|
14281
|
-
const sumSocial = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(
|
|
14529
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and12(
|
|
14282
14530
|
eq21(gaSocialReferrals.projectId, project.id),
|
|
14283
14531
|
sql6`${gaSocialReferrals.date} >= ${from}`,
|
|
14284
14532
|
sql6`${gaSocialReferrals.date} < ${to}`
|
|
@@ -14291,7 +14539,7 @@ async function ga4Routes(app, opts) {
|
|
|
14291
14539
|
const sourceCurrent = app.db.select({
|
|
14292
14540
|
source: gaSocialReferrals.source,
|
|
14293
14541
|
sessions: sql6`SUM(${gaSocialReferrals.sessions})`
|
|
14294
|
-
}).from(gaSocialReferrals).where(
|
|
14542
|
+
}).from(gaSocialReferrals).where(and12(
|
|
14295
14543
|
eq21(gaSocialReferrals.projectId, project.id),
|
|
14296
14544
|
sql6`${gaSocialReferrals.date} >= ${daysAgo2(7)}`,
|
|
14297
14545
|
sql6`${gaSocialReferrals.date} < ${fmt(today)}`
|
|
@@ -14299,7 +14547,7 @@ async function ga4Routes(app, opts) {
|
|
|
14299
14547
|
const sourcePrev = app.db.select({
|
|
14300
14548
|
source: gaSocialReferrals.source,
|
|
14301
14549
|
sessions: sql6`SUM(${gaSocialReferrals.sessions})`
|
|
14302
|
-
}).from(gaSocialReferrals).where(
|
|
14550
|
+
}).from(gaSocialReferrals).where(and12(
|
|
14303
14551
|
eq21(gaSocialReferrals.projectId, project.id),
|
|
14304
14552
|
sql6`${gaSocialReferrals.date} >= ${daysAgo2(14)}`,
|
|
14305
14553
|
sql6`${gaSocialReferrals.date} < ${daysAgo2(7)}`
|
|
@@ -14341,16 +14589,16 @@ async function ga4Routes(app, opts) {
|
|
|
14341
14589
|
return fmt(d);
|
|
14342
14590
|
};
|
|
14343
14591
|
const pct = (cur, prev) => prev === 0 ? null : Math.round((cur - prev) / prev * 100);
|
|
14344
|
-
const sumTotal = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)` }).from(gaTrafficSnapshots).where(
|
|
14345
|
-
const sumOrganic = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)` }).from(gaTrafficSnapshots).where(
|
|
14346
|
-
const sumDirect = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)` }).from(gaTrafficSnapshots).where(
|
|
14347
|
-
const sumAi = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(
|
|
14592
|
+
const sumTotal = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)` }).from(gaTrafficSnapshots).where(and12(eq21(gaTrafficSnapshots.projectId, project.id), sql6`${gaTrafficSnapshots.date} >= ${from}`, sql6`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
14593
|
+
const sumOrganic = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)` }).from(gaTrafficSnapshots).where(and12(eq21(gaTrafficSnapshots.projectId, project.id), sql6`${gaTrafficSnapshots.date} >= ${from}`, sql6`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
14594
|
+
const sumDirect = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)` }).from(gaTrafficSnapshots).where(and12(eq21(gaTrafficSnapshots.projectId, project.id), sql6`${gaTrafficSnapshots.date} >= ${from}`, sql6`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
14595
|
+
const sumAi = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and12(
|
|
14348
14596
|
eq21(gaAiReferrals.projectId, project.id),
|
|
14349
14597
|
sql6`${gaAiReferrals.date} >= ${from}`,
|
|
14350
14598
|
sql6`${gaAiReferrals.date} < ${to}`,
|
|
14351
14599
|
eq21(gaAiReferrals.sourceDimension, "session")
|
|
14352
14600
|
)).get();
|
|
14353
|
-
const sumSocial = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(
|
|
14601
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and12(eq21(gaSocialReferrals.projectId, project.id), sql6`${gaSocialReferrals.date} >= ${from}`, sql6`${gaSocialReferrals.date} < ${to}`)).get();
|
|
14354
14602
|
const todayStr = fmt(today);
|
|
14355
14603
|
const buildTrend = (sum) => {
|
|
14356
14604
|
const c7 = sum(daysAgo2(7), todayStr)?.sessions ?? 0;
|
|
@@ -14359,13 +14607,13 @@ async function ga4Routes(app, opts) {
|
|
|
14359
14607
|
const p30 = sum(daysAgo2(60), daysAgo2(30))?.sessions ?? 0;
|
|
14360
14608
|
return { sessions7d: c7, sessionsPrev7d: p7, trend7dPct: pct(c7, p7), sessions30d: c30, sessionsPrev30d: p30, trend30dPct: pct(c30, p30) };
|
|
14361
14609
|
};
|
|
14362
|
-
const aiSourceCurrent = app.db.select({ source: gaAiReferrals.source, sessions: sql6`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(
|
|
14610
|
+
const aiSourceCurrent = app.db.select({ source: gaAiReferrals.source, sessions: sql6`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and12(
|
|
14363
14611
|
eq21(gaAiReferrals.projectId, project.id),
|
|
14364
14612
|
sql6`${gaAiReferrals.date} >= ${daysAgo2(7)}`,
|
|
14365
14613
|
sql6`${gaAiReferrals.date} < ${todayStr}`,
|
|
14366
14614
|
eq21(gaAiReferrals.sourceDimension, "session")
|
|
14367
14615
|
)).groupBy(gaAiReferrals.source).all();
|
|
14368
|
-
const aiSourcePrev = app.db.select({ source: gaAiReferrals.source, sessions: sql6`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(
|
|
14616
|
+
const aiSourcePrev = app.db.select({ source: gaAiReferrals.source, sessions: sql6`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and12(
|
|
14369
14617
|
eq21(gaAiReferrals.projectId, project.id),
|
|
14370
14618
|
sql6`${gaAiReferrals.date} >= ${daysAgo2(14)}`,
|
|
14371
14619
|
sql6`${gaAiReferrals.date} < ${daysAgo2(7)}`,
|
|
@@ -14385,8 +14633,8 @@ async function ga4Routes(app, opts) {
|
|
|
14385
14633
|
}
|
|
14386
14634
|
return mover;
|
|
14387
14635
|
};
|
|
14388
|
-
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions: sql6`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(
|
|
14389
|
-
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions: sql6`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(
|
|
14636
|
+
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions: sql6`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and12(eq21(gaSocialReferrals.projectId, project.id), sql6`${gaSocialReferrals.date} >= ${daysAgo2(7)}`, sql6`${gaSocialReferrals.date} < ${todayStr}`)).groupBy(gaSocialReferrals.source).all();
|
|
14637
|
+
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions: sql6`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and12(eq21(gaSocialReferrals.projectId, project.id), sql6`${gaSocialReferrals.date} >= ${daysAgo2(14)}`, sql6`${gaSocialReferrals.date} < ${daysAgo2(7)}`)).groupBy(gaSocialReferrals.source).all();
|
|
14390
14638
|
return {
|
|
14391
14639
|
total: buildTrend(sumTotal),
|
|
14392
14640
|
organic: buildTrend(sumOrganic),
|
|
@@ -14408,7 +14656,7 @@ async function ga4Routes(app, opts) {
|
|
|
14408
14656
|
sessions: sql6`SUM(${gaTrafficSnapshots.sessions})`,
|
|
14409
14657
|
organicSessions: sql6`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
14410
14658
|
users: sql6`SUM(${gaTrafficSnapshots.users})`
|
|
14411
|
-
}).from(gaTrafficSnapshots).where(
|
|
14659
|
+
}).from(gaTrafficSnapshots).where(and12(...conditions)).groupBy(gaTrafficSnapshots.date).orderBy(gaTrafficSnapshots.date).all();
|
|
14412
14660
|
return rows.map((r) => ({
|
|
14413
14661
|
date: r.date,
|
|
14414
14662
|
sessions: r.sessions ?? 0,
|
|
@@ -16061,7 +16309,7 @@ async function wordpressRoutes(app, opts) {
|
|
|
16061
16309
|
|
|
16062
16310
|
// ../api-routes/src/backlinks.ts
|
|
16063
16311
|
import crypto18 from "crypto";
|
|
16064
|
-
import { and as
|
|
16312
|
+
import { and as and14, asc as asc2, desc as desc11, eq as eq22, sql as sql7 } from "drizzle-orm";
|
|
16065
16313
|
|
|
16066
16314
|
// ../integration-commoncrawl/src/constants.ts
|
|
16067
16315
|
import os3 from "os";
|
|
@@ -16236,7 +16484,7 @@ function parseContentLength2(value) {
|
|
|
16236
16484
|
|
|
16237
16485
|
// ../integration-commoncrawl/src/plugin-resolver.ts
|
|
16238
16486
|
import fs3 from "fs";
|
|
16239
|
-
import { createRequire as
|
|
16487
|
+
import { createRequire as createRequire3 } from "module";
|
|
16240
16488
|
import path4 from "path";
|
|
16241
16489
|
function pluginDirFor(pkgJson) {
|
|
16242
16490
|
return path4.dirname(pkgJson);
|
|
@@ -16255,7 +16503,7 @@ function loadDuckdb(opts = {}) {
|
|
|
16255
16503
|
);
|
|
16256
16504
|
}
|
|
16257
16505
|
try {
|
|
16258
|
-
const pluginRequire =
|
|
16506
|
+
const pluginRequire = createRequire3(duckdbPkg);
|
|
16259
16507
|
return pluginRequire("@duckdb/node-api");
|
|
16260
16508
|
} catch {
|
|
16261
16509
|
throw missingDependency(
|
|
@@ -16458,7 +16706,7 @@ function pruneCachedRelease(release, opts = {}) {
|
|
|
16458
16706
|
}
|
|
16459
16707
|
|
|
16460
16708
|
// ../api-routes/src/backlinks-filter.ts
|
|
16461
|
-
import { and as
|
|
16709
|
+
import { and as and13, ne as ne2, notLike } from "drizzle-orm";
|
|
16462
16710
|
var BACKLINK_FILTER_PATTERNS = [
|
|
16463
16711
|
"*.google.com",
|
|
16464
16712
|
"*.googleusercontent.com",
|
|
@@ -16481,7 +16729,7 @@ function backlinkCrawlerExclusionClause() {
|
|
|
16481
16729
|
conditions.push(ne2(backlinkDomains.linkingDomain, pattern));
|
|
16482
16730
|
}
|
|
16483
16731
|
}
|
|
16484
|
-
const combined =
|
|
16732
|
+
const combined = and13(...conditions);
|
|
16485
16733
|
if (!combined) throw new Error("BACKLINK_FILTER_PATTERNS is unexpectedly empty");
|
|
16486
16734
|
return combined;
|
|
16487
16735
|
}
|
|
@@ -16542,7 +16790,7 @@ function mapRunRow(row) {
|
|
|
16542
16790
|
};
|
|
16543
16791
|
}
|
|
16544
16792
|
function latestSummaryForProject(db, projectId, release) {
|
|
16545
|
-
const condition = release ?
|
|
16793
|
+
const condition = release ? and14(eq22(backlinkSummaries.projectId, projectId), eq22(backlinkSummaries.release, release)) : eq22(backlinkSummaries.projectId, projectId);
|
|
16546
16794
|
return db.select().from(backlinkSummaries).where(condition).orderBy(desc11(backlinkSummaries.queriedAt)).limit(1).get();
|
|
16547
16795
|
}
|
|
16548
16796
|
function parseExcludeCrawlers(value) {
|
|
@@ -16551,11 +16799,11 @@ function parseExcludeCrawlers(value) {
|
|
|
16551
16799
|
return lower === "1" || lower === "true" || lower === "yes";
|
|
16552
16800
|
}
|
|
16553
16801
|
function computeFilteredSummary(db, base) {
|
|
16554
|
-
const baseDomainCondition =
|
|
16802
|
+
const baseDomainCondition = and14(
|
|
16555
16803
|
eq22(backlinkDomains.projectId, base.projectId),
|
|
16556
16804
|
eq22(backlinkDomains.release, base.release)
|
|
16557
16805
|
);
|
|
16558
|
-
const filteredCondition =
|
|
16806
|
+
const filteredCondition = and14(baseDomainCondition, backlinkCrawlerExclusionClause());
|
|
16559
16807
|
const unfilteredAgg = db.select({
|
|
16560
16808
|
count: sql7`count(*)`,
|
|
16561
16809
|
total: sql7`coalesce(sum(${backlinkDomains.numHosts}), 0)`
|
|
@@ -16731,11 +16979,11 @@ async function backlinksRoutes(app, opts) {
|
|
|
16731
16979
|
const limit = Math.min(Math.max(parseInt(request.query.limit ?? "50", 10) || 50, 1), 500);
|
|
16732
16980
|
const offset = Math.max(parseInt(request.query.offset ?? "0", 10) || 0, 0);
|
|
16733
16981
|
const excludeCrawlers = parseExcludeCrawlers(request.query.excludeCrawlers);
|
|
16734
|
-
const baseDomainCondition =
|
|
16982
|
+
const baseDomainCondition = and14(
|
|
16735
16983
|
eq22(backlinkDomains.projectId, project.id),
|
|
16736
16984
|
eq22(backlinkDomains.release, targetRelease)
|
|
16737
16985
|
);
|
|
16738
|
-
const domainCondition = excludeCrawlers ?
|
|
16986
|
+
const domainCondition = excludeCrawlers ? and14(baseDomainCondition, backlinkCrawlerExclusionClause()) : baseDomainCondition;
|
|
16739
16987
|
const totalRow = app.db.select({ count: sql7`count(*)` }).from(backlinkDomains).where(domainCondition).get();
|
|
16740
16988
|
const rows = app.db.select({
|
|
16741
16989
|
linkingDomain: backlinkDomains.linkingDomain,
|
|
@@ -16771,7 +17019,7 @@ async function backlinksRoutes(app, opts) {
|
|
|
16771
17019
|
|
|
16772
17020
|
// ../api-routes/src/traffic.ts
|
|
16773
17021
|
import crypto20 from "crypto";
|
|
16774
|
-
import { and as
|
|
17022
|
+
import { and as and15, desc as desc12, eq as eq23, gte as gte2, lte as lte2, sql as sql8 } from "drizzle-orm";
|
|
16775
17023
|
|
|
16776
17024
|
// ../integration-cloud-run/src/auth.ts
|
|
16777
17025
|
import crypto19 from "crypto";
|
|
@@ -17839,7 +18087,7 @@ var DEFAULT_WP_MAX_PAGES = 20;
|
|
|
17839
18087
|
var DEFAULT_VERCEL_MAX_PAGES = 50;
|
|
17840
18088
|
var MAX_TRACKED_EVENT_IDS = 1e3;
|
|
17841
18089
|
var DEFAULT_BACKFILL_DAYS = 30;
|
|
17842
|
-
var MAX_BACKFILL_DAYS =
|
|
18090
|
+
var MAX_BACKFILL_DAYS = 90;
|
|
17843
18091
|
var BACKFILL_MAX_PAGES = 1e3;
|
|
17844
18092
|
var BACKFILL_SAMPLE_LIMIT = 500;
|
|
17845
18093
|
function parseSourceConfig(row) {
|
|
@@ -17919,21 +18167,21 @@ async function runBackfillTask(options) {
|
|
|
17919
18167
|
try {
|
|
17920
18168
|
app.db.transaction((tx) => {
|
|
17921
18169
|
tx.delete(crawlerEventsHourly).where(
|
|
17922
|
-
|
|
18170
|
+
and15(
|
|
17923
18171
|
eq23(crawlerEventsHourly.sourceId, sourceRow.id),
|
|
17924
18172
|
gte2(crawlerEventsHourly.tsHour, windowStartIso),
|
|
17925
18173
|
lte2(crawlerEventsHourly.tsHour, windowEndIso)
|
|
17926
18174
|
)
|
|
17927
18175
|
).run();
|
|
17928
18176
|
tx.delete(aiReferralEventsHourly).where(
|
|
17929
|
-
|
|
18177
|
+
and15(
|
|
17930
18178
|
eq23(aiReferralEventsHourly.sourceId, sourceRow.id),
|
|
17931
18179
|
gte2(aiReferralEventsHourly.tsHour, windowStartIso),
|
|
17932
18180
|
lte2(aiReferralEventsHourly.tsHour, windowEndIso)
|
|
17933
18181
|
)
|
|
17934
18182
|
).run();
|
|
17935
18183
|
tx.delete(rawEventSamples).where(
|
|
17936
|
-
|
|
18184
|
+
and15(
|
|
17937
18185
|
eq23(rawEventSamples.sourceId, sourceRow.id),
|
|
17938
18186
|
gte2(rawEventSamples.ts, windowStartIso),
|
|
17939
18187
|
lte2(rawEventSamples.ts, windowEndIso)
|
|
@@ -18356,7 +18604,8 @@ async function trafficRoutes(app, opts) {
|
|
|
18356
18604
|
endTime: windowEnd.toISOString(),
|
|
18357
18605
|
pageSize,
|
|
18358
18606
|
maxPages,
|
|
18359
|
-
firstSync: isFirstSync
|
|
18607
|
+
firstSync: isFirstSync,
|
|
18608
|
+
requestUrlSubstrings: [project.canonicalDomain]
|
|
18360
18609
|
});
|
|
18361
18610
|
allEvents = page.events;
|
|
18362
18611
|
} catch (e) {
|
|
@@ -18682,7 +18931,8 @@ async function trafficRoutes(app, opts) {
|
|
|
18682
18931
|
// ring-buffer reseed at the end takes the most-recent IDs from the
|
|
18683
18932
|
// dedupedEvents anyway.
|
|
18684
18933
|
firstSync: false,
|
|
18685
|
-
orderBy: "timestamp asc"
|
|
18934
|
+
orderBy: "timestamp asc",
|
|
18935
|
+
requestUrlSubstrings: [project.canonicalDomain]
|
|
18686
18936
|
});
|
|
18687
18937
|
return page.events;
|
|
18688
18938
|
};
|
|
@@ -18795,25 +19045,25 @@ async function trafficRoutes(app, opts) {
|
|
|
18795
19045
|
});
|
|
18796
19046
|
function buildSourceDetail(projectId, row, since) {
|
|
18797
19047
|
const crawlerTotals = app.db.select({ total: sql8`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
18798
|
-
|
|
19048
|
+
and15(
|
|
18799
19049
|
eq23(crawlerEventsHourly.sourceId, row.id),
|
|
18800
19050
|
gte2(crawlerEventsHourly.tsHour, since)
|
|
18801
19051
|
)
|
|
18802
19052
|
).get();
|
|
18803
19053
|
const aiTotals = app.db.select({ total: sql8`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
18804
|
-
|
|
19054
|
+
and15(
|
|
18805
19055
|
eq23(aiReferralEventsHourly.sourceId, row.id),
|
|
18806
19056
|
gte2(aiReferralEventsHourly.tsHour, since)
|
|
18807
19057
|
)
|
|
18808
19058
|
).get();
|
|
18809
19059
|
const sampleTotals = app.db.select({ total: sql8`COUNT(*)` }).from(rawEventSamples).where(
|
|
18810
|
-
|
|
19060
|
+
and15(
|
|
18811
19061
|
eq23(rawEventSamples.sourceId, row.id),
|
|
18812
19062
|
gte2(rawEventSamples.ts, since)
|
|
18813
19063
|
)
|
|
18814
19064
|
).get();
|
|
18815
19065
|
const latestRun = app.db.select().from(runs).where(
|
|
18816
|
-
|
|
19066
|
+
and15(
|
|
18817
19067
|
eq23(runs.projectId, projectId),
|
|
18818
19068
|
eq23(runs.kind, RunKinds["traffic-sync"]),
|
|
18819
19069
|
eq23(runs.sourceId, row.id)
|
|
@@ -18907,7 +19157,7 @@ async function trafficRoutes(app, opts) {
|
|
|
18907
19157
|
lte2(crawlerEventsHourly.tsHour, untilIso)
|
|
18908
19158
|
];
|
|
18909
19159
|
if (sourceIdParam) crawlerFilters.push(eq23(crawlerEventsHourly.sourceId, sourceIdParam));
|
|
18910
|
-
const crawlerWhere =
|
|
19160
|
+
const crawlerWhere = and15(...crawlerFilters);
|
|
18911
19161
|
const total = app.db.select({ total: sql8`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(crawlerWhere).get();
|
|
18912
19162
|
crawlerTotal = Number(total?.total ?? 0);
|
|
18913
19163
|
const rows = app.db.select().from(crawlerEventsHourly).where(crawlerWhere).orderBy(desc12(crawlerEventsHourly.tsHour)).limit(limit).all();
|
|
@@ -18932,7 +19182,7 @@ async function trafficRoutes(app, opts) {
|
|
|
18932
19182
|
lte2(aiReferralEventsHourly.tsHour, untilIso)
|
|
18933
19183
|
];
|
|
18934
19184
|
if (sourceIdParam) aiFilters.push(eq23(aiReferralEventsHourly.sourceId, sourceIdParam));
|
|
18935
|
-
const aiWhere =
|
|
19185
|
+
const aiWhere = and15(...aiFilters);
|
|
18936
19186
|
const total = app.db.select({ total: sql8`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(aiWhere).get();
|
|
18937
19187
|
aiReferralTotal = Number(total?.total ?? 0);
|
|
18938
19188
|
const rows = app.db.select().from(aiReferralEventsHourly).where(aiWhere).orderBy(desc12(aiReferralEventsHourly.tsHour)).limit(limit).all();
|
|
@@ -19587,7 +19837,7 @@ var providersConfiguredCheck = {
|
|
|
19587
19837
|
var PROVIDERS_CHECKS = [providersConfiguredCheck];
|
|
19588
19838
|
|
|
19589
19839
|
// ../api-routes/src/doctor/checks/traffic-source.ts
|
|
19590
|
-
import { and as
|
|
19840
|
+
import { and as and16, eq as eq24, gte as gte3, ne as ne3, sql as sql9 } from "drizzle-orm";
|
|
19591
19841
|
var RECENT_DATA_WARN_DAYS = 7;
|
|
19592
19842
|
var RECENT_DATA_FAIL_DAYS = 30;
|
|
19593
19843
|
function skippedNoProject2() {
|
|
@@ -19601,7 +19851,7 @@ function skippedNoProject2() {
|
|
|
19601
19851
|
function loadProbes(ctx) {
|
|
19602
19852
|
if (!ctx.project) return [];
|
|
19603
19853
|
const rows = ctx.db.select().from(trafficSources).where(
|
|
19604
|
-
|
|
19854
|
+
and16(
|
|
19605
19855
|
eq24(trafficSources.projectId, ctx.project.id),
|
|
19606
19856
|
ne3(trafficSources.status, TrafficSourceStatuses.archived)
|
|
19607
19857
|
)
|
|
@@ -19682,7 +19932,7 @@ var recentDataCheck = {
|
|
|
19682
19932
|
const failCutoff = new Date(now.getTime() - RECENT_DATA_FAIL_DAYS * 24 * 60 * 6e4).toISOString();
|
|
19683
19933
|
const recentCrawlers = Number(
|
|
19684
19934
|
ctx.db.select({ total: sql9`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
19685
|
-
|
|
19935
|
+
and16(
|
|
19686
19936
|
eq24(crawlerEventsHourly.projectId, ctx.project.id),
|
|
19687
19937
|
gte3(crawlerEventsHourly.tsHour, warnCutoff)
|
|
19688
19938
|
)
|
|
@@ -19690,7 +19940,7 @@ var recentDataCheck = {
|
|
|
19690
19940
|
);
|
|
19691
19941
|
const recentReferrals = Number(
|
|
19692
19942
|
ctx.db.select({ total: sql9`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
19693
|
-
|
|
19943
|
+
and16(
|
|
19694
19944
|
eq24(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
19695
19945
|
gte3(aiReferralEventsHourly.tsHour, warnCutoff)
|
|
19696
19946
|
)
|
|
@@ -19706,7 +19956,7 @@ var recentDataCheck = {
|
|
|
19706
19956
|
}
|
|
19707
19957
|
const olderCrawlers = Number(
|
|
19708
19958
|
ctx.db.select({ total: sql9`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
19709
|
-
|
|
19959
|
+
and16(
|
|
19710
19960
|
eq24(crawlerEventsHourly.projectId, ctx.project.id),
|
|
19711
19961
|
gte3(crawlerEventsHourly.tsHour, failCutoff)
|
|
19712
19962
|
)
|
|
@@ -19714,7 +19964,7 @@ var recentDataCheck = {
|
|
|
19714
19964
|
);
|
|
19715
19965
|
const olderReferrals = Number(
|
|
19716
19966
|
ctx.db.select({ total: sql9`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
19717
|
-
|
|
19967
|
+
and16(
|
|
19718
19968
|
eq24(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
19719
19969
|
gte3(aiReferralEventsHourly.tsHour, failCutoff)
|
|
19720
19970
|
)
|
|
@@ -20012,7 +20262,7 @@ async function doctorRoutes(app, opts) {
|
|
|
20012
20262
|
|
|
20013
20263
|
// ../api-routes/src/discovery/routes.ts
|
|
20014
20264
|
import crypto21 from "crypto";
|
|
20015
|
-
import { and as
|
|
20265
|
+
import { and as and17, desc as desc13, eq as eq25, gte as gte4, inArray as inArray8 } from "drizzle-orm";
|
|
20016
20266
|
var MAX_INFLIGHT_DISCOVERY_AGE_MS = 2 * 60 * 60 * 1e3;
|
|
20017
20267
|
async function discoveryRoutes(app, opts) {
|
|
20018
20268
|
app.post("/projects/:name/discover/run", async (request, reply) => {
|
|
@@ -20044,7 +20294,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
20044
20294
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
20045
20295
|
const ageFloorIso = new Date(Date.now() - MAX_INFLIGHT_DISCOVERY_AGE_MS).toISOString();
|
|
20046
20296
|
const decision = app.db.transaction((tx) => {
|
|
20047
|
-
const existing = tx.select({ id: discoverySessions.id, runId: discoverySessions.runId }).from(discoverySessions).where(
|
|
20297
|
+
const existing = tx.select({ id: discoverySessions.id, runId: discoverySessions.runId }).from(discoverySessions).where(and17(
|
|
20048
20298
|
eq25(discoverySessions.projectId, project.id),
|
|
20049
20299
|
eq25(discoverySessions.icpDescription, icpDescription),
|
|
20050
20300
|
inArray8(discoverySessions.status, [
|
|
@@ -20508,6 +20758,7 @@ async function apiRoutes(app, opts) {
|
|
|
20508
20758
|
await api.register(projectRoutes, {
|
|
20509
20759
|
onProjectDeleted: opts.onProjectDeleted,
|
|
20510
20760
|
onProjectUpserted: opts.onProjectUpserted,
|
|
20761
|
+
onAliasesChanged: opts.onAliasesChanged,
|
|
20511
20762
|
validProviderNames: opts.providerAdapters?.map((a) => a.name)
|
|
20512
20763
|
});
|
|
20513
20764
|
await api.register(queryRoutes, {
|
|
@@ -20522,7 +20773,9 @@ async function apiRoutes(app, opts) {
|
|
|
20522
20773
|
await api.register(applyRoutes, {
|
|
20523
20774
|
onScheduleUpdated: opts.onScheduleUpdated,
|
|
20524
20775
|
onProjectUpserted: opts.onProjectUpserted,
|
|
20776
|
+
onAliasesChanged: opts.onAliasesChanged,
|
|
20525
20777
|
validProviderNames: opts.providerAdapters?.map((a) => a.name),
|
|
20778
|
+
allowLoopbackWebhooks: opts.allowLoopbackWebhooks,
|
|
20526
20779
|
onGoogleConnectionPropertyUpdated: (domain, connectionType, propertyId) => {
|
|
20527
20780
|
opts.googleConnectionStore?.updateConnection(domain, connectionType, {
|
|
20528
20781
|
propertyId,
|
|
@@ -20553,7 +20806,9 @@ async function apiRoutes(app, opts) {
|
|
|
20553
20806
|
onScheduleUpdated: opts.onScheduleUpdated,
|
|
20554
20807
|
validProviderNames: opts.providerAdapters?.map((a) => a.name)
|
|
20555
20808
|
});
|
|
20556
|
-
await api.register(notificationRoutes
|
|
20809
|
+
await api.register(notificationRoutes, {
|
|
20810
|
+
allowLoopbackWebhooks: opts.allowLoopbackWebhooks
|
|
20811
|
+
});
|
|
20557
20812
|
await api.register(telemetryRoutes, {
|
|
20558
20813
|
getTelemetryStatus: opts.getTelemetryStatus,
|
|
20559
20814
|
setTelemetryEnabled: opts.setTelemetryEnabled
|
|
@@ -20808,7 +21063,7 @@ function isVertexConfig(config) {
|
|
|
20808
21063
|
function resolveModel(config) {
|
|
20809
21064
|
return config.model || DEFAULT_MODEL;
|
|
20810
21065
|
}
|
|
20811
|
-
function
|
|
21066
|
+
function createClient2(config) {
|
|
20812
21067
|
if (isVertexConfig(config)) {
|
|
20813
21068
|
return new GoogleGenAI({
|
|
20814
21069
|
vertexai: true,
|
|
@@ -20848,7 +21103,7 @@ async function healthcheck(config) {
|
|
|
20848
21103
|
if (!validation.ok) return validation;
|
|
20849
21104
|
try {
|
|
20850
21105
|
const model = resolveModel(config);
|
|
20851
|
-
const client =
|
|
21106
|
+
const client = createClient2(config);
|
|
20852
21107
|
const result = await withRetry(
|
|
20853
21108
|
() => client.models.generateContent({
|
|
20854
21109
|
model,
|
|
@@ -20875,7 +21130,7 @@ async function healthcheck(config) {
|
|
|
20875
21130
|
async function executeTrackedQuery(input) {
|
|
20876
21131
|
const model = resolveModel(input.config);
|
|
20877
21132
|
const prompt = buildPrompt(input.query, input.location);
|
|
20878
|
-
const client =
|
|
21133
|
+
const client = createClient2(input.config);
|
|
20879
21134
|
try {
|
|
20880
21135
|
const result = await withRetry(
|
|
20881
21136
|
() => client.models.generateContent({
|
|
@@ -21040,7 +21295,7 @@ function extractDomainFromUri(uri) {
|
|
|
21040
21295
|
}
|
|
21041
21296
|
async function generateText(prompt, config) {
|
|
21042
21297
|
const model = resolveModel(config);
|
|
21043
|
-
const client =
|
|
21298
|
+
const client = createClient2(config);
|
|
21044
21299
|
const result = await withRetry(
|
|
21045
21300
|
() => client.models.generateContent({
|
|
21046
21301
|
model,
|
|
@@ -23360,7 +23615,7 @@ import crypto24 from "crypto";
|
|
|
23360
23615
|
import fs7 from "fs";
|
|
23361
23616
|
import path9 from "path";
|
|
23362
23617
|
import os5 from "os";
|
|
23363
|
-
import { and as
|
|
23618
|
+
import { and as and18, eq as eq27, inArray as inArray9, sql as sql10 } from "drizzle-orm";
|
|
23364
23619
|
|
|
23365
23620
|
// src/run-telemetry.ts
|
|
23366
23621
|
import crypto23 from "crypto";
|
|
@@ -23467,11 +23722,15 @@ function computeCompetitorOverlap(normalized, competitorDomains) {
|
|
|
23467
23722
|
function escapeRegExp5(value) {
|
|
23468
23723
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
23469
23724
|
}
|
|
23470
|
-
function extractRecommendedCompetitors(answerText, ownDomains, citedDomains, competitorDomains) {
|
|
23725
|
+
function extractRecommendedCompetitors(answerText, ownDomains, citedDomains, competitorDomains, ownBrandNames = []) {
|
|
23471
23726
|
if (!answerText || answerText.length < 20) return [];
|
|
23472
23727
|
const ownBrandKeys = new Set(
|
|
23473
23728
|
ownDomains.flatMap((domain) => collectBrandKeysFromDomain(domain))
|
|
23474
23729
|
);
|
|
23730
|
+
for (const name of ownBrandNames) {
|
|
23731
|
+
const key = brandKeyFromText(name);
|
|
23732
|
+
if (key.length >= 4) ownBrandKeys.add(key);
|
|
23733
|
+
}
|
|
23475
23734
|
const knownCompetitorKeys = new Set(
|
|
23476
23735
|
[...citedDomains, ...competitorDomains].flatMap((domain) => collectBrandKeysFromDomain(domain)).filter((key) => !ownBrandKeys.has(key))
|
|
23477
23736
|
);
|
|
@@ -23739,7 +23998,7 @@ var JobRunner = class {
|
|
|
23739
23998
|
throw new Error(`Run ${runId} is not executable from status '${existingRun.status}'`);
|
|
23740
23999
|
}
|
|
23741
24000
|
if (existingRun.status === "queued") {
|
|
23742
|
-
this.db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
24001
|
+
this.db.update(runs).set({ status: "running", startedAt: now }).where(and18(eq27(runs.id, runId), eq27(runs.status, "queued"))).run();
|
|
23743
24002
|
}
|
|
23744
24003
|
this.throwIfRunCancelled(runId);
|
|
23745
24004
|
const project = this.db.select().from(projects).where(eq27(projects.id, projectId)).get();
|
|
@@ -23764,13 +24023,17 @@ var JobRunner = class {
|
|
|
23764
24023
|
}
|
|
23765
24024
|
log.info("run.dispatch", { runId, providerCount: activeProviders.length, providers: activeProviders.map((p) => p.adapter.name) });
|
|
23766
24025
|
const scopedQueryNames = parseJsonColumn(existingRun.queries, null);
|
|
23767
|
-
projectQueries = scopedQueryNames ? this.db.select().from(queries).where(
|
|
24026
|
+
projectQueries = scopedQueryNames ? this.db.select().from(queries).where(and18(eq27(queries.projectId, projectId), inArray9(queries.query, scopedQueryNames))).all() : this.db.select().from(queries).where(eq27(queries.projectId, projectId)).all();
|
|
23768
24027
|
const projectCompetitors = this.db.select().from(competitors).where(eq27(competitors.projectId, projectId)).all();
|
|
23769
24028
|
const competitorDomains = projectCompetitors.map((c) => c.domain);
|
|
23770
24029
|
const allDomains = effectiveDomains({
|
|
23771
24030
|
canonicalDomain: project.canonicalDomain,
|
|
23772
24031
|
ownedDomains: parseJsonColumn(project.ownedDomains, [])
|
|
23773
24032
|
});
|
|
24033
|
+
const allBrandNames = effectiveBrandNames({
|
|
24034
|
+
displayName: project.displayName,
|
|
24035
|
+
aliases: parseJsonColumn(project.aliases, [])
|
|
24036
|
+
});
|
|
23774
24037
|
const executionContext = {
|
|
23775
24038
|
providerCount: activeProviders.length,
|
|
23776
24039
|
providers: activeProviders.map((provider) => provider.adapter.name),
|
|
@@ -23831,7 +24094,7 @@ var JobRunner = class {
|
|
|
23831
24094
|
const citationState = determineCitationState(normalized, allDomains);
|
|
23832
24095
|
const answerMentioned = determineAnswerMentioned(
|
|
23833
24096
|
normalized.answerText,
|
|
23834
|
-
|
|
24097
|
+
allBrandNames,
|
|
23835
24098
|
allDomains
|
|
23836
24099
|
);
|
|
23837
24100
|
const overlap = computeCompetitorOverlap(normalized, competitorDomains);
|
|
@@ -23839,7 +24102,8 @@ var JobRunner = class {
|
|
|
23839
24102
|
normalized.answerText,
|
|
23840
24103
|
allDomains,
|
|
23841
24104
|
normalized.citedDomains,
|
|
23842
|
-
competitorDomains
|
|
24105
|
+
competitorDomains,
|
|
24106
|
+
allBrandNames
|
|
23843
24107
|
);
|
|
23844
24108
|
let screenshotRelPath = null;
|
|
23845
24109
|
if (raw.screenshotPath && fs7.existsSync(raw.screenshotPath)) {
|
|
@@ -24102,7 +24366,7 @@ function buildPhases(input) {
|
|
|
24102
24366
|
|
|
24103
24367
|
// src/gsc-sync.ts
|
|
24104
24368
|
import crypto25 from "crypto";
|
|
24105
|
-
import { eq as eq28, and as
|
|
24369
|
+
import { eq as eq28, and as and19, sql as sql11 } from "drizzle-orm";
|
|
24106
24370
|
var log2 = createLogger("GscSync");
|
|
24107
24371
|
function formatDate3(d) {
|
|
24108
24372
|
return d.toISOString().split("T")[0];
|
|
@@ -24154,7 +24418,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
24154
24418
|
});
|
|
24155
24419
|
log2.info("fetch.complete", { runId, projectId, rowCount: rows.length });
|
|
24156
24420
|
db.delete(gscSearchData).where(
|
|
24157
|
-
|
|
24421
|
+
and19(
|
|
24158
24422
|
eq28(gscSearchData.projectId, projectId),
|
|
24159
24423
|
sql11`${gscSearchData.date} >= ${startDate}`,
|
|
24160
24424
|
sql11`${gscSearchData.date} <= ${endDate}`
|
|
@@ -24243,7 +24507,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
24243
24507
|
}
|
|
24244
24508
|
}
|
|
24245
24509
|
const snapshotDate = formatDate3(/* @__PURE__ */ new Date());
|
|
24246
|
-
db.delete(gscCoverageSnapshots).where(
|
|
24510
|
+
db.delete(gscCoverageSnapshots).where(and19(eq28(gscCoverageSnapshots.projectId, projectId), eq28(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
24247
24511
|
db.insert(gscCoverageSnapshots).values({
|
|
24248
24512
|
id: crypto25.randomUUID(),
|
|
24249
24513
|
projectId,
|
|
@@ -24266,7 +24530,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
24266
24530
|
|
|
24267
24531
|
// src/gsc-inspect-sitemap.ts
|
|
24268
24532
|
import crypto26 from "crypto";
|
|
24269
|
-
import { eq as eq29, and as
|
|
24533
|
+
import { eq as eq29, and as and20 } from "drizzle-orm";
|
|
24270
24534
|
|
|
24271
24535
|
// src/sitemap-parser.ts
|
|
24272
24536
|
var log3 = createLogger("SitemapParser");
|
|
@@ -24482,7 +24746,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
24482
24746
|
}
|
|
24483
24747
|
}
|
|
24484
24748
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
24485
|
-
db.delete(gscCoverageSnapshots).where(
|
|
24749
|
+
db.delete(gscCoverageSnapshots).where(and20(eq29(gscCoverageSnapshots.projectId, projectId), eq29(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
24486
24750
|
db.insert(gscCoverageSnapshots).values({
|
|
24487
24751
|
id: crypto26.randomUUID(),
|
|
24488
24752
|
projectId,
|
|
@@ -24693,7 +24957,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
24693
24957
|
// src/commoncrawl-sync.ts
|
|
24694
24958
|
import crypto28 from "crypto";
|
|
24695
24959
|
import path10 from "path";
|
|
24696
|
-
import { and as
|
|
24960
|
+
import { and as and21, eq as eq31, sql as sql12 } from "drizzle-orm";
|
|
24697
24961
|
var log6 = createLogger("CommonCrawlSync");
|
|
24698
24962
|
var INSERT_CHUNK_SIZE = 1e4;
|
|
24699
24963
|
function defaultDeps() {
|
|
@@ -24884,7 +25148,7 @@ function computeSummary(rows) {
|
|
|
24884
25148
|
// src/backlink-extract.ts
|
|
24885
25149
|
import crypto29 from "crypto";
|
|
24886
25150
|
import fs8 from "fs";
|
|
24887
|
-
import { and as
|
|
25151
|
+
import { and as and22, desc as desc15, eq as eq32 } from "drizzle-orm";
|
|
24888
25152
|
var log7 = createLogger("BacklinkExtract");
|
|
24889
25153
|
function defaultDeps2() {
|
|
24890
25154
|
return {
|
|
@@ -24930,7 +25194,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
24930
25194
|
const targetDomain = project.canonicalDomain;
|
|
24931
25195
|
db.transaction((tx) => {
|
|
24932
25196
|
tx.delete(backlinkDomains).where(
|
|
24933
|
-
|
|
25197
|
+
and22(eq32(backlinkDomains.projectId, projectId), eq32(backlinkDomains.release, release))
|
|
24934
25198
|
).run();
|
|
24935
25199
|
if (rows.length > 0) {
|
|
24936
25200
|
const values = rows.map((r) => ({
|
|
@@ -25001,9 +25265,10 @@ function computeSummary2(rows) {
|
|
|
25001
25265
|
|
|
25002
25266
|
// src/discovery-run.ts
|
|
25003
25267
|
import crypto30 from "crypto";
|
|
25004
|
-
import { eq as eq33 } from "drizzle-orm";
|
|
25268
|
+
import { and as and23, eq as eq33 } from "drizzle-orm";
|
|
25005
25269
|
var log8 = createLogger("DiscoveryRun");
|
|
25006
25270
|
var DEFAULT_SEED_COUNT = 30;
|
|
25271
|
+
var QUERIES_PER_INTENT_BUCKET = 6;
|
|
25007
25272
|
async function executeDiscoveryRun(opts) {
|
|
25008
25273
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
25009
25274
|
opts.db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq33(runs.id, opts.runId)).run();
|
|
@@ -25208,6 +25473,7 @@ function buildLocationConstraint(locations) {
|
|
|
25208
25473
|
}
|
|
25209
25474
|
function buildSeedPrompt(input) {
|
|
25210
25475
|
const locationConstraint = buildLocationConstraint(input.locations ?? []);
|
|
25476
|
+
const currentYear = (/* @__PURE__ */ new Date()).getFullYear();
|
|
25211
25477
|
return [
|
|
25212
25478
|
"You are an AEO (Answer Engine Optimization) analyst expanding a tracked-query basket for a customer.",
|
|
25213
25479
|
"",
|
|
@@ -25215,14 +25481,15 @@ function buildSeedPrompt(input) {
|
|
|
25215
25481
|
`ICP: ${input.icpDescription}`,
|
|
25216
25482
|
...locationConstraint.length > 0 ? ["", ...locationConstraint] : [],
|
|
25217
25483
|
"",
|
|
25218
|
-
"Brainstorm
|
|
25219
|
-
|
|
25220
|
-
"
|
|
25221
|
-
"
|
|
25222
|
-
"
|
|
25223
|
-
|
|
25484
|
+
"Brainstorm queries a member of this ICP would type into an AI answer engine (Gemini, ChatGPT, Perplexity). Generate candidates across the five intent buckets below \u2014 these are SEMANTICALLY DISTINCT search intents, not stylistic variants. A query should fit one bucket cleanly. Diversity across buckets is the point: it keeps the list from collapsing into near-synonyms of a single intent.",
|
|
25485
|
+
"",
|
|
25486
|
+
' 1. Informational \u2014 the searcher wants to understand a concept, market, or problem. Templates: "what is X", "how does X work", "X explained", "why X matters".',
|
|
25487
|
+
` 2. Commercial \u2014 the searcher is researching category leaders before a purchase. Templates: "best X for Y", "top X ${currentYear}", "leading X providers", "X for [use case]".`,
|
|
25488
|
+
' 3. Navigational \u2014 the searcher is looking for a specific brand, place, or directory. Templates: "X near me", "X reviews", "X website", "X directory".',
|
|
25489
|
+
' 4. Comparative \u2014 the searcher is weighing named alternatives head-to-head. Templates: "X vs Y", "X or Y for Z", "alternatives to X".',
|
|
25490
|
+
' 5. Transactional \u2014 the searcher is ready to act on a purchase or booking. Templates: "book X", "X pricing", "X discount code", "buy X online".',
|
|
25224
25491
|
"",
|
|
25225
|
-
|
|
25492
|
+
`Generate EXACTLY ${QUERIES_PER_INTENT_BUCKET} queries per bucket \u2014 ${QUERIES_PER_INTENT_BUCKET * 5} total. Return ONE query per line. Plain text only \u2014 no numbering, bullets, quotes, bucket labels, or commentary.`
|
|
25226
25493
|
].join("\n");
|
|
25227
25494
|
}
|
|
25228
25495
|
function parseQueryLines(text, max) {
|
|
@@ -25257,33 +25524,40 @@ function writeDiscoveryInsight(db, input) {
|
|
|
25257
25524
|
aspirational: buckets.aspirational,
|
|
25258
25525
|
totalProbes
|
|
25259
25526
|
});
|
|
25260
|
-
db.
|
|
25261
|
-
|
|
25262
|
-
|
|
25263
|
-
|
|
25264
|
-
|
|
25265
|
-
|
|
25266
|
-
|
|
25267
|
-
|
|
25268
|
-
|
|
25269
|
-
|
|
25270
|
-
|
|
25271
|
-
|
|
25272
|
-
|
|
25273
|
-
|
|
25274
|
-
|
|
25275
|
-
|
|
25276
|
-
|
|
25277
|
-
|
|
25278
|
-
|
|
25279
|
-
|
|
25280
|
-
|
|
25281
|
-
|
|
25282
|
-
|
|
25283
|
-
|
|
25284
|
-
|
|
25285
|
-
|
|
25286
|
-
|
|
25527
|
+
db.transaction((tx) => {
|
|
25528
|
+
tx.update(insights).set({ dismissed: true }).where(and23(
|
|
25529
|
+
eq33(insights.projectId, input.projectId),
|
|
25530
|
+
eq33(insights.type, "discovery.basket-divergence"),
|
|
25531
|
+
eq33(insights.dismissed, false)
|
|
25532
|
+
)).run();
|
|
25533
|
+
tx.insert(insights).values({
|
|
25534
|
+
id: crypto30.randomUUID(),
|
|
25535
|
+
projectId: input.projectId,
|
|
25536
|
+
runId: input.runId,
|
|
25537
|
+
type: "discovery.basket-divergence",
|
|
25538
|
+
severity,
|
|
25539
|
+
title,
|
|
25540
|
+
// query/provider fields don't fit the visibility-snapshot model for
|
|
25541
|
+
// a session-level insight. Use the session marker so the
|
|
25542
|
+
// (query, provider) index stays distinct across sessions; PR 5 will
|
|
25543
|
+
// formalize a session-scoped insight subtype.
|
|
25544
|
+
query: `discovery:${input.sessionId}`,
|
|
25545
|
+
provider: input.seedProvider,
|
|
25546
|
+
recommendation: JSON.stringify({
|
|
25547
|
+
action: "review-discovered-basket",
|
|
25548
|
+
summary: `Run \`canonry discover show ${input.sessionId} --format json\` to inspect the per-query breakdown, then \`canonry discover promote <project> ${input.sessionId}\` to merge cited + aspirational findings into the project.`,
|
|
25549
|
+
bucketCounts: buckets,
|
|
25550
|
+
topCompetitors
|
|
25551
|
+
}),
|
|
25552
|
+
cause: JSON.stringify({
|
|
25553
|
+
sessionId: input.sessionId,
|
|
25554
|
+
totalProbes,
|
|
25555
|
+
seedProvider: input.seedProvider
|
|
25556
|
+
}),
|
|
25557
|
+
dismissed: false,
|
|
25558
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
25559
|
+
}).run();
|
|
25560
|
+
});
|
|
25287
25561
|
}
|
|
25288
25562
|
function buildDiscoveryInsightTitle(input) {
|
|
25289
25563
|
const parts = [];
|
|
@@ -25294,6 +25568,512 @@ function buildDiscoveryInsightTitle(input) {
|
|
|
25294
25568
|
return parts.join(" \u2022 ");
|
|
25295
25569
|
}
|
|
25296
25570
|
|
|
25571
|
+
// src/commands/backfill.ts
|
|
25572
|
+
import { and as and24, eq as eq34, inArray as inArray10 } from "drizzle-orm";
|
|
25573
|
+
var SNAPSHOT_BATCH_SIZE = 500;
|
|
25574
|
+
async function backfillAnswerVisibilityCommand(opts) {
|
|
25575
|
+
const config = loadConfig();
|
|
25576
|
+
const db = createClient(config.database);
|
|
25577
|
+
migrate(db);
|
|
25578
|
+
const projectFilter = opts?.project?.trim();
|
|
25579
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq34(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
25580
|
+
let examined = 0;
|
|
25581
|
+
let updated = 0;
|
|
25582
|
+
let mentioned = 0;
|
|
25583
|
+
let reparsed = 0;
|
|
25584
|
+
let providerErrors = 0;
|
|
25585
|
+
if (scopedProjects.length > 0) {
|
|
25586
|
+
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(and24(
|
|
25587
|
+
eq34(runs.kind, RunKinds["answer-visibility"]),
|
|
25588
|
+
inArray10(runs.projectId, scopedProjects.map((project) => project.id))
|
|
25589
|
+
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(eq34(runs.kind, RunKinds["answer-visibility"])).all();
|
|
25590
|
+
const runIdsByProject = /* @__PURE__ */ new Map();
|
|
25591
|
+
for (const run of runRows) {
|
|
25592
|
+
const existing = runIdsByProject.get(run.projectId);
|
|
25593
|
+
if (existing) existing.push(run.id);
|
|
25594
|
+
else runIdsByProject.set(run.projectId, [run.id]);
|
|
25595
|
+
}
|
|
25596
|
+
for (const project of scopedProjects) {
|
|
25597
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq34(competitors.projectId, project.id)).all().map((row) => row.domain);
|
|
25598
|
+
const runIds = runIdsByProject.get(project.id) ?? [];
|
|
25599
|
+
if (runIds.length === 0) continue;
|
|
25600
|
+
const projectDomains = effectiveDomains({
|
|
25601
|
+
canonicalDomain: project.canonicalDomain,
|
|
25602
|
+
ownedDomains: parseJsonColumn(project.ownedDomains, [])
|
|
25603
|
+
});
|
|
25604
|
+
const projectBrandNames = effectiveBrandNames({
|
|
25605
|
+
displayName: project.displayName,
|
|
25606
|
+
aliases: parseJsonColumn(project.aliases, [])
|
|
25607
|
+
});
|
|
25608
|
+
for (let offset = 0; offset < runIds.length; offset += SNAPSHOT_BATCH_SIZE) {
|
|
25609
|
+
const batchRunIds = runIds.slice(offset, offset + SNAPSHOT_BATCH_SIZE);
|
|
25610
|
+
const snapshotRows = db.select({
|
|
25611
|
+
id: querySnapshots.id,
|
|
25612
|
+
provider: querySnapshots.provider,
|
|
25613
|
+
citationState: querySnapshots.citationState,
|
|
25614
|
+
answerMentioned: querySnapshots.answerMentioned,
|
|
25615
|
+
answerText: querySnapshots.answerText,
|
|
25616
|
+
citedDomains: querySnapshots.citedDomains,
|
|
25617
|
+
competitorOverlap: querySnapshots.competitorOverlap,
|
|
25618
|
+
recommendedCompetitors: querySnapshots.recommendedCompetitors,
|
|
25619
|
+
rawResponse: querySnapshots.rawResponse
|
|
25620
|
+
}).from(querySnapshots).where(inArray10(querySnapshots.runId, batchRunIds)).all();
|
|
25621
|
+
const pendingUpdates = [];
|
|
25622
|
+
for (const snapshot of snapshotRows) {
|
|
25623
|
+
examined++;
|
|
25624
|
+
const reparsedResult = reparseProviderSnapshot(snapshot.provider, snapshot.rawResponse);
|
|
25625
|
+
if (reparsedResult) reparsed++;
|
|
25626
|
+
if (reparsedResult?.providerError) providerErrors++;
|
|
25627
|
+
const answerText = reparsedResult?.answerText ?? snapshot.answerText ?? "";
|
|
25628
|
+
const nextValue = determineAnswerMentioned(answerText, projectBrandNames, projectDomains);
|
|
25629
|
+
if (nextValue) mentioned++;
|
|
25630
|
+
const nextPatch = {};
|
|
25631
|
+
if (snapshot.answerMentioned !== nextValue) {
|
|
25632
|
+
nextPatch.answerMentioned = nextValue;
|
|
25633
|
+
}
|
|
25634
|
+
if ((snapshot.answerText ?? "") !== answerText) {
|
|
25635
|
+
nextPatch.answerText = answerText;
|
|
25636
|
+
}
|
|
25637
|
+
if (reparsedResult) {
|
|
25638
|
+
const normalized = {
|
|
25639
|
+
provider: snapshot.provider,
|
|
25640
|
+
answerText,
|
|
25641
|
+
citedDomains: reparsedResult.citedDomains,
|
|
25642
|
+
groundingSources: reparsedResult.groundingSources,
|
|
25643
|
+
searchQueries: reparsedResult.searchQueries
|
|
25644
|
+
};
|
|
25645
|
+
const nextCitationState = determineCitationState(normalized, projectDomains);
|
|
25646
|
+
const nextCitedDomains = JSON.stringify(reparsedResult.citedDomains);
|
|
25647
|
+
const nextCompetitorOverlap = JSON.stringify(
|
|
25648
|
+
computeCompetitorOverlap(normalized, competitorDomains)
|
|
25649
|
+
);
|
|
25650
|
+
const nextRecommendedCompetitors = JSON.stringify(
|
|
25651
|
+
extractRecommendedCompetitors(
|
|
25652
|
+
normalized.answerText,
|
|
25653
|
+
projectDomains,
|
|
25654
|
+
normalized.citedDomains,
|
|
25655
|
+
competitorDomains,
|
|
25656
|
+
projectBrandNames
|
|
25657
|
+
)
|
|
25658
|
+
);
|
|
25659
|
+
const nextRawResponse = stringifyStoredSnapshotEnvelope(
|
|
25660
|
+
snapshot.rawResponse,
|
|
25661
|
+
reparsedResult
|
|
25662
|
+
);
|
|
25663
|
+
if (snapshot.citationState !== nextCitationState) {
|
|
25664
|
+
nextPatch.citationState = nextCitationState;
|
|
25665
|
+
}
|
|
25666
|
+
if (snapshot.citedDomains !== nextCitedDomains) {
|
|
25667
|
+
nextPatch.citedDomains = nextCitedDomains;
|
|
25668
|
+
}
|
|
25669
|
+
if (snapshot.competitorOverlap !== nextCompetitorOverlap) {
|
|
25670
|
+
nextPatch.competitorOverlap = nextCompetitorOverlap;
|
|
25671
|
+
}
|
|
25672
|
+
if (snapshot.recommendedCompetitors !== nextRecommendedCompetitors) {
|
|
25673
|
+
nextPatch.recommendedCompetitors = nextRecommendedCompetitors;
|
|
25674
|
+
}
|
|
25675
|
+
if (snapshot.rawResponse !== nextRawResponse) {
|
|
25676
|
+
nextPatch.rawResponse = nextRawResponse;
|
|
25677
|
+
}
|
|
25678
|
+
}
|
|
25679
|
+
if (Object.keys(nextPatch).length > 0) {
|
|
25680
|
+
pendingUpdates.push({ id: snapshot.id, patch: nextPatch });
|
|
25681
|
+
}
|
|
25682
|
+
}
|
|
25683
|
+
if (pendingUpdates.length > 0) {
|
|
25684
|
+
db.transaction((tx) => {
|
|
25685
|
+
for (const update of pendingUpdates) {
|
|
25686
|
+
tx.update(querySnapshots).set(update.patch).where(eq34(querySnapshots.id, update.id)).run();
|
|
25687
|
+
}
|
|
25688
|
+
});
|
|
25689
|
+
updated += pendingUpdates.length;
|
|
25690
|
+
}
|
|
25691
|
+
}
|
|
25692
|
+
}
|
|
25693
|
+
}
|
|
25694
|
+
const result = {
|
|
25695
|
+
project: projectFilter ?? null,
|
|
25696
|
+
projects: scopedProjects.length,
|
|
25697
|
+
examined,
|
|
25698
|
+
updated,
|
|
25699
|
+
mentioned,
|
|
25700
|
+
reparsed,
|
|
25701
|
+
providerErrors
|
|
25702
|
+
};
|
|
25703
|
+
if (opts?.format === "json") {
|
|
25704
|
+
console.log(JSON.stringify(result, null, 2));
|
|
25705
|
+
return;
|
|
25706
|
+
}
|
|
25707
|
+
console.log("Answer visibility backfill complete.\n");
|
|
25708
|
+
if (projectFilter) {
|
|
25709
|
+
console.log(` Project: ${projectFilter}`);
|
|
25710
|
+
}
|
|
25711
|
+
console.log(` Projects: ${scopedProjects.length}`);
|
|
25712
|
+
console.log(` Examined: ${examined}`);
|
|
25713
|
+
console.log(` Updated: ${updated}`);
|
|
25714
|
+
console.log(` Mentioned: ${mentioned}`);
|
|
25715
|
+
console.log(` Reparsed: ${reparsed}`);
|
|
25716
|
+
console.log(` Errors: ${providerErrors}`);
|
|
25717
|
+
}
|
|
25718
|
+
function backfillNormalizedPaths(db, opts) {
|
|
25719
|
+
const baseConditions = [];
|
|
25720
|
+
if (opts?.projectId) {
|
|
25721
|
+
baseConditions.push(eq34(gaTrafficSnapshots.projectId, opts.projectId));
|
|
25722
|
+
}
|
|
25723
|
+
const rows = db.select({
|
|
25724
|
+
id: gaTrafficSnapshots.id,
|
|
25725
|
+
landingPage: gaTrafficSnapshots.landingPage,
|
|
25726
|
+
landingPageNormalized: gaTrafficSnapshots.landingPageNormalized
|
|
25727
|
+
}).from(gaTrafficSnapshots).where(baseConditions.length > 0 ? and24(...baseConditions) : void 0).all();
|
|
25728
|
+
let updated = 0;
|
|
25729
|
+
let unchanged = 0;
|
|
25730
|
+
if (rows.length > 0) {
|
|
25731
|
+
db.transaction((tx) => {
|
|
25732
|
+
for (const row of rows) {
|
|
25733
|
+
const next = normalizeUrlPath(row.landingPage);
|
|
25734
|
+
if (next === null) {
|
|
25735
|
+
unchanged++;
|
|
25736
|
+
continue;
|
|
25737
|
+
}
|
|
25738
|
+
if (row.landingPageNormalized === next) {
|
|
25739
|
+
unchanged++;
|
|
25740
|
+
continue;
|
|
25741
|
+
}
|
|
25742
|
+
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(eq34(gaTrafficSnapshots.id, row.id)).run();
|
|
25743
|
+
updated++;
|
|
25744
|
+
}
|
|
25745
|
+
});
|
|
25746
|
+
}
|
|
25747
|
+
return { examined: rows.length, updated, unchanged };
|
|
25748
|
+
}
|
|
25749
|
+
async function backfillNormalizedPathsCommand(opts) {
|
|
25750
|
+
const config = loadConfig();
|
|
25751
|
+
const db = createClient(config.database);
|
|
25752
|
+
migrate(db);
|
|
25753
|
+
const projectFilter = opts?.project?.trim();
|
|
25754
|
+
let projectId;
|
|
25755
|
+
if (projectFilter) {
|
|
25756
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq34(projects.name, projectFilter)).get();
|
|
25757
|
+
if (!project) {
|
|
25758
|
+
const result2 = {
|
|
25759
|
+
project: projectFilter,
|
|
25760
|
+
examined: 0,
|
|
25761
|
+
updated: 0,
|
|
25762
|
+
unchanged: 0
|
|
25763
|
+
};
|
|
25764
|
+
if (opts?.format === "json") {
|
|
25765
|
+
console.log(JSON.stringify(result2, null, 2));
|
|
25766
|
+
return;
|
|
25767
|
+
}
|
|
25768
|
+
console.log(`Backfill normalized-paths: project "${projectFilter}" not found.`);
|
|
25769
|
+
return;
|
|
25770
|
+
}
|
|
25771
|
+
projectId = project.id;
|
|
25772
|
+
}
|
|
25773
|
+
const { examined, updated, unchanged } = backfillNormalizedPaths(db, { projectId });
|
|
25774
|
+
const result = {
|
|
25775
|
+
project: projectFilter ?? null,
|
|
25776
|
+
examined,
|
|
25777
|
+
updated,
|
|
25778
|
+
unchanged
|
|
25779
|
+
};
|
|
25780
|
+
if (opts?.format === "json") {
|
|
25781
|
+
console.log(JSON.stringify(result, null, 2));
|
|
25782
|
+
return;
|
|
25783
|
+
}
|
|
25784
|
+
console.log("Normalized-path backfill complete.\n");
|
|
25785
|
+
if (projectFilter) console.log(` Project: ${projectFilter}`);
|
|
25786
|
+
console.log(` Examined: ${examined}`);
|
|
25787
|
+
console.log(` Updated: ${updated}`);
|
|
25788
|
+
console.log(` Unchanged: ${unchanged}`);
|
|
25789
|
+
}
|
|
25790
|
+
function backfillAiReferralPaths(db, opts) {
|
|
25791
|
+
const baseConditions = [];
|
|
25792
|
+
if (opts?.projectId) {
|
|
25793
|
+
baseConditions.push(eq34(gaAiReferrals.projectId, opts.projectId));
|
|
25794
|
+
}
|
|
25795
|
+
const rows = db.select({
|
|
25796
|
+
id: gaAiReferrals.id,
|
|
25797
|
+
landingPage: gaAiReferrals.landingPage,
|
|
25798
|
+
landingPageNormalized: gaAiReferrals.landingPageNormalized
|
|
25799
|
+
}).from(gaAiReferrals).where(baseConditions.length > 0 ? and24(...baseConditions) : void 0).all();
|
|
25800
|
+
let updated = 0;
|
|
25801
|
+
let unchanged = 0;
|
|
25802
|
+
if (rows.length > 0) {
|
|
25803
|
+
db.transaction((tx) => {
|
|
25804
|
+
for (const row of rows) {
|
|
25805
|
+
const next = normalizeUrlPath(row.landingPage);
|
|
25806
|
+
if (next === null) {
|
|
25807
|
+
unchanged++;
|
|
25808
|
+
continue;
|
|
25809
|
+
}
|
|
25810
|
+
if (row.landingPageNormalized === next) {
|
|
25811
|
+
unchanged++;
|
|
25812
|
+
continue;
|
|
25813
|
+
}
|
|
25814
|
+
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(eq34(gaAiReferrals.id, row.id)).run();
|
|
25815
|
+
updated++;
|
|
25816
|
+
}
|
|
25817
|
+
});
|
|
25818
|
+
}
|
|
25819
|
+
return { examined: rows.length, updated, unchanged };
|
|
25820
|
+
}
|
|
25821
|
+
async function backfillAiReferralPathsCommand(opts) {
|
|
25822
|
+
const config = loadConfig();
|
|
25823
|
+
const db = createClient(config.database);
|
|
25824
|
+
migrate(db);
|
|
25825
|
+
const projectFilter = opts?.project?.trim();
|
|
25826
|
+
let projectId;
|
|
25827
|
+
if (projectFilter) {
|
|
25828
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq34(projects.name, projectFilter)).get();
|
|
25829
|
+
if (!project) {
|
|
25830
|
+
const result2 = {
|
|
25831
|
+
project: projectFilter,
|
|
25832
|
+
examined: 0,
|
|
25833
|
+
updated: 0,
|
|
25834
|
+
unchanged: 0
|
|
25835
|
+
};
|
|
25836
|
+
if (opts?.format === "json") {
|
|
25837
|
+
console.log(JSON.stringify(result2, null, 2));
|
|
25838
|
+
return;
|
|
25839
|
+
}
|
|
25840
|
+
console.log(`Backfill ai-referral-paths: project "${projectFilter}" not found.`);
|
|
25841
|
+
return;
|
|
25842
|
+
}
|
|
25843
|
+
projectId = project.id;
|
|
25844
|
+
}
|
|
25845
|
+
const { examined, updated, unchanged } = backfillAiReferralPaths(db, { projectId });
|
|
25846
|
+
const result = {
|
|
25847
|
+
project: projectFilter ?? null,
|
|
25848
|
+
examined,
|
|
25849
|
+
updated,
|
|
25850
|
+
unchanged
|
|
25851
|
+
};
|
|
25852
|
+
if (opts?.format === "json") {
|
|
25853
|
+
console.log(JSON.stringify(result, null, 2));
|
|
25854
|
+
return;
|
|
25855
|
+
}
|
|
25856
|
+
console.log("AI referral landing-page backfill complete.\n");
|
|
25857
|
+
if (projectFilter) console.log(` Project: ${projectFilter}`);
|
|
25858
|
+
console.log(` Examined: ${examined}`);
|
|
25859
|
+
console.log(` Updated: ${updated}`);
|
|
25860
|
+
console.log(` Unchanged: ${unchanged}`);
|
|
25861
|
+
}
|
|
25862
|
+
function backfillProjectAnswerMentions(db, projectId) {
|
|
25863
|
+
const project = db.select().from(projects).where(eq34(projects.id, projectId)).get();
|
|
25864
|
+
if (!project) return { examined: 0, updated: 0, mentioned: 0 };
|
|
25865
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq34(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
25866
|
+
const runRows = db.select({ id: runs.id }).from(runs).where(and24(eq34(runs.kind, RunKinds["answer-visibility"]), eq34(runs.projectId, projectId))).all();
|
|
25867
|
+
const runIds = runRows.map((r) => r.id);
|
|
25868
|
+
let examined = 0;
|
|
25869
|
+
let updated = 0;
|
|
25870
|
+
let mentioned = 0;
|
|
25871
|
+
if (runIds.length === 0) return { examined, updated, mentioned };
|
|
25872
|
+
const projectDomains = effectiveDomains({
|
|
25873
|
+
canonicalDomain: project.canonicalDomain,
|
|
25874
|
+
ownedDomains: parseJsonColumn(project.ownedDomains, [])
|
|
25875
|
+
});
|
|
25876
|
+
const projectBrandNames = effectiveBrandNames({
|
|
25877
|
+
displayName: project.displayName,
|
|
25878
|
+
aliases: parseJsonColumn(project.aliases, [])
|
|
25879
|
+
});
|
|
25880
|
+
for (let offset = 0; offset < runIds.length; offset += SNAPSHOT_BATCH_SIZE) {
|
|
25881
|
+
const batchRunIds = runIds.slice(offset, offset + SNAPSHOT_BATCH_SIZE);
|
|
25882
|
+
const snapshotRows = db.select({
|
|
25883
|
+
id: querySnapshots.id,
|
|
25884
|
+
provider: querySnapshots.provider,
|
|
25885
|
+
answerMentioned: querySnapshots.answerMentioned,
|
|
25886
|
+
answerText: querySnapshots.answerText,
|
|
25887
|
+
citedDomains: querySnapshots.citedDomains,
|
|
25888
|
+
competitorOverlap: querySnapshots.competitorOverlap,
|
|
25889
|
+
recommendedCompetitors: querySnapshots.recommendedCompetitors,
|
|
25890
|
+
rawResponse: querySnapshots.rawResponse
|
|
25891
|
+
}).from(querySnapshots).where(inArray10(querySnapshots.runId, batchRunIds)).all();
|
|
25892
|
+
const pendingUpdates = [];
|
|
25893
|
+
for (const snapshot of snapshotRows) {
|
|
25894
|
+
examined++;
|
|
25895
|
+
const answerText = snapshot.answerText ?? "";
|
|
25896
|
+
const nextAnswerMentioned = determineAnswerMentioned(answerText, projectBrandNames, projectDomains);
|
|
25897
|
+
if (nextAnswerMentioned) mentioned++;
|
|
25898
|
+
const citedDomains = parseJsonColumn(snapshot.citedDomains, []);
|
|
25899
|
+
const groundingSources = readStoredGroundingSources(snapshot.rawResponse);
|
|
25900
|
+
const normalized = {
|
|
25901
|
+
provider: snapshot.provider,
|
|
25902
|
+
answerText,
|
|
25903
|
+
citedDomains,
|
|
25904
|
+
groundingSources,
|
|
25905
|
+
searchQueries: []
|
|
25906
|
+
};
|
|
25907
|
+
const nextCompetitorOverlap = JSON.stringify(
|
|
25908
|
+
computeCompetitorOverlap(normalized, competitorDomains)
|
|
25909
|
+
);
|
|
25910
|
+
const nextRecommendedCompetitors = JSON.stringify(
|
|
25911
|
+
extractRecommendedCompetitors(
|
|
25912
|
+
answerText,
|
|
25913
|
+
projectDomains,
|
|
25914
|
+
citedDomains,
|
|
25915
|
+
competitorDomains,
|
|
25916
|
+
projectBrandNames
|
|
25917
|
+
)
|
|
25918
|
+
);
|
|
25919
|
+
const nextPatch = {};
|
|
25920
|
+
if (snapshot.answerMentioned !== nextAnswerMentioned) {
|
|
25921
|
+
nextPatch.answerMentioned = nextAnswerMentioned;
|
|
25922
|
+
}
|
|
25923
|
+
if (snapshot.competitorOverlap !== nextCompetitorOverlap) {
|
|
25924
|
+
nextPatch.competitorOverlap = nextCompetitorOverlap;
|
|
25925
|
+
}
|
|
25926
|
+
if (snapshot.recommendedCompetitors !== nextRecommendedCompetitors) {
|
|
25927
|
+
nextPatch.recommendedCompetitors = nextRecommendedCompetitors;
|
|
25928
|
+
}
|
|
25929
|
+
if (Object.keys(nextPatch).length > 0) {
|
|
25930
|
+
pendingUpdates.push({ id: snapshot.id, patch: nextPatch });
|
|
25931
|
+
}
|
|
25932
|
+
}
|
|
25933
|
+
if (pendingUpdates.length > 0) {
|
|
25934
|
+
db.transaction((tx) => {
|
|
25935
|
+
for (const update of pendingUpdates) {
|
|
25936
|
+
tx.update(querySnapshots).set(update.patch).where(eq34(querySnapshots.id, update.id)).run();
|
|
25937
|
+
}
|
|
25938
|
+
});
|
|
25939
|
+
updated += pendingUpdates.length;
|
|
25940
|
+
}
|
|
25941
|
+
}
|
|
25942
|
+
return { examined, updated, mentioned };
|
|
25943
|
+
}
|
|
25944
|
+
async function backfillAnswerMentionsCommand(opts) {
|
|
25945
|
+
const config = loadConfig();
|
|
25946
|
+
const db = createClient(config.database);
|
|
25947
|
+
migrate(db);
|
|
25948
|
+
const projectFilter = opts?.project?.trim();
|
|
25949
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq34(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
25950
|
+
let examined = 0;
|
|
25951
|
+
let updated = 0;
|
|
25952
|
+
let mentioned = 0;
|
|
25953
|
+
for (const project of scopedProjects) {
|
|
25954
|
+
const result2 = backfillProjectAnswerMentions(db, project.id);
|
|
25955
|
+
examined += result2.examined;
|
|
25956
|
+
updated += result2.updated;
|
|
25957
|
+
mentioned += result2.mentioned;
|
|
25958
|
+
}
|
|
25959
|
+
const result = {
|
|
25960
|
+
project: projectFilter ?? null,
|
|
25961
|
+
projects: scopedProjects.length,
|
|
25962
|
+
examined,
|
|
25963
|
+
updated,
|
|
25964
|
+
mentioned
|
|
25965
|
+
};
|
|
25966
|
+
if (opts?.format === "json") {
|
|
25967
|
+
console.log(JSON.stringify(result, null, 2));
|
|
25968
|
+
return;
|
|
25969
|
+
}
|
|
25970
|
+
console.log("Answer mentions backfill complete.\n");
|
|
25971
|
+
if (projectFilter) console.log(` Project: ${projectFilter}`);
|
|
25972
|
+
console.log(` Projects: ${scopedProjects.length}`);
|
|
25973
|
+
console.log(` Examined: ${examined}`);
|
|
25974
|
+
console.log(` Updated: ${updated}`);
|
|
25975
|
+
console.log(` Mentioned: ${mentioned}`);
|
|
25976
|
+
}
|
|
25977
|
+
function readStoredGroundingSources(rawResponse) {
|
|
25978
|
+
const envelope = parseJsonColumn(rawResponse, {});
|
|
25979
|
+
const sources = envelope.groundingSources;
|
|
25980
|
+
if (!Array.isArray(sources)) return [];
|
|
25981
|
+
const result = [];
|
|
25982
|
+
for (const source of sources) {
|
|
25983
|
+
if (source && typeof source === "object") {
|
|
25984
|
+
const uri = source.uri;
|
|
25985
|
+
const title = source.title;
|
|
25986
|
+
if (typeof uri === "string") {
|
|
25987
|
+
result.push({ uri, title: typeof title === "string" ? title : "" });
|
|
25988
|
+
}
|
|
25989
|
+
}
|
|
25990
|
+
}
|
|
25991
|
+
return result;
|
|
25992
|
+
}
|
|
25993
|
+
async function backfillInsightsCommand(project, opts) {
|
|
25994
|
+
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-WAJOEOJV.js");
|
|
25995
|
+
const config = loadConfig();
|
|
25996
|
+
const db = createClient(config.database);
|
|
25997
|
+
migrate(db);
|
|
25998
|
+
const service = new IntelligenceService2(db);
|
|
25999
|
+
const isJson = opts?.format === "json";
|
|
26000
|
+
if (!isJson) {
|
|
26001
|
+
process.stderr.write(`Backfilling insights for "${project}"...
|
|
26002
|
+
`);
|
|
26003
|
+
}
|
|
26004
|
+
const result = service.backfill(project, {
|
|
26005
|
+
fromRunId: opts?.fromRun,
|
|
26006
|
+
toRunId: opts?.toRun
|
|
26007
|
+
}, (info) => {
|
|
26008
|
+
if (!isJson) {
|
|
26009
|
+
process.stderr.write(` [${info.index}/${info.total}] ${info.runId} \u2014 ${info.insights} insights
|
|
26010
|
+
`);
|
|
26011
|
+
}
|
|
26012
|
+
});
|
|
26013
|
+
const output = {
|
|
26014
|
+
project,
|
|
26015
|
+
processed: result.processed,
|
|
26016
|
+
skipped: result.skipped,
|
|
26017
|
+
totalInsights: result.totalInsights
|
|
26018
|
+
};
|
|
26019
|
+
if (isJson) {
|
|
26020
|
+
console.log(JSON.stringify(output, null, 2));
|
|
26021
|
+
return;
|
|
26022
|
+
}
|
|
26023
|
+
console.log(`
|
|
26024
|
+
Backfill complete.`);
|
|
26025
|
+
console.log(` Processed: ${result.processed}`);
|
|
26026
|
+
console.log(` Skipped: ${result.skipped}`);
|
|
26027
|
+
console.log(` Insights: ${result.totalInsights}`);
|
|
26028
|
+
}
|
|
26029
|
+
function reparseProviderSnapshot(provider, rawResponse) {
|
|
26030
|
+
const envelope = parseJsonColumn(rawResponse, {});
|
|
26031
|
+
const apiResponse = resolveStoredApiResponse(envelope);
|
|
26032
|
+
if (!apiResponse) return null;
|
|
26033
|
+
switch (provider) {
|
|
26034
|
+
case ProviderNames.openai:
|
|
26035
|
+
return reparseStoredResult2(apiResponse);
|
|
26036
|
+
case ProviderNames.claude:
|
|
26037
|
+
return reparseStoredResult3(apiResponse);
|
|
26038
|
+
case ProviderNames.gemini:
|
|
26039
|
+
return reparseStoredResult(apiResponse);
|
|
26040
|
+
case ProviderNames.perplexity:
|
|
26041
|
+
return reparseStoredResult4(apiResponse);
|
|
26042
|
+
default:
|
|
26043
|
+
return null;
|
|
26044
|
+
}
|
|
26045
|
+
}
|
|
26046
|
+
function resolveStoredApiResponse(parsed) {
|
|
26047
|
+
const nested = parsed.apiResponse;
|
|
26048
|
+
if (nested !== null && typeof nested === "object" && !Array.isArray(nested)) {
|
|
26049
|
+
return nested;
|
|
26050
|
+
}
|
|
26051
|
+
if (looksLikeProviderApiResponse(parsed)) {
|
|
26052
|
+
return parsed;
|
|
26053
|
+
}
|
|
26054
|
+
return null;
|
|
26055
|
+
}
|
|
26056
|
+
function looksLikeProviderApiResponse(value) {
|
|
26057
|
+
return Array.isArray(value.output) || Array.isArray(value.content) || Array.isArray(value.candidates) || Array.isArray(value.choices);
|
|
26058
|
+
}
|
|
26059
|
+
function stringifyStoredSnapshotEnvelope(rawResponse, reparsed) {
|
|
26060
|
+
const parsed = parseJsonColumn(rawResponse, {});
|
|
26061
|
+
const apiResponse = resolveStoredApiResponse(parsed);
|
|
26062
|
+
const envelope = apiResponse === parsed ? {} : { ...parsed };
|
|
26063
|
+
delete envelope.answerText;
|
|
26064
|
+
delete envelope.citedDomains;
|
|
26065
|
+
delete envelope.competitorOverlap;
|
|
26066
|
+
delete envelope.recommendedCompetitors;
|
|
26067
|
+
delete envelope.providerError;
|
|
26068
|
+
return JSON.stringify({
|
|
26069
|
+
...envelope,
|
|
26070
|
+
groundingSources: reparsed.groundingSources,
|
|
26071
|
+
searchQueries: reparsed.searchQueries,
|
|
26072
|
+
...reparsed.providerError ? { providerError: reparsed.providerError } : {},
|
|
26073
|
+
...apiResponse ? { apiResponse } : {}
|
|
26074
|
+
});
|
|
26075
|
+
}
|
|
26076
|
+
|
|
25297
26077
|
// src/provider-registry.ts
|
|
25298
26078
|
var ProviderRegistry = class {
|
|
25299
26079
|
providers = /* @__PURE__ */ new Map();
|
|
@@ -25347,7 +26127,7 @@ var ProviderRegistry = class {
|
|
|
25347
26127
|
|
|
25348
26128
|
// src/scheduler.ts
|
|
25349
26129
|
import cron from "node-cron";
|
|
25350
|
-
import { and as
|
|
26130
|
+
import { and as and25, eq as eq35 } from "drizzle-orm";
|
|
25351
26131
|
var log9 = createLogger("Scheduler");
|
|
25352
26132
|
function taskKey(projectId, kind) {
|
|
25353
26133
|
return `${projectId}::${kind}`;
|
|
@@ -25362,7 +26142,7 @@ var Scheduler = class {
|
|
|
25362
26142
|
}
|
|
25363
26143
|
/** Load all enabled schedules from DB and register cron jobs. */
|
|
25364
26144
|
start() {
|
|
25365
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
26145
|
+
const allSchedules = this.db.select().from(schedules).where(eq35(schedules.enabled, 1)).all();
|
|
25366
26146
|
for (const schedule of allSchedules) {
|
|
25367
26147
|
const missedRunAt = schedule.nextRunAt;
|
|
25368
26148
|
this.registerCronTask(schedule);
|
|
@@ -25392,7 +26172,7 @@ var Scheduler = class {
|
|
|
25392
26172
|
this.stopTask(key, existing, "Stopped");
|
|
25393
26173
|
this.tasks.delete(key);
|
|
25394
26174
|
}
|
|
25395
|
-
const schedule = this.db.select().from(schedules).where(
|
|
26175
|
+
const schedule = this.db.select().from(schedules).where(and25(eq35(schedules.projectId, projectId), eq35(schedules.kind, kind))).get();
|
|
25396
26176
|
if (schedule && schedule.enabled === 1) {
|
|
25397
26177
|
this.registerCronTask(schedule);
|
|
25398
26178
|
}
|
|
@@ -25433,14 +26213,14 @@ var Scheduler = class {
|
|
|
25433
26213
|
this.db.update(schedules).set({
|
|
25434
26214
|
nextRunAt: task.getNextRun()?.toISOString() ?? null,
|
|
25435
26215
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
25436
|
-
}).where(
|
|
26216
|
+
}).where(eq35(schedules.id, scheduleId)).run();
|
|
25437
26217
|
const label = schedule.preset ?? cronExpr;
|
|
25438
26218
|
log9.info("cron.registered", { projectId, kind, schedule: label, timezone });
|
|
25439
26219
|
}
|
|
25440
26220
|
triggerRun(scheduleId, projectId, kind) {
|
|
25441
26221
|
try {
|
|
25442
26222
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
25443
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
26223
|
+
const currentSchedule = this.db.select().from(schedules).where(eq35(schedules.id, scheduleId)).get();
|
|
25444
26224
|
if (!currentSchedule || currentSchedule.enabled !== 1) {
|
|
25445
26225
|
log9.warn("schedule.stale", { scheduleId, projectId, kind, msg: "schedule no longer exists or is disabled" });
|
|
25446
26226
|
this.remove(projectId, kind);
|
|
@@ -25448,7 +26228,7 @@ var Scheduler = class {
|
|
|
25448
26228
|
}
|
|
25449
26229
|
const task = this.tasks.get(taskKey(projectId, kind));
|
|
25450
26230
|
const nextRunAt = task?.getNextRun()?.toISOString() ?? null;
|
|
25451
|
-
const project = this.db.select().from(projects).where(
|
|
26231
|
+
const project = this.db.select().from(projects).where(eq35(projects.id, projectId)).get();
|
|
25452
26232
|
if (!project) {
|
|
25453
26233
|
log9.error("project.not-found", { projectId, kind, msg: "skipping scheduled run" });
|
|
25454
26234
|
this.remove(projectId, kind);
|
|
@@ -25468,7 +26248,7 @@ var Scheduler = class {
|
|
|
25468
26248
|
lastRunAt: now,
|
|
25469
26249
|
nextRunAt,
|
|
25470
26250
|
updatedAt: now
|
|
25471
|
-
}).where(
|
|
26251
|
+
}).where(eq35(schedules.id, currentSchedule.id)).run();
|
|
25472
26252
|
log9.info("traffic-sync.triggered", { projectName: project.name, sourceId });
|
|
25473
26253
|
this.callbacks.onTrafficSyncRequested(project.name, sourceId);
|
|
25474
26254
|
return;
|
|
@@ -25496,7 +26276,7 @@ var Scheduler = class {
|
|
|
25496
26276
|
this.db.update(schedules).set({
|
|
25497
26277
|
nextRunAt,
|
|
25498
26278
|
updatedAt: now
|
|
25499
|
-
}).where(
|
|
26279
|
+
}).where(eq35(schedules.id, currentSchedule.id)).run();
|
|
25500
26280
|
return;
|
|
25501
26281
|
}
|
|
25502
26282
|
const runId = queueResult.runId;
|
|
@@ -25504,7 +26284,7 @@ var Scheduler = class {
|
|
|
25504
26284
|
lastRunAt: now,
|
|
25505
26285
|
nextRunAt,
|
|
25506
26286
|
updatedAt: now
|
|
25507
|
-
}).where(
|
|
26287
|
+
}).where(eq35(schedules.id, currentSchedule.id)).run();
|
|
25508
26288
|
const scheduleProviders = parseJsonColumn(currentSchedule.providers, []);
|
|
25509
26289
|
const providers = scheduleProviders.length > 0 ? scheduleProviders : void 0;
|
|
25510
26290
|
log9.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
@@ -25516,7 +26296,7 @@ var Scheduler = class {
|
|
|
25516
26296
|
};
|
|
25517
26297
|
|
|
25518
26298
|
// src/notifier.ts
|
|
25519
|
-
import { eq as
|
|
26299
|
+
import { eq as eq36, desc as desc16, and as and26, inArray as inArray11, or as or5 } from "drizzle-orm";
|
|
25520
26300
|
import crypto31 from "crypto";
|
|
25521
26301
|
var log10 = createLogger("Notifier");
|
|
25522
26302
|
var Notifier = class {
|
|
@@ -25529,18 +26309,18 @@ var Notifier = class {
|
|
|
25529
26309
|
/** Called after a run completes (success, partial, or failed). */
|
|
25530
26310
|
async onRunCompleted(runId, projectId) {
|
|
25531
26311
|
log10.info("run.completed", { runId, projectId });
|
|
25532
|
-
const notifs = this.db.select().from(notifications).where(
|
|
26312
|
+
const notifs = this.db.select().from(notifications).where(eq36(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
25533
26313
|
if (notifs.length === 0) {
|
|
25534
26314
|
log10.info("notifications.none-enabled", { projectId });
|
|
25535
26315
|
return;
|
|
25536
26316
|
}
|
|
25537
26317
|
log10.info("notifications.found", { projectId, count: notifs.length });
|
|
25538
|
-
const run = this.db.select().from(runs).where(
|
|
26318
|
+
const run = this.db.select().from(runs).where(eq36(runs.id, runId)).get();
|
|
25539
26319
|
if (!run) {
|
|
25540
26320
|
log10.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
25541
26321
|
return;
|
|
25542
26322
|
}
|
|
25543
|
-
const project = this.db.select().from(projects).where(
|
|
26323
|
+
const project = this.db.select().from(projects).where(eq36(projects.id, projectId)).get();
|
|
25544
26324
|
if (!project) {
|
|
25545
26325
|
log10.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
25546
26326
|
return;
|
|
@@ -25587,11 +26367,11 @@ var Notifier = class {
|
|
|
25587
26367
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
25588
26368
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
25589
26369
|
if (insightEvents.length === 0) return;
|
|
25590
|
-
const notifs = this.db.select().from(notifications).where(
|
|
26370
|
+
const notifs = this.db.select().from(notifications).where(eq36(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
25591
26371
|
if (notifs.length === 0) return;
|
|
25592
|
-
const run = this.db.select().from(runs).where(
|
|
26372
|
+
const run = this.db.select().from(runs).where(eq36(runs.id, runId)).get();
|
|
25593
26373
|
if (!run) return;
|
|
25594
|
-
const project = this.db.select().from(projects).where(
|
|
26374
|
+
const project = this.db.select().from(projects).where(eq36(projects.id, projectId)).get();
|
|
25595
26375
|
if (!project) return;
|
|
25596
26376
|
for (const notif of notifs) {
|
|
25597
26377
|
const config = parseJsonColumn(notif.config, { url: "", events: [] });
|
|
@@ -25621,12 +26401,12 @@ var Notifier = class {
|
|
|
25621
26401
|
}
|
|
25622
26402
|
}
|
|
25623
26403
|
computeTransitions(runId, projectId) {
|
|
25624
|
-
const thisRun = this.db.select().from(runs).where(
|
|
26404
|
+
const thisRun = this.db.select().from(runs).where(eq36(runs.id, runId)).get();
|
|
25625
26405
|
if (!thisRun) return [];
|
|
25626
|
-
const groupSiblings = this.db.select().from(runs).where(
|
|
25627
|
-
|
|
25628
|
-
|
|
25629
|
-
|
|
26406
|
+
const groupSiblings = this.db.select().from(runs).where(and26(
|
|
26407
|
+
eq36(runs.projectId, projectId),
|
|
26408
|
+
eq36(runs.kind, thisRun.kind),
|
|
26409
|
+
eq36(runs.createdAt, thisRun.createdAt)
|
|
25630
26410
|
)).all();
|
|
25631
26411
|
const stillPending = groupSiblings.some((r) => r.status === "queued" || r.status === "running");
|
|
25632
26412
|
if (stillPending) return [];
|
|
@@ -25642,17 +26422,17 @@ var Notifier = class {
|
|
|
25642
26422
|
return candidate.id > best.id ? candidate : best;
|
|
25643
26423
|
});
|
|
25644
26424
|
if (winner.id !== runId) return [];
|
|
25645
|
-
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
26425
|
+
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(eq36(projects.id, projectId)).get();
|
|
25646
26426
|
const locationCount = Math.max(
|
|
25647
26427
|
1,
|
|
25648
26428
|
parseJsonColumn(projectLocations?.locations ?? null, []).length
|
|
25649
26429
|
);
|
|
25650
26430
|
const RECENT_FETCH_LIMIT = Math.max(8, locationCount * 4);
|
|
25651
26431
|
const recentRuns = this.db.select().from(runs).where(
|
|
25652
|
-
|
|
25653
|
-
|
|
25654
|
-
|
|
25655
|
-
|
|
26432
|
+
and26(
|
|
26433
|
+
eq36(runs.projectId, projectId),
|
|
26434
|
+
eq36(runs.kind, thisRun.kind),
|
|
26435
|
+
or5(eq36(runs.status, "completed"), eq36(runs.status, "partial"))
|
|
25656
26436
|
)
|
|
25657
26437
|
).orderBy(desc16(runs.createdAt), desc16(runs.id)).limit(RECENT_FETCH_LIMIT).all();
|
|
25658
26438
|
const groups = groupRunsByCreatedAt(recentRuns);
|
|
@@ -25669,13 +26449,13 @@ var Notifier = class {
|
|
|
25669
26449
|
provider: querySnapshots.provider,
|
|
25670
26450
|
location: querySnapshots.location,
|
|
25671
26451
|
citationState: querySnapshots.citationState
|
|
25672
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
26452
|
+
}).from(querySnapshots).leftJoin(queries, eq36(querySnapshots.queryId, queries.id)).where(inArray11(querySnapshots.runId, currentRunIds)).all();
|
|
25673
26453
|
const previousSnapshots = this.db.select({
|
|
25674
26454
|
queryId: querySnapshots.queryId,
|
|
25675
26455
|
provider: querySnapshots.provider,
|
|
25676
26456
|
location: querySnapshots.location,
|
|
25677
26457
|
citationState: querySnapshots.citationState
|
|
25678
|
-
}).from(querySnapshots).where(
|
|
26458
|
+
}).from(querySnapshots).where(inArray11(querySnapshots.runId, previousRunIds)).all();
|
|
25679
26459
|
const prevMap = /* @__PURE__ */ new Map();
|
|
25680
26460
|
for (const s of previousSnapshots) {
|
|
25681
26461
|
if (s.queryId == null) continue;
|
|
@@ -25749,7 +26529,7 @@ var Notifier = class {
|
|
|
25749
26529
|
};
|
|
25750
26530
|
|
|
25751
26531
|
// src/run-coordinator.ts
|
|
25752
|
-
import { eq as
|
|
26532
|
+
import { eq as eq37 } from "drizzle-orm";
|
|
25753
26533
|
var log11 = createLogger("RunCoordinator");
|
|
25754
26534
|
var RunCoordinator = class {
|
|
25755
26535
|
constructor(db, notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
@@ -25760,7 +26540,7 @@ var RunCoordinator = class {
|
|
|
25760
26540
|
this.onAeroEvent = onAeroEvent;
|
|
25761
26541
|
}
|
|
25762
26542
|
async onRunCompleted(runId, projectId) {
|
|
25763
|
-
const runRow = this.db.select().from(runs).where(
|
|
26543
|
+
const runRow = this.db.select().from(runs).where(eq37(runs.id, runId)).get();
|
|
25764
26544
|
const kind = runRow?.kind ?? RunKinds["answer-visibility"];
|
|
25765
26545
|
let insightCount = 0;
|
|
25766
26546
|
let criticalOrHigh = 0;
|
|
@@ -25815,7 +26595,7 @@ var RunCoordinator = class {
|
|
|
25815
26595
|
* so the Aero queue is never starved of a follow-up.
|
|
25816
26596
|
*/
|
|
25817
26597
|
buildDiscoveryAeroContext(runId, projectId, status, error) {
|
|
25818
|
-
const session = this.db.select().from(discoverySessions).where(
|
|
26598
|
+
const session = this.db.select().from(discoverySessions).where(eq37(discoverySessions.runId, runId)).get();
|
|
25819
26599
|
const competitorMap = session ? parseJsonColumn(session.competitorMap, []) : [];
|
|
25820
26600
|
return {
|
|
25821
26601
|
kind: RunKinds["aeo-discover-probe"],
|
|
@@ -25838,7 +26618,7 @@ var RunCoordinator = class {
|
|
|
25838
26618
|
|
|
25839
26619
|
// src/agent/session-registry.ts
|
|
25840
26620
|
import crypto33 from "crypto";
|
|
25841
|
-
import { eq as
|
|
26621
|
+
import { eq as eq39 } from "drizzle-orm";
|
|
25842
26622
|
|
|
25843
26623
|
// src/agent/session.ts
|
|
25844
26624
|
import fs11 from "fs";
|
|
@@ -26188,7 +26968,7 @@ function resolveSessionProviderAndModel(config, opts) {
|
|
|
26188
26968
|
|
|
26189
26969
|
// src/agent/memory-store.ts
|
|
26190
26970
|
import crypto32 from "crypto";
|
|
26191
|
-
import { and as
|
|
26971
|
+
import { and as and27, desc as desc17, eq as eq38, like as like2, sql as sql13 } from "drizzle-orm";
|
|
26192
26972
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
26193
26973
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
26194
26974
|
function rowToDto2(row) {
|
|
@@ -26202,7 +26982,7 @@ function rowToDto2(row) {
|
|
|
26202
26982
|
};
|
|
26203
26983
|
}
|
|
26204
26984
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
26205
|
-
const query = db.select().from(agentMemory).where(
|
|
26985
|
+
const query = db.select().from(agentMemory).where(eq38(agentMemory.projectId, projectId)).orderBy(desc17(agentMemory.updatedAt));
|
|
26206
26986
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
26207
26987
|
return rows.map(rowToDto2);
|
|
26208
26988
|
}
|
|
@@ -26233,12 +27013,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
26233
27013
|
updatedAt: now
|
|
26234
27014
|
}
|
|
26235
27015
|
}).run();
|
|
26236
|
-
const row = db.select().from(agentMemory).where(
|
|
27016
|
+
const row = db.select().from(agentMemory).where(and27(eq38(agentMemory.projectId, args.projectId), eq38(agentMemory.key, args.key))).get();
|
|
26237
27017
|
if (!row) throw new Error("memory upsert produced no row");
|
|
26238
27018
|
return rowToDto2(row);
|
|
26239
27019
|
}
|
|
26240
27020
|
function deleteMemoryEntry(db, projectId, key) {
|
|
26241
|
-
const result = db.delete(agentMemory).where(
|
|
27021
|
+
const result = db.delete(agentMemory).where(and27(eq38(agentMemory.projectId, projectId), eq38(agentMemory.key, key))).run();
|
|
26242
27022
|
const changes = result.changes ?? 0;
|
|
26243
27023
|
return changes > 0;
|
|
26244
27024
|
}
|
|
@@ -26267,8 +27047,8 @@ function writeCompactionNote(db, args) {
|
|
|
26267
27047
|
}).run();
|
|
26268
27048
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
26269
27049
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
26270
|
-
|
|
26271
|
-
|
|
27050
|
+
and27(
|
|
27051
|
+
eq38(agentMemory.projectId, args.projectId),
|
|
26272
27052
|
like2(agentMemory.key, `${sessionPrefix}%`)
|
|
26273
27053
|
)
|
|
26274
27054
|
).orderBy(desc17(agentMemory.updatedAt)).all();
|
|
@@ -26276,7 +27056,7 @@ function writeCompactionNote(db, args) {
|
|
|
26276
27056
|
if (stale.length > 0) {
|
|
26277
27057
|
tx.delete(agentMemory).where(sql13`${agentMemory.id} IN (${sql13.join(stale.map((s) => sql13`${s}`), sql13`, `)})`).run();
|
|
26278
27058
|
}
|
|
26279
|
-
const row = tx.select().from(agentMemory).where(
|
|
27059
|
+
const row = tx.select().from(agentMemory).where(and27(eq38(agentMemory.projectId, args.projectId), eq38(agentMemory.key, key))).get();
|
|
26280
27060
|
if (row) inserted = rowToDto2(row);
|
|
26281
27061
|
});
|
|
26282
27062
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -26458,7 +27238,7 @@ var SessionRegistry = class {
|
|
|
26458
27238
|
modelProvider: effectiveProvider,
|
|
26459
27239
|
modelId: effectiveModelId,
|
|
26460
27240
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
26461
|
-
}).where(
|
|
27241
|
+
}).where(eq39(agentSessions.projectId, projectId)).run();
|
|
26462
27242
|
}
|
|
26463
27243
|
const agent2 = createAeroSession({
|
|
26464
27244
|
projectName,
|
|
@@ -26672,7 +27452,7 @@ ${lines.join("\n")}
|
|
|
26672
27452
|
modelProvider: nextProvider,
|
|
26673
27453
|
modelId: nextModelId,
|
|
26674
27454
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
26675
|
-
}).where(
|
|
27455
|
+
}).where(eq39(agentSessions.projectId, projectId)).run();
|
|
26676
27456
|
}
|
|
26677
27457
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
26678
27458
|
save(projectName) {
|
|
@@ -26834,11 +27614,11 @@ ${lines.join("\n")}
|
|
|
26834
27614
|
return id;
|
|
26835
27615
|
}
|
|
26836
27616
|
tryResolveProjectId(projectName) {
|
|
26837
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
27617
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq39(projects.name, projectName)).get();
|
|
26838
27618
|
return row?.id;
|
|
26839
27619
|
}
|
|
26840
27620
|
loadRow(projectId) {
|
|
26841
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
27621
|
+
const row = this.opts.db.select().from(agentSessions).where(eq39(agentSessions.projectId, projectId)).get();
|
|
26842
27622
|
return row ?? null;
|
|
26843
27623
|
}
|
|
26844
27624
|
insertRow(params) {
|
|
@@ -26857,14 +27637,14 @@ ${lines.join("\n")}
|
|
|
26857
27637
|
}
|
|
26858
27638
|
updateRow(projectId, patch) {
|
|
26859
27639
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
26860
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
27640
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq39(agentSessions.projectId, projectId)).run();
|
|
26861
27641
|
}
|
|
26862
27642
|
};
|
|
26863
27643
|
|
|
26864
27644
|
// src/agent/agent-routes.ts
|
|
26865
|
-
import { eq as
|
|
27645
|
+
import { eq as eq40 } from "drizzle-orm";
|
|
26866
27646
|
function resolveProject2(db, name) {
|
|
26867
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
27647
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq40(projects.name, name)).get();
|
|
26868
27648
|
if (!row) throw notFound("project", name);
|
|
26869
27649
|
return row;
|
|
26870
27650
|
}
|
|
@@ -26873,7 +27653,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
26873
27653
|
"/projects/:name/agent/transcript",
|
|
26874
27654
|
async (request) => {
|
|
26875
27655
|
const project = resolveProject2(opts.db, request.params.name);
|
|
26876
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
27656
|
+
const row = opts.db.select().from(agentSessions).where(eq40(agentSessions.projectId, project.id)).get();
|
|
26877
27657
|
if (!row) {
|
|
26878
27658
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
26879
27659
|
}
|
|
@@ -26897,7 +27677,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
26897
27677
|
async (request) => {
|
|
26898
27678
|
const project = resolveProject2(opts.db, request.params.name);
|
|
26899
27679
|
opts.sessionRegistry.reset(project.name);
|
|
26900
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
27680
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq40(agentSessions.projectId, project.id)).run();
|
|
26901
27681
|
return { status: "reset" };
|
|
26902
27682
|
}
|
|
26903
27683
|
);
|
|
@@ -27015,7 +27795,7 @@ import { runAeoAudit } from "@ainyc/aeo-audit";
|
|
|
27015
27795
|
|
|
27016
27796
|
// src/site-fetch.ts
|
|
27017
27797
|
import https2 from "https";
|
|
27018
|
-
var
|
|
27798
|
+
var FETCH_TIMEOUT_MS2 = 1e4;
|
|
27019
27799
|
var MAX_TEXT_LENGTH = 4e3;
|
|
27020
27800
|
var MAX_BODY_BYTES = 512e3;
|
|
27021
27801
|
var USER_AGENT = "Canonry/1.0 (site-analysis)";
|
|
@@ -27040,7 +27820,7 @@ function fetchWithPinnedAddress(target) {
|
|
|
27040
27820
|
port,
|
|
27041
27821
|
path: path15,
|
|
27042
27822
|
method: "GET",
|
|
27043
|
-
timeout:
|
|
27823
|
+
timeout: FETCH_TIMEOUT_MS2,
|
|
27044
27824
|
servername: target.url.hostname,
|
|
27045
27825
|
// SNI for TLS
|
|
27046
27826
|
headers: {
|
|
@@ -27361,7 +28141,7 @@ var SnapshotService = class {
|
|
|
27361
28141
|
provider: provider.adapter.name,
|
|
27362
28142
|
displayName: provider.adapter.displayName,
|
|
27363
28143
|
model: raw.model,
|
|
27364
|
-
mentioned: determineAnswerMentioned(normalized.answerText, ctx.companyName, answerVisibilityDomains),
|
|
28144
|
+
mentioned: determineAnswerMentioned(normalized.answerText, [ctx.companyName], answerVisibilityDomains),
|
|
27365
28145
|
cited: citesTargetDomain(normalized.citedDomains, normalized.groundingSources, ctx.domain),
|
|
27366
28146
|
describedAccurately: "unknown",
|
|
27367
28147
|
accuracyNotes: null,
|
|
@@ -27728,8 +28508,8 @@ function clipText(value, length) {
|
|
|
27728
28508
|
}
|
|
27729
28509
|
|
|
27730
28510
|
// src/server.ts
|
|
27731
|
-
var
|
|
27732
|
-
var { version:
|
|
28511
|
+
var _require3 = createRequire4(import.meta.url);
|
|
28512
|
+
var { version: PKG_VERSION2 } = _require3("../package.json");
|
|
27733
28513
|
var log14 = createLogger("Server");
|
|
27734
28514
|
var DEFAULT_QUOTA = {
|
|
27735
28515
|
maxConcurrency: 2,
|
|
@@ -27920,7 +28700,7 @@ async function createServer(opts) {
|
|
|
27920
28700
|
intelligenceService,
|
|
27921
28701
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
27922
28702
|
async (ctx) => {
|
|
27923
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
28703
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq41(projects.id, ctx.projectId)).get();
|
|
27924
28704
|
if (!project) return;
|
|
27925
28705
|
let content;
|
|
27926
28706
|
if (ctx.kind === RunKinds["aeo-discover-probe"]) {
|
|
@@ -28121,7 +28901,7 @@ async function createServer(opts) {
|
|
|
28121
28901
|
const apiPrefix = basePath ? `${basePath}api/v1` : "/api/v1";
|
|
28122
28902
|
if (opts.config.apiKey) {
|
|
28123
28903
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
28124
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
28904
|
+
const existing = opts.db.select().from(apiKeys).where(eq41(apiKeys.keyHash, keyHash)).get();
|
|
28125
28905
|
if (!existing) {
|
|
28126
28906
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
28127
28907
|
opts.db.insert(apiKeys).values({
|
|
@@ -28173,7 +28953,7 @@ async function createServer(opts) {
|
|
|
28173
28953
|
};
|
|
28174
28954
|
const getDefaultApiKey = () => {
|
|
28175
28955
|
if (!opts.config.apiKey) return void 0;
|
|
28176
|
-
return opts.db.select().from(apiKeys).where(
|
|
28956
|
+
return opts.db.select().from(apiKeys).where(eq41(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
28177
28957
|
};
|
|
28178
28958
|
const createPasswordSession = (reply) => {
|
|
28179
28959
|
const key = getDefaultApiKey();
|
|
@@ -28230,12 +29010,12 @@ async function createServer(opts) {
|
|
|
28230
29010
|
return reply.send({ authenticated: true });
|
|
28231
29011
|
}
|
|
28232
29012
|
if (apiKey) {
|
|
28233
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
29013
|
+
const key = opts.db.select().from(apiKeys).where(eq41(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
28234
29014
|
if (!key || key.revokedAt) {
|
|
28235
29015
|
const err2 = authInvalid();
|
|
28236
29016
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
28237
29017
|
}
|
|
28238
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
29018
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq41(apiKeys.id, key.id)).run();
|
|
28239
29019
|
const sessionId = createSession(key.id);
|
|
28240
29020
|
reply.header("set-cookie", serializeSessionCookie({
|
|
28241
29021
|
name: SESSION_COOKIE_NAME,
|
|
@@ -28289,6 +29069,12 @@ async function createServer(opts) {
|
|
|
28289
29069
|
skipAuth: false,
|
|
28290
29070
|
sessionCookieName: SESSION_COOKIE_NAME,
|
|
28291
29071
|
resolveSessionApiKeyId,
|
|
29072
|
+
// Local canonry serve runs on the operator's machine, where pointing a
|
|
29073
|
+
// webhook at localhost (Discord test container, Pipedream-mock dev server,
|
|
29074
|
+
// etc.) is a legitimate workflow. Default to allowing it for the local
|
|
29075
|
+
// installer; cloud deployments inherit the secure default of `false` by
|
|
29076
|
+
// not passing this option. Override with CANONRY_ALLOW_LOOPBACK_WEBHOOKS=0.
|
|
29077
|
+
allowLoopbackWebhooks: process.env.CANONRY_ALLOW_LOOPBACK_WEBHOOKS !== "0",
|
|
28292
29078
|
// Local-only Aero agent routes. Registered here so they inherit api-routes'
|
|
28293
29079
|
// auth plugin — bare `registerAgentRoutes(app, ...)` would skip auth.
|
|
28294
29080
|
registerAuthenticatedRoutes: async (scope) => {
|
|
@@ -28409,7 +29195,7 @@ async function createServer(opts) {
|
|
|
28409
29195
|
discoverLatestRelease,
|
|
28410
29196
|
openApiInfo: {
|
|
28411
29197
|
title: "Canonry API",
|
|
28412
|
-
version:
|
|
29198
|
+
version: PKG_VERSION2,
|
|
28413
29199
|
includeCanonryLocal: true
|
|
28414
29200
|
},
|
|
28415
29201
|
providerSummary,
|
|
@@ -28556,6 +29342,19 @@ async function createServer(opts) {
|
|
|
28556
29342
|
onProjectDeleted: (projectId) => {
|
|
28557
29343
|
scheduler.removeAllForProject(projectId);
|
|
28558
29344
|
},
|
|
29345
|
+
onAliasesChanged: (projectId, projectName) => {
|
|
29346
|
+
setImmediate(() => {
|
|
29347
|
+
try {
|
|
29348
|
+
const result = backfillProjectAnswerMentions(opts.db, projectId);
|
|
29349
|
+
app.log.info(
|
|
29350
|
+
{ projectId, projectName, ...result },
|
|
29351
|
+
"aliases changed \u2014 recomputed mention fields on historical snapshots"
|
|
29352
|
+
);
|
|
29353
|
+
} catch (err) {
|
|
29354
|
+
app.log.error({ err, projectId, projectName }, "alias-triggered backfill failed");
|
|
29355
|
+
}
|
|
29356
|
+
});
|
|
29357
|
+
},
|
|
28559
29358
|
getTelemetryStatus: () => {
|
|
28560
29359
|
const enabled = isTelemetryEnabled();
|
|
28561
29360
|
return {
|
|
@@ -28689,16 +29488,21 @@ async function createServer(opts) {
|
|
|
28689
29488
|
return reply.status(404).send({ error: "Not found" });
|
|
28690
29489
|
});
|
|
28691
29490
|
}
|
|
28692
|
-
const healthHandler =
|
|
28693
|
-
|
|
28694
|
-
|
|
28695
|
-
|
|
28696
|
-
|
|
28697
|
-
|
|
29491
|
+
const healthHandler = () => {
|
|
29492
|
+
const update = checkLatestVersionForServer();
|
|
29493
|
+
return {
|
|
29494
|
+
status: "ok",
|
|
29495
|
+
service: "canonry",
|
|
29496
|
+
version: PKG_VERSION2,
|
|
29497
|
+
...basePath ? { basePath: basePath.replace(/\/$/, "") } : {},
|
|
29498
|
+
...update ? { updateAvailable: update } : {}
|
|
29499
|
+
};
|
|
29500
|
+
};
|
|
28698
29501
|
app.get("/health", healthHandler);
|
|
28699
29502
|
if (basePath) {
|
|
28700
29503
|
app.get(`${basePath}health`, healthHandler);
|
|
28701
29504
|
}
|
|
29505
|
+
checkLatestVersionForServer();
|
|
28702
29506
|
scheduler.start();
|
|
28703
29507
|
app.addHook("onClose", async () => {
|
|
28704
29508
|
scheduler.stop();
|
|
@@ -28759,16 +29563,17 @@ export {
|
|
|
28759
29563
|
showFirstRunNotice,
|
|
28760
29564
|
detectAndTrackUpgrade,
|
|
28761
29565
|
trackEvent,
|
|
28762
|
-
|
|
28763
|
-
|
|
28764
|
-
|
|
28765
|
-
|
|
28766
|
-
|
|
28767
|
-
|
|
28768
|
-
|
|
29566
|
+
backfillAnswerVisibilityCommand,
|
|
29567
|
+
backfillNormalizedPaths,
|
|
29568
|
+
backfillNormalizedPathsCommand,
|
|
29569
|
+
backfillAiReferralPaths,
|
|
29570
|
+
backfillAiReferralPathsCommand,
|
|
29571
|
+
backfillAnswerMentionsCommand,
|
|
29572
|
+
backfillInsightsCommand,
|
|
28769
29573
|
renderReportHtml,
|
|
28770
29574
|
setGoogleAuthConfig,
|
|
28771
29575
|
formatAuditFactorScore,
|
|
29576
|
+
checkLatestVersionForCli,
|
|
28772
29577
|
listAgentProviders,
|
|
28773
29578
|
coerceAgentProvider,
|
|
28774
29579
|
createServer
|