@ainyc/canonry 4.15.2 → 4.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/assets/assets/index-4fWsYFLp.css +1 -0
- package/assets/assets/{index-Qq_oMI-C.js → index-dLsgu2ck.js} +106 -106
- package/assets/index.html +2 -2
- package/dist/{chunk-IVNWS2YU.js → chunk-7VDM3JBI.js} +1111 -328
- package/dist/{chunk-MI33SQL6.js → chunk-BN2VQDZ2.js} +73 -3
- package/dist/{chunk-7SRKUAZO.js → chunk-P3SFTXHG.js} +18 -11
- package/dist/{chunk-ONI3TX2A.js → chunk-SBZTDECX.js} +36 -2
- package/dist/cli.js +53 -39
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-JYV3CO4H.js → intelligence-service-6CX5HH27.js} +2 -2
- package/dist/mcp.js +2 -2
- package/package.json +10 -10
- package/assets/assets/index-C1WW21tz.css +0 -1
|
@@ -5,7 +5,7 @@ import {
|
|
|
5
5
|
loadConfig,
|
|
6
6
|
loadConfigRaw,
|
|
7
7
|
saveConfigPatch
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-P3SFTXHG.js";
|
|
9
9
|
import {
|
|
10
10
|
DEFAULT_RUN_HISTORY_LIMIT,
|
|
11
11
|
IntelligenceService,
|
|
@@ -66,7 +66,7 @@ import {
|
|
|
66
66
|
schedules,
|
|
67
67
|
trafficSources,
|
|
68
68
|
usageCounters
|
|
69
|
-
} from "./chunk-
|
|
69
|
+
} from "./chunk-BN2VQDZ2.js";
|
|
70
70
|
import {
|
|
71
71
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
72
72
|
AGENT_PROVIDER_IDS,
|
|
@@ -81,12 +81,14 @@ import {
|
|
|
81
81
|
RunKinds,
|
|
82
82
|
RunStatuses,
|
|
83
83
|
RunTriggers,
|
|
84
|
+
SchedulableRunKinds,
|
|
84
85
|
TrafficEventConfidences,
|
|
85
86
|
TrafficEventKinds,
|
|
86
87
|
TrafficEvidenceKinds,
|
|
87
88
|
TrafficSourceAuthModes,
|
|
88
89
|
TrafficSourceStatuses,
|
|
89
90
|
TrafficSourceTypes,
|
|
91
|
+
VerificationStatuses,
|
|
90
92
|
absolutizeProjectUrl,
|
|
91
93
|
actionConfidenceLabel,
|
|
92
94
|
agentBusy,
|
|
@@ -105,6 +107,8 @@ import {
|
|
|
105
107
|
dedupeReportActions,
|
|
106
108
|
dedupeReportOpportunities,
|
|
107
109
|
deliveryFailed,
|
|
110
|
+
deltaPercent,
|
|
111
|
+
deltaTone,
|
|
108
112
|
determineAnswerMentioned,
|
|
109
113
|
effectiveDomains,
|
|
110
114
|
emptyCitationVisibility,
|
|
@@ -112,6 +116,7 @@ import {
|
|
|
112
116
|
findDuplicateLocationLabels,
|
|
113
117
|
formatDate,
|
|
114
118
|
formatDateRange,
|
|
119
|
+
formatDeltaCopy,
|
|
115
120
|
formatIsoDate,
|
|
116
121
|
formatNumber,
|
|
117
122
|
formatRatio,
|
|
@@ -144,6 +149,7 @@ import {
|
|
|
144
149
|
runInProgress,
|
|
145
150
|
runNotCancellable,
|
|
146
151
|
runTriggerRequestSchema,
|
|
152
|
+
schedulableRunKindSchema,
|
|
147
153
|
scheduleUpsertRequestSchema,
|
|
148
154
|
serializeRunError,
|
|
149
155
|
snapshotRequestSchema,
|
|
@@ -153,7 +159,7 @@ import {
|
|
|
153
159
|
visibilityStateFromAnswerMentioned,
|
|
154
160
|
windowCutoff,
|
|
155
161
|
wordpressEnvSchema
|
|
156
|
-
} from "./chunk-
|
|
162
|
+
} from "./chunk-SBZTDECX.js";
|
|
157
163
|
|
|
158
164
|
// src/telemetry.ts
|
|
159
165
|
import crypto from "crypto";
|
|
@@ -312,7 +318,7 @@ import crypto31 from "crypto";
|
|
|
312
318
|
import fs12 from "fs";
|
|
313
319
|
import path14 from "path";
|
|
314
320
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
315
|
-
import { eq as
|
|
321
|
+
import { eq as eq36 } from "drizzle-orm";
|
|
316
322
|
import Fastify from "fastify";
|
|
317
323
|
|
|
318
324
|
// ../api-routes/src/auth.ts
|
|
@@ -1436,7 +1442,7 @@ function loadRunDetail(app, run) {
|
|
|
1436
1442
|
|
|
1437
1443
|
// ../api-routes/src/apply.ts
|
|
1438
1444
|
import crypto10 from "crypto";
|
|
1439
|
-
import { eq as eq8 } from "drizzle-orm";
|
|
1445
|
+
import { and as and2, eq as eq8 } from "drizzle-orm";
|
|
1440
1446
|
|
|
1441
1447
|
// ../api-routes/src/schedule-utils.ts
|
|
1442
1448
|
var DAY_MAP = {
|
|
@@ -1841,8 +1847,9 @@ async function applyRoutes(app, opts) {
|
|
|
1841
1847
|
entityType: "competitor",
|
|
1842
1848
|
diff: { competitors: normalizedCompetitors }
|
|
1843
1849
|
});
|
|
1850
|
+
const AV_KIND = SchedulableRunKinds["answer-visibility"];
|
|
1844
1851
|
if (resolvedSchedule) {
|
|
1845
|
-
const existingSched = tx.select().from(schedules).where(eq8(schedules.projectId, projectId)).get();
|
|
1852
|
+
const existingSched = tx.select().from(schedules).where(and2(eq8(schedules.projectId, projectId), eq8(schedules.kind, AV_KIND))).get();
|
|
1846
1853
|
if (existingSched) {
|
|
1847
1854
|
tx.update(schedules).set({
|
|
1848
1855
|
cronExpr: resolvedSchedule.cronExpr,
|
|
@@ -1856,6 +1863,7 @@ async function applyRoutes(app, opts) {
|
|
|
1856
1863
|
tx.insert(schedules).values({
|
|
1857
1864
|
id: crypto10.randomUUID(),
|
|
1858
1865
|
projectId,
|
|
1866
|
+
kind: AV_KIND,
|
|
1859
1867
|
cronExpr: resolvedSchedule.cronExpr,
|
|
1860
1868
|
preset: resolvedSchedule.preset,
|
|
1861
1869
|
timezone: resolvedSchedule.timezone,
|
|
@@ -1867,9 +1875,9 @@ async function applyRoutes(app, opts) {
|
|
|
1867
1875
|
}
|
|
1868
1876
|
scheduleAction = "upsert";
|
|
1869
1877
|
} else if (deleteSchedule) {
|
|
1870
|
-
const existingSched = tx.select().from(schedules).where(eq8(schedules.projectId, projectId)).get();
|
|
1878
|
+
const existingSched = tx.select().from(schedules).where(and2(eq8(schedules.projectId, projectId), eq8(schedules.kind, AV_KIND))).get();
|
|
1871
1879
|
if (existingSched) {
|
|
1872
|
-
tx.delete(schedules).where(eq8(schedules.projectId, projectId)).run();
|
|
1880
|
+
tx.delete(schedules).where(and2(eq8(schedules.projectId, projectId), eq8(schedules.kind, AV_KIND))).run();
|
|
1873
1881
|
scheduleAction = "delete";
|
|
1874
1882
|
}
|
|
1875
1883
|
}
|
|
@@ -1897,7 +1905,7 @@ async function applyRoutes(app, opts) {
|
|
|
1897
1905
|
}
|
|
1898
1906
|
});
|
|
1899
1907
|
if (scheduleAction) {
|
|
1900
|
-
opts?.onScheduleUpdated?.(scheduleAction, projectId);
|
|
1908
|
+
opts?.onScheduleUpdated?.(scheduleAction, projectId, SchedulableRunKinds["answer-visibility"]);
|
|
1901
1909
|
}
|
|
1902
1910
|
if (!hasNotifications) {
|
|
1903
1911
|
opts?.onProjectUpserted?.(projectId, config.metadata.name);
|
|
@@ -2559,7 +2567,7 @@ function buildCategoryCounts(counts) {
|
|
|
2559
2567
|
}
|
|
2560
2568
|
|
|
2561
2569
|
// ../api-routes/src/intelligence.ts
|
|
2562
|
-
import { eq as eq11, desc as desc4, and as
|
|
2570
|
+
import { eq as eq11, desc as desc4, and as and3 } from "drizzle-orm";
|
|
2563
2571
|
function emptyHealthSnapshot(projectId) {
|
|
2564
2572
|
return {
|
|
2565
2573
|
id: `no-data:${projectId}`,
|
|
@@ -2610,7 +2618,7 @@ async function intelligenceRoutes(app) {
|
|
|
2610
2618
|
if (request.query.runId) {
|
|
2611
2619
|
conditions.push(eq11(insights.runId, request.query.runId));
|
|
2612
2620
|
}
|
|
2613
|
-
const rows = app.db.select().from(insights).where(conditions.length === 1 ? conditions[0] :
|
|
2621
|
+
const rows = app.db.select().from(insights).where(conditions.length === 1 ? conditions[0] : and3(...conditions)).orderBy(desc4(insights.createdAt)).all();
|
|
2614
2622
|
const showDismissed = request.query.dismissed === "true";
|
|
2615
2623
|
const result = rows.filter((r) => showDismissed || !r.dismissed).map(mapInsightRow);
|
|
2616
2624
|
return reply.send(result);
|
|
@@ -2650,7 +2658,7 @@ async function intelligenceRoutes(app) {
|
|
|
2650
2658
|
}
|
|
2651
2659
|
|
|
2652
2660
|
// ../api-routes/src/report.ts
|
|
2653
|
-
import { and as
|
|
2661
|
+
import { and as and5, desc as desc6, eq as eq13, gte, inArray as inArray4, lt, lte, ne, or as or2, sql as sql3 } from "drizzle-orm";
|
|
2654
2662
|
|
|
2655
2663
|
// ../api-routes/src/report-renderer.ts
|
|
2656
2664
|
var COLORS = {
|
|
@@ -4264,11 +4272,151 @@ function renderAiReferrals(report) {
|
|
|
4264
4272
|
</div>`
|
|
4265
4273
|
);
|
|
4266
4274
|
}
|
|
4275
|
+
function serverActivityHeading(audience, hasData) {
|
|
4276
|
+
const isClient = audience === "client";
|
|
4277
|
+
return {
|
|
4278
|
+
id: "server-activity",
|
|
4279
|
+
eyebrow: isClient ? "AI engine attention" : "Section 10",
|
|
4280
|
+
title: "AI Visibility \u2014 Server-Side",
|
|
4281
|
+
intro: isClient ? hasData ? "What AI engines actually do in your server logs over the last 7 days \u2014 the other half of citations." : "Live telemetry from your server logs." : "What AI engines actually do in your server logs \u2014 direct evidence, complementary to citations (which measure what they say)."
|
|
4282
|
+
};
|
|
4283
|
+
}
|
|
4284
|
+
function renderServerActivity(report, audience) {
|
|
4285
|
+
const sa = report.serverActivity;
|
|
4286
|
+
const isClient = audience === "client";
|
|
4287
|
+
if (!sa) {
|
|
4288
|
+
if (isClient) return "";
|
|
4289
|
+
return section(
|
|
4290
|
+
serverActivityHeading("agency", false),
|
|
4291
|
+
renderEmpty("Connect a server-side traffic source to surface what AI engines do directly in your server logs \u2014 distinct from GA4 click-throughs.")
|
|
4292
|
+
);
|
|
4293
|
+
}
|
|
4294
|
+
if (!sa.hasData) {
|
|
4295
|
+
return section(
|
|
4296
|
+
serverActivityHeading(audience, false),
|
|
4297
|
+
renderEmpty(isClient ? "Your server-side traffic source is connected. Numbers will appear after the next sync." : "Source connected \u2014 collecting your first data. Numbers will appear after the next sync.")
|
|
4298
|
+
);
|
|
4299
|
+
}
|
|
4300
|
+
const formatDelta = (d, suffix) => {
|
|
4301
|
+
const copy = formatDeltaCopy(d, suffix);
|
|
4302
|
+
if (!copy) return "";
|
|
4303
|
+
return `<span class="tone-${deltaTone(d.deltaPct)}">${escapeHtml(copy)}</span>`;
|
|
4304
|
+
};
|
|
4305
|
+
if (isClient) {
|
|
4306
|
+
const clientOperators = sa.byOperator.filter((o) => o.verifiedHits > 0 || o.referralArrivals > 0).slice(0, 5);
|
|
4307
|
+
const clientOperatorRows = clientOperators.map((o) => `
|
|
4308
|
+
<tr>
|
|
4309
|
+
<td>${escapeHtml(o.operator)}</td>
|
|
4310
|
+
<td class="numeric">${formatNumber(o.verifiedHits)}</td>
|
|
4311
|
+
<td class="numeric">${formatNumber(o.referralArrivals)}</td>
|
|
4312
|
+
</tr>`).join("");
|
|
4313
|
+
return section(
|
|
4314
|
+
serverActivityHeading("client", true),
|
|
4315
|
+
`<div class="metric-grid">
|
|
4316
|
+
<div class="metric">
|
|
4317
|
+
<div class="label">AI bots visited your site</div>
|
|
4318
|
+
<div class="value">${formatNumber(sa.verifiedCrawlerHits.current)}</div>
|
|
4319
|
+
<div class="subtitle">${formatDelta(sa.verifiedCrawlerHits, "crawls")}</div>
|
|
4320
|
+
</div>
|
|
4321
|
+
<div class="metric">
|
|
4322
|
+
<div class="label">People clicked through from AI</div>
|
|
4323
|
+
<div class="value">${formatNumber(sa.referralArrivals.current)}</div>
|
|
4324
|
+
<div class="subtitle">${formatDelta(sa.referralArrivals, "arrivals")}</div>
|
|
4325
|
+
</div>
|
|
4326
|
+
</div>
|
|
4327
|
+
${clientOperatorRows ? `<div class="chart-card"><h3>By AI tool</h3>
|
|
4328
|
+
<table class="report-table">
|
|
4329
|
+
<thead><tr><th>AI tool</th><th class="numeric">Bot visits (7d)</th><th class="numeric">Click-throughs</th></tr></thead>
|
|
4330
|
+
<tbody>${clientOperatorRows}</tbody>
|
|
4331
|
+
</table>
|
|
4332
|
+
<p class="meta">Verified visits only. We confirm each bot via reverse-DNS so the numbers above can't be inflated by anyone faking a user agent.</p>
|
|
4333
|
+
</div>` : ""}`
|
|
4334
|
+
);
|
|
4335
|
+
}
|
|
4336
|
+
const operatorRows = sa.byOperator.map((o) => {
|
|
4337
|
+
const deltaText = o.deltaPct === null ? "\u2014" : `${o.deltaPct > 0 ? "+" : ""}${o.deltaPct}%`;
|
|
4338
|
+
const toneClass = o.deltaPct === null ? "" : `tone-${deltaTone(o.deltaPct)}`;
|
|
4339
|
+
return `
|
|
4340
|
+
<tr>
|
|
4341
|
+
<td>${escapeHtml(o.operator)}</td>
|
|
4342
|
+
<td class="numeric">${formatNumber(o.verifiedHits)}</td>
|
|
4343
|
+
<td class="numeric meta">${formatNumber(o.unverifiedHits)}</td>
|
|
4344
|
+
<td class="numeric">${formatNumber(o.referralArrivals)}</td>
|
|
4345
|
+
<td class="numeric ${toneClass}">${deltaText}</td>
|
|
4346
|
+
</tr>`;
|
|
4347
|
+
}).join("");
|
|
4348
|
+
const pathRows = sa.topCrawledPaths.map((p) => `
|
|
4349
|
+
<tr>
|
|
4350
|
+
<td class="page-cell">${formatLandingPageHtml(p.path)}</td>
|
|
4351
|
+
<td class="numeric">${formatNumber(p.verifiedHits)}</td>
|
|
4352
|
+
<td class="numeric">${p.distinctOperators}</td>
|
|
4353
|
+
</tr>`).join("");
|
|
4354
|
+
const referralProductRows = sa.referralProducts.map((p) => `
|
|
4355
|
+
<tr>
|
|
4356
|
+
<td>${escapeHtml(p.product)}</td>
|
|
4357
|
+
<td class="numeric">${formatNumber(p.arrivals)}</td>
|
|
4358
|
+
<td class="numeric">${p.distinctLandingPaths}</td>
|
|
4359
|
+
</tr>`).join("");
|
|
4360
|
+
const referralLandingRows = sa.topReferralLandingPaths.map((p) => `
|
|
4361
|
+
<tr>
|
|
4362
|
+
<td class="page-cell">${formatLandingPageHtml(p.path)}</td>
|
|
4363
|
+
<td class="numeric">${formatNumber(p.arrivals)}</td>
|
|
4364
|
+
<td class="numeric">${p.distinctProducts}</td>
|
|
4365
|
+
</tr>`).join("");
|
|
4366
|
+
const trendChart = sa.dailyTrend.length > 0 ? renderLineChart(
|
|
4367
|
+
sa.dailyTrend.map((d) => ({ x: d.date, y: d.verifiedCrawlerHits, label: d.date.slice(5) })),
|
|
4368
|
+
COLORS.series[1],
|
|
4369
|
+
"Verified crawler hits over time (last 14 days)"
|
|
4370
|
+
) : "";
|
|
4371
|
+
return section(
|
|
4372
|
+
serverActivityHeading("agency", true),
|
|
4373
|
+
`<div class="metric-grid">
|
|
4374
|
+
<div class="metric">
|
|
4375
|
+
<div class="label">Verified crawler hits (7d)</div>
|
|
4376
|
+
<div class="value">${formatNumber(sa.verifiedCrawlerHits.current)}</div>
|
|
4377
|
+
<div class="subtitle">${formatDelta(sa.verifiedCrawlerHits, "hits")}</div>
|
|
4378
|
+
</div>
|
|
4379
|
+
<div class="metric">
|
|
4380
|
+
<div class="label">AI-referral arrivals (7d)</div>
|
|
4381
|
+
<div class="value">${formatNumber(sa.referralArrivals.current)}</div>
|
|
4382
|
+
<div class="subtitle">${formatDelta(sa.referralArrivals, "arrivals")}</div>
|
|
4383
|
+
</div>
|
|
4384
|
+
</div>
|
|
4385
|
+
${trendChart}
|
|
4386
|
+
${operatorRows ? `<div class="chart-card"><h3>Per AI operator</h3>
|
|
4387
|
+
<p class="meta">Verified means rDNS-confirmed. Unverified bots claim the user-agent but couldn't be verified \u2014 could be the real bot or an imitator.</p>
|
|
4388
|
+
<table class="report-table">
|
|
4389
|
+
<thead><tr><th>Operator</th><th class="numeric">Verified hits</th><th class="numeric">Unverified</th><th class="numeric">Referral arrivals</th><th class="numeric">7d delta</th></tr></thead>
|
|
4390
|
+
<tbody>${operatorRows}</tbody>
|
|
4391
|
+
</table>
|
|
4392
|
+
</div>` : ""}
|
|
4393
|
+
${pathRows ? `<div class="chart-card"><h3>Top crawled paths</h3>
|
|
4394
|
+
<p class="meta">Pages AI bots fetched most often (verified only, last 7d).</p>
|
|
4395
|
+
<table class="report-table">
|
|
4396
|
+
<thead><tr><th>Path</th><th class="numeric">Verified hits</th><th class="numeric">Distinct operators</th></tr></thead>
|
|
4397
|
+
<tbody>${pathRows}</tbody>
|
|
4398
|
+
</table>
|
|
4399
|
+
</div>` : ""}
|
|
4400
|
+
${referralProductRows ? `<div class="chart-card"><h3>Click-throughs by AI product</h3>
|
|
4401
|
+
<p class="meta">Where humans landed coming from each AI product (chatgpt.com, claude.ai, \u2026).</p>
|
|
4402
|
+
<table class="report-table">
|
|
4403
|
+
<thead><tr><th>Product</th><th class="numeric">Arrivals</th><th class="numeric">Distinct landing paths</th></tr></thead>
|
|
4404
|
+
<tbody>${referralProductRows}</tbody>
|
|
4405
|
+
</table>
|
|
4406
|
+
</div>` : ""}
|
|
4407
|
+
${referralLandingRows ? `<div class="chart-card"><h3>Top AI-referral landing paths</h3>
|
|
4408
|
+
<table class="report-table">
|
|
4409
|
+
<thead><tr><th>Path</th><th class="numeric">Arrivals</th><th class="numeric">Distinct products</th></tr></thead>
|
|
4410
|
+
<tbody>${referralLandingRows}</tbody>
|
|
4411
|
+
</table>
|
|
4412
|
+
</div>` : ""}`
|
|
4413
|
+
);
|
|
4414
|
+
}
|
|
4267
4415
|
function renderIndexingHealth(report) {
|
|
4268
4416
|
const ih = report.indexingHealth;
|
|
4269
4417
|
if (!ih) {
|
|
4270
4418
|
return section(
|
|
4271
|
-
{ id: "indexing-health", eyebrow: "Section
|
|
4419
|
+
{ id: "indexing-health", eyebrow: "Section 11", title: "Indexing Health" },
|
|
4272
4420
|
renderEmpty("Connect Google Search Console or Bing Webmaster Tools and run a sitemap inspection.")
|
|
4273
4421
|
);
|
|
4274
4422
|
}
|
|
@@ -4290,7 +4438,7 @@ function renderIndexingHealth(report) {
|
|
|
4290
4438
|
}).join("");
|
|
4291
4439
|
const legend = segments.map((s) => `<span><span class="legend-swatch" style="background:${s.color}"></span>${escapeHtml(s.label)}: ${s.count}</span>`).join("");
|
|
4292
4440
|
return section(
|
|
4293
|
-
{ id: "indexing-health", eyebrow: "Section
|
|
4441
|
+
{ id: "indexing-health", eyebrow: "Section 11", title: "Indexing Health", intro: `Pages absent from ${ih.provider === "google" ? "Google" : "Bing"} are harder for AI engines to retrieve.` },
|
|
4294
4442
|
`<div class="metric-grid">
|
|
4295
4443
|
<div class="metric"><div class="label">Indexed</div><div class="value tone-positive">${formatNumber(ih.indexed)}</div></div>
|
|
4296
4444
|
<div class="metric"><div class="label">Total inspected</div><div class="value">${formatNumber(ih.total)}</div></div>
|
|
@@ -4307,13 +4455,13 @@ function renderCitationsTrend(report) {
|
|
|
4307
4455
|
const trend = report.citationsTrend;
|
|
4308
4456
|
if (trend.length === 0) {
|
|
4309
4457
|
return section(
|
|
4310
|
-
{ id: "citations-trend", eyebrow: "Section
|
|
4458
|
+
{ id: "citations-trend", eyebrow: "Section 12", title: "Citations Over Time" },
|
|
4311
4459
|
renderEmpty("Run multiple checks to see a trend.")
|
|
4312
4460
|
);
|
|
4313
4461
|
}
|
|
4314
4462
|
if (isTrendBaseline(trend)) {
|
|
4315
4463
|
return section(
|
|
4316
|
-
{ id: "citations-trend", eyebrow: "Section
|
|
4464
|
+
{ id: "citations-trend", eyebrow: "Section 12", title: "Citations Over Time" },
|
|
4317
4465
|
renderEmpty(`Building baseline (${trend.length} of ${MIN_TREND_POINTS} checks completed). Trend will appear once more checks are recorded.`)
|
|
4318
4466
|
);
|
|
4319
4467
|
}
|
|
@@ -4330,7 +4478,7 @@ function renderCitationsTrend(report) {
|
|
|
4330
4478
|
<td>${t.providerRates.map((r) => `${escapeHtml(r.provider)}: ${r.citationRate}%`).join(" \xB7 ")}</td>
|
|
4331
4479
|
</tr>`).join("");
|
|
4332
4480
|
return section(
|
|
4333
|
-
{ id: "citations-trend", eyebrow: "Section
|
|
4481
|
+
{ id: "citations-trend", eyebrow: "Section 12", title: "Citations Over Time", intro: "Citation coverage across recent checks." },
|
|
4334
4482
|
`${chart}
|
|
4335
4483
|
<div class="chart-card"><h3>Check-by-check breakdown</h3>
|
|
4336
4484
|
<table class="report-table">
|
|
@@ -4344,7 +4492,7 @@ function renderInsights(report) {
|
|
|
4344
4492
|
const list = report.insights;
|
|
4345
4493
|
if (list.length === 0) {
|
|
4346
4494
|
return section(
|
|
4347
|
-
{ id: "insights", eyebrow: "Section
|
|
4495
|
+
{ id: "insights", eyebrow: "Section 13", title: "Insights & Alerts" },
|
|
4348
4496
|
renderEmpty("No insights yet \u2014 run a check to generate alerts.")
|
|
4349
4497
|
);
|
|
4350
4498
|
}
|
|
@@ -4361,7 +4509,7 @@ function renderInsights(report) {
|
|
|
4361
4509
|
</tr>`;
|
|
4362
4510
|
}).join("");
|
|
4363
4511
|
return section(
|
|
4364
|
-
{ id: "insights", eyebrow: "Section
|
|
4512
|
+
{ id: "insights", eyebrow: "Section 13", title: "Insights & Alerts", intro: "Regressions, gains, and recurring alerts ordered by severity." },
|
|
4365
4513
|
`<table class="report-table insights-table">
|
|
4366
4514
|
<thead><tr>
|
|
4367
4515
|
<th class="col-severity">Severity</th>
|
|
@@ -4403,7 +4551,7 @@ function renderOpportunities(report) {
|
|
|
4403
4551
|
return section(
|
|
4404
4552
|
{
|
|
4405
4553
|
id: "content-opportunities",
|
|
4406
|
-
eyebrow: "Section
|
|
4554
|
+
eyebrow: "Section 14",
|
|
4407
4555
|
title: "Content Opportunities",
|
|
4408
4556
|
intro: "Queries where content work has the clearest path to more AI citations. Opportunity score is 0\u2013100, higher = stronger."
|
|
4409
4557
|
},
|
|
@@ -4429,7 +4577,7 @@ function renderContentGaps(report) {
|
|
|
4429
4577
|
return section(
|
|
4430
4578
|
{
|
|
4431
4579
|
id: "content-gaps",
|
|
4432
|
-
eyebrow: "Section
|
|
4580
|
+
eyebrow: "Section 15",
|
|
4433
4581
|
title: "Content Gaps",
|
|
4434
4582
|
intro: "Tracked queries where competitors are cited and the client is missing."
|
|
4435
4583
|
},
|
|
@@ -4443,7 +4591,7 @@ function renderRecommendedNextSteps(report) {
|
|
|
4443
4591
|
const steps = report.recommendedNextSteps;
|
|
4444
4592
|
if (steps.length === 0) {
|
|
4445
4593
|
return section(
|
|
4446
|
-
{ id: "recommended-next-steps", eyebrow: "Section
|
|
4594
|
+
{ id: "recommended-next-steps", eyebrow: "Section 16", title: "Recommended Next Steps", intro: "Action items bucketed by timing." },
|
|
4447
4595
|
renderEmpty("No outstanding actions.")
|
|
4448
4596
|
);
|
|
4449
4597
|
}
|
|
@@ -4454,7 +4602,7 @@ function renderRecommendedNextSteps(report) {
|
|
|
4454
4602
|
<span class="rationale">${escapeHtml(s.rationale)}</span>
|
|
4455
4603
|
</div>`).join("");
|
|
4456
4604
|
return section(
|
|
4457
|
-
{ id: "recommended-next-steps", eyebrow: "Section
|
|
4605
|
+
{ id: "recommended-next-steps", eyebrow: "Section 16", title: "Recommended Next Steps", intro: "Action items bucketed by timing." },
|
|
4458
4606
|
`<div class="steps">${items}</div>`
|
|
4459
4607
|
);
|
|
4460
4608
|
}
|
|
@@ -4679,6 +4827,10 @@ function renderReportHtml(report, opts = {}) {
|
|
|
4679
4827
|
const sections = audience === "client" ? [
|
|
4680
4828
|
renderClientSummary(report),
|
|
4681
4829
|
renderWhatsChanged(report, "client"),
|
|
4830
|
+
// Server-side AI visibility runs between WhatsChanged and the action
|
|
4831
|
+
// plan in BOTH the SPA and HTML so clients see the same ordered set
|
|
4832
|
+
// of sections in either surface (per the report-parity rule).
|
|
4833
|
+
renderServerActivity(report, "client"),
|
|
4682
4834
|
renderAudienceActionPlan(report, "client"),
|
|
4683
4835
|
renderClientEvidenceSummary(report)
|
|
4684
4836
|
].join("\n") : [
|
|
@@ -4693,6 +4845,7 @@ function renderReportHtml(report, opts = {}) {
|
|
|
4693
4845
|
renderGa(report),
|
|
4694
4846
|
renderSocial(report),
|
|
4695
4847
|
renderAiReferrals(report),
|
|
4848
|
+
renderServerActivity(report, "agency"),
|
|
4696
4849
|
renderIndexingHealth(report),
|
|
4697
4850
|
renderCitationsTrend(report),
|
|
4698
4851
|
renderInsights(report),
|
|
@@ -4725,7 +4878,7 @@ function renderReportHtml(report, opts = {}) {
|
|
|
4725
4878
|
}
|
|
4726
4879
|
|
|
4727
4880
|
// ../api-routes/src/content-data.ts
|
|
4728
|
-
import { and as
|
|
4881
|
+
import { and as and4, eq as eq12, desc as desc5, inArray as inArray3 } from "drizzle-orm";
|
|
4729
4882
|
var RECENT_RUNS_WINDOW = 5;
|
|
4730
4883
|
function loadOrchestratorInput(db, project, locationFilter = void 0) {
|
|
4731
4884
|
const projectId = project.id;
|
|
@@ -4849,7 +5002,7 @@ function listCompetitorDomains(db, projectId) {
|
|
|
4849
5002
|
}
|
|
4850
5003
|
function listRecentAnswerVisibilityRunIds(db, projectId, limit, locationFilter) {
|
|
4851
5004
|
const rows = db.select({ id: runs.id, location: runs.location }).from(runs).where(
|
|
4852
|
-
|
|
5005
|
+
and4(
|
|
4853
5006
|
eq12(runs.projectId, projectId),
|
|
4854
5007
|
eq12(runs.kind, RunKinds["answer-visibility"]),
|
|
4855
5008
|
// Queued/running/failed/cancelled runs may have partial or no
|
|
@@ -5078,6 +5231,9 @@ var TOP_AI_REFERRAL_PAGES_LIMIT = 10;
|
|
|
5078
5231
|
var TOP_CAMPAIGN_LIMIT = 10;
|
|
5079
5232
|
var INSIGHT_LOOKBACK_RUNS = 5;
|
|
5080
5233
|
var REPORT_WINDOW_DAYS = 30;
|
|
5234
|
+
var SERVER_ACTIVITY_HEADLINE_DAYS = 7;
|
|
5235
|
+
var SERVER_ACTIVITY_TREND_DAYS = 14;
|
|
5236
|
+
var SERVER_ACTIVITY_TOP_PATHS_LIMIT = 10;
|
|
5081
5237
|
function windowStartDate(endDate, windowDays) {
|
|
5082
5238
|
const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(endDate);
|
|
5083
5239
|
if (!m) return endDate;
|
|
@@ -5193,7 +5349,7 @@ function buildGscSection(db, projectId, projectDisplayName, canonicalDomain, tra
|
|
|
5193
5349
|
}
|
|
5194
5350
|
function buildGaSection(db, projectId) {
|
|
5195
5351
|
const windowSummary = db.select().from(gaTrafficWindowSummaries).where(
|
|
5196
|
-
|
|
5352
|
+
and5(
|
|
5197
5353
|
eq13(gaTrafficWindowSummaries.projectId, projectId),
|
|
5198
5354
|
eq13(gaTrafficWindowSummaries.windowKey, "30d")
|
|
5199
5355
|
)
|
|
@@ -5346,6 +5502,209 @@ function buildAiReferrals(db, projectId) {
|
|
|
5346
5502
|
const topLandingPages = [...pageAgg.entries()].map(([page, data]) => ({ page, sessions: data.sessions, users: data.users })).sort((a, b) => b.sessions - a.sessions).slice(0, TOP_AI_REFERRAL_PAGES_LIMIT);
|
|
5347
5503
|
return { totalSessions: total, totalUsers, bySource, trend, topLandingPages };
|
|
5348
5504
|
}
|
|
5505
|
+
function buildServerActivity(db, projectId) {
|
|
5506
|
+
const sourceRows = db.select({ id: trafficSources.id }).from(trafficSources).where(
|
|
5507
|
+
and5(
|
|
5508
|
+
eq13(trafficSources.projectId, projectId),
|
|
5509
|
+
ne(trafficSources.status, TrafficSourceStatuses.archived)
|
|
5510
|
+
)
|
|
5511
|
+
).all();
|
|
5512
|
+
if (sourceRows.length === 0) return null;
|
|
5513
|
+
const now = /* @__PURE__ */ new Date();
|
|
5514
|
+
const headlineEnd = now.toISOString();
|
|
5515
|
+
const headlineStartMs = now.getTime() - SERVER_ACTIVITY_HEADLINE_DAYS * 24 * 60 * 6e4;
|
|
5516
|
+
const priorStartMs = headlineStartMs - SERVER_ACTIVITY_HEADLINE_DAYS * 24 * 60 * 6e4;
|
|
5517
|
+
const trendStartMs = now.getTime() - SERVER_ACTIVITY_TREND_DAYS * 24 * 60 * 6e4;
|
|
5518
|
+
const headlineStart = new Date(headlineStartMs).toISOString();
|
|
5519
|
+
const priorStart = new Date(priorStartMs).toISOString();
|
|
5520
|
+
const trendStart = new Date(trendStartMs).toISOString();
|
|
5521
|
+
const sumVerifiedCrawlers = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
5522
|
+
db.select({ total: sql3`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
5523
|
+
and5(
|
|
5524
|
+
eq13(crawlerEventsHourly.projectId, projectId),
|
|
5525
|
+
eq13(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
5526
|
+
gte(crawlerEventsHourly.tsHour, windowStartIso),
|
|
5527
|
+
exclusiveEnd ? lt(crawlerEventsHourly.tsHour, windowEndIso) : lte(crawlerEventsHourly.tsHour, windowEndIso)
|
|
5528
|
+
)
|
|
5529
|
+
).get()?.total ?? 0
|
|
5530
|
+
);
|
|
5531
|
+
const sumReferrals = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
5532
|
+
db.select({ total: sql3`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
5533
|
+
and5(
|
|
5534
|
+
eq13(aiReferralEventsHourly.projectId, projectId),
|
|
5535
|
+
gte(aiReferralEventsHourly.tsHour, windowStartIso),
|
|
5536
|
+
exclusiveEnd ? lt(aiReferralEventsHourly.tsHour, windowEndIso) : lte(aiReferralEventsHourly.tsHour, windowEndIso)
|
|
5537
|
+
)
|
|
5538
|
+
).get()?.total ?? 0
|
|
5539
|
+
);
|
|
5540
|
+
const verifiedCurrent = sumVerifiedCrawlers(headlineStart, headlineEnd);
|
|
5541
|
+
const verifiedPrior = sumVerifiedCrawlers(priorStart, headlineStart, true);
|
|
5542
|
+
const referralCurrent = sumReferrals(headlineStart, headlineEnd);
|
|
5543
|
+
const referralPrior = sumReferrals(priorStart, headlineStart, true);
|
|
5544
|
+
const crawlerByOperatorRows = db.select({
|
|
5545
|
+
operator: crawlerEventsHourly.operator,
|
|
5546
|
+
verificationStatus: crawlerEventsHourly.verificationStatus,
|
|
5547
|
+
hits: sql3`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
5548
|
+
}).from(crawlerEventsHourly).where(
|
|
5549
|
+
and5(
|
|
5550
|
+
eq13(crawlerEventsHourly.projectId, projectId),
|
|
5551
|
+
gte(crawlerEventsHourly.tsHour, headlineStart),
|
|
5552
|
+
lte(crawlerEventsHourly.tsHour, headlineEnd)
|
|
5553
|
+
)
|
|
5554
|
+
).groupBy(crawlerEventsHourly.operator, crawlerEventsHourly.verificationStatus).all();
|
|
5555
|
+
const crawlerByOperatorPriorRows = db.select({
|
|
5556
|
+
operator: crawlerEventsHourly.operator,
|
|
5557
|
+
hits: sql3`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
5558
|
+
}).from(crawlerEventsHourly).where(
|
|
5559
|
+
and5(
|
|
5560
|
+
eq13(crawlerEventsHourly.projectId, projectId),
|
|
5561
|
+
eq13(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
5562
|
+
gte(crawlerEventsHourly.tsHour, priorStart),
|
|
5563
|
+
lt(crawlerEventsHourly.tsHour, headlineStart)
|
|
5564
|
+
)
|
|
5565
|
+
).groupBy(crawlerEventsHourly.operator).all();
|
|
5566
|
+
const referralByOperatorRows = db.select({
|
|
5567
|
+
operator: aiReferralEventsHourly.operator,
|
|
5568
|
+
hits: sql3`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`
|
|
5569
|
+
}).from(aiReferralEventsHourly).where(
|
|
5570
|
+
and5(
|
|
5571
|
+
eq13(aiReferralEventsHourly.projectId, projectId),
|
|
5572
|
+
gte(aiReferralEventsHourly.tsHour, headlineStart),
|
|
5573
|
+
lte(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
5574
|
+
)
|
|
5575
|
+
).groupBy(aiReferralEventsHourly.operator).all();
|
|
5576
|
+
const operatorAgg = /* @__PURE__ */ new Map();
|
|
5577
|
+
const ensureOp = (op) => {
|
|
5578
|
+
let entry = operatorAgg.get(op);
|
|
5579
|
+
if (!entry) {
|
|
5580
|
+
entry = { verified: 0, unverified: 0, referrals: 0, prior: 0 };
|
|
5581
|
+
operatorAgg.set(op, entry);
|
|
5582
|
+
}
|
|
5583
|
+
return entry;
|
|
5584
|
+
};
|
|
5585
|
+
for (const r of crawlerByOperatorRows) {
|
|
5586
|
+
const entry = ensureOp(r.operator);
|
|
5587
|
+
if (r.verificationStatus === VerificationStatuses.verified) entry.verified += Number(r.hits);
|
|
5588
|
+
else entry.unverified += Number(r.hits);
|
|
5589
|
+
}
|
|
5590
|
+
for (const r of crawlerByOperatorPriorRows) {
|
|
5591
|
+
ensureOp(r.operator).prior += Number(r.hits);
|
|
5592
|
+
}
|
|
5593
|
+
for (const r of referralByOperatorRows) {
|
|
5594
|
+
ensureOp(r.operator).referrals += Number(r.hits);
|
|
5595
|
+
}
|
|
5596
|
+
const byOperator = [...operatorAgg.entries()].map(([operator, v]) => ({
|
|
5597
|
+
operator,
|
|
5598
|
+
verifiedHits: v.verified,
|
|
5599
|
+
unverifiedHits: v.unverified,
|
|
5600
|
+
referralArrivals: v.referrals,
|
|
5601
|
+
deltaPct: deltaPercent(v.verified, v.prior)
|
|
5602
|
+
})).sort(
|
|
5603
|
+
(a, b) => b.verifiedHits - a.verifiedHits || b.referralArrivals - a.referralArrivals
|
|
5604
|
+
);
|
|
5605
|
+
const topPathsRows = db.select({
|
|
5606
|
+
path: crawlerEventsHourly.pathNormalized,
|
|
5607
|
+
hits: sql3`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
5608
|
+
operators: sql3`COUNT(DISTINCT ${crawlerEventsHourly.operator})`
|
|
5609
|
+
}).from(crawlerEventsHourly).where(
|
|
5610
|
+
and5(
|
|
5611
|
+
eq13(crawlerEventsHourly.projectId, projectId),
|
|
5612
|
+
eq13(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
5613
|
+
gte(crawlerEventsHourly.tsHour, headlineStart),
|
|
5614
|
+
lte(crawlerEventsHourly.tsHour, headlineEnd)
|
|
5615
|
+
)
|
|
5616
|
+
).groupBy(crawlerEventsHourly.pathNormalized).orderBy(desc6(sql3`SUM(${crawlerEventsHourly.hits})`)).limit(SERVER_ACTIVITY_TOP_PATHS_LIMIT).all();
|
|
5617
|
+
const topCrawledPaths = topPathsRows.map((r) => ({
|
|
5618
|
+
path: r.path,
|
|
5619
|
+
verifiedHits: Number(r.hits),
|
|
5620
|
+
distinctOperators: Number(r.operators)
|
|
5621
|
+
}));
|
|
5622
|
+
const referralProductsRows = db.select({
|
|
5623
|
+
product: aiReferralEventsHourly.product,
|
|
5624
|
+
arrivals: sql3`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
5625
|
+
landingPaths: sql3`COUNT(DISTINCT ${aiReferralEventsHourly.landingPathNormalized})`
|
|
5626
|
+
}).from(aiReferralEventsHourly).where(
|
|
5627
|
+
and5(
|
|
5628
|
+
eq13(aiReferralEventsHourly.projectId, projectId),
|
|
5629
|
+
gte(aiReferralEventsHourly.tsHour, headlineStart),
|
|
5630
|
+
lte(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
5631
|
+
)
|
|
5632
|
+
).groupBy(aiReferralEventsHourly.product).orderBy(desc6(sql3`SUM(${aiReferralEventsHourly.sessionsOrHits})`)).all();
|
|
5633
|
+
const referralProducts = referralProductsRows.map((r) => ({
|
|
5634
|
+
product: r.product,
|
|
5635
|
+
arrivals: Number(r.arrivals),
|
|
5636
|
+
distinctLandingPaths: Number(r.landingPaths)
|
|
5637
|
+
}));
|
|
5638
|
+
const topReferralRows = db.select({
|
|
5639
|
+
path: aiReferralEventsHourly.landingPathNormalized,
|
|
5640
|
+
arrivals: sql3`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
5641
|
+
products: sql3`COUNT(DISTINCT ${aiReferralEventsHourly.product})`
|
|
5642
|
+
}).from(aiReferralEventsHourly).where(
|
|
5643
|
+
and5(
|
|
5644
|
+
eq13(aiReferralEventsHourly.projectId, projectId),
|
|
5645
|
+
gte(aiReferralEventsHourly.tsHour, headlineStart),
|
|
5646
|
+
lte(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
5647
|
+
)
|
|
5648
|
+
).groupBy(aiReferralEventsHourly.landingPathNormalized).orderBy(desc6(sql3`SUM(${aiReferralEventsHourly.sessionsOrHits})`)).limit(SERVER_ACTIVITY_TOP_PATHS_LIMIT).all();
|
|
5649
|
+
const topReferralLandingPaths = topReferralRows.map((r) => ({
|
|
5650
|
+
path: r.path,
|
|
5651
|
+
arrivals: Number(r.arrivals),
|
|
5652
|
+
distinctProducts: Number(r.products)
|
|
5653
|
+
}));
|
|
5654
|
+
const crawlerTrendRows = db.select({
|
|
5655
|
+
date: sql3`SUBSTR(${crawlerEventsHourly.tsHour}, 1, 10)`,
|
|
5656
|
+
hits: sql3`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
5657
|
+
}).from(crawlerEventsHourly).where(
|
|
5658
|
+
and5(
|
|
5659
|
+
eq13(crawlerEventsHourly.projectId, projectId),
|
|
5660
|
+
eq13(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
5661
|
+
gte(crawlerEventsHourly.tsHour, trendStart),
|
|
5662
|
+
lte(crawlerEventsHourly.tsHour, headlineEnd)
|
|
5663
|
+
)
|
|
5664
|
+
).groupBy(sql3`SUBSTR(${crawlerEventsHourly.tsHour}, 1, 10)`).all();
|
|
5665
|
+
const referralTrendRows = db.select({
|
|
5666
|
+
date: sql3`SUBSTR(${aiReferralEventsHourly.tsHour}, 1, 10)`,
|
|
5667
|
+
hits: sql3`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`
|
|
5668
|
+
}).from(aiReferralEventsHourly).where(
|
|
5669
|
+
and5(
|
|
5670
|
+
eq13(aiReferralEventsHourly.projectId, projectId),
|
|
5671
|
+
gte(aiReferralEventsHourly.tsHour, trendStart),
|
|
5672
|
+
lte(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
5673
|
+
)
|
|
5674
|
+
).groupBy(sql3`SUBSTR(${aiReferralEventsHourly.tsHour}, 1, 10)`).all();
|
|
5675
|
+
const dailyTrendMap = /* @__PURE__ */ new Map();
|
|
5676
|
+
for (const r of crawlerTrendRows) {
|
|
5677
|
+
const e = dailyTrendMap.get(r.date) ?? { verifiedCrawlerHits: 0, referralArrivals: 0 };
|
|
5678
|
+
e.verifiedCrawlerHits += Number(r.hits);
|
|
5679
|
+
dailyTrendMap.set(r.date, e);
|
|
5680
|
+
}
|
|
5681
|
+
for (const r of referralTrendRows) {
|
|
5682
|
+
const e = dailyTrendMap.get(r.date) ?? { verifiedCrawlerHits: 0, referralArrivals: 0 };
|
|
5683
|
+
e.referralArrivals += Number(r.hits);
|
|
5684
|
+
dailyTrendMap.set(r.date, e);
|
|
5685
|
+
}
|
|
5686
|
+
const dailyTrend = [...dailyTrendMap.entries()].map(([date, v]) => ({ date, ...v })).sort((a, b) => a.date.localeCompare(b.date));
|
|
5687
|
+
return {
|
|
5688
|
+
windowStart: headlineStart,
|
|
5689
|
+
windowEnd: headlineEnd,
|
|
5690
|
+
hasData: verifiedCurrent + referralCurrent + verifiedPrior + referralPrior > 0 || byOperator.length > 0 || topCrawledPaths.length > 0 || referralProducts.length > 0,
|
|
5691
|
+
verifiedCrawlerHits: {
|
|
5692
|
+
current: verifiedCurrent,
|
|
5693
|
+
prior: verifiedPrior,
|
|
5694
|
+
deltaPct: deltaPercent(verifiedCurrent, verifiedPrior)
|
|
5695
|
+
},
|
|
5696
|
+
referralArrivals: {
|
|
5697
|
+
current: referralCurrent,
|
|
5698
|
+
prior: referralPrior,
|
|
5699
|
+
deltaPct: deltaPercent(referralCurrent, referralPrior)
|
|
5700
|
+
},
|
|
5701
|
+
byOperator,
|
|
5702
|
+
topCrawledPaths,
|
|
5703
|
+
referralProducts,
|
|
5704
|
+
dailyTrend,
|
|
5705
|
+
topReferralLandingPaths
|
|
5706
|
+
};
|
|
5707
|
+
}
|
|
5349
5708
|
function buildIndexingHealth(db, projectId) {
|
|
5350
5709
|
const gsc = db.select().from(gscCoverageSnapshots).where(eq13(gscCoverageSnapshots.projectId, projectId)).orderBy(desc6(gscCoverageSnapshots.date)).limit(1).get();
|
|
5351
5710
|
if (gsc) {
|
|
@@ -5376,7 +5735,7 @@ function buildIndexingHealth(db, projectId) {
|
|
|
5376
5735
|
return null;
|
|
5377
5736
|
}
|
|
5378
5737
|
function buildCitationsTrend(db, projectId, queryLookup, locationFilter) {
|
|
5379
|
-
const visibilityRuns = db.select().from(runs).where(
|
|
5738
|
+
const visibilityRuns = db.select().from(runs).where(and5(eq13(runs.projectId, projectId), eq13(runs.kind, RunKinds["answer-visibility"]))).all().filter((r) => locationFilter === void 0 || (r.location ?? null) === locationFilter);
|
|
5380
5739
|
const totalQueries = queryLookup.byId.size;
|
|
5381
5740
|
const points = [];
|
|
5382
5741
|
for (const run of visibilityRuns) {
|
|
@@ -5422,14 +5781,14 @@ function buildCitationsTrend(db, projectId, queryLookup, locationFilter) {
|
|
|
5422
5781
|
}
|
|
5423
5782
|
function buildInsightList(db, projectId, locationFilter) {
|
|
5424
5783
|
const recentRunIds = db.select({ id: runs.id, location: runs.location }).from(runs).where(
|
|
5425
|
-
|
|
5784
|
+
and5(
|
|
5426
5785
|
eq13(runs.projectId, projectId),
|
|
5427
5786
|
eq13(runs.kind, RunKinds["answer-visibility"]),
|
|
5428
5787
|
or2(eq13(runs.status, RunStatuses.completed), eq13(runs.status, RunStatuses.partial))
|
|
5429
5788
|
)
|
|
5430
5789
|
).orderBy(desc6(runs.createdAt)).all().filter((r) => locationFilter === void 0 || (r.location ?? null) === locationFilter).slice(0, INSIGHT_LOOKBACK_RUNS).map((r) => r.id);
|
|
5431
5790
|
if (recentRunIds.length === 0) return [];
|
|
5432
|
-
const rows = db.select().from(insights).where(
|
|
5791
|
+
const rows = db.select().from(insights).where(and5(eq13(insights.projectId, projectId), inArray4(insights.runId, recentRunIds))).orderBy(desc6(insights.createdAt)).all();
|
|
5433
5792
|
const severityRank = { critical: 0, high: 1, medium: 2, low: 3 };
|
|
5434
5793
|
const flat = rows.filter((r) => !r.dismissed).map((r) => {
|
|
5435
5794
|
const recommendation = parseJsonColumn(r.recommendation, null);
|
|
@@ -6005,6 +6364,7 @@ function buildProjectReport(db, projectName) {
|
|
|
6005
6364
|
const gaSection = buildGaSection(db, project.id);
|
|
6006
6365
|
const socialSection = buildSocialReferrals(db, project.id);
|
|
6007
6366
|
const aiReferralsSection = buildAiReferrals(db, project.id);
|
|
6367
|
+
const serverActivitySection = buildServerActivity(db, project.id);
|
|
6008
6368
|
const indexingHealthSection = buildIndexingHealth(db, project.id);
|
|
6009
6369
|
const citationsTrend = buildCitationsTrend(db, project.id, queryLookup, latestRunLocation);
|
|
6010
6370
|
const insightList = buildInsightList(db, project.id, latestRunLocation);
|
|
@@ -6148,6 +6508,7 @@ function buildProjectReport(db, projectName) {
|
|
|
6148
6508
|
ga: gaSection,
|
|
6149
6509
|
socialReferrals: socialSection,
|
|
6150
6510
|
aiReferrals: aiReferralsSection,
|
|
6511
|
+
serverActivity: serverActivitySection,
|
|
6151
6512
|
indexingHealth: indexingHealthSection,
|
|
6152
6513
|
citationsTrend,
|
|
6153
6514
|
whatsChanged,
|
|
@@ -6351,7 +6712,7 @@ function normalizeDomain2(domain) {
|
|
|
6351
6712
|
}
|
|
6352
6713
|
|
|
6353
6714
|
// ../api-routes/src/composites.ts
|
|
6354
|
-
import { eq as eq15, and as
|
|
6715
|
+
import { eq as eq15, and as and6, desc as desc7, sql as sql4, like, or as or3, inArray as inArray6 } from "drizzle-orm";
|
|
6355
6716
|
var TOP_INSIGHT_LIMIT = 5;
|
|
6356
6717
|
var SEARCH_HIT_HARD_LIMIT = 50;
|
|
6357
6718
|
var SEARCH_SNIPPET_RADIUS = 80;
|
|
@@ -6454,24 +6815,24 @@ async function compositeRoutes(app) {
|
|
|
6454
6815
|
rawResponse: querySnapshots.rawResponse,
|
|
6455
6816
|
createdAt: querySnapshots.createdAt
|
|
6456
6817
|
}).from(querySnapshots).innerJoin(queries, eq15(querySnapshots.queryId, queries.id)).where(
|
|
6457
|
-
|
|
6818
|
+
and6(
|
|
6458
6819
|
eq15(queries.projectId, project.id),
|
|
6459
6820
|
or3(
|
|
6460
|
-
|
|
6461
|
-
|
|
6462
|
-
|
|
6821
|
+
sql4`${querySnapshots.answerText} LIKE ${pattern} ESCAPE '\\'`,
|
|
6822
|
+
sql4`${querySnapshots.citedDomains} LIKE ${pattern} ESCAPE '\\'`,
|
|
6823
|
+
sql4`${querySnapshots.rawResponse} LIKE ${pattern} ESCAPE '\\'`,
|
|
6463
6824
|
like(queries.query, pattern)
|
|
6464
6825
|
)
|
|
6465
6826
|
)
|
|
6466
6827
|
).orderBy(desc7(querySnapshots.createdAt)).limit(limit + 1).all();
|
|
6467
6828
|
const insightMatches = app.db.select().from(insights).where(
|
|
6468
|
-
|
|
6829
|
+
and6(
|
|
6469
6830
|
eq15(insights.projectId, project.id),
|
|
6470
6831
|
or3(
|
|
6471
6832
|
like(insights.title, pattern),
|
|
6472
6833
|
like(insights.query, pattern),
|
|
6473
|
-
|
|
6474
|
-
|
|
6834
|
+
sql4`${insights.recommendation} LIKE ${pattern} ESCAPE '\\'`,
|
|
6835
|
+
sql4`${insights.cause} LIKE ${pattern} ESCAPE '\\'`
|
|
6475
6836
|
)
|
|
6476
6837
|
)
|
|
6477
6838
|
).orderBy(desc7(insights.createdAt)).limit(limit + 1).all();
|
|
@@ -7037,6 +7398,12 @@ var locationQueryParameter = {
|
|
|
7037
7398
|
description: "Filter by location label. Use an empty value to request locationless results.",
|
|
7038
7399
|
schema: stringSchema
|
|
7039
7400
|
};
|
|
7401
|
+
var scheduleKindQueryParameter = {
|
|
7402
|
+
name: "kind",
|
|
7403
|
+
in: "query",
|
|
7404
|
+
description: 'Schedulable run kind. Defaults to "answer-visibility" for backward compatibility.',
|
|
7405
|
+
schema: { type: "string", enum: ["answer-visibility", "traffic-sync"] }
|
|
7406
|
+
};
|
|
7040
7407
|
var reportAudienceQueryParameter = {
|
|
7041
7408
|
name: "audience",
|
|
7042
7409
|
in: "query",
|
|
@@ -7894,7 +8261,7 @@ var routeCatalog = [
|
|
|
7894
8261
|
path: "/api/v1/projects/{name}/schedule",
|
|
7895
8262
|
summary: "Create or update a schedule",
|
|
7896
8263
|
tags: ["schedules"],
|
|
7897
|
-
parameters: [nameParameter],
|
|
8264
|
+
parameters: [nameParameter, scheduleKindQueryParameter],
|
|
7898
8265
|
requestBody: {
|
|
7899
8266
|
required: true,
|
|
7900
8267
|
content: {
|
|
@@ -7902,11 +8269,13 @@ var routeCatalog = [
|
|
|
7902
8269
|
schema: {
|
|
7903
8270
|
type: "object",
|
|
7904
8271
|
properties: {
|
|
8272
|
+
kind: { type: "string", enum: ["answer-visibility", "traffic-sync"] },
|
|
7905
8273
|
preset: stringSchema,
|
|
7906
8274
|
cron: stringSchema,
|
|
7907
8275
|
timezone: stringSchema,
|
|
7908
8276
|
providers: stringArraySchema,
|
|
7909
|
-
enabled: booleanSchema
|
|
8277
|
+
enabled: booleanSchema,
|
|
8278
|
+
sourceId: stringSchema
|
|
7910
8279
|
}
|
|
7911
8280
|
}
|
|
7912
8281
|
}
|
|
@@ -7914,7 +8283,8 @@ var routeCatalog = [
|
|
|
7914
8283
|
},
|
|
7915
8284
|
responses: {
|
|
7916
8285
|
200: { description: "Schedule updated." },
|
|
7917
|
-
201: { description: "Schedule created." }
|
|
8286
|
+
201: { description: "Schedule created." },
|
|
8287
|
+
400: { description: "Invalid payload (e.g. sourceId missing for kind=traffic-sync, or providers set for kind=traffic-sync)." }
|
|
7918
8288
|
}
|
|
7919
8289
|
},
|
|
7920
8290
|
{
|
|
@@ -7922,7 +8292,7 @@ var routeCatalog = [
|
|
|
7922
8292
|
path: "/api/v1/projects/{name}/schedule",
|
|
7923
8293
|
summary: "Get a schedule",
|
|
7924
8294
|
tags: ["schedules"],
|
|
7925
|
-
parameters: [nameParameter],
|
|
8295
|
+
parameters: [nameParameter, scheduleKindQueryParameter],
|
|
7926
8296
|
responses: {
|
|
7927
8297
|
200: { description: "Schedule returned." },
|
|
7928
8298
|
404: { description: "Schedule not found." }
|
|
@@ -7933,7 +8303,7 @@ var routeCatalog = [
|
|
|
7933
8303
|
path: "/api/v1/projects/{name}/schedule",
|
|
7934
8304
|
summary: "Delete a schedule",
|
|
7935
8305
|
tags: ["schedules"],
|
|
7936
|
-
parameters: [nameParameter],
|
|
8306
|
+
parameters: [nameParameter, scheduleKindQueryParameter],
|
|
7937
8307
|
responses: {
|
|
7938
8308
|
204: { description: "Schedule deleted." },
|
|
7939
8309
|
404: { description: "Schedule not found." }
|
|
@@ -10089,7 +10459,15 @@ async function telemetryRoutes(app, opts) {
|
|
|
10089
10459
|
|
|
10090
10460
|
// ../api-routes/src/schedules.ts
|
|
10091
10461
|
import crypto11 from "crypto";
|
|
10092
|
-
import { eq as eq16 } from "drizzle-orm";
|
|
10462
|
+
import { and as and7, eq as eq16 } from "drizzle-orm";
|
|
10463
|
+
function parseKindParam(raw) {
|
|
10464
|
+
if (raw === void 0 || raw === null || raw === "") return SchedulableRunKinds["answer-visibility"];
|
|
10465
|
+
const parsed = schedulableRunKindSchema.safeParse(raw);
|
|
10466
|
+
if (!parsed.success) {
|
|
10467
|
+
throw validationError(`Invalid kind "${String(raw)}". Must be one of: ${Object.values(SchedulableRunKinds).join(", ")}`);
|
|
10468
|
+
}
|
|
10469
|
+
return parsed.data;
|
|
10470
|
+
}
|
|
10093
10471
|
async function scheduleRoutes(app, opts) {
|
|
10094
10472
|
app.put("/projects/:name/schedule", async (request, reply) => {
|
|
10095
10473
|
const project = resolveProject(app.db, request.params.name);
|
|
@@ -10102,7 +10480,22 @@ async function scheduleRoutes(app, opts) {
|
|
|
10102
10480
|
}))
|
|
10103
10481
|
});
|
|
10104
10482
|
}
|
|
10105
|
-
const
|
|
10483
|
+
const kind = parsedBody.data.kind ?? parseKindParam(request.query?.kind);
|
|
10484
|
+
const { preset, cron: cron2, timezone, providers, enabled, sourceId } = parsedBody.data;
|
|
10485
|
+
if (kind === SchedulableRunKinds["traffic-sync"]) {
|
|
10486
|
+
if (!sourceId) {
|
|
10487
|
+
throw validationError('"sourceId" is required when kind is "traffic-sync"');
|
|
10488
|
+
}
|
|
10489
|
+
const sourceRow = app.db.select().from(trafficSources).where(eq16(trafficSources.id, sourceId)).get();
|
|
10490
|
+
if (!sourceRow || sourceRow.projectId !== project.id) {
|
|
10491
|
+
throw notFound("Traffic source", sourceId);
|
|
10492
|
+
}
|
|
10493
|
+
if (providers && providers.length > 0) {
|
|
10494
|
+
throw validationError('"providers" is not valid for kind "traffic-sync"');
|
|
10495
|
+
}
|
|
10496
|
+
} else if (sourceId) {
|
|
10497
|
+
throw validationError(`"sourceId" is only valid when kind is "traffic-sync"`);
|
|
10498
|
+
}
|
|
10106
10499
|
const validNames = opts.validProviderNames ?? [];
|
|
10107
10500
|
if (validNames.length && providers?.length) {
|
|
10108
10501
|
const invalid = providers.filter((p) => !validNames.includes(p));
|
|
@@ -10132,13 +10525,14 @@ async function scheduleRoutes(app, opts) {
|
|
|
10132
10525
|
}
|
|
10133
10526
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
10134
10527
|
const enabledInt = enabled === false ? 0 : 1;
|
|
10135
|
-
const existing = app.db.select().from(schedules).where(eq16(schedules.projectId, project.id)).get();
|
|
10528
|
+
const existing = app.db.select().from(schedules).where(and7(eq16(schedules.projectId, project.id), eq16(schedules.kind, kind))).get();
|
|
10136
10529
|
if (existing) {
|
|
10137
10530
|
app.db.update(schedules).set({
|
|
10138
10531
|
cronExpr,
|
|
10139
10532
|
preset: preset ?? null,
|
|
10140
10533
|
timezone,
|
|
10141
|
-
providers: JSON.stringify(providers),
|
|
10534
|
+
providers: JSON.stringify(providers ?? []),
|
|
10535
|
+
sourceId: sourceId ?? null,
|
|
10142
10536
|
enabled: enabledInt,
|
|
10143
10537
|
updatedAt: now
|
|
10144
10538
|
}).where(eq16(schedules.id, existing.id)).run();
|
|
@@ -10146,11 +10540,13 @@ async function scheduleRoutes(app, opts) {
|
|
|
10146
10540
|
app.db.insert(schedules).values({
|
|
10147
10541
|
id: crypto11.randomUUID(),
|
|
10148
10542
|
projectId: project.id,
|
|
10543
|
+
kind,
|
|
10149
10544
|
cronExpr,
|
|
10150
10545
|
preset: preset ?? null,
|
|
10151
10546
|
timezone,
|
|
10152
10547
|
enabled: enabledInt,
|
|
10153
|
-
providers: JSON.stringify(providers),
|
|
10548
|
+
providers: JSON.stringify(providers ?? []),
|
|
10549
|
+
sourceId: sourceId ?? null,
|
|
10154
10550
|
createdAt: now,
|
|
10155
10551
|
updatedAt: now
|
|
10156
10552
|
}).run();
|
|
@@ -10160,25 +10556,27 @@ async function scheduleRoutes(app, opts) {
|
|
|
10160
10556
|
actor: "api",
|
|
10161
10557
|
action: existing ? "schedule.updated" : "schedule.created",
|
|
10162
10558
|
entityType: "schedule",
|
|
10163
|
-
diff: { cronExpr, preset, timezone, providers }
|
|
10559
|
+
diff: { kind, cronExpr, preset, timezone, providers, sourceId }
|
|
10164
10560
|
});
|
|
10165
|
-
opts.onScheduleUpdated?.("upsert", project.id);
|
|
10166
|
-
const schedule = app.db.select().from(schedules).where(eq16(schedules.projectId, project.id)).get();
|
|
10561
|
+
opts.onScheduleUpdated?.("upsert", project.id, kind);
|
|
10562
|
+
const schedule = app.db.select().from(schedules).where(and7(eq16(schedules.projectId, project.id), eq16(schedules.kind, kind))).get();
|
|
10167
10563
|
return reply.status(existing ? 200 : 201).send(formatSchedule(schedule));
|
|
10168
10564
|
});
|
|
10169
10565
|
app.get("/projects/:name/schedule", async (request, reply) => {
|
|
10170
10566
|
const project = resolveProject(app.db, request.params.name);
|
|
10171
|
-
const
|
|
10567
|
+
const kind = parseKindParam(request.query?.kind);
|
|
10568
|
+
const schedule = app.db.select().from(schedules).where(and7(eq16(schedules.projectId, project.id), eq16(schedules.kind, kind))).get();
|
|
10172
10569
|
if (!schedule) {
|
|
10173
|
-
throw notFound("Schedule", request.params.name);
|
|
10570
|
+
throw notFound("Schedule", `${request.params.name} (kind=${kind})`);
|
|
10174
10571
|
}
|
|
10175
10572
|
return reply.send(formatSchedule(schedule));
|
|
10176
10573
|
});
|
|
10177
10574
|
app.delete("/projects/:name/schedule", async (request, reply) => {
|
|
10178
10575
|
const project = resolveProject(app.db, request.params.name);
|
|
10179
|
-
const
|
|
10576
|
+
const kind = parseKindParam(request.query?.kind);
|
|
10577
|
+
const schedule = app.db.select().from(schedules).where(and7(eq16(schedules.projectId, project.id), eq16(schedules.kind, kind))).get();
|
|
10180
10578
|
if (!schedule) {
|
|
10181
|
-
throw notFound("Schedule", request.params.name);
|
|
10579
|
+
throw notFound("Schedule", `${request.params.name} (kind=${kind})`);
|
|
10182
10580
|
}
|
|
10183
10581
|
app.db.delete(schedules).where(eq16(schedules.id, schedule.id)).run();
|
|
10184
10582
|
writeAuditLog(app.db, {
|
|
@@ -10186,9 +10584,10 @@ async function scheduleRoutes(app, opts) {
|
|
|
10186
10584
|
actor: "api",
|
|
10187
10585
|
action: "schedule.deleted",
|
|
10188
10586
|
entityType: "schedule",
|
|
10189
|
-
entityId: schedule.id
|
|
10587
|
+
entityId: schedule.id,
|
|
10588
|
+
diff: { kind }
|
|
10190
10589
|
});
|
|
10191
|
-
opts.onScheduleUpdated?.("delete", project.id);
|
|
10590
|
+
opts.onScheduleUpdated?.("delete", project.id, kind);
|
|
10192
10591
|
return reply.status(204).send();
|
|
10193
10592
|
});
|
|
10194
10593
|
}
|
|
@@ -10196,11 +10595,13 @@ function formatSchedule(row) {
|
|
|
10196
10595
|
return {
|
|
10197
10596
|
id: row.id,
|
|
10198
10597
|
projectId: row.projectId,
|
|
10598
|
+
kind: row.kind,
|
|
10199
10599
|
cronExpr: row.cronExpr,
|
|
10200
10600
|
preset: row.preset,
|
|
10201
10601
|
timezone: row.timezone,
|
|
10202
10602
|
enabled: row.enabled === 1,
|
|
10203
10603
|
providers: parseJsonColumn(row.providers, []),
|
|
10604
|
+
sourceId: row.sourceId,
|
|
10204
10605
|
lastRunAt: row.lastRunAt,
|
|
10205
10606
|
nextRunAt: row.nextRunAt,
|
|
10206
10607
|
createdAt: row.createdAt,
|
|
@@ -10329,7 +10730,7 @@ function formatNotification(row) {
|
|
|
10329
10730
|
|
|
10330
10731
|
// ../api-routes/src/google.ts
|
|
10331
10732
|
import crypto14 from "crypto";
|
|
10332
|
-
import { eq as eq18, and as
|
|
10733
|
+
import { eq as eq18, and as and8, desc as desc8, sql as sql5 } from "drizzle-orm";
|
|
10333
10734
|
|
|
10334
10735
|
// ../integration-google/src/constants.ts
|
|
10335
10736
|
var GOOGLE_AUTH_URL = "https://accounts.google.com/o/oauth2/v2/auth";
|
|
@@ -11539,12 +11940,12 @@ async function googleRoutes(app, opts) {
|
|
|
11539
11940
|
const { startDate, endDate, query, page, limit } = request.query;
|
|
11540
11941
|
const cutoffDate = !startDate ? windowCutoff(parseWindow(request.query.window))?.slice(0, 10) ?? null : null;
|
|
11541
11942
|
const conditions = [eq18(gscSearchData.projectId, project.id)];
|
|
11542
|
-
if (startDate) conditions.push(
|
|
11543
|
-
else if (cutoffDate) conditions.push(
|
|
11544
|
-
if (endDate) conditions.push(
|
|
11545
|
-
if (query) conditions.push(
|
|
11546
|
-
if (page) conditions.push(
|
|
11547
|
-
const rows = app.db.select().from(gscSearchData).where(
|
|
11943
|
+
if (startDate) conditions.push(sql5`${gscSearchData.date} >= ${startDate}`);
|
|
11944
|
+
else if (cutoffDate) conditions.push(sql5`${gscSearchData.date} >= ${cutoffDate}`);
|
|
11945
|
+
if (endDate) conditions.push(sql5`${gscSearchData.date} <= ${endDate}`);
|
|
11946
|
+
if (query) conditions.push(sql5`${gscSearchData.query} LIKE ${"%" + query + "%"}`);
|
|
11947
|
+
if (page) conditions.push(sql5`${gscSearchData.page} LIKE ${"%" + page + "%"}`);
|
|
11948
|
+
const rows = app.db.select().from(gscSearchData).where(and8(...conditions)).orderBy(desc8(gscSearchData.date)).limit(parseInt(limit ?? "500", 10)).all();
|
|
11548
11949
|
return rows.map((r) => ({
|
|
11549
11950
|
date: r.date,
|
|
11550
11951
|
query: r.query,
|
|
@@ -11618,7 +12019,7 @@ async function googleRoutes(app, opts) {
|
|
|
11618
12019
|
const { url, limit } = request.query;
|
|
11619
12020
|
const conditions = [eq18(gscUrlInspections.projectId, project.id)];
|
|
11620
12021
|
if (url) conditions.push(eq18(gscUrlInspections.url, url));
|
|
11621
|
-
const rows = app.db.select().from(gscUrlInspections).where(
|
|
12022
|
+
const rows = app.db.select().from(gscUrlInspections).where(and8(...conditions)).orderBy(desc8(gscUrlInspections.inspectedAt)).limit(parseInt(limit ?? "100", 10)).all();
|
|
11622
12023
|
return rows.map((r) => ({
|
|
11623
12024
|
id: r.id,
|
|
11624
12025
|
url: r.url,
|
|
@@ -11970,7 +12371,7 @@ async function googleRoutes(app, opts) {
|
|
|
11970
12371
|
|
|
11971
12372
|
// ../api-routes/src/bing.ts
|
|
11972
12373
|
import crypto15 from "crypto";
|
|
11973
|
-
import { eq as eq19, and as
|
|
12374
|
+
import { eq as eq19, and as and9, desc as desc9 } from "drizzle-orm";
|
|
11974
12375
|
|
|
11975
12376
|
// ../integration-bing/src/constants.ts
|
|
11976
12377
|
var BING_WMT_API_BASE = "https://ssl.bing.com/webmaster/api.svc/json";
|
|
@@ -12384,7 +12785,7 @@ async function bingRoutes(app, opts) {
|
|
|
12384
12785
|
requireConnectionStore();
|
|
12385
12786
|
const project = resolveProject(app.db, request.params.name);
|
|
12386
12787
|
const { url, limit } = request.query;
|
|
12387
|
-
const whereClause = url ?
|
|
12788
|
+
const whereClause = url ? and9(eq19(bingUrlInspections.projectId, project.id), eq19(bingUrlInspections.url, url)) : eq19(bingUrlInspections.projectId, project.id);
|
|
12388
12789
|
const filtered = app.db.select().from(bingUrlInspections).where(whereClause).orderBy(desc9(bingUrlInspections.inspectedAt)).limit(Math.max(1, Math.min(parseInt(limit ?? "100", 10) || 100, 1e3))).all();
|
|
12389
12790
|
return filtered.map((r) => ({
|
|
12390
12791
|
id: r.id,
|
|
@@ -12616,7 +13017,7 @@ async function bingRoutes(app, opts) {
|
|
|
12616
13017
|
import fs from "fs";
|
|
12617
13018
|
import path from "path";
|
|
12618
13019
|
import os2 from "os";
|
|
12619
|
-
import { eq as eq20, and as
|
|
13020
|
+
import { eq as eq20, and as and10 } from "drizzle-orm";
|
|
12620
13021
|
function getScreenshotDir() {
|
|
12621
13022
|
return path.join(os2.homedir(), ".canonry", "screenshots");
|
|
12622
13023
|
}
|
|
@@ -12689,7 +13090,7 @@ async function cdpRoutes(app, opts) {
|
|
|
12689
13090
|
async (request, reply) => {
|
|
12690
13091
|
const project = resolveProject(app.db, request.params.name);
|
|
12691
13092
|
const { runId } = request.params;
|
|
12692
|
-
const run = app.db.select().from(runs).where(
|
|
13093
|
+
const run = app.db.select().from(runs).where(and10(eq20(runs.id, runId), eq20(runs.projectId, project.id))).get();
|
|
12693
13094
|
if (!run) {
|
|
12694
13095
|
const err = notFound("Run", runId);
|
|
12695
13096
|
return reply.code(err.statusCode).send(err.toJSON());
|
|
@@ -12786,7 +13187,7 @@ async function cdpRoutes(app, opts) {
|
|
|
12786
13187
|
|
|
12787
13188
|
// ../api-routes/src/ga.ts
|
|
12788
13189
|
import crypto16 from "crypto";
|
|
12789
|
-
import { eq as eq21, desc as desc10, and as
|
|
13190
|
+
import { eq as eq21, desc as desc10, and as and11, sql as sql6 } from "drizzle-orm";
|
|
12790
13191
|
function gaLog(level, action, ctx) {
|
|
12791
13192
|
const entry = { ts: (/* @__PURE__ */ new Date()).toISOString(), level, module: "GA4Routes", action, ...ctx };
|
|
12792
13193
|
const stream = level === "error" ? process.stderr : process.stdout;
|
|
@@ -13081,10 +13482,10 @@ async function ga4Routes(app, opts) {
|
|
|
13081
13482
|
app.db.transaction((tx) => {
|
|
13082
13483
|
if (syncTraffic) {
|
|
13083
13484
|
tx.delete(gaTrafficSnapshots).where(
|
|
13084
|
-
|
|
13485
|
+
and11(
|
|
13085
13486
|
eq21(gaTrafficSnapshots.projectId, project.id),
|
|
13086
|
-
|
|
13087
|
-
|
|
13487
|
+
sql6`${gaTrafficSnapshots.date} >= ${summary.periodStart}`,
|
|
13488
|
+
sql6`${gaTrafficSnapshots.date} <= ${summary.periodEnd}`
|
|
13088
13489
|
)
|
|
13089
13490
|
).run();
|
|
13090
13491
|
for (const row of rows) {
|
|
@@ -13105,10 +13506,10 @@ async function ga4Routes(app, opts) {
|
|
|
13105
13506
|
}
|
|
13106
13507
|
if (syncAi) {
|
|
13107
13508
|
tx.delete(gaAiReferrals).where(
|
|
13108
|
-
|
|
13509
|
+
and11(
|
|
13109
13510
|
eq21(gaAiReferrals.projectId, project.id),
|
|
13110
|
-
|
|
13111
|
-
|
|
13511
|
+
sql6`${gaAiReferrals.date} >= ${summary.periodStart}`,
|
|
13512
|
+
sql6`${gaAiReferrals.date} <= ${summary.periodEnd}`
|
|
13112
13513
|
)
|
|
13113
13514
|
).run();
|
|
13114
13515
|
for (const row of aiReferrals) {
|
|
@@ -13131,10 +13532,10 @@ async function ga4Routes(app, opts) {
|
|
|
13131
13532
|
}
|
|
13132
13533
|
if (syncSocial) {
|
|
13133
13534
|
tx.delete(gaSocialReferrals).where(
|
|
13134
|
-
|
|
13535
|
+
and11(
|
|
13135
13536
|
eq21(gaSocialReferrals.projectId, project.id),
|
|
13136
|
-
|
|
13137
|
-
|
|
13537
|
+
sql6`${gaSocialReferrals.date} >= ${summary.periodStart}`,
|
|
13538
|
+
sql6`${gaSocialReferrals.date} <= ${summary.periodEnd}`
|
|
13138
13539
|
)
|
|
13139
13540
|
).run();
|
|
13140
13541
|
for (const row of socialReferrals) {
|
|
@@ -13224,65 +13625,65 @@ async function ga4Routes(app, opts) {
|
|
|
13224
13625
|
const cutoff = windowCutoff(window);
|
|
13225
13626
|
const cutoffDate = cutoff?.slice(0, 10) ?? null;
|
|
13226
13627
|
const snapshotConditions = [eq21(gaTrafficSnapshots.projectId, project.id)];
|
|
13227
|
-
if (cutoffDate) snapshotConditions.push(
|
|
13628
|
+
if (cutoffDate) snapshotConditions.push(sql6`${gaTrafficSnapshots.date} >= ${cutoffDate}`);
|
|
13228
13629
|
const aiConditions = [eq21(gaAiReferrals.projectId, project.id)];
|
|
13229
|
-
if (cutoffDate) aiConditions.push(
|
|
13630
|
+
if (cutoffDate) aiConditions.push(sql6`${gaAiReferrals.date} >= ${cutoffDate}`);
|
|
13230
13631
|
const socialConditions = [eq21(gaSocialReferrals.projectId, project.id)];
|
|
13231
|
-
if (cutoffDate) socialConditions.push(
|
|
13632
|
+
if (cutoffDate) socialConditions.push(sql6`${gaSocialReferrals.date} >= ${cutoffDate}`);
|
|
13232
13633
|
const windowSummaryRow = cutoffDate ? app.db.select({
|
|
13233
13634
|
totalSessions: gaTrafficWindowSummaries.totalSessions,
|
|
13234
13635
|
totalOrganicSessions: gaTrafficWindowSummaries.totalOrganicSessions,
|
|
13235
13636
|
totalDirectSessions: gaTrafficWindowSummaries.totalDirectSessions,
|
|
13236
13637
|
totalUsers: gaTrafficWindowSummaries.totalUsers
|
|
13237
13638
|
}).from(gaTrafficWindowSummaries).where(
|
|
13238
|
-
|
|
13639
|
+
and11(
|
|
13239
13640
|
eq21(gaTrafficWindowSummaries.projectId, project.id),
|
|
13240
13641
|
eq21(gaTrafficWindowSummaries.windowKey, window)
|
|
13241
13642
|
)
|
|
13242
13643
|
).get() : null;
|
|
13243
13644
|
const snapshotTotalsRow = cutoffDate && !windowSummaryRow ? app.db.select({
|
|
13244
|
-
totalSessions:
|
|
13245
|
-
totalOrganicSessions:
|
|
13246
|
-
totalUsers:
|
|
13247
|
-
}).from(gaTrafficSnapshots).where(
|
|
13645
|
+
totalSessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)`,
|
|
13646
|
+
totalOrganicSessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)`,
|
|
13647
|
+
totalUsers: sql6`COALESCE(SUM(${gaTrafficSnapshots.users}), 0)`
|
|
13648
|
+
}).from(gaTrafficSnapshots).where(and11(...snapshotConditions)).get() : null;
|
|
13248
13649
|
const summaryRow = cutoffDate ? windowSummaryRow ?? snapshotTotalsRow : app.db.select({
|
|
13249
13650
|
totalSessions: gaTrafficSummaries.totalSessions,
|
|
13250
13651
|
totalOrganicSessions: gaTrafficSummaries.totalOrganicSessions,
|
|
13251
13652
|
totalUsers: gaTrafficSummaries.totalUsers
|
|
13252
13653
|
}).from(gaTrafficSummaries).where(eq21(gaTrafficSummaries.projectId, project.id)).get();
|
|
13253
13654
|
const directTotalRow = windowSummaryRow ? { totalDirectSessions: windowSummaryRow.totalDirectSessions } : app.db.select({
|
|
13254
|
-
totalDirectSessions:
|
|
13255
|
-
}).from(gaTrafficSnapshots).where(
|
|
13655
|
+
totalDirectSessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)`
|
|
13656
|
+
}).from(gaTrafficSnapshots).where(and11(...snapshotConditions)).get();
|
|
13256
13657
|
const summaryMeta = app.db.select({
|
|
13257
13658
|
periodStart: gaTrafficSummaries.periodStart,
|
|
13258
13659
|
periodEnd: gaTrafficSummaries.periodEnd
|
|
13259
13660
|
}).from(gaTrafficSummaries).where(eq21(gaTrafficSummaries.projectId, project.id)).get();
|
|
13260
13661
|
const rows = app.db.select({
|
|
13261
|
-
landingPage:
|
|
13262
|
-
sessions:
|
|
13263
|
-
organicSessions:
|
|
13264
|
-
directSessions:
|
|
13265
|
-
users:
|
|
13266
|
-
}).from(gaTrafficSnapshots).where(
|
|
13662
|
+
landingPage: sql6`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`,
|
|
13663
|
+
sessions: sql6`SUM(${gaTrafficSnapshots.sessions})`,
|
|
13664
|
+
organicSessions: sql6`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
13665
|
+
directSessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)`,
|
|
13666
|
+
users: sql6`SUM(${gaTrafficSnapshots.users})`
|
|
13667
|
+
}).from(gaTrafficSnapshots).where(and11(...snapshotConditions)).groupBy(sql6`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`).orderBy(sql6`SUM(${gaTrafficSnapshots.sessions}) DESC`).limit(limit).all();
|
|
13267
13668
|
const aiReferralRows = app.db.select({
|
|
13268
13669
|
source: gaAiReferrals.source,
|
|
13269
13670
|
medium: gaAiReferrals.medium,
|
|
13270
13671
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
13271
|
-
sessions:
|
|
13272
|
-
users:
|
|
13273
|
-
}).from(gaAiReferrals).where(
|
|
13672
|
+
sessions: sql6`SUM(${gaAiReferrals.sessions})`,
|
|
13673
|
+
users: sql6`SUM(${gaAiReferrals.users})`
|
|
13674
|
+
}).from(gaAiReferrals).where(and11(...aiConditions)).groupBy(gaAiReferrals.source, gaAiReferrals.medium, gaAiReferrals.sourceDimension).all();
|
|
13274
13675
|
const aiReferralLandingPageRows = app.db.select({
|
|
13275
13676
|
source: gaAiReferrals.source,
|
|
13276
13677
|
medium: gaAiReferrals.medium,
|
|
13277
13678
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
13278
|
-
landingPage:
|
|
13279
|
-
sessions:
|
|
13280
|
-
users:
|
|
13281
|
-
}).from(gaAiReferrals).where(
|
|
13679
|
+
landingPage: sql6`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`,
|
|
13680
|
+
sessions: sql6`SUM(${gaAiReferrals.sessions})`,
|
|
13681
|
+
users: sql6`SUM(${gaAiReferrals.users})`
|
|
13682
|
+
}).from(gaAiReferrals).where(and11(...aiConditions)).groupBy(
|
|
13282
13683
|
gaAiReferrals.source,
|
|
13283
13684
|
gaAiReferrals.medium,
|
|
13284
13685
|
gaAiReferrals.sourceDimension,
|
|
13285
|
-
|
|
13686
|
+
sql6`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`
|
|
13286
13687
|
).all();
|
|
13287
13688
|
const aiReferrals = pickWinningDimension(
|
|
13288
13689
|
aiReferralRows,
|
|
@@ -13293,10 +13694,10 @@ async function ga4Routes(app, opts) {
|
|
|
13293
13694
|
(r) => `${r.source}\0${r.medium}\0${r.landingPage}`
|
|
13294
13695
|
);
|
|
13295
13696
|
const aiDeduped = app.db.select({
|
|
13296
|
-
sessions:
|
|
13297
|
-
users:
|
|
13697
|
+
sessions: sql6`COALESCE(SUM(max_sessions), 0)`,
|
|
13698
|
+
users: sql6`COALESCE(SUM(max_users), 0)`
|
|
13298
13699
|
}).from(
|
|
13299
|
-
|
|
13700
|
+
sql6`(
|
|
13300
13701
|
SELECT date, source, medium,
|
|
13301
13702
|
MAX(dimension_sessions) AS max_sessions,
|
|
13302
13703
|
MAX(dimension_users) AS max_users
|
|
@@ -13305,7 +13706,7 @@ async function ga4Routes(app, opts) {
|
|
|
13305
13706
|
SUM(sessions) AS dimension_sessions,
|
|
13306
13707
|
SUM(users) AS dimension_users
|
|
13307
13708
|
FROM ga_ai_referrals
|
|
13308
|
-
WHERE project_id = ${project.id}${cutoffDate ?
|
|
13709
|
+
WHERE project_id = ${project.id}${cutoffDate ? sql6` AND date >= ${cutoffDate}` : sql6``}
|
|
13309
13710
|
GROUP BY date, source, medium, source_dimension
|
|
13310
13711
|
)
|
|
13311
13712
|
GROUP BY date, source, medium
|
|
@@ -13313,9 +13714,9 @@ async function ga4Routes(app, opts) {
|
|
|
13313
13714
|
).get();
|
|
13314
13715
|
const aiBySessionRows = app.db.select({
|
|
13315
13716
|
channelGroup: gaAiReferrals.channelGroup,
|
|
13316
|
-
sessions:
|
|
13317
|
-
users:
|
|
13318
|
-
}).from(gaAiReferrals).where(
|
|
13717
|
+
sessions: sql6`COALESCE(SUM(${gaAiReferrals.sessions}), 0)`,
|
|
13718
|
+
users: sql6`COALESCE(SUM(${gaAiReferrals.users}), 0)`
|
|
13719
|
+
}).from(gaAiReferrals).where(and11(...aiConditions, eq21(gaAiReferrals.sourceDimension, "session"))).groupBy(gaAiReferrals.channelGroup).all();
|
|
13319
13720
|
const aiSessionsByChannelGroup = /* @__PURE__ */ new Map();
|
|
13320
13721
|
let aiBySessionUsers = 0;
|
|
13321
13722
|
for (const row of aiBySessionRows) {
|
|
@@ -13327,13 +13728,13 @@ async function ga4Routes(app, opts) {
|
|
|
13327
13728
|
source: gaSocialReferrals.source,
|
|
13328
13729
|
medium: gaSocialReferrals.medium,
|
|
13329
13730
|
channelGroup: gaSocialReferrals.channelGroup,
|
|
13330
|
-
sessions:
|
|
13331
|
-
users:
|
|
13332
|
-
}).from(gaSocialReferrals).where(
|
|
13731
|
+
sessions: sql6`SUM(${gaSocialReferrals.sessions})`,
|
|
13732
|
+
users: sql6`SUM(${gaSocialReferrals.users})`
|
|
13733
|
+
}).from(gaSocialReferrals).where(and11(...socialConditions)).groupBy(gaSocialReferrals.source, gaSocialReferrals.medium, gaSocialReferrals.channelGroup).orderBy(sql6`SUM(${gaSocialReferrals.sessions}) DESC`).all();
|
|
13333
13734
|
const socialTotals = app.db.select({
|
|
13334
|
-
sessions:
|
|
13335
|
-
users:
|
|
13336
|
-
}).from(gaSocialReferrals).where(
|
|
13735
|
+
sessions: sql6`SUM(${gaSocialReferrals.sessions})`,
|
|
13736
|
+
users: sql6`SUM(${gaSocialReferrals.users})`
|
|
13737
|
+
}).from(gaSocialReferrals).where(and11(...socialConditions)).get();
|
|
13337
13738
|
const latestSync = app.db.select({ syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(eq21(gaTrafficSummaries.projectId, project.id)).orderBy(desc10(gaTrafficSummaries.syncedAt)).limit(1).get();
|
|
13338
13739
|
const total = summaryRow?.totalSessions ?? 0;
|
|
13339
13740
|
const totalDirectSessions = directTotalRow?.totalDirectSessions ?? 0;
|
|
@@ -13415,21 +13816,21 @@ async function ga4Routes(app, opts) {
|
|
|
13415
13816
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
13416
13817
|
const cutoffDate = windowCutoff(parseWindow(request.query.window))?.slice(0, 10) ?? null;
|
|
13417
13818
|
const conditions = [eq21(gaAiReferrals.projectId, project.id)];
|
|
13418
|
-
if (cutoffDate) conditions.push(
|
|
13819
|
+
if (cutoffDate) conditions.push(sql6`${gaAiReferrals.date} >= ${cutoffDate}`);
|
|
13419
13820
|
const rows = app.db.select({
|
|
13420
13821
|
date: gaAiReferrals.date,
|
|
13421
13822
|
source: gaAiReferrals.source,
|
|
13422
13823
|
medium: gaAiReferrals.medium,
|
|
13423
|
-
landingPage:
|
|
13824
|
+
landingPage: sql6`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`,
|
|
13424
13825
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
13425
|
-
sessions:
|
|
13426
|
-
users:
|
|
13427
|
-
}).from(gaAiReferrals).where(
|
|
13826
|
+
sessions: sql6`SUM(${gaAiReferrals.sessions})`,
|
|
13827
|
+
users: sql6`SUM(${gaAiReferrals.users})`
|
|
13828
|
+
}).from(gaAiReferrals).where(and11(...conditions)).groupBy(
|
|
13428
13829
|
gaAiReferrals.date,
|
|
13429
13830
|
gaAiReferrals.source,
|
|
13430
13831
|
gaAiReferrals.medium,
|
|
13431
13832
|
gaAiReferrals.sourceDimension,
|
|
13432
|
-
|
|
13833
|
+
sql6`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`
|
|
13433
13834
|
).orderBy(gaAiReferrals.date).all();
|
|
13434
13835
|
return rows;
|
|
13435
13836
|
});
|
|
@@ -13438,7 +13839,7 @@ async function ga4Routes(app, opts) {
|
|
|
13438
13839
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
13439
13840
|
const cutoffDate = windowCutoff(parseWindow(request.query.window))?.slice(0, 10) ?? null;
|
|
13440
13841
|
const conditions = [eq21(gaSocialReferrals.projectId, project.id)];
|
|
13441
|
-
if (cutoffDate) conditions.push(
|
|
13842
|
+
if (cutoffDate) conditions.push(sql6`${gaSocialReferrals.date} >= ${cutoffDate}`);
|
|
13442
13843
|
const rows = app.db.select({
|
|
13443
13844
|
date: gaSocialReferrals.date,
|
|
13444
13845
|
source: gaSocialReferrals.source,
|
|
@@ -13446,7 +13847,7 @@ async function ga4Routes(app, opts) {
|
|
|
13446
13847
|
channelGroup: gaSocialReferrals.channelGroup,
|
|
13447
13848
|
sessions: gaSocialReferrals.sessions,
|
|
13448
13849
|
users: gaSocialReferrals.users
|
|
13449
|
-
}).from(gaSocialReferrals).where(
|
|
13850
|
+
}).from(gaSocialReferrals).where(and11(...conditions)).orderBy(gaSocialReferrals.date).all();
|
|
13450
13851
|
return rows;
|
|
13451
13852
|
});
|
|
13452
13853
|
app.get("/projects/:name/ga/social-referral-trend", async (request, _reply) => {
|
|
@@ -13459,10 +13860,10 @@ async function ga4Routes(app, opts) {
|
|
|
13459
13860
|
d.setDate(d.getDate() - n);
|
|
13460
13861
|
return fmt(d);
|
|
13461
13862
|
};
|
|
13462
|
-
const sumSocial = (from, to) => app.db.select({ sessions:
|
|
13863
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and11(
|
|
13463
13864
|
eq21(gaSocialReferrals.projectId, project.id),
|
|
13464
|
-
|
|
13465
|
-
|
|
13865
|
+
sql6`${gaSocialReferrals.date} >= ${from}`,
|
|
13866
|
+
sql6`${gaSocialReferrals.date} < ${to}`
|
|
13466
13867
|
)).get();
|
|
13467
13868
|
const current7d = sumSocial(daysAgo2(7), fmt(today));
|
|
13468
13869
|
const prev7d = sumSocial(daysAgo2(14), daysAgo2(7));
|
|
@@ -13471,19 +13872,19 @@ async function ga4Routes(app, opts) {
|
|
|
13471
13872
|
const pct = (cur, prev) => prev === 0 ? null : Math.round((cur - prev) / prev * 100);
|
|
13472
13873
|
const sourceCurrent = app.db.select({
|
|
13473
13874
|
source: gaSocialReferrals.source,
|
|
13474
|
-
sessions:
|
|
13475
|
-
}).from(gaSocialReferrals).where(
|
|
13875
|
+
sessions: sql6`SUM(${gaSocialReferrals.sessions})`
|
|
13876
|
+
}).from(gaSocialReferrals).where(and11(
|
|
13476
13877
|
eq21(gaSocialReferrals.projectId, project.id),
|
|
13477
|
-
|
|
13478
|
-
|
|
13878
|
+
sql6`${gaSocialReferrals.date} >= ${daysAgo2(7)}`,
|
|
13879
|
+
sql6`${gaSocialReferrals.date} < ${fmt(today)}`
|
|
13479
13880
|
)).groupBy(gaSocialReferrals.source).all();
|
|
13480
13881
|
const sourcePrev = app.db.select({
|
|
13481
13882
|
source: gaSocialReferrals.source,
|
|
13482
|
-
sessions:
|
|
13483
|
-
}).from(gaSocialReferrals).where(
|
|
13883
|
+
sessions: sql6`SUM(${gaSocialReferrals.sessions})`
|
|
13884
|
+
}).from(gaSocialReferrals).where(and11(
|
|
13484
13885
|
eq21(gaSocialReferrals.projectId, project.id),
|
|
13485
|
-
|
|
13486
|
-
|
|
13886
|
+
sql6`${gaSocialReferrals.date} >= ${daysAgo2(14)}`,
|
|
13887
|
+
sql6`${gaSocialReferrals.date} < ${daysAgo2(7)}`
|
|
13487
13888
|
)).groupBy(gaSocialReferrals.source).all();
|
|
13488
13889
|
const prevMap = new Map(sourcePrev.map((r) => [r.source, r.sessions]));
|
|
13489
13890
|
let biggestMover = null;
|
|
@@ -13522,16 +13923,16 @@ async function ga4Routes(app, opts) {
|
|
|
13522
13923
|
return fmt(d);
|
|
13523
13924
|
};
|
|
13524
13925
|
const pct = (cur, prev) => prev === 0 ? null : Math.round((cur - prev) / prev * 100);
|
|
13525
|
-
const sumTotal = (from, to) => app.db.select({ sessions:
|
|
13526
|
-
const sumOrganic = (from, to) => app.db.select({ sessions:
|
|
13527
|
-
const sumDirect = (from, to) => app.db.select({ sessions:
|
|
13528
|
-
const sumAi = (from, to) => app.db.select({ sessions:
|
|
13926
|
+
const sumTotal = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)` }).from(gaTrafficSnapshots).where(and11(eq21(gaTrafficSnapshots.projectId, project.id), sql6`${gaTrafficSnapshots.date} >= ${from}`, sql6`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
13927
|
+
const sumOrganic = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)` }).from(gaTrafficSnapshots).where(and11(eq21(gaTrafficSnapshots.projectId, project.id), sql6`${gaTrafficSnapshots.date} >= ${from}`, sql6`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
13928
|
+
const sumDirect = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)` }).from(gaTrafficSnapshots).where(and11(eq21(gaTrafficSnapshots.projectId, project.id), sql6`${gaTrafficSnapshots.date} >= ${from}`, sql6`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
13929
|
+
const sumAi = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and11(
|
|
13529
13930
|
eq21(gaAiReferrals.projectId, project.id),
|
|
13530
|
-
|
|
13531
|
-
|
|
13931
|
+
sql6`${gaAiReferrals.date} >= ${from}`,
|
|
13932
|
+
sql6`${gaAiReferrals.date} < ${to}`,
|
|
13532
13933
|
eq21(gaAiReferrals.sourceDimension, "session")
|
|
13533
13934
|
)).get();
|
|
13534
|
-
const sumSocial = (from, to) => app.db.select({ sessions:
|
|
13935
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and11(eq21(gaSocialReferrals.projectId, project.id), sql6`${gaSocialReferrals.date} >= ${from}`, sql6`${gaSocialReferrals.date} < ${to}`)).get();
|
|
13535
13936
|
const todayStr = fmt(today);
|
|
13536
13937
|
const buildTrend = (sum) => {
|
|
13537
13938
|
const c7 = sum(daysAgo2(7), todayStr)?.sessions ?? 0;
|
|
@@ -13540,16 +13941,16 @@ async function ga4Routes(app, opts) {
|
|
|
13540
13941
|
const p30 = sum(daysAgo2(60), daysAgo2(30))?.sessions ?? 0;
|
|
13541
13942
|
return { sessions7d: c7, sessionsPrev7d: p7, trend7dPct: pct(c7, p7), sessions30d: c30, sessionsPrev30d: p30, trend30dPct: pct(c30, p30) };
|
|
13542
13943
|
};
|
|
13543
|
-
const aiSourceCurrent = app.db.select({ source: gaAiReferrals.source, sessions:
|
|
13944
|
+
const aiSourceCurrent = app.db.select({ source: gaAiReferrals.source, sessions: sql6`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and11(
|
|
13544
13945
|
eq21(gaAiReferrals.projectId, project.id),
|
|
13545
|
-
|
|
13546
|
-
|
|
13946
|
+
sql6`${gaAiReferrals.date} >= ${daysAgo2(7)}`,
|
|
13947
|
+
sql6`${gaAiReferrals.date} < ${todayStr}`,
|
|
13547
13948
|
eq21(gaAiReferrals.sourceDimension, "session")
|
|
13548
13949
|
)).groupBy(gaAiReferrals.source).all();
|
|
13549
|
-
const aiSourcePrev = app.db.select({ source: gaAiReferrals.source, sessions:
|
|
13950
|
+
const aiSourcePrev = app.db.select({ source: gaAiReferrals.source, sessions: sql6`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and11(
|
|
13550
13951
|
eq21(gaAiReferrals.projectId, project.id),
|
|
13551
|
-
|
|
13552
|
-
|
|
13952
|
+
sql6`${gaAiReferrals.date} >= ${daysAgo2(14)}`,
|
|
13953
|
+
sql6`${gaAiReferrals.date} < ${daysAgo2(7)}`,
|
|
13553
13954
|
eq21(gaAiReferrals.sourceDimension, "session")
|
|
13554
13955
|
)).groupBy(gaAiReferrals.source).all();
|
|
13555
13956
|
const findBiggestMover = (current, prev) => {
|
|
@@ -13566,8 +13967,8 @@ async function ga4Routes(app, opts) {
|
|
|
13566
13967
|
}
|
|
13567
13968
|
return mover;
|
|
13568
13969
|
};
|
|
13569
|
-
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions:
|
|
13570
|
-
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions:
|
|
13970
|
+
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions: sql6`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and11(eq21(gaSocialReferrals.projectId, project.id), sql6`${gaSocialReferrals.date} >= ${daysAgo2(7)}`, sql6`${gaSocialReferrals.date} < ${todayStr}`)).groupBy(gaSocialReferrals.source).all();
|
|
13971
|
+
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions: sql6`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and11(eq21(gaSocialReferrals.projectId, project.id), sql6`${gaSocialReferrals.date} >= ${daysAgo2(14)}`, sql6`${gaSocialReferrals.date} < ${daysAgo2(7)}`)).groupBy(gaSocialReferrals.source).all();
|
|
13571
13972
|
return {
|
|
13572
13973
|
total: buildTrend(sumTotal),
|
|
13573
13974
|
organic: buildTrend(sumOrganic),
|
|
@@ -13583,13 +13984,13 @@ async function ga4Routes(app, opts) {
|
|
|
13583
13984
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
13584
13985
|
const cutoffDate = windowCutoff(parseWindow(request.query.window))?.slice(0, 10) ?? null;
|
|
13585
13986
|
const conditions = [eq21(gaTrafficSnapshots.projectId, project.id)];
|
|
13586
|
-
if (cutoffDate) conditions.push(
|
|
13987
|
+
if (cutoffDate) conditions.push(sql6`${gaTrafficSnapshots.date} >= ${cutoffDate}`);
|
|
13587
13988
|
const rows = app.db.select({
|
|
13588
13989
|
date: gaTrafficSnapshots.date,
|
|
13589
|
-
sessions:
|
|
13590
|
-
organicSessions:
|
|
13591
|
-
users:
|
|
13592
|
-
}).from(gaTrafficSnapshots).where(
|
|
13990
|
+
sessions: sql6`SUM(${gaTrafficSnapshots.sessions})`,
|
|
13991
|
+
organicSessions: sql6`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
13992
|
+
users: sql6`SUM(${gaTrafficSnapshots.users})`
|
|
13993
|
+
}).from(gaTrafficSnapshots).where(and11(...conditions)).groupBy(gaTrafficSnapshots.date).orderBy(gaTrafficSnapshots.date).all();
|
|
13593
13994
|
return rows.map((r) => ({
|
|
13594
13995
|
date: r.date,
|
|
13595
13996
|
sessions: r.sessions ?? 0,
|
|
@@ -13601,11 +14002,11 @@ async function ga4Routes(app, opts) {
|
|
|
13601
14002
|
const project = resolveProject(app.db, request.params.name);
|
|
13602
14003
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
13603
14004
|
const trafficPages = app.db.select({
|
|
13604
|
-
landingPage:
|
|
13605
|
-
sessions:
|
|
13606
|
-
organicSessions:
|
|
13607
|
-
users:
|
|
13608
|
-
}).from(gaTrafficSnapshots).where(eq21(gaTrafficSnapshots.projectId, project.id)).groupBy(
|
|
14005
|
+
landingPage: sql6`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`,
|
|
14006
|
+
sessions: sql6`SUM(${gaTrafficSnapshots.sessions})`,
|
|
14007
|
+
organicSessions: sql6`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
14008
|
+
users: sql6`SUM(${gaTrafficSnapshots.users})`
|
|
14009
|
+
}).from(gaTrafficSnapshots).where(eq21(gaTrafficSnapshots.projectId, project.id)).groupBy(sql6`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`).orderBy(sql6`SUM(${gaTrafficSnapshots.sessions}) DESC`).all();
|
|
13609
14010
|
return {
|
|
13610
14011
|
pages: trafficPages.map((r) => ({
|
|
13611
14012
|
landingPage: r.landingPage,
|
|
@@ -15242,7 +15643,7 @@ async function wordpressRoutes(app, opts) {
|
|
|
15242
15643
|
|
|
15243
15644
|
// ../api-routes/src/backlinks.ts
|
|
15244
15645
|
import crypto18 from "crypto";
|
|
15245
|
-
import { and as
|
|
15646
|
+
import { and as and13, asc as asc2, desc as desc11, eq as eq22, sql as sql7 } from "drizzle-orm";
|
|
15246
15647
|
|
|
15247
15648
|
// ../integration-commoncrawl/src/constants.ts
|
|
15248
15649
|
import os3 from "os";
|
|
@@ -15526,7 +15927,7 @@ async function queryBacklinks(opts) {
|
|
|
15526
15927
|
const reversed = opts.targets.map(reverseDomain);
|
|
15527
15928
|
const targetList = reversed.map(quote).join(", ");
|
|
15528
15929
|
const limitClause = opts.limitPerTarget ? `QUALIFY row_number() OVER (PARTITION BY t.target_rev_domain ORDER BY v.num_hosts DESC) <= ${Math.floor(opts.limitPerTarget)}` : "";
|
|
15529
|
-
const
|
|
15930
|
+
const sql14 = `
|
|
15530
15931
|
WITH vertices AS (
|
|
15531
15932
|
SELECT * FROM read_csv(
|
|
15532
15933
|
${quote(opts.vertexPath)},
|
|
@@ -15562,7 +15963,7 @@ async function queryBacklinks(opts) {
|
|
|
15562
15963
|
const conn = await instance.connect();
|
|
15563
15964
|
let rows;
|
|
15564
15965
|
try {
|
|
15565
|
-
const reader = await conn.runAndReadAll(
|
|
15966
|
+
const reader = await conn.runAndReadAll(sql14);
|
|
15566
15967
|
rows = reader.getRowObjects();
|
|
15567
15968
|
} finally {
|
|
15568
15969
|
conn.disconnectSync?.();
|
|
@@ -15639,7 +16040,7 @@ function pruneCachedRelease(release, opts = {}) {
|
|
|
15639
16040
|
}
|
|
15640
16041
|
|
|
15641
16042
|
// ../api-routes/src/backlinks-filter.ts
|
|
15642
|
-
import { and as
|
|
16043
|
+
import { and as and12, ne as ne2, notLike } from "drizzle-orm";
|
|
15643
16044
|
var BACKLINK_FILTER_PATTERNS = [
|
|
15644
16045
|
"*.google.com",
|
|
15645
16046
|
"*.googleusercontent.com",
|
|
@@ -15656,13 +16057,13 @@ function backlinkCrawlerExclusionClause() {
|
|
|
15656
16057
|
for (const pattern of BACKLINK_FILTER_PATTERNS) {
|
|
15657
16058
|
if (pattern.startsWith("*.")) {
|
|
15658
16059
|
const suffix = pattern.slice(2);
|
|
15659
|
-
conditions.push(
|
|
16060
|
+
conditions.push(ne2(backlinkDomains.linkingDomain, suffix));
|
|
15660
16061
|
conditions.push(notLike(backlinkDomains.linkingDomain, `%.${suffix}`));
|
|
15661
16062
|
} else {
|
|
15662
|
-
conditions.push(
|
|
16063
|
+
conditions.push(ne2(backlinkDomains.linkingDomain, pattern));
|
|
15663
16064
|
}
|
|
15664
16065
|
}
|
|
15665
|
-
const combined =
|
|
16066
|
+
const combined = and12(...conditions);
|
|
15666
16067
|
if (!combined) throw new Error("BACKLINK_FILTER_PATTERNS is unexpectedly empty");
|
|
15667
16068
|
return combined;
|
|
15668
16069
|
}
|
|
@@ -15723,7 +16124,7 @@ function mapRunRow(row) {
|
|
|
15723
16124
|
};
|
|
15724
16125
|
}
|
|
15725
16126
|
function latestSummaryForProject(db, projectId, release) {
|
|
15726
|
-
const condition = release ?
|
|
16127
|
+
const condition = release ? and13(eq22(backlinkSummaries.projectId, projectId), eq22(backlinkSummaries.release, release)) : eq22(backlinkSummaries.projectId, projectId);
|
|
15727
16128
|
return db.select().from(backlinkSummaries).where(condition).orderBy(desc11(backlinkSummaries.queriedAt)).limit(1).get();
|
|
15728
16129
|
}
|
|
15729
16130
|
function parseExcludeCrawlers(value) {
|
|
@@ -15732,18 +16133,18 @@ function parseExcludeCrawlers(value) {
|
|
|
15732
16133
|
return lower === "1" || lower === "true" || lower === "yes";
|
|
15733
16134
|
}
|
|
15734
16135
|
function computeFilteredSummary(db, base) {
|
|
15735
|
-
const baseDomainCondition =
|
|
16136
|
+
const baseDomainCondition = and13(
|
|
15736
16137
|
eq22(backlinkDomains.projectId, base.projectId),
|
|
15737
16138
|
eq22(backlinkDomains.release, base.release)
|
|
15738
16139
|
);
|
|
15739
|
-
const filteredCondition =
|
|
16140
|
+
const filteredCondition = and13(baseDomainCondition, backlinkCrawlerExclusionClause());
|
|
15740
16141
|
const unfilteredAgg = db.select({
|
|
15741
|
-
count:
|
|
15742
|
-
total:
|
|
16142
|
+
count: sql7`count(*)`,
|
|
16143
|
+
total: sql7`coalesce(sum(${backlinkDomains.numHosts}), 0)`
|
|
15743
16144
|
}).from(backlinkDomains).where(baseDomainCondition).get();
|
|
15744
16145
|
const filteredAgg = db.select({
|
|
15745
|
-
count:
|
|
15746
|
-
total:
|
|
16146
|
+
count: sql7`count(*)`,
|
|
16147
|
+
total: sql7`coalesce(sum(${backlinkDomains.numHosts}), 0)`
|
|
15747
16148
|
}).from(backlinkDomains).where(filteredCondition).get();
|
|
15748
16149
|
const top10Rows = db.select({ numHosts: backlinkDomains.numHosts }).from(backlinkDomains).where(filteredCondition).orderBy(desc11(backlinkDomains.numHosts)).limit(10).all();
|
|
15749
16150
|
const totalLinkingDomains = Number(filteredAgg?.count ?? 0);
|
|
@@ -15912,12 +16313,12 @@ async function backlinksRoutes(app, opts) {
|
|
|
15912
16313
|
const limit = Math.min(Math.max(parseInt(request.query.limit ?? "50", 10) || 50, 1), 500);
|
|
15913
16314
|
const offset = Math.max(parseInt(request.query.offset ?? "0", 10) || 0, 0);
|
|
15914
16315
|
const excludeCrawlers = parseExcludeCrawlers(request.query.excludeCrawlers);
|
|
15915
|
-
const baseDomainCondition =
|
|
16316
|
+
const baseDomainCondition = and13(
|
|
15916
16317
|
eq22(backlinkDomains.projectId, project.id),
|
|
15917
16318
|
eq22(backlinkDomains.release, targetRelease)
|
|
15918
16319
|
);
|
|
15919
|
-
const domainCondition = excludeCrawlers ?
|
|
15920
|
-
const totalRow = app.db.select({ count:
|
|
16320
|
+
const domainCondition = excludeCrawlers ? and13(baseDomainCondition, backlinkCrawlerExclusionClause()) : baseDomainCondition;
|
|
16321
|
+
const totalRow = app.db.select({ count: sql7`count(*)` }).from(backlinkDomains).where(domainCondition).get();
|
|
15921
16322
|
const rows = app.db.select({
|
|
15922
16323
|
linkingDomain: backlinkDomains.linkingDomain,
|
|
15923
16324
|
numHosts: backlinkDomains.numHosts
|
|
@@ -15952,7 +16353,7 @@ async function backlinksRoutes(app, opts) {
|
|
|
15952
16353
|
|
|
15953
16354
|
// ../api-routes/src/traffic.ts
|
|
15954
16355
|
import crypto20 from "crypto";
|
|
15955
|
-
import { and as
|
|
16356
|
+
import { and as and14, desc as desc12, eq as eq23, gte as gte2, lte as lte2, sql as sql8 } from "drizzle-orm";
|
|
15956
16357
|
|
|
15957
16358
|
// ../integration-cloud-run/src/auth.ts
|
|
15958
16359
|
import crypto19 from "crypto";
|
|
@@ -16805,7 +17206,7 @@ async function trafficRoutes(app, opts) {
|
|
|
16805
17206
|
crawlerEventsHourly.status
|
|
16806
17207
|
],
|
|
16807
17208
|
set: {
|
|
16808
|
-
hits:
|
|
17209
|
+
hits: sql8`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
16809
17210
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
16810
17211
|
updatedAt: finishedAt
|
|
16811
17212
|
}
|
|
@@ -16840,7 +17241,7 @@ async function trafficRoutes(app, opts) {
|
|
|
16840
17241
|
aiReferralEventsHourly.status
|
|
16841
17242
|
],
|
|
16842
17243
|
set: {
|
|
16843
|
-
sessionsOrHits:
|
|
17244
|
+
sessionsOrHits: sql8`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
16844
17245
|
updatedAt: finishedAt
|
|
16845
17246
|
}
|
|
16846
17247
|
}).run();
|
|
@@ -16908,26 +17309,26 @@ async function trafficRoutes(app, opts) {
|
|
|
16908
17309
|
return response;
|
|
16909
17310
|
});
|
|
16910
17311
|
function buildSourceDetail(projectId, row, since) {
|
|
16911
|
-
const crawlerTotals = app.db.select({ total:
|
|
16912
|
-
|
|
17312
|
+
const crawlerTotals = app.db.select({ total: sql8`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
17313
|
+
and14(
|
|
16913
17314
|
eq23(crawlerEventsHourly.sourceId, row.id),
|
|
16914
|
-
|
|
17315
|
+
gte2(crawlerEventsHourly.tsHour, since)
|
|
16915
17316
|
)
|
|
16916
17317
|
).get();
|
|
16917
|
-
const aiTotals = app.db.select({ total:
|
|
16918
|
-
|
|
17318
|
+
const aiTotals = app.db.select({ total: sql8`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
17319
|
+
and14(
|
|
16919
17320
|
eq23(aiReferralEventsHourly.sourceId, row.id),
|
|
16920
|
-
|
|
17321
|
+
gte2(aiReferralEventsHourly.tsHour, since)
|
|
16921
17322
|
)
|
|
16922
17323
|
).get();
|
|
16923
|
-
const sampleTotals = app.db.select({ total:
|
|
16924
|
-
|
|
17324
|
+
const sampleTotals = app.db.select({ total: sql8`COUNT(*)` }).from(rawEventSamples).where(
|
|
17325
|
+
and14(
|
|
16925
17326
|
eq23(rawEventSamples.sourceId, row.id),
|
|
16926
|
-
|
|
17327
|
+
gte2(rawEventSamples.ts, since)
|
|
16927
17328
|
)
|
|
16928
17329
|
).get();
|
|
16929
17330
|
const latestRun = app.db.select().from(runs).where(
|
|
16930
|
-
|
|
17331
|
+
and14(
|
|
16931
17332
|
eq23(runs.projectId, projectId),
|
|
16932
17333
|
eq23(runs.kind, RunKinds["traffic-sync"]),
|
|
16933
17334
|
eq23(runs.sourceId, row.id)
|
|
@@ -17017,12 +17418,12 @@ async function trafficRoutes(app, opts) {
|
|
|
17017
17418
|
if (kind === "all" || kind === TrafficEventKinds.crawler) {
|
|
17018
17419
|
const crawlerFilters = [
|
|
17019
17420
|
eq23(crawlerEventsHourly.projectId, project.id),
|
|
17020
|
-
|
|
17021
|
-
|
|
17421
|
+
gte2(crawlerEventsHourly.tsHour, sinceIso),
|
|
17422
|
+
lte2(crawlerEventsHourly.tsHour, untilIso)
|
|
17022
17423
|
];
|
|
17023
17424
|
if (sourceIdParam) crawlerFilters.push(eq23(crawlerEventsHourly.sourceId, sourceIdParam));
|
|
17024
|
-
const crawlerWhere =
|
|
17025
|
-
const total = app.db.select({ total:
|
|
17425
|
+
const crawlerWhere = and14(...crawlerFilters);
|
|
17426
|
+
const total = app.db.select({ total: sql8`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(crawlerWhere).get();
|
|
17026
17427
|
crawlerTotal = Number(total?.total ?? 0);
|
|
17027
17428
|
const rows = app.db.select().from(crawlerEventsHourly).where(crawlerWhere).orderBy(desc12(crawlerEventsHourly.tsHour)).limit(limit).all();
|
|
17028
17429
|
for (const r of rows) {
|
|
@@ -17042,12 +17443,12 @@ async function trafficRoutes(app, opts) {
|
|
|
17042
17443
|
if (kind === "all" || kind === TrafficEventKinds["ai-referral"]) {
|
|
17043
17444
|
const aiFilters = [
|
|
17044
17445
|
eq23(aiReferralEventsHourly.projectId, project.id),
|
|
17045
|
-
|
|
17046
|
-
|
|
17446
|
+
gte2(aiReferralEventsHourly.tsHour, sinceIso),
|
|
17447
|
+
lte2(aiReferralEventsHourly.tsHour, untilIso)
|
|
17047
17448
|
];
|
|
17048
17449
|
if (sourceIdParam) aiFilters.push(eq23(aiReferralEventsHourly.sourceId, sourceIdParam));
|
|
17049
|
-
const aiWhere =
|
|
17050
|
-
const total = app.db.select({ total:
|
|
17450
|
+
const aiWhere = and14(...aiFilters);
|
|
17451
|
+
const total = app.db.select({ total: sql8`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(aiWhere).get();
|
|
17051
17452
|
aiReferralTotal = Number(total?.total ?? 0);
|
|
17052
17453
|
const rows = app.db.select().from(aiReferralEventsHourly).where(aiWhere).orderBy(desc12(aiReferralEventsHourly.tsHour)).limit(limit).all();
|
|
17053
17454
|
for (const r of rows) {
|
|
@@ -17700,12 +18101,308 @@ var providersConfiguredCheck = {
|
|
|
17700
18101
|
};
|
|
17701
18102
|
var PROVIDERS_CHECKS = [providersConfiguredCheck];
|
|
17702
18103
|
|
|
18104
|
+
// ../api-routes/src/doctor/checks/traffic-source.ts
|
|
18105
|
+
import { and as and15, eq as eq24, gte as gte3, ne as ne3, sql as sql9 } from "drizzle-orm";
|
|
18106
|
+
var RECENT_DATA_WARN_DAYS = 7;
|
|
18107
|
+
var RECENT_DATA_FAIL_DAYS = 30;
|
|
18108
|
+
function skippedNoProject2() {
|
|
18109
|
+
return {
|
|
18110
|
+
status: CheckStatuses.skipped,
|
|
18111
|
+
code: "traffic.no-project",
|
|
18112
|
+
summary: "Project context required for traffic source checks.",
|
|
18113
|
+
remediation: "Run `canonry doctor --project <name>` to scope this check to a project."
|
|
18114
|
+
};
|
|
18115
|
+
}
|
|
18116
|
+
function loadProbes(ctx) {
|
|
18117
|
+
if (!ctx.project) return [];
|
|
18118
|
+
const rows = ctx.db.select().from(trafficSources).where(
|
|
18119
|
+
and15(
|
|
18120
|
+
eq24(trafficSources.projectId, ctx.project.id),
|
|
18121
|
+
ne3(trafficSources.status, TrafficSourceStatuses.archived)
|
|
18122
|
+
)
|
|
18123
|
+
).all();
|
|
18124
|
+
return rows.map((r) => ({
|
|
18125
|
+
id: r.id,
|
|
18126
|
+
projectId: r.projectId,
|
|
18127
|
+
projectName: ctx.project.name,
|
|
18128
|
+
sourceType: r.sourceType,
|
|
18129
|
+
displayName: r.displayName,
|
|
18130
|
+
status: r.status,
|
|
18131
|
+
lastSyncedAt: r.lastSyncedAt,
|
|
18132
|
+
lastError: r.lastError,
|
|
18133
|
+
configJson: r.configJson
|
|
18134
|
+
}));
|
|
18135
|
+
}
|
|
18136
|
+
var sourceConnectedCheck = {
|
|
18137
|
+
id: "traffic.source.connected",
|
|
18138
|
+
category: CheckCategories.integrations,
|
|
18139
|
+
scope: CheckScopes.project,
|
|
18140
|
+
title: "Traffic source connected",
|
|
18141
|
+
run: (ctx) => {
|
|
18142
|
+
if (!ctx.project) return skippedNoProject2();
|
|
18143
|
+
const sources = loadProbes(ctx);
|
|
18144
|
+
if (sources.length === 0) {
|
|
18145
|
+
return {
|
|
18146
|
+
status: CheckStatuses.skipped,
|
|
18147
|
+
code: "traffic.source.none",
|
|
18148
|
+
summary: "No server-side traffic source connected \u2014 server-log AI visibility data unavailable for this project.",
|
|
18149
|
+
remediation: "Connect a traffic source via `canonry traffic connect <type> <project>` to surface crawler hits and AI-referral arrivals from your server logs.",
|
|
18150
|
+
details: { sourceCount: 0 }
|
|
18151
|
+
};
|
|
18152
|
+
}
|
|
18153
|
+
const errored = sources.filter((s) => s.status === "error");
|
|
18154
|
+
if (errored.length > 0 && errored.length === sources.length) {
|
|
18155
|
+
return {
|
|
18156
|
+
status: CheckStatuses.fail,
|
|
18157
|
+
code: "traffic.source.all-errored",
|
|
18158
|
+
summary: `All ${sources.length} traffic source(s) are in error state. No data is being ingested.`,
|
|
18159
|
+
remediation: errored[0].lastError ? `Latest error: "${errored[0].lastError}". Re-connect the source or run \`canonry traffic sync <project> --source <id>\` to retry.` : "Run `canonry traffic sources <project>` to inspect the failing source(s) and re-connect.",
|
|
18160
|
+
details: { sourceCount: sources.length, erroredIds: errored.map((s) => s.id) }
|
|
18161
|
+
};
|
|
18162
|
+
}
|
|
18163
|
+
if (errored.length > 0) {
|
|
18164
|
+
return {
|
|
18165
|
+
status: CheckStatuses.warn,
|
|
18166
|
+
code: "traffic.source.partially-errored",
|
|
18167
|
+
summary: `${errored.length} of ${sources.length} traffic source(s) are in error state.`,
|
|
18168
|
+
remediation: "Run `canonry traffic sources <project>` to inspect the failing sources individually.",
|
|
18169
|
+
details: { sourceCount: sources.length, erroredIds: errored.map((s) => s.id) }
|
|
18170
|
+
};
|
|
18171
|
+
}
|
|
18172
|
+
return {
|
|
18173
|
+
status: CheckStatuses.ok,
|
|
18174
|
+
code: "traffic.source.connected",
|
|
18175
|
+
summary: `${sources.length} traffic source(s) connected: ${sources.map((s) => s.displayName).join(", ")}.`,
|
|
18176
|
+
details: { sourceCount: sources.length, sourceTypes: [...new Set(sources.map((s) => s.sourceType))] }
|
|
18177
|
+
};
|
|
18178
|
+
}
|
|
18179
|
+
};
|
|
18180
|
+
var recentDataCheck = {
|
|
18181
|
+
id: "traffic.source.recent-data",
|
|
18182
|
+
category: CheckCategories.integrations,
|
|
18183
|
+
scope: CheckScopes.project,
|
|
18184
|
+
title: "Traffic source recent data",
|
|
18185
|
+
run: (ctx) => {
|
|
18186
|
+
if (!ctx.project) return skippedNoProject2();
|
|
18187
|
+
const sources = loadProbes(ctx);
|
|
18188
|
+
if (sources.length === 0) {
|
|
18189
|
+
return {
|
|
18190
|
+
status: CheckStatuses.skipped,
|
|
18191
|
+
code: "traffic.recent-data.no-source",
|
|
18192
|
+
summary: "No traffic source connected \u2014 recent-data check skipped."
|
|
18193
|
+
};
|
|
18194
|
+
}
|
|
18195
|
+
const now = /* @__PURE__ */ new Date();
|
|
18196
|
+
const warnCutoff = new Date(now.getTime() - RECENT_DATA_WARN_DAYS * 24 * 60 * 6e4).toISOString();
|
|
18197
|
+
const failCutoff = new Date(now.getTime() - RECENT_DATA_FAIL_DAYS * 24 * 60 * 6e4).toISOString();
|
|
18198
|
+
const recentCrawlers = Number(
|
|
18199
|
+
ctx.db.select({ total: sql9`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
18200
|
+
and15(
|
|
18201
|
+
eq24(crawlerEventsHourly.projectId, ctx.project.id),
|
|
18202
|
+
gte3(crawlerEventsHourly.tsHour, warnCutoff)
|
|
18203
|
+
)
|
|
18204
|
+
).get()?.total ?? 0
|
|
18205
|
+
);
|
|
18206
|
+
const recentReferrals = Number(
|
|
18207
|
+
ctx.db.select({ total: sql9`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
18208
|
+
and15(
|
|
18209
|
+
eq24(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
18210
|
+
gte3(aiReferralEventsHourly.tsHour, warnCutoff)
|
|
18211
|
+
)
|
|
18212
|
+
).get()?.total ?? 0
|
|
18213
|
+
);
|
|
18214
|
+
if (recentCrawlers > 0 || recentReferrals > 0) {
|
|
18215
|
+
return {
|
|
18216
|
+
status: CheckStatuses.ok,
|
|
18217
|
+
code: "traffic.recent-data.fresh",
|
|
18218
|
+
summary: `${recentCrawlers} crawler hit(s) and ${recentReferrals} AI-referral arrival(s) in the last ${RECENT_DATA_WARN_DAYS} days.`,
|
|
18219
|
+
details: { crawlerHits: recentCrawlers, referralArrivals: recentReferrals, windowDays: RECENT_DATA_WARN_DAYS }
|
|
18220
|
+
};
|
|
18221
|
+
}
|
|
18222
|
+
const olderCrawlers = Number(
|
|
18223
|
+
ctx.db.select({ total: sql9`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
18224
|
+
and15(
|
|
18225
|
+
eq24(crawlerEventsHourly.projectId, ctx.project.id),
|
|
18226
|
+
gte3(crawlerEventsHourly.tsHour, failCutoff)
|
|
18227
|
+
)
|
|
18228
|
+
).get()?.total ?? 0
|
|
18229
|
+
);
|
|
18230
|
+
const lastSyncedAt = sources.map((s) => s.lastSyncedAt).filter(Boolean).sort().at(-1) ?? null;
|
|
18231
|
+
if (olderCrawlers > 0 || lastSyncedAt) {
|
|
18232
|
+
return {
|
|
18233
|
+
status: CheckStatuses.warn,
|
|
18234
|
+
code: "traffic.recent-data.stale",
|
|
18235
|
+
summary: `No crawler hits or AI-referral arrivals in the last ${RECENT_DATA_WARN_DAYS} days, though older data exists.`,
|
|
18236
|
+
remediation: lastSyncedAt ? `Last sync: ${lastSyncedAt}. Run \`canonry traffic sync <project>\` to refresh, or check the source connection.` : "Run `canonry traffic sync <project>` to pull recent events.",
|
|
18237
|
+
details: { lastSyncedAt, sourceCount: sources.length }
|
|
18238
|
+
};
|
|
18239
|
+
}
|
|
18240
|
+
return {
|
|
18241
|
+
status: CheckStatuses.fail,
|
|
18242
|
+
code: "traffic.recent-data.empty",
|
|
18243
|
+
summary: `No traffic data in the last ${RECENT_DATA_FAIL_DAYS} days. The source is connected but isn't ingesting.`,
|
|
18244
|
+
remediation: "Verify the source's configuration with `canonry traffic sources <project>` and run a manual sync to confirm credentials + scopes are still valid.",
|
|
18245
|
+
details: { sourceCount: sources.length }
|
|
18246
|
+
};
|
|
18247
|
+
}
|
|
18248
|
+
};
|
|
18249
|
+
async function runValidator(source, validator, fallbackId, fallbackLabel) {
|
|
18250
|
+
if (!validator) {
|
|
18251
|
+
return {
|
|
18252
|
+
source,
|
|
18253
|
+
output: {
|
|
18254
|
+
status: CheckStatuses.skipped,
|
|
18255
|
+
code: `traffic.${fallbackId}.no-validator`,
|
|
18256
|
+
summary: `No ${fallbackLabel} validator registered for source type "${source.sourceType}".`
|
|
18257
|
+
}
|
|
18258
|
+
};
|
|
18259
|
+
}
|
|
18260
|
+
try {
|
|
18261
|
+
const result = await validator(source);
|
|
18262
|
+
if (!result) {
|
|
18263
|
+
return {
|
|
18264
|
+
source,
|
|
18265
|
+
output: {
|
|
18266
|
+
status: CheckStatuses.skipped,
|
|
18267
|
+
code: `traffic.${fallbackId}.unsupported`,
|
|
18268
|
+
summary: `Validator for "${source.sourceType}" does not implement ${fallbackLabel} validation.`
|
|
18269
|
+
}
|
|
18270
|
+
};
|
|
18271
|
+
}
|
|
18272
|
+
return { source, output: result };
|
|
18273
|
+
} catch (e) {
|
|
18274
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
18275
|
+
return {
|
|
18276
|
+
source,
|
|
18277
|
+
output: {
|
|
18278
|
+
status: CheckStatuses.fail,
|
|
18279
|
+
code: `traffic.${fallbackId}.validator-error`,
|
|
18280
|
+
summary: `${fallbackLabel} validator threw: ${msg}.`,
|
|
18281
|
+
remediation: "Check the source configuration and credentials, then re-run the doctor."
|
|
18282
|
+
}
|
|
18283
|
+
};
|
|
18284
|
+
}
|
|
18285
|
+
}
|
|
18286
|
+
function summarizePerSourceResults(fallbackId, fallbackLabel, results) {
|
|
18287
|
+
const failed = results.filter((r) => r.output.status === CheckStatuses.fail);
|
|
18288
|
+
const warned = results.filter((r) => r.output.status === CheckStatuses.warn);
|
|
18289
|
+
const skipped = results.filter((r) => r.output.status === CheckStatuses.skipped);
|
|
18290
|
+
const ok = results.filter((r) => r.output.status === CheckStatuses.ok);
|
|
18291
|
+
const detail = {
|
|
18292
|
+
sources: results.map((r) => ({
|
|
18293
|
+
id: r.source.id,
|
|
18294
|
+
sourceType: r.source.sourceType,
|
|
18295
|
+
displayName: r.source.displayName,
|
|
18296
|
+
status: r.output.status,
|
|
18297
|
+
code: r.output.code,
|
|
18298
|
+
summary: r.output.summary
|
|
18299
|
+
}))
|
|
18300
|
+
};
|
|
18301
|
+
if (failed.length > 0) {
|
|
18302
|
+
return {
|
|
18303
|
+
status: CheckStatuses.fail,
|
|
18304
|
+
code: `traffic.${fallbackId}.failed`,
|
|
18305
|
+
summary: `${failed.length} of ${results.length} source(s) failed ${fallbackLabel} validation: ${failed.map((r) => `${r.source.displayName} (${r.output.summary})`).join("; ")}.`,
|
|
18306
|
+
remediation: failed[0].output.remediation ?? `Inspect the failing source(s) \u2014 see details.sources for per-source codes.`,
|
|
18307
|
+
details: detail
|
|
18308
|
+
};
|
|
18309
|
+
}
|
|
18310
|
+
if (warned.length > 0) {
|
|
18311
|
+
return {
|
|
18312
|
+
status: CheckStatuses.warn,
|
|
18313
|
+
code: `traffic.${fallbackId}.warned`,
|
|
18314
|
+
summary: `${warned.length} of ${results.length} source(s) raised warnings during ${fallbackLabel} validation.`,
|
|
18315
|
+
remediation: warned[0].output.remediation ?? `Review the warning(s) \u2014 see details.sources.`,
|
|
18316
|
+
details: detail
|
|
18317
|
+
};
|
|
18318
|
+
}
|
|
18319
|
+
if (ok.length > 0) {
|
|
18320
|
+
return {
|
|
18321
|
+
status: CheckStatuses.ok,
|
|
18322
|
+
code: `traffic.${fallbackId}.ok`,
|
|
18323
|
+
summary: `${ok.length} source(s) passed ${fallbackLabel} validation${skipped.length > 0 ? ` (${skipped.length} skipped)` : ""}.`,
|
|
18324
|
+
details: detail
|
|
18325
|
+
};
|
|
18326
|
+
}
|
|
18327
|
+
return {
|
|
18328
|
+
status: CheckStatuses.skipped,
|
|
18329
|
+
code: `traffic.${fallbackId}.all-skipped`,
|
|
18330
|
+
summary: `No source-type validator was available for any of the ${results.length} connected source(s).`,
|
|
18331
|
+
details: detail
|
|
18332
|
+
};
|
|
18333
|
+
}
|
|
18334
|
+
var credentialsCheck = {
|
|
18335
|
+
id: "traffic.source.credentials",
|
|
18336
|
+
category: CheckCategories.auth,
|
|
18337
|
+
scope: CheckScopes.project,
|
|
18338
|
+
title: "Traffic source credentials",
|
|
18339
|
+
run: async (ctx) => {
|
|
18340
|
+
if (!ctx.project) return skippedNoProject2();
|
|
18341
|
+
const sources = loadProbes(ctx);
|
|
18342
|
+
if (sources.length === 0) {
|
|
18343
|
+
return {
|
|
18344
|
+
status: CheckStatuses.skipped,
|
|
18345
|
+
code: "traffic.credentials.no-source",
|
|
18346
|
+
summary: "No traffic source connected \u2014 credentials check skipped."
|
|
18347
|
+
};
|
|
18348
|
+
}
|
|
18349
|
+
const validators = ctx.trafficSourceValidators ?? {};
|
|
18350
|
+
const results = await Promise.all(
|
|
18351
|
+
sources.map(
|
|
18352
|
+
(s) => runValidator(
|
|
18353
|
+
s,
|
|
18354
|
+
validators[s.sourceType]?.validateCredentials?.bind(validators[s.sourceType]),
|
|
18355
|
+
"credentials",
|
|
18356
|
+
"credentials"
|
|
18357
|
+
)
|
|
18358
|
+
)
|
|
18359
|
+
);
|
|
18360
|
+
return summarizePerSourceResults("credentials", "credentials", results);
|
|
18361
|
+
}
|
|
18362
|
+
};
|
|
18363
|
+
var scopesCheck2 = {
|
|
18364
|
+
id: "traffic.source.scopes",
|
|
18365
|
+
category: CheckCategories.auth,
|
|
18366
|
+
scope: CheckScopes.project,
|
|
18367
|
+
title: "Traffic source scopes",
|
|
18368
|
+
run: async (ctx) => {
|
|
18369
|
+
if (!ctx.project) return skippedNoProject2();
|
|
18370
|
+
const sources = loadProbes(ctx);
|
|
18371
|
+
if (sources.length === 0) {
|
|
18372
|
+
return {
|
|
18373
|
+
status: CheckStatuses.skipped,
|
|
18374
|
+
code: "traffic.scopes.no-source",
|
|
18375
|
+
summary: "No traffic source connected \u2014 scopes check skipped."
|
|
18376
|
+
};
|
|
18377
|
+
}
|
|
18378
|
+
const validators = ctx.trafficSourceValidators ?? {};
|
|
18379
|
+
const results = await Promise.all(
|
|
18380
|
+
sources.map(
|
|
18381
|
+
(s) => runValidator(
|
|
18382
|
+
s,
|
|
18383
|
+
validators[s.sourceType]?.validateScopes?.bind(validators[s.sourceType]),
|
|
18384
|
+
"scopes",
|
|
18385
|
+
"scopes"
|
|
18386
|
+
)
|
|
18387
|
+
)
|
|
18388
|
+
);
|
|
18389
|
+
return summarizePerSourceResults("scopes", "scopes", results);
|
|
18390
|
+
}
|
|
18391
|
+
};
|
|
18392
|
+
var TRAFFIC_SOURCE_CHECKS = [
|
|
18393
|
+
sourceConnectedCheck,
|
|
18394
|
+
recentDataCheck,
|
|
18395
|
+
credentialsCheck,
|
|
18396
|
+
scopesCheck2
|
|
18397
|
+
];
|
|
18398
|
+
|
|
17703
18399
|
// ../api-routes/src/doctor/registry.ts
|
|
17704
18400
|
var ALL_CHECKS = [
|
|
17705
18401
|
...GOOGLE_AUTH_CHECKS,
|
|
17706
18402
|
...BING_AUTH_CHECKS,
|
|
17707
18403
|
...GA_AUTH_CHECKS,
|
|
17708
|
-
...PROVIDERS_CHECKS
|
|
18404
|
+
...PROVIDERS_CHECKS,
|
|
18405
|
+
...TRAFFIC_SOURCE_CHECKS
|
|
17709
18406
|
];
|
|
17710
18407
|
var CHECK_BY_ID = Object.fromEntries(
|
|
17711
18408
|
ALL_CHECKS.map((check) => [check.id, check])
|
|
@@ -17792,7 +18489,8 @@ async function doctorRoutes(app, opts) {
|
|
|
17792
18489
|
ga4CredentialStore: opts.ga4CredentialStore,
|
|
17793
18490
|
getGoogleAuthConfig: opts.getGoogleAuthConfig,
|
|
17794
18491
|
redirectUri,
|
|
17795
|
-
providerSummary: opts.providerSummary
|
|
18492
|
+
providerSummary: opts.providerSummary,
|
|
18493
|
+
trafficSourceValidators: opts.trafficSourceValidators
|
|
17796
18494
|
};
|
|
17797
18495
|
return runChecks(ctx, ALL_CHECKS, { checkIds });
|
|
17798
18496
|
});
|
|
@@ -17812,7 +18510,8 @@ async function doctorRoutes(app, opts) {
|
|
|
17812
18510
|
ga4CredentialStore: opts.ga4CredentialStore,
|
|
17813
18511
|
getGoogleAuthConfig: opts.getGoogleAuthConfig,
|
|
17814
18512
|
redirectUri,
|
|
17815
|
-
providerSummary: opts.providerSummary
|
|
18513
|
+
providerSummary: opts.providerSummary,
|
|
18514
|
+
trafficSourceValidators: opts.trafficSourceValidators
|
|
17816
18515
|
};
|
|
17817
18516
|
return runChecks(ctx, ALL_CHECKS, { checkIds });
|
|
17818
18517
|
});
|
|
@@ -17951,13 +18650,57 @@ async function apiRoutes(app, opts) {
|
|
|
17951
18650
|
ga4CredentialStore: opts.ga4CredentialStore,
|
|
17952
18651
|
getGoogleAuthConfig: opts.getGoogleAuthConfig,
|
|
17953
18652
|
publicUrl: opts.publicUrl,
|
|
17954
|
-
providerSummary: opts.providerSummary
|
|
18653
|
+
providerSummary: opts.providerSummary,
|
|
18654
|
+
trafficSourceValidators: buildTrafficSourceValidators(opts)
|
|
17955
18655
|
});
|
|
17956
18656
|
if (opts.registerAuthenticatedRoutes) {
|
|
17957
18657
|
await opts.registerAuthenticatedRoutes(api);
|
|
17958
18658
|
}
|
|
17959
18659
|
}, { prefix: opts.routePrefix ?? "/api/v1" });
|
|
17960
18660
|
}
|
|
18661
|
+
function buildTrafficSourceValidators(opts) {
|
|
18662
|
+
const validators = {};
|
|
18663
|
+
if (opts.cloudRunCredentialStore) {
|
|
18664
|
+
const store = opts.cloudRunCredentialStore;
|
|
18665
|
+
const resolveToken = opts.resolveCloudRunAccessToken ?? defaultResolveAccessToken;
|
|
18666
|
+
validators["cloud-run"] = {
|
|
18667
|
+
validateCredentials: async (source) => {
|
|
18668
|
+
const record = store.getConnection(source.projectName);
|
|
18669
|
+
if (!record) {
|
|
18670
|
+
return {
|
|
18671
|
+
status: CheckStatuses.fail,
|
|
18672
|
+
code: "traffic.credentials.missing",
|
|
18673
|
+
summary: `No Cloud Run credential found in ~/.canonry/config.yaml for project "${source.projectName}".`,
|
|
18674
|
+
remediation: "Re-run `canonry traffic connect cloud-run <project> --gcp-project <id> --service-account-key <path>`."
|
|
18675
|
+
};
|
|
18676
|
+
}
|
|
18677
|
+
try {
|
|
18678
|
+
await resolveToken(record);
|
|
18679
|
+
return {
|
|
18680
|
+
status: CheckStatuses.ok,
|
|
18681
|
+
code: "traffic.credentials.resolved",
|
|
18682
|
+
summary: `Cloud Run access token resolves for "${source.displayName}" (project ${record.gcpProjectId}).`
|
|
18683
|
+
};
|
|
18684
|
+
} catch (e) {
|
|
18685
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
18686
|
+
return {
|
|
18687
|
+
status: CheckStatuses.fail,
|
|
18688
|
+
code: "traffic.credentials.resolve-failed",
|
|
18689
|
+
summary: `Failed to resolve Cloud Run access token: ${msg}.`,
|
|
18690
|
+
remediation: "Verify the service-account key in ~/.canonry/config.yaml is unexpired and well-formed. Re-connect the source if needed."
|
|
18691
|
+
};
|
|
18692
|
+
}
|
|
18693
|
+
},
|
|
18694
|
+
// Cloud Run scopes are implicit in the service-account key — Cloud
|
|
18695
|
+
// Logging viewer is the only required scope today, and it's enforced
|
|
18696
|
+
// at the IAM layer rather than baked into the token. We surface a
|
|
18697
|
+
// skipped result so the framework is uniform without producing a
|
|
18698
|
+
// false signal.
|
|
18699
|
+
validateScopes: () => null
|
|
18700
|
+
};
|
|
18701
|
+
}
|
|
18702
|
+
return Object.keys(validators).length > 0 ? validators : void 0;
|
|
18703
|
+
}
|
|
17961
18704
|
|
|
17962
18705
|
// src/server.ts
|
|
17963
18706
|
import os6 from "os";
|
|
@@ -20449,7 +21192,7 @@ import crypto22 from "crypto";
|
|
|
20449
21192
|
import fs7 from "fs";
|
|
20450
21193
|
import path9 from "path";
|
|
20451
21194
|
import os5 from "os";
|
|
20452
|
-
import { and as
|
|
21195
|
+
import { and as and16, eq as eq25, inArray as inArray7, sql as sql10 } from "drizzle-orm";
|
|
20453
21196
|
|
|
20454
21197
|
// src/run-telemetry.ts
|
|
20455
21198
|
import crypto21 from "crypto";
|
|
@@ -20794,7 +21537,7 @@ var JobRunner = class {
|
|
|
20794
21537
|
if (stale.length === 0) return;
|
|
20795
21538
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
20796
21539
|
for (const run of stale) {
|
|
20797
|
-
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(
|
|
21540
|
+
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(eq25(runs.id, run.id)).run();
|
|
20798
21541
|
log.warn("run.recovered-stale", { runId: run.id, previousStatus: run.status });
|
|
20799
21542
|
}
|
|
20800
21543
|
}
|
|
@@ -20828,10 +21571,10 @@ var JobRunner = class {
|
|
|
20828
21571
|
throw new Error(`Run ${runId} is not executable from status '${existingRun.status}'`);
|
|
20829
21572
|
}
|
|
20830
21573
|
if (existingRun.status === "queued") {
|
|
20831
|
-
this.db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
21574
|
+
this.db.update(runs).set({ status: "running", startedAt: now }).where(and16(eq25(runs.id, runId), eq25(runs.status, "queued"))).run();
|
|
20832
21575
|
}
|
|
20833
21576
|
this.throwIfRunCancelled(runId);
|
|
20834
|
-
const project = this.db.select().from(projects).where(
|
|
21577
|
+
const project = this.db.select().from(projects).where(eq25(projects.id, projectId)).get();
|
|
20835
21578
|
if (!project) {
|
|
20836
21579
|
throw new Error(`Project ${projectId} not found`);
|
|
20837
21580
|
}
|
|
@@ -20852,8 +21595,8 @@ var JobRunner = class {
|
|
|
20852
21595
|
throw new Error("No providers configured. Add at least one provider API key.");
|
|
20853
21596
|
}
|
|
20854
21597
|
log.info("run.dispatch", { runId, providerCount: activeProviders.length, providers: activeProviders.map((p) => p.adapter.name) });
|
|
20855
|
-
projectQueries = this.db.select().from(queries).where(
|
|
20856
|
-
const projectCompetitors = this.db.select().from(competitors).where(
|
|
21598
|
+
projectQueries = this.db.select().from(queries).where(eq25(queries.projectId, projectId)).all();
|
|
21599
|
+
const projectCompetitors = this.db.select().from(competitors).where(eq25(competitors.projectId, projectId)).all();
|
|
20857
21600
|
const competitorDomains = projectCompetitors.map((c) => c.domain);
|
|
20858
21601
|
const allDomains = effectiveDomains({
|
|
20859
21602
|
canonicalDomain: project.canonicalDomain,
|
|
@@ -20871,7 +21614,7 @@ var JobRunner = class {
|
|
|
20871
21614
|
const todayPeriod = getCurrentUsageDay();
|
|
20872
21615
|
for (const p of activeProviders) {
|
|
20873
21616
|
const providerScope = `${projectId}:${p.adapter.name}`;
|
|
20874
|
-
const providerUsage = this.db.select().from(usageCounters).where(
|
|
21617
|
+
const providerUsage = this.db.select().from(usageCounters).where(eq25(usageCounters.scope, providerScope)).all().filter((r) => r.period === todayPeriod && r.metric === "queries").reduce((sum, r) => sum + r.count, 0);
|
|
20875
21618
|
const limit = p.config.quotaPolicy.maxRequestsPerDay;
|
|
20876
21619
|
if (providerUsage + queriesPerProvider > limit) {
|
|
20877
21620
|
throw new Error(
|
|
@@ -21014,12 +21757,12 @@ var JobRunner = class {
|
|
|
21014
21757
|
const someFailed = providerErrors.size > 0;
|
|
21015
21758
|
if (allFailed) {
|
|
21016
21759
|
const errorDetail = serializeRunError(buildRunErrorFromMessages(providerErrors));
|
|
21017
|
-
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
21760
|
+
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq25(runs.id, runId)).run();
|
|
21018
21761
|
} else if (someFailed) {
|
|
21019
21762
|
const errorDetail = serializeRunError(buildRunErrorFromMessages(providerErrors));
|
|
21020
|
-
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
21763
|
+
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq25(runs.id, runId)).run();
|
|
21021
21764
|
} else {
|
|
21022
|
-
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
21765
|
+
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq25(runs.id, runId)).run();
|
|
21023
21766
|
}
|
|
21024
21767
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
21025
21768
|
const finalStatus = allFailed ? "failed" : someFailed ? "partial" : "completed";
|
|
@@ -21065,7 +21808,7 @@ var JobRunner = class {
|
|
|
21065
21808
|
status: "failed",
|
|
21066
21809
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21067
21810
|
error: errorMessage
|
|
21068
|
-
}).where(
|
|
21811
|
+
}).where(eq25(runs.id, runId)).run();
|
|
21069
21812
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
21070
21813
|
const abortReason = classifyRunAbortReason(errorMessage);
|
|
21071
21814
|
const phases = buildPhases({ startTime, providerCallStart, providerCallEnd });
|
|
@@ -21118,7 +21861,7 @@ var JobRunner = class {
|
|
|
21118
21861
|
updatedAt: now
|
|
21119
21862
|
}).onConflictDoUpdate({
|
|
21120
21863
|
target: [usageCounters.scope, usageCounters.period, usageCounters.metric],
|
|
21121
|
-
set: { count:
|
|
21864
|
+
set: { count: sql10`${usageCounters.count} + ${count}`, updatedAt: now }
|
|
21122
21865
|
}).run();
|
|
21123
21866
|
}
|
|
21124
21867
|
flushProviderUsage(projectId, providerDispatchCounts) {
|
|
@@ -21133,7 +21876,7 @@ var JobRunner = class {
|
|
|
21133
21876
|
finishedAt: runs.finishedAt,
|
|
21134
21877
|
error: runs.error,
|
|
21135
21878
|
trigger: runs.trigger
|
|
21136
|
-
}).from(runs).where(
|
|
21879
|
+
}).from(runs).where(eq25(runs.id, runId)).get();
|
|
21137
21880
|
}
|
|
21138
21881
|
isRunCancelled(runId) {
|
|
21139
21882
|
return this.getRunState(runId)?.status === "cancelled";
|
|
@@ -21149,7 +21892,7 @@ var JobRunner = class {
|
|
|
21149
21892
|
this.db.update(runs).set({
|
|
21150
21893
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21151
21894
|
error: currentRun.error ?? "Cancelled by user"
|
|
21152
|
-
}).where(
|
|
21895
|
+
}).where(eq25(runs.id, runId)).run();
|
|
21153
21896
|
}
|
|
21154
21897
|
trackEvent(
|
|
21155
21898
|
"run.completed",
|
|
@@ -21187,7 +21930,7 @@ function buildPhases(input) {
|
|
|
21187
21930
|
|
|
21188
21931
|
// src/gsc-sync.ts
|
|
21189
21932
|
import crypto23 from "crypto";
|
|
21190
|
-
import { eq as
|
|
21933
|
+
import { eq as eq26, and as and17, sql as sql11 } from "drizzle-orm";
|
|
21191
21934
|
var log2 = createLogger("GscSync");
|
|
21192
21935
|
function formatDate3(d) {
|
|
21193
21936
|
return d.toISOString().split("T")[0];
|
|
@@ -21199,13 +21942,13 @@ function daysAgo(n) {
|
|
|
21199
21942
|
}
|
|
21200
21943
|
async function executeGscSync(db, runId, projectId, opts) {
|
|
21201
21944
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
21202
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
21945
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq26(runs.id, runId)).run();
|
|
21203
21946
|
try {
|
|
21204
21947
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
21205
21948
|
if (!googleClientId || !googleClientSecret) {
|
|
21206
21949
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
21207
21950
|
}
|
|
21208
|
-
const project = db.select().from(projects).where(
|
|
21951
|
+
const project = db.select().from(projects).where(eq26(projects.id, projectId)).get();
|
|
21209
21952
|
if (!project) {
|
|
21210
21953
|
throw new Error(`Project not found: ${projectId}`);
|
|
21211
21954
|
}
|
|
@@ -21239,10 +21982,10 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
21239
21982
|
});
|
|
21240
21983
|
log2.info("fetch.complete", { runId, projectId, rowCount: rows.length });
|
|
21241
21984
|
db.delete(gscSearchData).where(
|
|
21242
|
-
|
|
21243
|
-
|
|
21244
|
-
|
|
21245
|
-
|
|
21985
|
+
and17(
|
|
21986
|
+
eq26(gscSearchData.projectId, projectId),
|
|
21987
|
+
sql11`${gscSearchData.date} >= ${startDate}`,
|
|
21988
|
+
sql11`${gscSearchData.date} <= ${endDate}`
|
|
21246
21989
|
)
|
|
21247
21990
|
).run();
|
|
21248
21991
|
const batchSize = 500;
|
|
@@ -21307,7 +22050,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
21307
22050
|
log2.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
|
|
21308
22051
|
}
|
|
21309
22052
|
}
|
|
21310
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
22053
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq26(gscUrlInspections.projectId, projectId)).all();
|
|
21311
22054
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
21312
22055
|
for (const row of allInspections) {
|
|
21313
22056
|
const existing = latestByUrl.get(row.url);
|
|
@@ -21328,7 +22071,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
21328
22071
|
}
|
|
21329
22072
|
}
|
|
21330
22073
|
const snapshotDate = formatDate3(/* @__PURE__ */ new Date());
|
|
21331
|
-
db.delete(gscCoverageSnapshots).where(
|
|
22074
|
+
db.delete(gscCoverageSnapshots).where(and17(eq26(gscCoverageSnapshots.projectId, projectId), eq26(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
21332
22075
|
db.insert(gscCoverageSnapshots).values({
|
|
21333
22076
|
id: crypto23.randomUUID(),
|
|
21334
22077
|
projectId,
|
|
@@ -21339,11 +22082,11 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
21339
22082
|
reasonBreakdown: JSON.stringify(reasonCounts),
|
|
21340
22083
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
21341
22084
|
}).run();
|
|
21342
|
-
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
22085
|
+
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq26(runs.id, runId)).run();
|
|
21343
22086
|
log2.info("sync.completed", { runId, projectId, searchDataRows: rows.length, urlInspections: topPages.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
21344
22087
|
} catch (err) {
|
|
21345
22088
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
21346
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
22089
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq26(runs.id, runId)).run();
|
|
21347
22090
|
log2.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
21348
22091
|
throw err;
|
|
21349
22092
|
}
|
|
@@ -21351,7 +22094,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
21351
22094
|
|
|
21352
22095
|
// src/gsc-inspect-sitemap.ts
|
|
21353
22096
|
import crypto24 from "crypto";
|
|
21354
|
-
import { eq as
|
|
22097
|
+
import { eq as eq27, and as and18 } from "drizzle-orm";
|
|
21355
22098
|
|
|
21356
22099
|
// src/sitemap-parser.ts
|
|
21357
22100
|
var log3 = createLogger("SitemapParser");
|
|
@@ -21472,13 +22215,13 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
21472
22215
|
var log4 = createLogger("InspectSitemap");
|
|
21473
22216
|
async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
21474
22217
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
21475
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
22218
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq27(runs.id, runId)).run();
|
|
21476
22219
|
try {
|
|
21477
22220
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
21478
22221
|
if (!googleClientId || !googleClientSecret) {
|
|
21479
22222
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
21480
22223
|
}
|
|
21481
|
-
const project = db.select().from(projects).where(
|
|
22224
|
+
const project = db.select().from(projects).where(eq27(projects.id, projectId)).get();
|
|
21482
22225
|
if (!project) {
|
|
21483
22226
|
throw new Error(`Project not found: ${projectId}`);
|
|
21484
22227
|
}
|
|
@@ -21546,7 +22289,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
21546
22289
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
21547
22290
|
}
|
|
21548
22291
|
}
|
|
21549
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
22292
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq27(gscUrlInspections.projectId, projectId)).all();
|
|
21550
22293
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
21551
22294
|
for (const row of allInspections) {
|
|
21552
22295
|
const existing = latestByUrl.get(row.url);
|
|
@@ -21567,7 +22310,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
21567
22310
|
}
|
|
21568
22311
|
}
|
|
21569
22312
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
21570
|
-
db.delete(gscCoverageSnapshots).where(
|
|
22313
|
+
db.delete(gscCoverageSnapshots).where(and18(eq27(gscCoverageSnapshots.projectId, projectId), eq27(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
21571
22314
|
db.insert(gscCoverageSnapshots).values({
|
|
21572
22315
|
id: crypto24.randomUUID(),
|
|
21573
22316
|
projectId,
|
|
@@ -21579,11 +22322,11 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
21579
22322
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
21580
22323
|
}).run();
|
|
21581
22324
|
const status = errors > 0 && inspected > 0 ? "partial" : errors === urls.length ? "failed" : "completed";
|
|
21582
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
22325
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq27(runs.id, runId)).run();
|
|
21583
22326
|
log4.info("inspect.completed", { runId, projectId, inspected, errors, total: urls.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
21584
22327
|
} catch (err) {
|
|
21585
22328
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
21586
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
22329
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq27(runs.id, runId)).run();
|
|
21587
22330
|
log4.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
21588
22331
|
throw err;
|
|
21589
22332
|
}
|
|
@@ -21591,7 +22334,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
21591
22334
|
|
|
21592
22335
|
// src/bing-inspect-sitemap.ts
|
|
21593
22336
|
import crypto25 from "crypto";
|
|
21594
|
-
import { eq as
|
|
22337
|
+
import { eq as eq28, desc as desc13 } from "drizzle-orm";
|
|
21595
22338
|
var log5 = createLogger("BingInspectSitemap");
|
|
21596
22339
|
function parseBingDate2(value) {
|
|
21597
22340
|
if (!value) return null;
|
|
@@ -21609,9 +22352,9 @@ function isBlockingIssueType2(issueType) {
|
|
|
21609
22352
|
}
|
|
21610
22353
|
async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
21611
22354
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
21612
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
22355
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq28(runs.id, runId)).run();
|
|
21613
22356
|
try {
|
|
21614
|
-
const project = db.select().from(projects).where(
|
|
22357
|
+
const project = db.select().from(projects).where(eq28(projects.id, projectId)).get();
|
|
21615
22358
|
if (!project) {
|
|
21616
22359
|
throw new Error(`Project not found: ${projectId}`);
|
|
21617
22360
|
}
|
|
@@ -21629,7 +22372,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
21629
22372
|
if (sitemapUrls.length === 0) {
|
|
21630
22373
|
throw new Error("No URLs found in sitemap");
|
|
21631
22374
|
}
|
|
21632
|
-
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(
|
|
22375
|
+
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(eq28(bingUrlInspections.projectId, projectId)).all();
|
|
21633
22376
|
const trackedUrls = new Set(trackedRows.map((r) => r.url));
|
|
21634
22377
|
const discovered = sitemapUrls.filter((u) => !trackedUrls.has(u));
|
|
21635
22378
|
log5.info("sitemap.diff", {
|
|
@@ -21712,7 +22455,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
21712
22455
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
21713
22456
|
}
|
|
21714
22457
|
}
|
|
21715
|
-
const allInspections = db.select().from(bingUrlInspections).where(
|
|
22458
|
+
const allInspections = db.select().from(bingUrlInspections).where(eq28(bingUrlInspections.projectId, projectId)).orderBy(desc13(bingUrlInspections.inspectedAt)).all();
|
|
21716
22459
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
21717
22460
|
const definitiveByUrl = /* @__PURE__ */ new Map();
|
|
21718
22461
|
for (const row of allInspections) {
|
|
@@ -21755,7 +22498,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
21755
22498
|
}
|
|
21756
22499
|
}).run();
|
|
21757
22500
|
const status = errors === sitemapUrls.length ? RunStatuses.failed : errors > 0 ? RunStatuses.partial : RunStatuses.completed;
|
|
21758
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
22501
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq28(runs.id, runId)).run();
|
|
21759
22502
|
log5.info("inspect.completed", {
|
|
21760
22503
|
runId,
|
|
21761
22504
|
projectId,
|
|
@@ -21769,7 +22512,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
21769
22512
|
});
|
|
21770
22513
|
} catch (err) {
|
|
21771
22514
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
21772
|
-
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
22515
|
+
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq28(runs.id, runId)).run();
|
|
21773
22516
|
log5.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
21774
22517
|
throw err;
|
|
21775
22518
|
}
|
|
@@ -21778,7 +22521,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
21778
22521
|
// src/commoncrawl-sync.ts
|
|
21779
22522
|
import crypto26 from "crypto";
|
|
21780
22523
|
import path10 from "path";
|
|
21781
|
-
import { and as
|
|
22524
|
+
import { and as and19, eq as eq29, sql as sql12 } from "drizzle-orm";
|
|
21782
22525
|
var log6 = createLogger("CommonCrawlSync");
|
|
21783
22526
|
var INSERT_CHUNK_SIZE = 1e4;
|
|
21784
22527
|
function defaultDeps() {
|
|
@@ -21804,7 +22547,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
21804
22547
|
phaseDetail: "downloading vertices + edges",
|
|
21805
22548
|
updatedAt: downloadStartedAt,
|
|
21806
22549
|
error: null
|
|
21807
|
-
}).where(
|
|
22550
|
+
}).where(eq29(ccReleaseSyncs.id, syncId)).run();
|
|
21808
22551
|
const paths = ccReleasePaths(release);
|
|
21809
22552
|
const releaseCacheDir = path10.join(deps.cacheDir, release);
|
|
21810
22553
|
const vertexPath = path10.join(releaseCacheDir, paths.vertexFilename);
|
|
@@ -21827,7 +22570,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
21827
22570
|
vertexSha256: vertex.sha256,
|
|
21828
22571
|
edgesSha256: edges.sha256,
|
|
21829
22572
|
updatedAt: downloadFinishedAt
|
|
21830
|
-
}).where(
|
|
22573
|
+
}).where(eq29(ccReleaseSyncs.id, syncId)).run();
|
|
21831
22574
|
const allProjects = db.select().from(projects).all();
|
|
21832
22575
|
const targets = Array.from(new Set(allProjects.map((p) => p.canonicalDomain)));
|
|
21833
22576
|
let rows = [];
|
|
@@ -21843,8 +22586,8 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
21843
22586
|
}
|
|
21844
22587
|
const queriedAt = deps.now().toISOString();
|
|
21845
22588
|
db.transaction((tx) => {
|
|
21846
|
-
tx.delete(backlinkDomains).where(
|
|
21847
|
-
tx.delete(backlinkSummaries).where(
|
|
22589
|
+
tx.delete(backlinkDomains).where(eq29(backlinkDomains.releaseSyncId, syncId)).run();
|
|
22590
|
+
tx.delete(backlinkSummaries).where(eq29(backlinkSummaries.releaseSyncId, syncId)).run();
|
|
21848
22591
|
const expanded = [];
|
|
21849
22592
|
for (const r of rows) {
|
|
21850
22593
|
const projectIds = projectsByDomain.get(r.targetDomain);
|
|
@@ -21903,7 +22646,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
21903
22646
|
domainsDiscovered: rows.length,
|
|
21904
22647
|
updatedAt: finishedAt,
|
|
21905
22648
|
error: null
|
|
21906
|
-
}).where(
|
|
22649
|
+
}).where(eq29(ccReleaseSyncs.id, syncId)).run();
|
|
21907
22650
|
log6.info("sync.completed", {
|
|
21908
22651
|
syncId,
|
|
21909
22652
|
release,
|
|
@@ -21933,7 +22676,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
21933
22676
|
error: errorMsg,
|
|
21934
22677
|
phaseDetail: null,
|
|
21935
22678
|
updatedAt: finishedAt
|
|
21936
|
-
}).where(
|
|
22679
|
+
}).where(eq29(ccReleaseSyncs.id, syncId)).run();
|
|
21937
22680
|
log6.error("sync.failed", { syncId, release, error: errorMsg });
|
|
21938
22681
|
throw err;
|
|
21939
22682
|
}
|
|
@@ -21969,7 +22712,7 @@ function computeSummary(rows) {
|
|
|
21969
22712
|
// src/backlink-extract.ts
|
|
21970
22713
|
import crypto27 from "crypto";
|
|
21971
22714
|
import fs8 from "fs";
|
|
21972
|
-
import { and as
|
|
22715
|
+
import { and as and20, desc as desc14, eq as eq30 } from "drizzle-orm";
|
|
21973
22716
|
var log7 = createLogger("BacklinkExtract");
|
|
21974
22717
|
function defaultDeps2() {
|
|
21975
22718
|
return {
|
|
@@ -21981,13 +22724,13 @@ function defaultDeps2() {
|
|
|
21981
22724
|
async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
21982
22725
|
const deps = { ...defaultDeps2(), ...opts.deps };
|
|
21983
22726
|
const startedAt = deps.now().toISOString();
|
|
21984
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
22727
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq30(runs.id, runId)).run();
|
|
21985
22728
|
try {
|
|
21986
|
-
const project = db.select().from(projects).where(
|
|
22729
|
+
const project = db.select().from(projects).where(eq30(projects.id, projectId)).get();
|
|
21987
22730
|
if (!project) {
|
|
21988
22731
|
throw new Error(`Project not found: ${projectId}`);
|
|
21989
22732
|
}
|
|
21990
|
-
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(
|
|
22733
|
+
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(eq30(ccReleaseSyncs.release, opts.release)).get() : db.select().from(ccReleaseSyncs).where(eq30(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).orderBy(desc14(ccReleaseSyncs.createdAt)).limit(1).get();
|
|
21991
22734
|
if (!sync) {
|
|
21992
22735
|
throw new Error("No ready release sync available \u2014 run `canonry backlinks sync` first");
|
|
21993
22736
|
}
|
|
@@ -22015,7 +22758,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
22015
22758
|
const targetDomain = project.canonicalDomain;
|
|
22016
22759
|
db.transaction((tx) => {
|
|
22017
22760
|
tx.delete(backlinkDomains).where(
|
|
22018
|
-
|
|
22761
|
+
and20(eq30(backlinkDomains.projectId, projectId), eq30(backlinkDomains.release, release))
|
|
22019
22762
|
).run();
|
|
22020
22763
|
if (rows.length > 0) {
|
|
22021
22764
|
const values = rows.map((r) => ({
|
|
@@ -22055,7 +22798,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
22055
22798
|
}).run();
|
|
22056
22799
|
});
|
|
22057
22800
|
const finishedAt = deps.now().toISOString();
|
|
22058
|
-
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
22801
|
+
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq30(runs.id, runId)).run();
|
|
22059
22802
|
log7.info("extract.completed", { runId, projectId, release, rows: rows.length });
|
|
22060
22803
|
} catch (err) {
|
|
22061
22804
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
@@ -22064,7 +22807,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
22064
22807
|
status: RunStatuses.failed,
|
|
22065
22808
|
error: errorMsg,
|
|
22066
22809
|
finishedAt
|
|
22067
|
-
}).where(
|
|
22810
|
+
}).where(eq30(runs.id, runId)).run();
|
|
22068
22811
|
log7.error("extract.failed", { runId, projectId, error: errorMsg });
|
|
22069
22812
|
throw err;
|
|
22070
22813
|
}
|
|
@@ -22137,8 +22880,11 @@ var ProviderRegistry = class {
|
|
|
22137
22880
|
|
|
22138
22881
|
// src/scheduler.ts
|
|
22139
22882
|
import cron from "node-cron";
|
|
22140
|
-
import { eq as
|
|
22883
|
+
import { and as and21, eq as eq31 } from "drizzle-orm";
|
|
22141
22884
|
var log8 = createLogger("Scheduler");
|
|
22885
|
+
function taskKey(projectId, kind) {
|
|
22886
|
+
return `${projectId}::${kind}`;
|
|
22887
|
+
}
|
|
22142
22888
|
var Scheduler = class {
|
|
22143
22889
|
db;
|
|
22144
22890
|
callbacks;
|
|
@@ -22149,83 +22895,115 @@ var Scheduler = class {
|
|
|
22149
22895
|
}
|
|
22150
22896
|
/** Load all enabled schedules from DB and register cron jobs. */
|
|
22151
22897
|
start() {
|
|
22152
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
22898
|
+
const allSchedules = this.db.select().from(schedules).where(eq31(schedules.enabled, 1)).all();
|
|
22153
22899
|
for (const schedule of allSchedules) {
|
|
22154
22900
|
const missedRunAt = schedule.nextRunAt;
|
|
22155
22901
|
this.registerCronTask(schedule);
|
|
22156
22902
|
if (missedRunAt && new Date(missedRunAt) < /* @__PURE__ */ new Date()) {
|
|
22157
|
-
log8.info("run.catch-up", { projectId: schedule.projectId, missedRunAt });
|
|
22158
|
-
this.triggerRun(schedule.id, schedule.projectId);
|
|
22903
|
+
log8.info("run.catch-up", { projectId: schedule.projectId, kind: schedule.kind, missedRunAt });
|
|
22904
|
+
this.triggerRun(schedule.id, schedule.projectId, schedule.kind);
|
|
22159
22905
|
}
|
|
22160
22906
|
}
|
|
22161
22907
|
log8.info("started", { scheduleCount: allSchedules.length });
|
|
22162
22908
|
}
|
|
22163
22909
|
/** Stop all cron tasks for graceful shutdown. */
|
|
22164
22910
|
stop() {
|
|
22165
|
-
for (const [
|
|
22166
|
-
this.stopTask(
|
|
22911
|
+
for (const [key, task] of this.tasks) {
|
|
22912
|
+
this.stopTask(key, task, "Stopped");
|
|
22167
22913
|
}
|
|
22168
22914
|
this.tasks.clear();
|
|
22169
22915
|
}
|
|
22170
|
-
/**
|
|
22171
|
-
|
|
22172
|
-
|
|
22916
|
+
/**
|
|
22917
|
+
* Add or update a cron registration at runtime (called when schedule API
|
|
22918
|
+
* is used). Keyed by `(projectId, kind)` so a project's traffic-sync and
|
|
22919
|
+
* answer-visibility schedules can coexist independently.
|
|
22920
|
+
*/
|
|
22921
|
+
upsert(projectId, kind) {
|
|
22922
|
+
const key = taskKey(projectId, kind);
|
|
22923
|
+
const existing = this.tasks.get(key);
|
|
22173
22924
|
if (existing) {
|
|
22174
|
-
this.stopTask(
|
|
22175
|
-
this.tasks.delete(
|
|
22925
|
+
this.stopTask(key, existing, "Stopped");
|
|
22926
|
+
this.tasks.delete(key);
|
|
22176
22927
|
}
|
|
22177
|
-
const schedule = this.db.select().from(schedules).where(
|
|
22928
|
+
const schedule = this.db.select().from(schedules).where(and21(eq31(schedules.projectId, projectId), eq31(schedules.kind, kind))).get();
|
|
22178
22929
|
if (schedule && schedule.enabled === 1) {
|
|
22179
22930
|
this.registerCronTask(schedule);
|
|
22180
22931
|
}
|
|
22181
22932
|
}
|
|
22182
|
-
/** Remove a cron registration (
|
|
22183
|
-
remove(projectId) {
|
|
22184
|
-
const
|
|
22933
|
+
/** Remove a single cron registration (kind-scoped). */
|
|
22934
|
+
remove(projectId, kind) {
|
|
22935
|
+
const key = taskKey(projectId, kind);
|
|
22936
|
+
const existing = this.tasks.get(key);
|
|
22185
22937
|
if (existing) {
|
|
22186
|
-
this.stopTask(
|
|
22187
|
-
this.tasks.delete(
|
|
22938
|
+
this.stopTask(key, existing, "Removed");
|
|
22939
|
+
this.tasks.delete(key);
|
|
22188
22940
|
}
|
|
22189
22941
|
}
|
|
22190
|
-
|
|
22942
|
+
/** Remove ALL cron registrations for a project (used on project delete). */
|
|
22943
|
+
removeAllForProject(projectId) {
|
|
22944
|
+
for (const kind of Object.values(SchedulableRunKinds)) {
|
|
22945
|
+
this.remove(projectId, kind);
|
|
22946
|
+
}
|
|
22947
|
+
}
|
|
22948
|
+
stopTask(key, task, verb) {
|
|
22191
22949
|
task.stop();
|
|
22192
22950
|
task.destroy();
|
|
22193
|
-
log8.info(`task.${verb.toLowerCase()}`, {
|
|
22951
|
+
log8.info(`task.${verb.toLowerCase()}`, { key });
|
|
22194
22952
|
}
|
|
22195
22953
|
registerCronTask(schedule) {
|
|
22196
22954
|
const { id: scheduleId, projectId, cronExpr, timezone } = schedule;
|
|
22955
|
+
const kind = schedule.kind;
|
|
22197
22956
|
if (!cron.validate(cronExpr)) {
|
|
22198
|
-
log8.error("cron.invalid", { projectId, cronExpr });
|
|
22957
|
+
log8.error("cron.invalid", { projectId, kind, cronExpr });
|
|
22199
22958
|
return;
|
|
22200
22959
|
}
|
|
22201
22960
|
const task = cron.schedule(cronExpr, () => {
|
|
22202
|
-
this.triggerRun(scheduleId, projectId);
|
|
22961
|
+
this.triggerRun(scheduleId, projectId, kind);
|
|
22203
22962
|
}, {
|
|
22204
22963
|
timezone
|
|
22205
22964
|
});
|
|
22206
|
-
this.tasks.set(projectId, task);
|
|
22965
|
+
this.tasks.set(taskKey(projectId, kind), task);
|
|
22207
22966
|
this.db.update(schedules).set({
|
|
22208
22967
|
nextRunAt: task.getNextRun()?.toISOString() ?? null,
|
|
22209
22968
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
22210
|
-
}).where(
|
|
22969
|
+
}).where(eq31(schedules.id, scheduleId)).run();
|
|
22211
22970
|
const label = schedule.preset ?? cronExpr;
|
|
22212
|
-
log8.info("cron.registered", { projectId, schedule: label, timezone });
|
|
22971
|
+
log8.info("cron.registered", { projectId, kind, schedule: label, timezone });
|
|
22213
22972
|
}
|
|
22214
|
-
triggerRun(scheduleId, projectId) {
|
|
22973
|
+
triggerRun(scheduleId, projectId, kind) {
|
|
22215
22974
|
try {
|
|
22216
22975
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
22217
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
22976
|
+
const currentSchedule = this.db.select().from(schedules).where(eq31(schedules.id, scheduleId)).get();
|
|
22218
22977
|
if (!currentSchedule || currentSchedule.enabled !== 1) {
|
|
22219
|
-
log8.warn("schedule.stale", { scheduleId, projectId, msg: "schedule no longer exists or is disabled" });
|
|
22220
|
-
this.remove(projectId);
|
|
22978
|
+
log8.warn("schedule.stale", { scheduleId, projectId, kind, msg: "schedule no longer exists or is disabled" });
|
|
22979
|
+
this.remove(projectId, kind);
|
|
22221
22980
|
return;
|
|
22222
22981
|
}
|
|
22223
|
-
const task = this.tasks.get(projectId);
|
|
22982
|
+
const task = this.tasks.get(taskKey(projectId, kind));
|
|
22224
22983
|
const nextRunAt = task?.getNextRun()?.toISOString() ?? null;
|
|
22225
|
-
const project = this.db.select().from(projects).where(
|
|
22984
|
+
const project = this.db.select().from(projects).where(eq31(projects.id, projectId)).get();
|
|
22226
22985
|
if (!project) {
|
|
22227
|
-
log8.error("project.not-found", { projectId, msg: "skipping scheduled run" });
|
|
22228
|
-
this.remove(projectId);
|
|
22986
|
+
log8.error("project.not-found", { projectId, kind, msg: "skipping scheduled run" });
|
|
22987
|
+
this.remove(projectId, kind);
|
|
22988
|
+
return;
|
|
22989
|
+
}
|
|
22990
|
+
if (kind === SchedulableRunKinds["traffic-sync"]) {
|
|
22991
|
+
const sourceId = currentSchedule.sourceId;
|
|
22992
|
+
if (!sourceId) {
|
|
22993
|
+
log8.warn("traffic-sync.missing-source", { scheduleId, projectId });
|
|
22994
|
+
return;
|
|
22995
|
+
}
|
|
22996
|
+
if (!this.callbacks.onTrafficSyncRequested) {
|
|
22997
|
+
log8.warn("traffic-sync.no-callback", { scheduleId, projectId, msg: "host did not register onTrafficSyncRequested" });
|
|
22998
|
+
return;
|
|
22999
|
+
}
|
|
23000
|
+
this.db.update(schedules).set({
|
|
23001
|
+
lastRunAt: now,
|
|
23002
|
+
nextRunAt,
|
|
23003
|
+
updatedAt: now
|
|
23004
|
+
}).where(eq31(schedules.id, currentSchedule.id)).run();
|
|
23005
|
+
log8.info("traffic-sync.triggered", { projectName: project.name, sourceId });
|
|
23006
|
+
this.callbacks.onTrafficSyncRequested(project.name, sourceId);
|
|
22229
23007
|
return;
|
|
22230
23008
|
}
|
|
22231
23009
|
const projectLocations = parseJsonColumn(project.locations, []);
|
|
@@ -22251,7 +23029,7 @@ var Scheduler = class {
|
|
|
22251
23029
|
this.db.update(schedules).set({
|
|
22252
23030
|
nextRunAt,
|
|
22253
23031
|
updatedAt: now
|
|
22254
|
-
}).where(
|
|
23032
|
+
}).where(eq31(schedules.id, currentSchedule.id)).run();
|
|
22255
23033
|
return;
|
|
22256
23034
|
}
|
|
22257
23035
|
const runId = queueResult.runId;
|
|
@@ -22259,19 +23037,19 @@ var Scheduler = class {
|
|
|
22259
23037
|
lastRunAt: now,
|
|
22260
23038
|
nextRunAt,
|
|
22261
23039
|
updatedAt: now
|
|
22262
|
-
}).where(
|
|
23040
|
+
}).where(eq31(schedules.id, currentSchedule.id)).run();
|
|
22263
23041
|
const scheduleProviders = parseJsonColumn(currentSchedule.providers, []);
|
|
22264
23042
|
const providers = scheduleProviders.length > 0 ? scheduleProviders : void 0;
|
|
22265
23043
|
log8.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
22266
23044
|
this.callbacks.onRunCreated(runId, projectId, providers, resolvedLocation);
|
|
22267
23045
|
} catch (err) {
|
|
22268
|
-
log8.error("trigger.error", { scheduleId, projectId, error: err instanceof Error ? err.message : String(err) });
|
|
23046
|
+
log8.error("trigger.error", { scheduleId, projectId, kind, error: err instanceof Error ? err.message : String(err) });
|
|
22269
23047
|
}
|
|
22270
23048
|
}
|
|
22271
23049
|
};
|
|
22272
23050
|
|
|
22273
23051
|
// src/notifier.ts
|
|
22274
|
-
import { eq as
|
|
23052
|
+
import { eq as eq32, desc as desc15, and as and22, or as or4 } from "drizzle-orm";
|
|
22275
23053
|
import crypto28 from "crypto";
|
|
22276
23054
|
var log9 = createLogger("Notifier");
|
|
22277
23055
|
var Notifier = class {
|
|
@@ -22284,18 +23062,18 @@ var Notifier = class {
|
|
|
22284
23062
|
/** Called after a run completes (success, partial, or failed). */
|
|
22285
23063
|
async onRunCompleted(runId, projectId) {
|
|
22286
23064
|
log9.info("run.completed", { runId, projectId });
|
|
22287
|
-
const notifs = this.db.select().from(notifications).where(
|
|
23065
|
+
const notifs = this.db.select().from(notifications).where(eq32(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
22288
23066
|
if (notifs.length === 0) {
|
|
22289
23067
|
log9.info("notifications.none-enabled", { projectId });
|
|
22290
23068
|
return;
|
|
22291
23069
|
}
|
|
22292
23070
|
log9.info("notifications.found", { projectId, count: notifs.length });
|
|
22293
|
-
const run = this.db.select().from(runs).where(
|
|
23071
|
+
const run = this.db.select().from(runs).where(eq32(runs.id, runId)).get();
|
|
22294
23072
|
if (!run) {
|
|
22295
23073
|
log9.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
22296
23074
|
return;
|
|
22297
23075
|
}
|
|
22298
|
-
const project = this.db.select().from(projects).where(
|
|
23076
|
+
const project = this.db.select().from(projects).where(eq32(projects.id, projectId)).get();
|
|
22299
23077
|
if (!project) {
|
|
22300
23078
|
log9.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
22301
23079
|
return;
|
|
@@ -22342,11 +23120,11 @@ var Notifier = class {
|
|
|
22342
23120
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
22343
23121
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
22344
23122
|
if (insightEvents.length === 0) return;
|
|
22345
|
-
const notifs = this.db.select().from(notifications).where(
|
|
23123
|
+
const notifs = this.db.select().from(notifications).where(eq32(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
22346
23124
|
if (notifs.length === 0) return;
|
|
22347
|
-
const run = this.db.select().from(runs).where(
|
|
23125
|
+
const run = this.db.select().from(runs).where(eq32(runs.id, runId)).get();
|
|
22348
23126
|
if (!run) return;
|
|
22349
|
-
const project = this.db.select().from(projects).where(
|
|
23127
|
+
const project = this.db.select().from(projects).where(eq32(projects.id, projectId)).get();
|
|
22350
23128
|
if (!project) return;
|
|
22351
23129
|
for (const notif of notifs) {
|
|
22352
23130
|
const config = parseJsonColumn(notif.config, { url: "", events: [] });
|
|
@@ -22377,9 +23155,9 @@ var Notifier = class {
|
|
|
22377
23155
|
}
|
|
22378
23156
|
computeTransitions(runId, projectId) {
|
|
22379
23157
|
const recentRuns = this.db.select().from(runs).where(
|
|
22380
|
-
|
|
22381
|
-
|
|
22382
|
-
or4(
|
|
23158
|
+
and22(
|
|
23159
|
+
eq32(runs.projectId, projectId),
|
|
23160
|
+
or4(eq32(runs.status, "completed"), eq32(runs.status, "partial"))
|
|
22383
23161
|
)
|
|
22384
23162
|
).orderBy(desc15(runs.createdAt)).limit(2).all();
|
|
22385
23163
|
if (recentRuns.length < 2) return [];
|
|
@@ -22391,12 +23169,12 @@ var Notifier = class {
|
|
|
22391
23169
|
query: queries.query,
|
|
22392
23170
|
provider: querySnapshots.provider,
|
|
22393
23171
|
citationState: querySnapshots.citationState
|
|
22394
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
23172
|
+
}).from(querySnapshots).leftJoin(queries, eq32(querySnapshots.queryId, queries.id)).where(eq32(querySnapshots.runId, currentRunId)).all();
|
|
22395
23173
|
const previousSnapshots = this.db.select({
|
|
22396
23174
|
queryId: querySnapshots.queryId,
|
|
22397
23175
|
provider: querySnapshots.provider,
|
|
22398
23176
|
citationState: querySnapshots.citationState
|
|
22399
|
-
}).from(querySnapshots).where(
|
|
23177
|
+
}).from(querySnapshots).where(eq32(querySnapshots.runId, previousRunId)).all();
|
|
22400
23178
|
const prevMap = /* @__PURE__ */ new Map();
|
|
22401
23179
|
for (const s of previousSnapshots) {
|
|
22402
23180
|
prevMap.set(`${s.queryId}:${s.provider}`, s.citationState);
|
|
@@ -22513,7 +23291,7 @@ var RunCoordinator = class {
|
|
|
22513
23291
|
|
|
22514
23292
|
// src/agent/session-registry.ts
|
|
22515
23293
|
import crypto30 from "crypto";
|
|
22516
|
-
import { eq as
|
|
23294
|
+
import { eq as eq34 } from "drizzle-orm";
|
|
22517
23295
|
|
|
22518
23296
|
// src/agent/session.ts
|
|
22519
23297
|
import fs11 from "fs";
|
|
@@ -22863,7 +23641,7 @@ function resolveSessionProviderAndModel(config, opts) {
|
|
|
22863
23641
|
|
|
22864
23642
|
// src/agent/memory-store.ts
|
|
22865
23643
|
import crypto29 from "crypto";
|
|
22866
|
-
import { and as
|
|
23644
|
+
import { and as and23, desc as desc16, eq as eq33, like as like2, sql as sql13 } from "drizzle-orm";
|
|
22867
23645
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
22868
23646
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
22869
23647
|
function rowToDto2(row) {
|
|
@@ -22877,7 +23655,7 @@ function rowToDto2(row) {
|
|
|
22877
23655
|
};
|
|
22878
23656
|
}
|
|
22879
23657
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
22880
|
-
const query = db.select().from(agentMemory).where(
|
|
23658
|
+
const query = db.select().from(agentMemory).where(eq33(agentMemory.projectId, projectId)).orderBy(desc16(agentMemory.updatedAt));
|
|
22881
23659
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
22882
23660
|
return rows.map(rowToDto2);
|
|
22883
23661
|
}
|
|
@@ -22908,12 +23686,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
22908
23686
|
updatedAt: now
|
|
22909
23687
|
}
|
|
22910
23688
|
}).run();
|
|
22911
|
-
const row = db.select().from(agentMemory).where(
|
|
23689
|
+
const row = db.select().from(agentMemory).where(and23(eq33(agentMemory.projectId, args.projectId), eq33(agentMemory.key, args.key))).get();
|
|
22912
23690
|
if (!row) throw new Error("memory upsert produced no row");
|
|
22913
23691
|
return rowToDto2(row);
|
|
22914
23692
|
}
|
|
22915
23693
|
function deleteMemoryEntry(db, projectId, key) {
|
|
22916
|
-
const result = db.delete(agentMemory).where(
|
|
23694
|
+
const result = db.delete(agentMemory).where(and23(eq33(agentMemory.projectId, projectId), eq33(agentMemory.key, key))).run();
|
|
22917
23695
|
const changes = result.changes ?? 0;
|
|
22918
23696
|
return changes > 0;
|
|
22919
23697
|
}
|
|
@@ -22942,16 +23720,16 @@ function writeCompactionNote(db, args) {
|
|
|
22942
23720
|
}).run();
|
|
22943
23721
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
22944
23722
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
22945
|
-
|
|
22946
|
-
|
|
23723
|
+
and23(
|
|
23724
|
+
eq33(agentMemory.projectId, args.projectId),
|
|
22947
23725
|
like2(agentMemory.key, `${sessionPrefix}%`)
|
|
22948
23726
|
)
|
|
22949
23727
|
).orderBy(desc16(agentMemory.updatedAt)).all();
|
|
22950
23728
|
const stale = existing.slice(COMPACTION_NOTES_PER_SESSION).map((r) => r.id);
|
|
22951
23729
|
if (stale.length > 0) {
|
|
22952
|
-
tx.delete(agentMemory).where(
|
|
23730
|
+
tx.delete(agentMemory).where(sql13`${agentMemory.id} IN (${sql13.join(stale.map((s) => sql13`${s}`), sql13`, `)})`).run();
|
|
22953
23731
|
}
|
|
22954
|
-
const row = tx.select().from(agentMemory).where(
|
|
23732
|
+
const row = tx.select().from(agentMemory).where(and23(eq33(agentMemory.projectId, args.projectId), eq33(agentMemory.key, key))).get();
|
|
22955
23733
|
if (row) inserted = rowToDto2(row);
|
|
22956
23734
|
});
|
|
22957
23735
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -23133,7 +23911,7 @@ var SessionRegistry = class {
|
|
|
23133
23911
|
modelProvider: effectiveProvider,
|
|
23134
23912
|
modelId: effectiveModelId,
|
|
23135
23913
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
23136
|
-
}).where(
|
|
23914
|
+
}).where(eq34(agentSessions.projectId, projectId)).run();
|
|
23137
23915
|
}
|
|
23138
23916
|
const agent2 = createAeroSession({
|
|
23139
23917
|
projectName,
|
|
@@ -23347,7 +24125,7 @@ ${lines.join("\n")}
|
|
|
23347
24125
|
modelProvider: nextProvider,
|
|
23348
24126
|
modelId: nextModelId,
|
|
23349
24127
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
23350
|
-
}).where(
|
|
24128
|
+
}).where(eq34(agentSessions.projectId, projectId)).run();
|
|
23351
24129
|
}
|
|
23352
24130
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
23353
24131
|
save(projectName) {
|
|
@@ -23509,11 +24287,11 @@ ${lines.join("\n")}
|
|
|
23509
24287
|
return id;
|
|
23510
24288
|
}
|
|
23511
24289
|
tryResolveProjectId(projectName) {
|
|
23512
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
24290
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq34(projects.name, projectName)).get();
|
|
23513
24291
|
return row?.id;
|
|
23514
24292
|
}
|
|
23515
24293
|
loadRow(projectId) {
|
|
23516
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
24294
|
+
const row = this.opts.db.select().from(agentSessions).where(eq34(agentSessions.projectId, projectId)).get();
|
|
23517
24295
|
return row ?? null;
|
|
23518
24296
|
}
|
|
23519
24297
|
insertRow(params) {
|
|
@@ -23532,14 +24310,14 @@ ${lines.join("\n")}
|
|
|
23532
24310
|
}
|
|
23533
24311
|
updateRow(projectId, patch) {
|
|
23534
24312
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
23535
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
24313
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq34(agentSessions.projectId, projectId)).run();
|
|
23536
24314
|
}
|
|
23537
24315
|
};
|
|
23538
24316
|
|
|
23539
24317
|
// src/agent/agent-routes.ts
|
|
23540
|
-
import { eq as
|
|
24318
|
+
import { eq as eq35 } from "drizzle-orm";
|
|
23541
24319
|
function resolveProject2(db, name) {
|
|
23542
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
24320
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq35(projects.name, name)).get();
|
|
23543
24321
|
if (!row) throw notFound("project", name);
|
|
23544
24322
|
return row;
|
|
23545
24323
|
}
|
|
@@ -23548,7 +24326,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
23548
24326
|
"/projects/:name/agent/transcript",
|
|
23549
24327
|
async (request) => {
|
|
23550
24328
|
const project = resolveProject2(opts.db, request.params.name);
|
|
23551
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
24329
|
+
const row = opts.db.select().from(agentSessions).where(eq35(agentSessions.projectId, project.id)).get();
|
|
23552
24330
|
if (!row) {
|
|
23553
24331
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
23554
24332
|
}
|
|
@@ -23572,7 +24350,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
23572
24350
|
async (request) => {
|
|
23573
24351
|
const project = resolveProject2(opts.db, request.params.name);
|
|
23574
24352
|
opts.sessionRegistry.reset(project.name);
|
|
23575
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
24353
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq35(agentSessions.projectId, project.id)).run();
|
|
23576
24354
|
return { status: "reset" };
|
|
23577
24355
|
}
|
|
23578
24356
|
);
|
|
@@ -24594,7 +25372,7 @@ async function createServer(opts) {
|
|
|
24594
25372
|
intelligenceService,
|
|
24595
25373
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
24596
25374
|
async ({ runId, projectId, insightCount, criticalOrHigh }) => {
|
|
24597
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
25375
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq36(projects.id, projectId)).get();
|
|
24598
25376
|
if (!project) return;
|
|
24599
25377
|
sessionRegistry.queueFollowUp(project.name, {
|
|
24600
25378
|
role: "user",
|
|
@@ -24618,6 +25396,11 @@ async function createServer(opts) {
|
|
|
24618
25396
|
jobRunner.executeRun(runId, projectId, providers2, location).catch((err) => {
|
|
24619
25397
|
app.log.error({ runId, err }, "Scheduled job runner failed");
|
|
24620
25398
|
});
|
|
25399
|
+
},
|
|
25400
|
+
onTrafficSyncRequested: (projectName, sourceId) => {
|
|
25401
|
+
aeroClient.trafficSync(projectName, sourceId).catch((err) => {
|
|
25402
|
+
app.log.error({ projectName, sourceId, err: err instanceof Error ? err.message : String(err) }, "Scheduled traffic sync failed");
|
|
25403
|
+
});
|
|
24621
25404
|
}
|
|
24622
25405
|
});
|
|
24623
25406
|
const providerSummary = API_ADAPTERS.map((adapter) => ({
|
|
@@ -24749,7 +25532,7 @@ async function createServer(opts) {
|
|
|
24749
25532
|
const apiPrefix = basePath ? `${basePath}api/v1` : "/api/v1";
|
|
24750
25533
|
if (opts.config.apiKey) {
|
|
24751
25534
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
24752
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
25535
|
+
const existing = opts.db.select().from(apiKeys).where(eq36(apiKeys.keyHash, keyHash)).get();
|
|
24753
25536
|
if (!existing) {
|
|
24754
25537
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
24755
25538
|
opts.db.insert(apiKeys).values({
|
|
@@ -24801,7 +25584,7 @@ async function createServer(opts) {
|
|
|
24801
25584
|
};
|
|
24802
25585
|
const getDefaultApiKey = () => {
|
|
24803
25586
|
if (!opts.config.apiKey) return void 0;
|
|
24804
|
-
return opts.db.select().from(apiKeys).where(
|
|
25587
|
+
return opts.db.select().from(apiKeys).where(eq36(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
24805
25588
|
};
|
|
24806
25589
|
const createPasswordSession = (reply) => {
|
|
24807
25590
|
const key = getDefaultApiKey();
|
|
@@ -24858,12 +25641,12 @@ async function createServer(opts) {
|
|
|
24858
25641
|
return reply.send({ authenticated: true });
|
|
24859
25642
|
}
|
|
24860
25643
|
if (apiKey) {
|
|
24861
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
25644
|
+
const key = opts.db.select().from(apiKeys).where(eq36(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
24862
25645
|
if (!key || key.revokedAt) {
|
|
24863
25646
|
const err2 = authInvalid();
|
|
24864
25647
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
24865
25648
|
}
|
|
24866
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
25649
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq36(apiKeys.id, key.id)).run();
|
|
24867
25650
|
const sessionId = createSession(key.id);
|
|
24868
25651
|
reply.header("set-cookie", serializeSessionCookie({
|
|
24869
25652
|
name: SESSION_COOKIE_NAME,
|
|
@@ -25149,12 +25932,12 @@ async function createServer(opts) {
|
|
|
25149
25932
|
return null;
|
|
25150
25933
|
}
|
|
25151
25934
|
},
|
|
25152
|
-
onScheduleUpdated: (action, projectId) => {
|
|
25153
|
-
if (action === "upsert") scheduler.upsert(projectId);
|
|
25154
|
-
if (action === "delete") scheduler.remove(projectId);
|
|
25935
|
+
onScheduleUpdated: (action, projectId, kind) => {
|
|
25936
|
+
if (action === "upsert") scheduler.upsert(projectId, kind);
|
|
25937
|
+
if (action === "delete") scheduler.remove(projectId, kind);
|
|
25155
25938
|
},
|
|
25156
25939
|
onProjectDeleted: (projectId) => {
|
|
25157
|
-
scheduler.
|
|
25940
|
+
scheduler.removeAllForProject(projectId);
|
|
25158
25941
|
},
|
|
25159
25942
|
getTelemetryStatus: () => {
|
|
25160
25943
|
const enabled = isTelemetryEnabled();
|