@ainyc/canonry 4.17.1 → 4.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/assets/assets/{index-C5-Gvl6o.js → index-dLsgu2ck.js} +106 -106
- package/assets/index.html +1 -1
- package/dist/{chunk-ZGHD3IAV.js → chunk-7VDM3JBI.js} +930 -231
- package/dist/{chunk-PAZCY4FF.js → chunk-BN2VQDZ2.js} +1 -1
- package/dist/{chunk-6TWKC3DP.js → chunk-P3SFTXHG.js} +1 -1
- package/dist/{chunk-Q2OED5JQ.js → chunk-SBZTDECX.js} +23 -1
- package/dist/cli.js +5 -5
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-X3PQLBUV.js → intelligence-service-6CX5HH27.js} +2 -2
- package/dist/mcp.js +2 -2
- package/package.json +9 -9
|
@@ -5,7 +5,7 @@ import {
|
|
|
5
5
|
loadConfig,
|
|
6
6
|
loadConfigRaw,
|
|
7
7
|
saveConfigPatch
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-P3SFTXHG.js";
|
|
9
9
|
import {
|
|
10
10
|
DEFAULT_RUN_HISTORY_LIMIT,
|
|
11
11
|
IntelligenceService,
|
|
@@ -66,7 +66,7 @@ import {
|
|
|
66
66
|
schedules,
|
|
67
67
|
trafficSources,
|
|
68
68
|
usageCounters
|
|
69
|
-
} from "./chunk-
|
|
69
|
+
} from "./chunk-BN2VQDZ2.js";
|
|
70
70
|
import {
|
|
71
71
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
72
72
|
AGENT_PROVIDER_IDS,
|
|
@@ -88,6 +88,7 @@ import {
|
|
|
88
88
|
TrafficSourceAuthModes,
|
|
89
89
|
TrafficSourceStatuses,
|
|
90
90
|
TrafficSourceTypes,
|
|
91
|
+
VerificationStatuses,
|
|
91
92
|
absolutizeProjectUrl,
|
|
92
93
|
actionConfidenceLabel,
|
|
93
94
|
agentBusy,
|
|
@@ -106,6 +107,8 @@ import {
|
|
|
106
107
|
dedupeReportActions,
|
|
107
108
|
dedupeReportOpportunities,
|
|
108
109
|
deliveryFailed,
|
|
110
|
+
deltaPercent,
|
|
111
|
+
deltaTone,
|
|
109
112
|
determineAnswerMentioned,
|
|
110
113
|
effectiveDomains,
|
|
111
114
|
emptyCitationVisibility,
|
|
@@ -113,6 +116,7 @@ import {
|
|
|
113
116
|
findDuplicateLocationLabels,
|
|
114
117
|
formatDate,
|
|
115
118
|
formatDateRange,
|
|
119
|
+
formatDeltaCopy,
|
|
116
120
|
formatIsoDate,
|
|
117
121
|
formatNumber,
|
|
118
122
|
formatRatio,
|
|
@@ -155,7 +159,7 @@ import {
|
|
|
155
159
|
visibilityStateFromAnswerMentioned,
|
|
156
160
|
windowCutoff,
|
|
157
161
|
wordpressEnvSchema
|
|
158
|
-
} from "./chunk-
|
|
162
|
+
} from "./chunk-SBZTDECX.js";
|
|
159
163
|
|
|
160
164
|
// src/telemetry.ts
|
|
161
165
|
import crypto from "crypto";
|
|
@@ -314,7 +318,7 @@ import crypto31 from "crypto";
|
|
|
314
318
|
import fs12 from "fs";
|
|
315
319
|
import path14 from "path";
|
|
316
320
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
317
|
-
import { eq as
|
|
321
|
+
import { eq as eq36 } from "drizzle-orm";
|
|
318
322
|
import Fastify from "fastify";
|
|
319
323
|
|
|
320
324
|
// ../api-routes/src/auth.ts
|
|
@@ -2654,7 +2658,7 @@ async function intelligenceRoutes(app) {
|
|
|
2654
2658
|
}
|
|
2655
2659
|
|
|
2656
2660
|
// ../api-routes/src/report.ts
|
|
2657
|
-
import { and as and5, desc as desc6, eq as eq13, inArray as inArray4, or as or2 } from "drizzle-orm";
|
|
2661
|
+
import { and as and5, desc as desc6, eq as eq13, gte, inArray as inArray4, lt, lte, ne, or as or2, sql as sql3 } from "drizzle-orm";
|
|
2658
2662
|
|
|
2659
2663
|
// ../api-routes/src/report-renderer.ts
|
|
2660
2664
|
var COLORS = {
|
|
@@ -4268,11 +4272,151 @@ function renderAiReferrals(report) {
|
|
|
4268
4272
|
</div>`
|
|
4269
4273
|
);
|
|
4270
4274
|
}
|
|
4275
|
+
function serverActivityHeading(audience, hasData) {
|
|
4276
|
+
const isClient = audience === "client";
|
|
4277
|
+
return {
|
|
4278
|
+
id: "server-activity",
|
|
4279
|
+
eyebrow: isClient ? "AI engine attention" : "Section 10",
|
|
4280
|
+
title: "AI Visibility \u2014 Server-Side",
|
|
4281
|
+
intro: isClient ? hasData ? "What AI engines actually do in your server logs over the last 7 days \u2014 the other half of citations." : "Live telemetry from your server logs." : "What AI engines actually do in your server logs \u2014 direct evidence, complementary to citations (which measure what they say)."
|
|
4282
|
+
};
|
|
4283
|
+
}
|
|
4284
|
+
function renderServerActivity(report, audience) {
|
|
4285
|
+
const sa = report.serverActivity;
|
|
4286
|
+
const isClient = audience === "client";
|
|
4287
|
+
if (!sa) {
|
|
4288
|
+
if (isClient) return "";
|
|
4289
|
+
return section(
|
|
4290
|
+
serverActivityHeading("agency", false),
|
|
4291
|
+
renderEmpty("Connect a server-side traffic source to surface what AI engines do directly in your server logs \u2014 distinct from GA4 click-throughs.")
|
|
4292
|
+
);
|
|
4293
|
+
}
|
|
4294
|
+
if (!sa.hasData) {
|
|
4295
|
+
return section(
|
|
4296
|
+
serverActivityHeading(audience, false),
|
|
4297
|
+
renderEmpty(isClient ? "Your server-side traffic source is connected. Numbers will appear after the next sync." : "Source connected \u2014 collecting your first data. Numbers will appear after the next sync.")
|
|
4298
|
+
);
|
|
4299
|
+
}
|
|
4300
|
+
const formatDelta = (d, suffix) => {
|
|
4301
|
+
const copy = formatDeltaCopy(d, suffix);
|
|
4302
|
+
if (!copy) return "";
|
|
4303
|
+
return `<span class="tone-${deltaTone(d.deltaPct)}">${escapeHtml(copy)}</span>`;
|
|
4304
|
+
};
|
|
4305
|
+
if (isClient) {
|
|
4306
|
+
const clientOperators = sa.byOperator.filter((o) => o.verifiedHits > 0 || o.referralArrivals > 0).slice(0, 5);
|
|
4307
|
+
const clientOperatorRows = clientOperators.map((o) => `
|
|
4308
|
+
<tr>
|
|
4309
|
+
<td>${escapeHtml(o.operator)}</td>
|
|
4310
|
+
<td class="numeric">${formatNumber(o.verifiedHits)}</td>
|
|
4311
|
+
<td class="numeric">${formatNumber(o.referralArrivals)}</td>
|
|
4312
|
+
</tr>`).join("");
|
|
4313
|
+
return section(
|
|
4314
|
+
serverActivityHeading("client", true),
|
|
4315
|
+
`<div class="metric-grid">
|
|
4316
|
+
<div class="metric">
|
|
4317
|
+
<div class="label">AI bots visited your site</div>
|
|
4318
|
+
<div class="value">${formatNumber(sa.verifiedCrawlerHits.current)}</div>
|
|
4319
|
+
<div class="subtitle">${formatDelta(sa.verifiedCrawlerHits, "crawls")}</div>
|
|
4320
|
+
</div>
|
|
4321
|
+
<div class="metric">
|
|
4322
|
+
<div class="label">People clicked through from AI</div>
|
|
4323
|
+
<div class="value">${formatNumber(sa.referralArrivals.current)}</div>
|
|
4324
|
+
<div class="subtitle">${formatDelta(sa.referralArrivals, "arrivals")}</div>
|
|
4325
|
+
</div>
|
|
4326
|
+
</div>
|
|
4327
|
+
${clientOperatorRows ? `<div class="chart-card"><h3>By AI tool</h3>
|
|
4328
|
+
<table class="report-table">
|
|
4329
|
+
<thead><tr><th>AI tool</th><th class="numeric">Bot visits (7d)</th><th class="numeric">Click-throughs</th></tr></thead>
|
|
4330
|
+
<tbody>${clientOperatorRows}</tbody>
|
|
4331
|
+
</table>
|
|
4332
|
+
<p class="meta">Verified visits only. We confirm each bot via reverse-DNS so the numbers above can't be inflated by anyone faking a user agent.</p>
|
|
4333
|
+
</div>` : ""}`
|
|
4334
|
+
);
|
|
4335
|
+
}
|
|
4336
|
+
const operatorRows = sa.byOperator.map((o) => {
|
|
4337
|
+
const deltaText = o.deltaPct === null ? "\u2014" : `${o.deltaPct > 0 ? "+" : ""}${o.deltaPct}%`;
|
|
4338
|
+
const toneClass = o.deltaPct === null ? "" : `tone-${deltaTone(o.deltaPct)}`;
|
|
4339
|
+
return `
|
|
4340
|
+
<tr>
|
|
4341
|
+
<td>${escapeHtml(o.operator)}</td>
|
|
4342
|
+
<td class="numeric">${formatNumber(o.verifiedHits)}</td>
|
|
4343
|
+
<td class="numeric meta">${formatNumber(o.unverifiedHits)}</td>
|
|
4344
|
+
<td class="numeric">${formatNumber(o.referralArrivals)}</td>
|
|
4345
|
+
<td class="numeric ${toneClass}">${deltaText}</td>
|
|
4346
|
+
</tr>`;
|
|
4347
|
+
}).join("");
|
|
4348
|
+
const pathRows = sa.topCrawledPaths.map((p) => `
|
|
4349
|
+
<tr>
|
|
4350
|
+
<td class="page-cell">${formatLandingPageHtml(p.path)}</td>
|
|
4351
|
+
<td class="numeric">${formatNumber(p.verifiedHits)}</td>
|
|
4352
|
+
<td class="numeric">${p.distinctOperators}</td>
|
|
4353
|
+
</tr>`).join("");
|
|
4354
|
+
const referralProductRows = sa.referralProducts.map((p) => `
|
|
4355
|
+
<tr>
|
|
4356
|
+
<td>${escapeHtml(p.product)}</td>
|
|
4357
|
+
<td class="numeric">${formatNumber(p.arrivals)}</td>
|
|
4358
|
+
<td class="numeric">${p.distinctLandingPaths}</td>
|
|
4359
|
+
</tr>`).join("");
|
|
4360
|
+
const referralLandingRows = sa.topReferralLandingPaths.map((p) => `
|
|
4361
|
+
<tr>
|
|
4362
|
+
<td class="page-cell">${formatLandingPageHtml(p.path)}</td>
|
|
4363
|
+
<td class="numeric">${formatNumber(p.arrivals)}</td>
|
|
4364
|
+
<td class="numeric">${p.distinctProducts}</td>
|
|
4365
|
+
</tr>`).join("");
|
|
4366
|
+
const trendChart = sa.dailyTrend.length > 0 ? renderLineChart(
|
|
4367
|
+
sa.dailyTrend.map((d) => ({ x: d.date, y: d.verifiedCrawlerHits, label: d.date.slice(5) })),
|
|
4368
|
+
COLORS.series[1],
|
|
4369
|
+
"Verified crawler hits over time (last 14 days)"
|
|
4370
|
+
) : "";
|
|
4371
|
+
return section(
|
|
4372
|
+
serverActivityHeading("agency", true),
|
|
4373
|
+
`<div class="metric-grid">
|
|
4374
|
+
<div class="metric">
|
|
4375
|
+
<div class="label">Verified crawler hits (7d)</div>
|
|
4376
|
+
<div class="value">${formatNumber(sa.verifiedCrawlerHits.current)}</div>
|
|
4377
|
+
<div class="subtitle">${formatDelta(sa.verifiedCrawlerHits, "hits")}</div>
|
|
4378
|
+
</div>
|
|
4379
|
+
<div class="metric">
|
|
4380
|
+
<div class="label">AI-referral arrivals (7d)</div>
|
|
4381
|
+
<div class="value">${formatNumber(sa.referralArrivals.current)}</div>
|
|
4382
|
+
<div class="subtitle">${formatDelta(sa.referralArrivals, "arrivals")}</div>
|
|
4383
|
+
</div>
|
|
4384
|
+
</div>
|
|
4385
|
+
${trendChart}
|
|
4386
|
+
${operatorRows ? `<div class="chart-card"><h3>Per AI operator</h3>
|
|
4387
|
+
<p class="meta">Verified means rDNS-confirmed. Unverified bots claim the user-agent but couldn't be verified \u2014 could be the real bot or an imitator.</p>
|
|
4388
|
+
<table class="report-table">
|
|
4389
|
+
<thead><tr><th>Operator</th><th class="numeric">Verified hits</th><th class="numeric">Unverified</th><th class="numeric">Referral arrivals</th><th class="numeric">7d delta</th></tr></thead>
|
|
4390
|
+
<tbody>${operatorRows}</tbody>
|
|
4391
|
+
</table>
|
|
4392
|
+
</div>` : ""}
|
|
4393
|
+
${pathRows ? `<div class="chart-card"><h3>Top crawled paths</h3>
|
|
4394
|
+
<p class="meta">Pages AI bots fetched most often (verified only, last 7d).</p>
|
|
4395
|
+
<table class="report-table">
|
|
4396
|
+
<thead><tr><th>Path</th><th class="numeric">Verified hits</th><th class="numeric">Distinct operators</th></tr></thead>
|
|
4397
|
+
<tbody>${pathRows}</tbody>
|
|
4398
|
+
</table>
|
|
4399
|
+
</div>` : ""}
|
|
4400
|
+
${referralProductRows ? `<div class="chart-card"><h3>Click-throughs by AI product</h3>
|
|
4401
|
+
<p class="meta">Where humans landed coming from each AI product (chatgpt.com, claude.ai, \u2026).</p>
|
|
4402
|
+
<table class="report-table">
|
|
4403
|
+
<thead><tr><th>Product</th><th class="numeric">Arrivals</th><th class="numeric">Distinct landing paths</th></tr></thead>
|
|
4404
|
+
<tbody>${referralProductRows}</tbody>
|
|
4405
|
+
</table>
|
|
4406
|
+
</div>` : ""}
|
|
4407
|
+
${referralLandingRows ? `<div class="chart-card"><h3>Top AI-referral landing paths</h3>
|
|
4408
|
+
<table class="report-table">
|
|
4409
|
+
<thead><tr><th>Path</th><th class="numeric">Arrivals</th><th class="numeric">Distinct products</th></tr></thead>
|
|
4410
|
+
<tbody>${referralLandingRows}</tbody>
|
|
4411
|
+
</table>
|
|
4412
|
+
</div>` : ""}`
|
|
4413
|
+
);
|
|
4414
|
+
}
|
|
4271
4415
|
function renderIndexingHealth(report) {
|
|
4272
4416
|
const ih = report.indexingHealth;
|
|
4273
4417
|
if (!ih) {
|
|
4274
4418
|
return section(
|
|
4275
|
-
{ id: "indexing-health", eyebrow: "Section
|
|
4419
|
+
{ id: "indexing-health", eyebrow: "Section 11", title: "Indexing Health" },
|
|
4276
4420
|
renderEmpty("Connect Google Search Console or Bing Webmaster Tools and run a sitemap inspection.")
|
|
4277
4421
|
);
|
|
4278
4422
|
}
|
|
@@ -4294,7 +4438,7 @@ function renderIndexingHealth(report) {
|
|
|
4294
4438
|
}).join("");
|
|
4295
4439
|
const legend = segments.map((s) => `<span><span class="legend-swatch" style="background:${s.color}"></span>${escapeHtml(s.label)}: ${s.count}</span>`).join("");
|
|
4296
4440
|
return section(
|
|
4297
|
-
{ id: "indexing-health", eyebrow: "Section
|
|
4441
|
+
{ id: "indexing-health", eyebrow: "Section 11", title: "Indexing Health", intro: `Pages absent from ${ih.provider === "google" ? "Google" : "Bing"} are harder for AI engines to retrieve.` },
|
|
4298
4442
|
`<div class="metric-grid">
|
|
4299
4443
|
<div class="metric"><div class="label">Indexed</div><div class="value tone-positive">${formatNumber(ih.indexed)}</div></div>
|
|
4300
4444
|
<div class="metric"><div class="label">Total inspected</div><div class="value">${formatNumber(ih.total)}</div></div>
|
|
@@ -4311,13 +4455,13 @@ function renderCitationsTrend(report) {
|
|
|
4311
4455
|
const trend = report.citationsTrend;
|
|
4312
4456
|
if (trend.length === 0) {
|
|
4313
4457
|
return section(
|
|
4314
|
-
{ id: "citations-trend", eyebrow: "Section
|
|
4458
|
+
{ id: "citations-trend", eyebrow: "Section 12", title: "Citations Over Time" },
|
|
4315
4459
|
renderEmpty("Run multiple checks to see a trend.")
|
|
4316
4460
|
);
|
|
4317
4461
|
}
|
|
4318
4462
|
if (isTrendBaseline(trend)) {
|
|
4319
4463
|
return section(
|
|
4320
|
-
{ id: "citations-trend", eyebrow: "Section
|
|
4464
|
+
{ id: "citations-trend", eyebrow: "Section 12", title: "Citations Over Time" },
|
|
4321
4465
|
renderEmpty(`Building baseline (${trend.length} of ${MIN_TREND_POINTS} checks completed). Trend will appear once more checks are recorded.`)
|
|
4322
4466
|
);
|
|
4323
4467
|
}
|
|
@@ -4334,7 +4478,7 @@ function renderCitationsTrend(report) {
|
|
|
4334
4478
|
<td>${t.providerRates.map((r) => `${escapeHtml(r.provider)}: ${r.citationRate}%`).join(" \xB7 ")}</td>
|
|
4335
4479
|
</tr>`).join("");
|
|
4336
4480
|
return section(
|
|
4337
|
-
{ id: "citations-trend", eyebrow: "Section
|
|
4481
|
+
{ id: "citations-trend", eyebrow: "Section 12", title: "Citations Over Time", intro: "Citation coverage across recent checks." },
|
|
4338
4482
|
`${chart}
|
|
4339
4483
|
<div class="chart-card"><h3>Check-by-check breakdown</h3>
|
|
4340
4484
|
<table class="report-table">
|
|
@@ -4348,7 +4492,7 @@ function renderInsights(report) {
|
|
|
4348
4492
|
const list = report.insights;
|
|
4349
4493
|
if (list.length === 0) {
|
|
4350
4494
|
return section(
|
|
4351
|
-
{ id: "insights", eyebrow: "Section
|
|
4495
|
+
{ id: "insights", eyebrow: "Section 13", title: "Insights & Alerts" },
|
|
4352
4496
|
renderEmpty("No insights yet \u2014 run a check to generate alerts.")
|
|
4353
4497
|
);
|
|
4354
4498
|
}
|
|
@@ -4365,7 +4509,7 @@ function renderInsights(report) {
|
|
|
4365
4509
|
</tr>`;
|
|
4366
4510
|
}).join("");
|
|
4367
4511
|
return section(
|
|
4368
|
-
{ id: "insights", eyebrow: "Section
|
|
4512
|
+
{ id: "insights", eyebrow: "Section 13", title: "Insights & Alerts", intro: "Regressions, gains, and recurring alerts ordered by severity." },
|
|
4369
4513
|
`<table class="report-table insights-table">
|
|
4370
4514
|
<thead><tr>
|
|
4371
4515
|
<th class="col-severity">Severity</th>
|
|
@@ -4407,7 +4551,7 @@ function renderOpportunities(report) {
|
|
|
4407
4551
|
return section(
|
|
4408
4552
|
{
|
|
4409
4553
|
id: "content-opportunities",
|
|
4410
|
-
eyebrow: "Section
|
|
4554
|
+
eyebrow: "Section 14",
|
|
4411
4555
|
title: "Content Opportunities",
|
|
4412
4556
|
intro: "Queries where content work has the clearest path to more AI citations. Opportunity score is 0\u2013100, higher = stronger."
|
|
4413
4557
|
},
|
|
@@ -4433,7 +4577,7 @@ function renderContentGaps(report) {
|
|
|
4433
4577
|
return section(
|
|
4434
4578
|
{
|
|
4435
4579
|
id: "content-gaps",
|
|
4436
|
-
eyebrow: "Section
|
|
4580
|
+
eyebrow: "Section 15",
|
|
4437
4581
|
title: "Content Gaps",
|
|
4438
4582
|
intro: "Tracked queries where competitors are cited and the client is missing."
|
|
4439
4583
|
},
|
|
@@ -4447,7 +4591,7 @@ function renderRecommendedNextSteps(report) {
|
|
|
4447
4591
|
const steps = report.recommendedNextSteps;
|
|
4448
4592
|
if (steps.length === 0) {
|
|
4449
4593
|
return section(
|
|
4450
|
-
{ id: "recommended-next-steps", eyebrow: "Section
|
|
4594
|
+
{ id: "recommended-next-steps", eyebrow: "Section 16", title: "Recommended Next Steps", intro: "Action items bucketed by timing." },
|
|
4451
4595
|
renderEmpty("No outstanding actions.")
|
|
4452
4596
|
);
|
|
4453
4597
|
}
|
|
@@ -4458,7 +4602,7 @@ function renderRecommendedNextSteps(report) {
|
|
|
4458
4602
|
<span class="rationale">${escapeHtml(s.rationale)}</span>
|
|
4459
4603
|
</div>`).join("");
|
|
4460
4604
|
return section(
|
|
4461
|
-
{ id: "recommended-next-steps", eyebrow: "Section
|
|
4605
|
+
{ id: "recommended-next-steps", eyebrow: "Section 16", title: "Recommended Next Steps", intro: "Action items bucketed by timing." },
|
|
4462
4606
|
`<div class="steps">${items}</div>`
|
|
4463
4607
|
);
|
|
4464
4608
|
}
|
|
@@ -4683,6 +4827,10 @@ function renderReportHtml(report, opts = {}) {
|
|
|
4683
4827
|
const sections = audience === "client" ? [
|
|
4684
4828
|
renderClientSummary(report),
|
|
4685
4829
|
renderWhatsChanged(report, "client"),
|
|
4830
|
+
// Server-side AI visibility runs between WhatsChanged and the action
|
|
4831
|
+
// plan in BOTH the SPA and HTML so clients see the same ordered set
|
|
4832
|
+
// of sections in either surface (per the report-parity rule).
|
|
4833
|
+
renderServerActivity(report, "client"),
|
|
4686
4834
|
renderAudienceActionPlan(report, "client"),
|
|
4687
4835
|
renderClientEvidenceSummary(report)
|
|
4688
4836
|
].join("\n") : [
|
|
@@ -4697,6 +4845,7 @@ function renderReportHtml(report, opts = {}) {
|
|
|
4697
4845
|
renderGa(report),
|
|
4698
4846
|
renderSocial(report),
|
|
4699
4847
|
renderAiReferrals(report),
|
|
4848
|
+
renderServerActivity(report, "agency"),
|
|
4700
4849
|
renderIndexingHealth(report),
|
|
4701
4850
|
renderCitationsTrend(report),
|
|
4702
4851
|
renderInsights(report),
|
|
@@ -5082,6 +5231,9 @@ var TOP_AI_REFERRAL_PAGES_LIMIT = 10;
|
|
|
5082
5231
|
var TOP_CAMPAIGN_LIMIT = 10;
|
|
5083
5232
|
var INSIGHT_LOOKBACK_RUNS = 5;
|
|
5084
5233
|
var REPORT_WINDOW_DAYS = 30;
|
|
5234
|
+
var SERVER_ACTIVITY_HEADLINE_DAYS = 7;
|
|
5235
|
+
var SERVER_ACTIVITY_TREND_DAYS = 14;
|
|
5236
|
+
var SERVER_ACTIVITY_TOP_PATHS_LIMIT = 10;
|
|
5085
5237
|
function windowStartDate(endDate, windowDays) {
|
|
5086
5238
|
const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(endDate);
|
|
5087
5239
|
if (!m) return endDate;
|
|
@@ -5350,6 +5502,209 @@ function buildAiReferrals(db, projectId) {
|
|
|
5350
5502
|
const topLandingPages = [...pageAgg.entries()].map(([page, data]) => ({ page, sessions: data.sessions, users: data.users })).sort((a, b) => b.sessions - a.sessions).slice(0, TOP_AI_REFERRAL_PAGES_LIMIT);
|
|
5351
5503
|
return { totalSessions: total, totalUsers, bySource, trend, topLandingPages };
|
|
5352
5504
|
}
|
|
5505
|
+
function buildServerActivity(db, projectId) {
|
|
5506
|
+
const sourceRows = db.select({ id: trafficSources.id }).from(trafficSources).where(
|
|
5507
|
+
and5(
|
|
5508
|
+
eq13(trafficSources.projectId, projectId),
|
|
5509
|
+
ne(trafficSources.status, TrafficSourceStatuses.archived)
|
|
5510
|
+
)
|
|
5511
|
+
).all();
|
|
5512
|
+
if (sourceRows.length === 0) return null;
|
|
5513
|
+
const now = /* @__PURE__ */ new Date();
|
|
5514
|
+
const headlineEnd = now.toISOString();
|
|
5515
|
+
const headlineStartMs = now.getTime() - SERVER_ACTIVITY_HEADLINE_DAYS * 24 * 60 * 6e4;
|
|
5516
|
+
const priorStartMs = headlineStartMs - SERVER_ACTIVITY_HEADLINE_DAYS * 24 * 60 * 6e4;
|
|
5517
|
+
const trendStartMs = now.getTime() - SERVER_ACTIVITY_TREND_DAYS * 24 * 60 * 6e4;
|
|
5518
|
+
const headlineStart = new Date(headlineStartMs).toISOString();
|
|
5519
|
+
const priorStart = new Date(priorStartMs).toISOString();
|
|
5520
|
+
const trendStart = new Date(trendStartMs).toISOString();
|
|
5521
|
+
const sumVerifiedCrawlers = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
5522
|
+
db.select({ total: sql3`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
5523
|
+
and5(
|
|
5524
|
+
eq13(crawlerEventsHourly.projectId, projectId),
|
|
5525
|
+
eq13(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
5526
|
+
gte(crawlerEventsHourly.tsHour, windowStartIso),
|
|
5527
|
+
exclusiveEnd ? lt(crawlerEventsHourly.tsHour, windowEndIso) : lte(crawlerEventsHourly.tsHour, windowEndIso)
|
|
5528
|
+
)
|
|
5529
|
+
).get()?.total ?? 0
|
|
5530
|
+
);
|
|
5531
|
+
const sumReferrals = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
5532
|
+
db.select({ total: sql3`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
5533
|
+
and5(
|
|
5534
|
+
eq13(aiReferralEventsHourly.projectId, projectId),
|
|
5535
|
+
gte(aiReferralEventsHourly.tsHour, windowStartIso),
|
|
5536
|
+
exclusiveEnd ? lt(aiReferralEventsHourly.tsHour, windowEndIso) : lte(aiReferralEventsHourly.tsHour, windowEndIso)
|
|
5537
|
+
)
|
|
5538
|
+
).get()?.total ?? 0
|
|
5539
|
+
);
|
|
5540
|
+
const verifiedCurrent = sumVerifiedCrawlers(headlineStart, headlineEnd);
|
|
5541
|
+
const verifiedPrior = sumVerifiedCrawlers(priorStart, headlineStart, true);
|
|
5542
|
+
const referralCurrent = sumReferrals(headlineStart, headlineEnd);
|
|
5543
|
+
const referralPrior = sumReferrals(priorStart, headlineStart, true);
|
|
5544
|
+
const crawlerByOperatorRows = db.select({
|
|
5545
|
+
operator: crawlerEventsHourly.operator,
|
|
5546
|
+
verificationStatus: crawlerEventsHourly.verificationStatus,
|
|
5547
|
+
hits: sql3`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
5548
|
+
}).from(crawlerEventsHourly).where(
|
|
5549
|
+
and5(
|
|
5550
|
+
eq13(crawlerEventsHourly.projectId, projectId),
|
|
5551
|
+
gte(crawlerEventsHourly.tsHour, headlineStart),
|
|
5552
|
+
lte(crawlerEventsHourly.tsHour, headlineEnd)
|
|
5553
|
+
)
|
|
5554
|
+
).groupBy(crawlerEventsHourly.operator, crawlerEventsHourly.verificationStatus).all();
|
|
5555
|
+
const crawlerByOperatorPriorRows = db.select({
|
|
5556
|
+
operator: crawlerEventsHourly.operator,
|
|
5557
|
+
hits: sql3`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
5558
|
+
}).from(crawlerEventsHourly).where(
|
|
5559
|
+
and5(
|
|
5560
|
+
eq13(crawlerEventsHourly.projectId, projectId),
|
|
5561
|
+
eq13(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
5562
|
+
gte(crawlerEventsHourly.tsHour, priorStart),
|
|
5563
|
+
lt(crawlerEventsHourly.tsHour, headlineStart)
|
|
5564
|
+
)
|
|
5565
|
+
).groupBy(crawlerEventsHourly.operator).all();
|
|
5566
|
+
const referralByOperatorRows = db.select({
|
|
5567
|
+
operator: aiReferralEventsHourly.operator,
|
|
5568
|
+
hits: sql3`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`
|
|
5569
|
+
}).from(aiReferralEventsHourly).where(
|
|
5570
|
+
and5(
|
|
5571
|
+
eq13(aiReferralEventsHourly.projectId, projectId),
|
|
5572
|
+
gte(aiReferralEventsHourly.tsHour, headlineStart),
|
|
5573
|
+
lte(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
5574
|
+
)
|
|
5575
|
+
).groupBy(aiReferralEventsHourly.operator).all();
|
|
5576
|
+
const operatorAgg = /* @__PURE__ */ new Map();
|
|
5577
|
+
const ensureOp = (op) => {
|
|
5578
|
+
let entry = operatorAgg.get(op);
|
|
5579
|
+
if (!entry) {
|
|
5580
|
+
entry = { verified: 0, unverified: 0, referrals: 0, prior: 0 };
|
|
5581
|
+
operatorAgg.set(op, entry);
|
|
5582
|
+
}
|
|
5583
|
+
return entry;
|
|
5584
|
+
};
|
|
5585
|
+
for (const r of crawlerByOperatorRows) {
|
|
5586
|
+
const entry = ensureOp(r.operator);
|
|
5587
|
+
if (r.verificationStatus === VerificationStatuses.verified) entry.verified += Number(r.hits);
|
|
5588
|
+
else entry.unverified += Number(r.hits);
|
|
5589
|
+
}
|
|
5590
|
+
for (const r of crawlerByOperatorPriorRows) {
|
|
5591
|
+
ensureOp(r.operator).prior += Number(r.hits);
|
|
5592
|
+
}
|
|
5593
|
+
for (const r of referralByOperatorRows) {
|
|
5594
|
+
ensureOp(r.operator).referrals += Number(r.hits);
|
|
5595
|
+
}
|
|
5596
|
+
const byOperator = [...operatorAgg.entries()].map(([operator, v]) => ({
|
|
5597
|
+
operator,
|
|
5598
|
+
verifiedHits: v.verified,
|
|
5599
|
+
unverifiedHits: v.unverified,
|
|
5600
|
+
referralArrivals: v.referrals,
|
|
5601
|
+
deltaPct: deltaPercent(v.verified, v.prior)
|
|
5602
|
+
})).sort(
|
|
5603
|
+
(a, b) => b.verifiedHits - a.verifiedHits || b.referralArrivals - a.referralArrivals
|
|
5604
|
+
);
|
|
5605
|
+
const topPathsRows = db.select({
|
|
5606
|
+
path: crawlerEventsHourly.pathNormalized,
|
|
5607
|
+
hits: sql3`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
5608
|
+
operators: sql3`COUNT(DISTINCT ${crawlerEventsHourly.operator})`
|
|
5609
|
+
}).from(crawlerEventsHourly).where(
|
|
5610
|
+
and5(
|
|
5611
|
+
eq13(crawlerEventsHourly.projectId, projectId),
|
|
5612
|
+
eq13(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
5613
|
+
gte(crawlerEventsHourly.tsHour, headlineStart),
|
|
5614
|
+
lte(crawlerEventsHourly.tsHour, headlineEnd)
|
|
5615
|
+
)
|
|
5616
|
+
).groupBy(crawlerEventsHourly.pathNormalized).orderBy(desc6(sql3`SUM(${crawlerEventsHourly.hits})`)).limit(SERVER_ACTIVITY_TOP_PATHS_LIMIT).all();
|
|
5617
|
+
const topCrawledPaths = topPathsRows.map((r) => ({
|
|
5618
|
+
path: r.path,
|
|
5619
|
+
verifiedHits: Number(r.hits),
|
|
5620
|
+
distinctOperators: Number(r.operators)
|
|
5621
|
+
}));
|
|
5622
|
+
const referralProductsRows = db.select({
|
|
5623
|
+
product: aiReferralEventsHourly.product,
|
|
5624
|
+
arrivals: sql3`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
5625
|
+
landingPaths: sql3`COUNT(DISTINCT ${aiReferralEventsHourly.landingPathNormalized})`
|
|
5626
|
+
}).from(aiReferralEventsHourly).where(
|
|
5627
|
+
and5(
|
|
5628
|
+
eq13(aiReferralEventsHourly.projectId, projectId),
|
|
5629
|
+
gte(aiReferralEventsHourly.tsHour, headlineStart),
|
|
5630
|
+
lte(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
5631
|
+
)
|
|
5632
|
+
).groupBy(aiReferralEventsHourly.product).orderBy(desc6(sql3`SUM(${aiReferralEventsHourly.sessionsOrHits})`)).all();
|
|
5633
|
+
const referralProducts = referralProductsRows.map((r) => ({
|
|
5634
|
+
product: r.product,
|
|
5635
|
+
arrivals: Number(r.arrivals),
|
|
5636
|
+
distinctLandingPaths: Number(r.landingPaths)
|
|
5637
|
+
}));
|
|
5638
|
+
const topReferralRows = db.select({
|
|
5639
|
+
path: aiReferralEventsHourly.landingPathNormalized,
|
|
5640
|
+
arrivals: sql3`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
5641
|
+
products: sql3`COUNT(DISTINCT ${aiReferralEventsHourly.product})`
|
|
5642
|
+
}).from(aiReferralEventsHourly).where(
|
|
5643
|
+
and5(
|
|
5644
|
+
eq13(aiReferralEventsHourly.projectId, projectId),
|
|
5645
|
+
gte(aiReferralEventsHourly.tsHour, headlineStart),
|
|
5646
|
+
lte(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
5647
|
+
)
|
|
5648
|
+
).groupBy(aiReferralEventsHourly.landingPathNormalized).orderBy(desc6(sql3`SUM(${aiReferralEventsHourly.sessionsOrHits})`)).limit(SERVER_ACTIVITY_TOP_PATHS_LIMIT).all();
|
|
5649
|
+
const topReferralLandingPaths = topReferralRows.map((r) => ({
|
|
5650
|
+
path: r.path,
|
|
5651
|
+
arrivals: Number(r.arrivals),
|
|
5652
|
+
distinctProducts: Number(r.products)
|
|
5653
|
+
}));
|
|
5654
|
+
const crawlerTrendRows = db.select({
|
|
5655
|
+
date: sql3`SUBSTR(${crawlerEventsHourly.tsHour}, 1, 10)`,
|
|
5656
|
+
hits: sql3`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
5657
|
+
}).from(crawlerEventsHourly).where(
|
|
5658
|
+
and5(
|
|
5659
|
+
eq13(crawlerEventsHourly.projectId, projectId),
|
|
5660
|
+
eq13(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
5661
|
+
gte(crawlerEventsHourly.tsHour, trendStart),
|
|
5662
|
+
lte(crawlerEventsHourly.tsHour, headlineEnd)
|
|
5663
|
+
)
|
|
5664
|
+
).groupBy(sql3`SUBSTR(${crawlerEventsHourly.tsHour}, 1, 10)`).all();
|
|
5665
|
+
const referralTrendRows = db.select({
|
|
5666
|
+
date: sql3`SUBSTR(${aiReferralEventsHourly.tsHour}, 1, 10)`,
|
|
5667
|
+
hits: sql3`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`
|
|
5668
|
+
}).from(aiReferralEventsHourly).where(
|
|
5669
|
+
and5(
|
|
5670
|
+
eq13(aiReferralEventsHourly.projectId, projectId),
|
|
5671
|
+
gte(aiReferralEventsHourly.tsHour, trendStart),
|
|
5672
|
+
lte(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
5673
|
+
)
|
|
5674
|
+
).groupBy(sql3`SUBSTR(${aiReferralEventsHourly.tsHour}, 1, 10)`).all();
|
|
5675
|
+
const dailyTrendMap = /* @__PURE__ */ new Map();
|
|
5676
|
+
for (const r of crawlerTrendRows) {
|
|
5677
|
+
const e = dailyTrendMap.get(r.date) ?? { verifiedCrawlerHits: 0, referralArrivals: 0 };
|
|
5678
|
+
e.verifiedCrawlerHits += Number(r.hits);
|
|
5679
|
+
dailyTrendMap.set(r.date, e);
|
|
5680
|
+
}
|
|
5681
|
+
for (const r of referralTrendRows) {
|
|
5682
|
+
const e = dailyTrendMap.get(r.date) ?? { verifiedCrawlerHits: 0, referralArrivals: 0 };
|
|
5683
|
+
e.referralArrivals += Number(r.hits);
|
|
5684
|
+
dailyTrendMap.set(r.date, e);
|
|
5685
|
+
}
|
|
5686
|
+
const dailyTrend = [...dailyTrendMap.entries()].map(([date, v]) => ({ date, ...v })).sort((a, b) => a.date.localeCompare(b.date));
|
|
5687
|
+
return {
|
|
5688
|
+
windowStart: headlineStart,
|
|
5689
|
+
windowEnd: headlineEnd,
|
|
5690
|
+
hasData: verifiedCurrent + referralCurrent + verifiedPrior + referralPrior > 0 || byOperator.length > 0 || topCrawledPaths.length > 0 || referralProducts.length > 0,
|
|
5691
|
+
verifiedCrawlerHits: {
|
|
5692
|
+
current: verifiedCurrent,
|
|
5693
|
+
prior: verifiedPrior,
|
|
5694
|
+
deltaPct: deltaPercent(verifiedCurrent, verifiedPrior)
|
|
5695
|
+
},
|
|
5696
|
+
referralArrivals: {
|
|
5697
|
+
current: referralCurrent,
|
|
5698
|
+
prior: referralPrior,
|
|
5699
|
+
deltaPct: deltaPercent(referralCurrent, referralPrior)
|
|
5700
|
+
},
|
|
5701
|
+
byOperator,
|
|
5702
|
+
topCrawledPaths,
|
|
5703
|
+
referralProducts,
|
|
5704
|
+
dailyTrend,
|
|
5705
|
+
topReferralLandingPaths
|
|
5706
|
+
};
|
|
5707
|
+
}
|
|
5353
5708
|
function buildIndexingHealth(db, projectId) {
|
|
5354
5709
|
const gsc = db.select().from(gscCoverageSnapshots).where(eq13(gscCoverageSnapshots.projectId, projectId)).orderBy(desc6(gscCoverageSnapshots.date)).limit(1).get();
|
|
5355
5710
|
if (gsc) {
|
|
@@ -6009,6 +6364,7 @@ function buildProjectReport(db, projectName) {
|
|
|
6009
6364
|
const gaSection = buildGaSection(db, project.id);
|
|
6010
6365
|
const socialSection = buildSocialReferrals(db, project.id);
|
|
6011
6366
|
const aiReferralsSection = buildAiReferrals(db, project.id);
|
|
6367
|
+
const serverActivitySection = buildServerActivity(db, project.id);
|
|
6012
6368
|
const indexingHealthSection = buildIndexingHealth(db, project.id);
|
|
6013
6369
|
const citationsTrend = buildCitationsTrend(db, project.id, queryLookup, latestRunLocation);
|
|
6014
6370
|
const insightList = buildInsightList(db, project.id, latestRunLocation);
|
|
@@ -6152,6 +6508,7 @@ function buildProjectReport(db, projectName) {
|
|
|
6152
6508
|
ga: gaSection,
|
|
6153
6509
|
socialReferrals: socialSection,
|
|
6154
6510
|
aiReferrals: aiReferralsSection,
|
|
6511
|
+
serverActivity: serverActivitySection,
|
|
6155
6512
|
indexingHealth: indexingHealthSection,
|
|
6156
6513
|
citationsTrend,
|
|
6157
6514
|
whatsChanged,
|
|
@@ -6355,7 +6712,7 @@ function normalizeDomain2(domain) {
|
|
|
6355
6712
|
}
|
|
6356
6713
|
|
|
6357
6714
|
// ../api-routes/src/composites.ts
|
|
6358
|
-
import { eq as eq15, and as and6, desc as desc7, sql as
|
|
6715
|
+
import { eq as eq15, and as and6, desc as desc7, sql as sql4, like, or as or3, inArray as inArray6 } from "drizzle-orm";
|
|
6359
6716
|
var TOP_INSIGHT_LIMIT = 5;
|
|
6360
6717
|
var SEARCH_HIT_HARD_LIMIT = 50;
|
|
6361
6718
|
var SEARCH_SNIPPET_RADIUS = 80;
|
|
@@ -6461,9 +6818,9 @@ async function compositeRoutes(app) {
|
|
|
6461
6818
|
and6(
|
|
6462
6819
|
eq15(queries.projectId, project.id),
|
|
6463
6820
|
or3(
|
|
6464
|
-
|
|
6465
|
-
|
|
6466
|
-
|
|
6821
|
+
sql4`${querySnapshots.answerText} LIKE ${pattern} ESCAPE '\\'`,
|
|
6822
|
+
sql4`${querySnapshots.citedDomains} LIKE ${pattern} ESCAPE '\\'`,
|
|
6823
|
+
sql4`${querySnapshots.rawResponse} LIKE ${pattern} ESCAPE '\\'`,
|
|
6467
6824
|
like(queries.query, pattern)
|
|
6468
6825
|
)
|
|
6469
6826
|
)
|
|
@@ -6474,8 +6831,8 @@ async function compositeRoutes(app) {
|
|
|
6474
6831
|
or3(
|
|
6475
6832
|
like(insights.title, pattern),
|
|
6476
6833
|
like(insights.query, pattern),
|
|
6477
|
-
|
|
6478
|
-
|
|
6834
|
+
sql4`${insights.recommendation} LIKE ${pattern} ESCAPE '\\'`,
|
|
6835
|
+
sql4`${insights.cause} LIKE ${pattern} ESCAPE '\\'`
|
|
6479
6836
|
)
|
|
6480
6837
|
)
|
|
6481
6838
|
).orderBy(desc7(insights.createdAt)).limit(limit + 1).all();
|
|
@@ -10373,7 +10730,7 @@ function formatNotification(row) {
|
|
|
10373
10730
|
|
|
10374
10731
|
// ../api-routes/src/google.ts
|
|
10375
10732
|
import crypto14 from "crypto";
|
|
10376
|
-
import { eq as eq18, and as and8, desc as desc8, sql as
|
|
10733
|
+
import { eq as eq18, and as and8, desc as desc8, sql as sql5 } from "drizzle-orm";
|
|
10377
10734
|
|
|
10378
10735
|
// ../integration-google/src/constants.ts
|
|
10379
10736
|
var GOOGLE_AUTH_URL = "https://accounts.google.com/o/oauth2/v2/auth";
|
|
@@ -11583,11 +11940,11 @@ async function googleRoutes(app, opts) {
|
|
|
11583
11940
|
const { startDate, endDate, query, page, limit } = request.query;
|
|
11584
11941
|
const cutoffDate = !startDate ? windowCutoff(parseWindow(request.query.window))?.slice(0, 10) ?? null : null;
|
|
11585
11942
|
const conditions = [eq18(gscSearchData.projectId, project.id)];
|
|
11586
|
-
if (startDate) conditions.push(
|
|
11587
|
-
else if (cutoffDate) conditions.push(
|
|
11588
|
-
if (endDate) conditions.push(
|
|
11589
|
-
if (query) conditions.push(
|
|
11590
|
-
if (page) conditions.push(
|
|
11943
|
+
if (startDate) conditions.push(sql5`${gscSearchData.date} >= ${startDate}`);
|
|
11944
|
+
else if (cutoffDate) conditions.push(sql5`${gscSearchData.date} >= ${cutoffDate}`);
|
|
11945
|
+
if (endDate) conditions.push(sql5`${gscSearchData.date} <= ${endDate}`);
|
|
11946
|
+
if (query) conditions.push(sql5`${gscSearchData.query} LIKE ${"%" + query + "%"}`);
|
|
11947
|
+
if (page) conditions.push(sql5`${gscSearchData.page} LIKE ${"%" + page + "%"}`);
|
|
11591
11948
|
const rows = app.db.select().from(gscSearchData).where(and8(...conditions)).orderBy(desc8(gscSearchData.date)).limit(parseInt(limit ?? "500", 10)).all();
|
|
11592
11949
|
return rows.map((r) => ({
|
|
11593
11950
|
date: r.date,
|
|
@@ -12830,7 +13187,7 @@ async function cdpRoutes(app, opts) {
|
|
|
12830
13187
|
|
|
12831
13188
|
// ../api-routes/src/ga.ts
|
|
12832
13189
|
import crypto16 from "crypto";
|
|
12833
|
-
import { eq as eq21, desc as desc10, and as and11, sql as
|
|
13190
|
+
import { eq as eq21, desc as desc10, and as and11, sql as sql6 } from "drizzle-orm";
|
|
12834
13191
|
function gaLog(level, action, ctx) {
|
|
12835
13192
|
const entry = { ts: (/* @__PURE__ */ new Date()).toISOString(), level, module: "GA4Routes", action, ...ctx };
|
|
12836
13193
|
const stream = level === "error" ? process.stderr : process.stdout;
|
|
@@ -13127,8 +13484,8 @@ async function ga4Routes(app, opts) {
|
|
|
13127
13484
|
tx.delete(gaTrafficSnapshots).where(
|
|
13128
13485
|
and11(
|
|
13129
13486
|
eq21(gaTrafficSnapshots.projectId, project.id),
|
|
13130
|
-
|
|
13131
|
-
|
|
13487
|
+
sql6`${gaTrafficSnapshots.date} >= ${summary.periodStart}`,
|
|
13488
|
+
sql6`${gaTrafficSnapshots.date} <= ${summary.periodEnd}`
|
|
13132
13489
|
)
|
|
13133
13490
|
).run();
|
|
13134
13491
|
for (const row of rows) {
|
|
@@ -13151,8 +13508,8 @@ async function ga4Routes(app, opts) {
|
|
|
13151
13508
|
tx.delete(gaAiReferrals).where(
|
|
13152
13509
|
and11(
|
|
13153
13510
|
eq21(gaAiReferrals.projectId, project.id),
|
|
13154
|
-
|
|
13155
|
-
|
|
13511
|
+
sql6`${gaAiReferrals.date} >= ${summary.periodStart}`,
|
|
13512
|
+
sql6`${gaAiReferrals.date} <= ${summary.periodEnd}`
|
|
13156
13513
|
)
|
|
13157
13514
|
).run();
|
|
13158
13515
|
for (const row of aiReferrals) {
|
|
@@ -13177,8 +13534,8 @@ async function ga4Routes(app, opts) {
|
|
|
13177
13534
|
tx.delete(gaSocialReferrals).where(
|
|
13178
13535
|
and11(
|
|
13179
13536
|
eq21(gaSocialReferrals.projectId, project.id),
|
|
13180
|
-
|
|
13181
|
-
|
|
13537
|
+
sql6`${gaSocialReferrals.date} >= ${summary.periodStart}`,
|
|
13538
|
+
sql6`${gaSocialReferrals.date} <= ${summary.periodEnd}`
|
|
13182
13539
|
)
|
|
13183
13540
|
).run();
|
|
13184
13541
|
for (const row of socialReferrals) {
|
|
@@ -13268,11 +13625,11 @@ async function ga4Routes(app, opts) {
|
|
|
13268
13625
|
const cutoff = windowCutoff(window);
|
|
13269
13626
|
const cutoffDate = cutoff?.slice(0, 10) ?? null;
|
|
13270
13627
|
const snapshotConditions = [eq21(gaTrafficSnapshots.projectId, project.id)];
|
|
13271
|
-
if (cutoffDate) snapshotConditions.push(
|
|
13628
|
+
if (cutoffDate) snapshotConditions.push(sql6`${gaTrafficSnapshots.date} >= ${cutoffDate}`);
|
|
13272
13629
|
const aiConditions = [eq21(gaAiReferrals.projectId, project.id)];
|
|
13273
|
-
if (cutoffDate) aiConditions.push(
|
|
13630
|
+
if (cutoffDate) aiConditions.push(sql6`${gaAiReferrals.date} >= ${cutoffDate}`);
|
|
13274
13631
|
const socialConditions = [eq21(gaSocialReferrals.projectId, project.id)];
|
|
13275
|
-
if (cutoffDate) socialConditions.push(
|
|
13632
|
+
if (cutoffDate) socialConditions.push(sql6`${gaSocialReferrals.date} >= ${cutoffDate}`);
|
|
13276
13633
|
const windowSummaryRow = cutoffDate ? app.db.select({
|
|
13277
13634
|
totalSessions: gaTrafficWindowSummaries.totalSessions,
|
|
13278
13635
|
totalOrganicSessions: gaTrafficWindowSummaries.totalOrganicSessions,
|
|
@@ -13285,9 +13642,9 @@ async function ga4Routes(app, opts) {
|
|
|
13285
13642
|
)
|
|
13286
13643
|
).get() : null;
|
|
13287
13644
|
const snapshotTotalsRow = cutoffDate && !windowSummaryRow ? app.db.select({
|
|
13288
|
-
totalSessions:
|
|
13289
|
-
totalOrganicSessions:
|
|
13290
|
-
totalUsers:
|
|
13645
|
+
totalSessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)`,
|
|
13646
|
+
totalOrganicSessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)`,
|
|
13647
|
+
totalUsers: sql6`COALESCE(SUM(${gaTrafficSnapshots.users}), 0)`
|
|
13291
13648
|
}).from(gaTrafficSnapshots).where(and11(...snapshotConditions)).get() : null;
|
|
13292
13649
|
const summaryRow = cutoffDate ? windowSummaryRow ?? snapshotTotalsRow : app.db.select({
|
|
13293
13650
|
totalSessions: gaTrafficSummaries.totalSessions,
|
|
@@ -13295,38 +13652,38 @@ async function ga4Routes(app, opts) {
|
|
|
13295
13652
|
totalUsers: gaTrafficSummaries.totalUsers
|
|
13296
13653
|
}).from(gaTrafficSummaries).where(eq21(gaTrafficSummaries.projectId, project.id)).get();
|
|
13297
13654
|
const directTotalRow = windowSummaryRow ? { totalDirectSessions: windowSummaryRow.totalDirectSessions } : app.db.select({
|
|
13298
|
-
totalDirectSessions:
|
|
13655
|
+
totalDirectSessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)`
|
|
13299
13656
|
}).from(gaTrafficSnapshots).where(and11(...snapshotConditions)).get();
|
|
13300
13657
|
const summaryMeta = app.db.select({
|
|
13301
13658
|
periodStart: gaTrafficSummaries.periodStart,
|
|
13302
13659
|
periodEnd: gaTrafficSummaries.periodEnd
|
|
13303
13660
|
}).from(gaTrafficSummaries).where(eq21(gaTrafficSummaries.projectId, project.id)).get();
|
|
13304
13661
|
const rows = app.db.select({
|
|
13305
|
-
landingPage:
|
|
13306
|
-
sessions:
|
|
13307
|
-
organicSessions:
|
|
13308
|
-
directSessions:
|
|
13309
|
-
users:
|
|
13310
|
-
}).from(gaTrafficSnapshots).where(and11(...snapshotConditions)).groupBy(
|
|
13662
|
+
landingPage: sql6`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`,
|
|
13663
|
+
sessions: sql6`SUM(${gaTrafficSnapshots.sessions})`,
|
|
13664
|
+
organicSessions: sql6`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
13665
|
+
directSessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)`,
|
|
13666
|
+
users: sql6`SUM(${gaTrafficSnapshots.users})`
|
|
13667
|
+
}).from(gaTrafficSnapshots).where(and11(...snapshotConditions)).groupBy(sql6`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`).orderBy(sql6`SUM(${gaTrafficSnapshots.sessions}) DESC`).limit(limit).all();
|
|
13311
13668
|
const aiReferralRows = app.db.select({
|
|
13312
13669
|
source: gaAiReferrals.source,
|
|
13313
13670
|
medium: gaAiReferrals.medium,
|
|
13314
13671
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
13315
|
-
sessions:
|
|
13316
|
-
users:
|
|
13672
|
+
sessions: sql6`SUM(${gaAiReferrals.sessions})`,
|
|
13673
|
+
users: sql6`SUM(${gaAiReferrals.users})`
|
|
13317
13674
|
}).from(gaAiReferrals).where(and11(...aiConditions)).groupBy(gaAiReferrals.source, gaAiReferrals.medium, gaAiReferrals.sourceDimension).all();
|
|
13318
13675
|
const aiReferralLandingPageRows = app.db.select({
|
|
13319
13676
|
source: gaAiReferrals.source,
|
|
13320
13677
|
medium: gaAiReferrals.medium,
|
|
13321
13678
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
13322
|
-
landingPage:
|
|
13323
|
-
sessions:
|
|
13324
|
-
users:
|
|
13679
|
+
landingPage: sql6`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`,
|
|
13680
|
+
sessions: sql6`SUM(${gaAiReferrals.sessions})`,
|
|
13681
|
+
users: sql6`SUM(${gaAiReferrals.users})`
|
|
13325
13682
|
}).from(gaAiReferrals).where(and11(...aiConditions)).groupBy(
|
|
13326
13683
|
gaAiReferrals.source,
|
|
13327
13684
|
gaAiReferrals.medium,
|
|
13328
13685
|
gaAiReferrals.sourceDimension,
|
|
13329
|
-
|
|
13686
|
+
sql6`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`
|
|
13330
13687
|
).all();
|
|
13331
13688
|
const aiReferrals = pickWinningDimension(
|
|
13332
13689
|
aiReferralRows,
|
|
@@ -13337,10 +13694,10 @@ async function ga4Routes(app, opts) {
|
|
|
13337
13694
|
(r) => `${r.source}\0${r.medium}\0${r.landingPage}`
|
|
13338
13695
|
);
|
|
13339
13696
|
const aiDeduped = app.db.select({
|
|
13340
|
-
sessions:
|
|
13341
|
-
users:
|
|
13697
|
+
sessions: sql6`COALESCE(SUM(max_sessions), 0)`,
|
|
13698
|
+
users: sql6`COALESCE(SUM(max_users), 0)`
|
|
13342
13699
|
}).from(
|
|
13343
|
-
|
|
13700
|
+
sql6`(
|
|
13344
13701
|
SELECT date, source, medium,
|
|
13345
13702
|
MAX(dimension_sessions) AS max_sessions,
|
|
13346
13703
|
MAX(dimension_users) AS max_users
|
|
@@ -13349,7 +13706,7 @@ async function ga4Routes(app, opts) {
|
|
|
13349
13706
|
SUM(sessions) AS dimension_sessions,
|
|
13350
13707
|
SUM(users) AS dimension_users
|
|
13351
13708
|
FROM ga_ai_referrals
|
|
13352
|
-
WHERE project_id = ${project.id}${cutoffDate ?
|
|
13709
|
+
WHERE project_id = ${project.id}${cutoffDate ? sql6` AND date >= ${cutoffDate}` : sql6``}
|
|
13353
13710
|
GROUP BY date, source, medium, source_dimension
|
|
13354
13711
|
)
|
|
13355
13712
|
GROUP BY date, source, medium
|
|
@@ -13357,8 +13714,8 @@ async function ga4Routes(app, opts) {
|
|
|
13357
13714
|
).get();
|
|
13358
13715
|
const aiBySessionRows = app.db.select({
|
|
13359
13716
|
channelGroup: gaAiReferrals.channelGroup,
|
|
13360
|
-
sessions:
|
|
13361
|
-
users:
|
|
13717
|
+
sessions: sql6`COALESCE(SUM(${gaAiReferrals.sessions}), 0)`,
|
|
13718
|
+
users: sql6`COALESCE(SUM(${gaAiReferrals.users}), 0)`
|
|
13362
13719
|
}).from(gaAiReferrals).where(and11(...aiConditions, eq21(gaAiReferrals.sourceDimension, "session"))).groupBy(gaAiReferrals.channelGroup).all();
|
|
13363
13720
|
const aiSessionsByChannelGroup = /* @__PURE__ */ new Map();
|
|
13364
13721
|
let aiBySessionUsers = 0;
|
|
@@ -13371,12 +13728,12 @@ async function ga4Routes(app, opts) {
|
|
|
13371
13728
|
source: gaSocialReferrals.source,
|
|
13372
13729
|
medium: gaSocialReferrals.medium,
|
|
13373
13730
|
channelGroup: gaSocialReferrals.channelGroup,
|
|
13374
|
-
sessions:
|
|
13375
|
-
users:
|
|
13376
|
-
}).from(gaSocialReferrals).where(and11(...socialConditions)).groupBy(gaSocialReferrals.source, gaSocialReferrals.medium, gaSocialReferrals.channelGroup).orderBy(
|
|
13731
|
+
sessions: sql6`SUM(${gaSocialReferrals.sessions})`,
|
|
13732
|
+
users: sql6`SUM(${gaSocialReferrals.users})`
|
|
13733
|
+
}).from(gaSocialReferrals).where(and11(...socialConditions)).groupBy(gaSocialReferrals.source, gaSocialReferrals.medium, gaSocialReferrals.channelGroup).orderBy(sql6`SUM(${gaSocialReferrals.sessions}) DESC`).all();
|
|
13377
13734
|
const socialTotals = app.db.select({
|
|
13378
|
-
sessions:
|
|
13379
|
-
users:
|
|
13735
|
+
sessions: sql6`SUM(${gaSocialReferrals.sessions})`,
|
|
13736
|
+
users: sql6`SUM(${gaSocialReferrals.users})`
|
|
13380
13737
|
}).from(gaSocialReferrals).where(and11(...socialConditions)).get();
|
|
13381
13738
|
const latestSync = app.db.select({ syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(eq21(gaTrafficSummaries.projectId, project.id)).orderBy(desc10(gaTrafficSummaries.syncedAt)).limit(1).get();
|
|
13382
13739
|
const total = summaryRow?.totalSessions ?? 0;
|
|
@@ -13459,21 +13816,21 @@ async function ga4Routes(app, opts) {
|
|
|
13459
13816
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
13460
13817
|
const cutoffDate = windowCutoff(parseWindow(request.query.window))?.slice(0, 10) ?? null;
|
|
13461
13818
|
const conditions = [eq21(gaAiReferrals.projectId, project.id)];
|
|
13462
|
-
if (cutoffDate) conditions.push(
|
|
13819
|
+
if (cutoffDate) conditions.push(sql6`${gaAiReferrals.date} >= ${cutoffDate}`);
|
|
13463
13820
|
const rows = app.db.select({
|
|
13464
13821
|
date: gaAiReferrals.date,
|
|
13465
13822
|
source: gaAiReferrals.source,
|
|
13466
13823
|
medium: gaAiReferrals.medium,
|
|
13467
|
-
landingPage:
|
|
13824
|
+
landingPage: sql6`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`,
|
|
13468
13825
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
13469
|
-
sessions:
|
|
13470
|
-
users:
|
|
13826
|
+
sessions: sql6`SUM(${gaAiReferrals.sessions})`,
|
|
13827
|
+
users: sql6`SUM(${gaAiReferrals.users})`
|
|
13471
13828
|
}).from(gaAiReferrals).where(and11(...conditions)).groupBy(
|
|
13472
13829
|
gaAiReferrals.date,
|
|
13473
13830
|
gaAiReferrals.source,
|
|
13474
13831
|
gaAiReferrals.medium,
|
|
13475
13832
|
gaAiReferrals.sourceDimension,
|
|
13476
|
-
|
|
13833
|
+
sql6`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`
|
|
13477
13834
|
).orderBy(gaAiReferrals.date).all();
|
|
13478
13835
|
return rows;
|
|
13479
13836
|
});
|
|
@@ -13482,7 +13839,7 @@ async function ga4Routes(app, opts) {
|
|
|
13482
13839
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
13483
13840
|
const cutoffDate = windowCutoff(parseWindow(request.query.window))?.slice(0, 10) ?? null;
|
|
13484
13841
|
const conditions = [eq21(gaSocialReferrals.projectId, project.id)];
|
|
13485
|
-
if (cutoffDate) conditions.push(
|
|
13842
|
+
if (cutoffDate) conditions.push(sql6`${gaSocialReferrals.date} >= ${cutoffDate}`);
|
|
13486
13843
|
const rows = app.db.select({
|
|
13487
13844
|
date: gaSocialReferrals.date,
|
|
13488
13845
|
source: gaSocialReferrals.source,
|
|
@@ -13503,10 +13860,10 @@ async function ga4Routes(app, opts) {
|
|
|
13503
13860
|
d.setDate(d.getDate() - n);
|
|
13504
13861
|
return fmt(d);
|
|
13505
13862
|
};
|
|
13506
|
-
const sumSocial = (from, to) => app.db.select({ sessions:
|
|
13863
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and11(
|
|
13507
13864
|
eq21(gaSocialReferrals.projectId, project.id),
|
|
13508
|
-
|
|
13509
|
-
|
|
13865
|
+
sql6`${gaSocialReferrals.date} >= ${from}`,
|
|
13866
|
+
sql6`${gaSocialReferrals.date} < ${to}`
|
|
13510
13867
|
)).get();
|
|
13511
13868
|
const current7d = sumSocial(daysAgo2(7), fmt(today));
|
|
13512
13869
|
const prev7d = sumSocial(daysAgo2(14), daysAgo2(7));
|
|
@@ -13515,19 +13872,19 @@ async function ga4Routes(app, opts) {
|
|
|
13515
13872
|
const pct = (cur, prev) => prev === 0 ? null : Math.round((cur - prev) / prev * 100);
|
|
13516
13873
|
const sourceCurrent = app.db.select({
|
|
13517
13874
|
source: gaSocialReferrals.source,
|
|
13518
|
-
sessions:
|
|
13875
|
+
sessions: sql6`SUM(${gaSocialReferrals.sessions})`
|
|
13519
13876
|
}).from(gaSocialReferrals).where(and11(
|
|
13520
13877
|
eq21(gaSocialReferrals.projectId, project.id),
|
|
13521
|
-
|
|
13522
|
-
|
|
13878
|
+
sql6`${gaSocialReferrals.date} >= ${daysAgo2(7)}`,
|
|
13879
|
+
sql6`${gaSocialReferrals.date} < ${fmt(today)}`
|
|
13523
13880
|
)).groupBy(gaSocialReferrals.source).all();
|
|
13524
13881
|
const sourcePrev = app.db.select({
|
|
13525
13882
|
source: gaSocialReferrals.source,
|
|
13526
|
-
sessions:
|
|
13883
|
+
sessions: sql6`SUM(${gaSocialReferrals.sessions})`
|
|
13527
13884
|
}).from(gaSocialReferrals).where(and11(
|
|
13528
13885
|
eq21(gaSocialReferrals.projectId, project.id),
|
|
13529
|
-
|
|
13530
|
-
|
|
13886
|
+
sql6`${gaSocialReferrals.date} >= ${daysAgo2(14)}`,
|
|
13887
|
+
sql6`${gaSocialReferrals.date} < ${daysAgo2(7)}`
|
|
13531
13888
|
)).groupBy(gaSocialReferrals.source).all();
|
|
13532
13889
|
const prevMap = new Map(sourcePrev.map((r) => [r.source, r.sessions]));
|
|
13533
13890
|
let biggestMover = null;
|
|
@@ -13566,16 +13923,16 @@ async function ga4Routes(app, opts) {
|
|
|
13566
13923
|
return fmt(d);
|
|
13567
13924
|
};
|
|
13568
13925
|
const pct = (cur, prev) => prev === 0 ? null : Math.round((cur - prev) / prev * 100);
|
|
13569
|
-
const sumTotal = (from, to) => app.db.select({ sessions:
|
|
13570
|
-
const sumOrganic = (from, to) => app.db.select({ sessions:
|
|
13571
|
-
const sumDirect = (from, to) => app.db.select({ sessions:
|
|
13572
|
-
const sumAi = (from, to) => app.db.select({ sessions:
|
|
13926
|
+
const sumTotal = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)` }).from(gaTrafficSnapshots).where(and11(eq21(gaTrafficSnapshots.projectId, project.id), sql6`${gaTrafficSnapshots.date} >= ${from}`, sql6`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
13927
|
+
const sumOrganic = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)` }).from(gaTrafficSnapshots).where(and11(eq21(gaTrafficSnapshots.projectId, project.id), sql6`${gaTrafficSnapshots.date} >= ${from}`, sql6`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
13928
|
+
const sumDirect = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)` }).from(gaTrafficSnapshots).where(and11(eq21(gaTrafficSnapshots.projectId, project.id), sql6`${gaTrafficSnapshots.date} >= ${from}`, sql6`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
13929
|
+
const sumAi = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and11(
|
|
13573
13930
|
eq21(gaAiReferrals.projectId, project.id),
|
|
13574
|
-
|
|
13575
|
-
|
|
13931
|
+
sql6`${gaAiReferrals.date} >= ${from}`,
|
|
13932
|
+
sql6`${gaAiReferrals.date} < ${to}`,
|
|
13576
13933
|
eq21(gaAiReferrals.sourceDimension, "session")
|
|
13577
13934
|
)).get();
|
|
13578
|
-
const sumSocial = (from, to) => app.db.select({ sessions:
|
|
13935
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql6`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and11(eq21(gaSocialReferrals.projectId, project.id), sql6`${gaSocialReferrals.date} >= ${from}`, sql6`${gaSocialReferrals.date} < ${to}`)).get();
|
|
13579
13936
|
const todayStr = fmt(today);
|
|
13580
13937
|
const buildTrend = (sum) => {
|
|
13581
13938
|
const c7 = sum(daysAgo2(7), todayStr)?.sessions ?? 0;
|
|
@@ -13584,16 +13941,16 @@ async function ga4Routes(app, opts) {
|
|
|
13584
13941
|
const p30 = sum(daysAgo2(60), daysAgo2(30))?.sessions ?? 0;
|
|
13585
13942
|
return { sessions7d: c7, sessionsPrev7d: p7, trend7dPct: pct(c7, p7), sessions30d: c30, sessionsPrev30d: p30, trend30dPct: pct(c30, p30) };
|
|
13586
13943
|
};
|
|
13587
|
-
const aiSourceCurrent = app.db.select({ source: gaAiReferrals.source, sessions:
|
|
13944
|
+
const aiSourceCurrent = app.db.select({ source: gaAiReferrals.source, sessions: sql6`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and11(
|
|
13588
13945
|
eq21(gaAiReferrals.projectId, project.id),
|
|
13589
|
-
|
|
13590
|
-
|
|
13946
|
+
sql6`${gaAiReferrals.date} >= ${daysAgo2(7)}`,
|
|
13947
|
+
sql6`${gaAiReferrals.date} < ${todayStr}`,
|
|
13591
13948
|
eq21(gaAiReferrals.sourceDimension, "session")
|
|
13592
13949
|
)).groupBy(gaAiReferrals.source).all();
|
|
13593
|
-
const aiSourcePrev = app.db.select({ source: gaAiReferrals.source, sessions:
|
|
13950
|
+
const aiSourcePrev = app.db.select({ source: gaAiReferrals.source, sessions: sql6`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and11(
|
|
13594
13951
|
eq21(gaAiReferrals.projectId, project.id),
|
|
13595
|
-
|
|
13596
|
-
|
|
13952
|
+
sql6`${gaAiReferrals.date} >= ${daysAgo2(14)}`,
|
|
13953
|
+
sql6`${gaAiReferrals.date} < ${daysAgo2(7)}`,
|
|
13597
13954
|
eq21(gaAiReferrals.sourceDimension, "session")
|
|
13598
13955
|
)).groupBy(gaAiReferrals.source).all();
|
|
13599
13956
|
const findBiggestMover = (current, prev) => {
|
|
@@ -13610,8 +13967,8 @@ async function ga4Routes(app, opts) {
|
|
|
13610
13967
|
}
|
|
13611
13968
|
return mover;
|
|
13612
13969
|
};
|
|
13613
|
-
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions:
|
|
13614
|
-
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions:
|
|
13970
|
+
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions: sql6`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and11(eq21(gaSocialReferrals.projectId, project.id), sql6`${gaSocialReferrals.date} >= ${daysAgo2(7)}`, sql6`${gaSocialReferrals.date} < ${todayStr}`)).groupBy(gaSocialReferrals.source).all();
|
|
13971
|
+
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions: sql6`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and11(eq21(gaSocialReferrals.projectId, project.id), sql6`${gaSocialReferrals.date} >= ${daysAgo2(14)}`, sql6`${gaSocialReferrals.date} < ${daysAgo2(7)}`)).groupBy(gaSocialReferrals.source).all();
|
|
13615
13972
|
return {
|
|
13616
13973
|
total: buildTrend(sumTotal),
|
|
13617
13974
|
organic: buildTrend(sumOrganic),
|
|
@@ -13627,12 +13984,12 @@ async function ga4Routes(app, opts) {
|
|
|
13627
13984
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
13628
13985
|
const cutoffDate = windowCutoff(parseWindow(request.query.window))?.slice(0, 10) ?? null;
|
|
13629
13986
|
const conditions = [eq21(gaTrafficSnapshots.projectId, project.id)];
|
|
13630
|
-
if (cutoffDate) conditions.push(
|
|
13987
|
+
if (cutoffDate) conditions.push(sql6`${gaTrafficSnapshots.date} >= ${cutoffDate}`);
|
|
13631
13988
|
const rows = app.db.select({
|
|
13632
13989
|
date: gaTrafficSnapshots.date,
|
|
13633
|
-
sessions:
|
|
13634
|
-
organicSessions:
|
|
13635
|
-
users:
|
|
13990
|
+
sessions: sql6`SUM(${gaTrafficSnapshots.sessions})`,
|
|
13991
|
+
organicSessions: sql6`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
13992
|
+
users: sql6`SUM(${gaTrafficSnapshots.users})`
|
|
13636
13993
|
}).from(gaTrafficSnapshots).where(and11(...conditions)).groupBy(gaTrafficSnapshots.date).orderBy(gaTrafficSnapshots.date).all();
|
|
13637
13994
|
return rows.map((r) => ({
|
|
13638
13995
|
date: r.date,
|
|
@@ -13645,11 +14002,11 @@ async function ga4Routes(app, opts) {
|
|
|
13645
14002
|
const project = resolveProject(app.db, request.params.name);
|
|
13646
14003
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
13647
14004
|
const trafficPages = app.db.select({
|
|
13648
|
-
landingPage:
|
|
13649
|
-
sessions:
|
|
13650
|
-
organicSessions:
|
|
13651
|
-
users:
|
|
13652
|
-
}).from(gaTrafficSnapshots).where(eq21(gaTrafficSnapshots.projectId, project.id)).groupBy(
|
|
14005
|
+
landingPage: sql6`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`,
|
|
14006
|
+
sessions: sql6`SUM(${gaTrafficSnapshots.sessions})`,
|
|
14007
|
+
organicSessions: sql6`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
14008
|
+
users: sql6`SUM(${gaTrafficSnapshots.users})`
|
|
14009
|
+
}).from(gaTrafficSnapshots).where(eq21(gaTrafficSnapshots.projectId, project.id)).groupBy(sql6`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`).orderBy(sql6`SUM(${gaTrafficSnapshots.sessions}) DESC`).all();
|
|
13653
14010
|
return {
|
|
13654
14011
|
pages: trafficPages.map((r) => ({
|
|
13655
14012
|
landingPage: r.landingPage,
|
|
@@ -15286,7 +15643,7 @@ async function wordpressRoutes(app, opts) {
|
|
|
15286
15643
|
|
|
15287
15644
|
// ../api-routes/src/backlinks.ts
|
|
15288
15645
|
import crypto18 from "crypto";
|
|
15289
|
-
import { and as and13, asc as asc2, desc as desc11, eq as eq22, sql as
|
|
15646
|
+
import { and as and13, asc as asc2, desc as desc11, eq as eq22, sql as sql7 } from "drizzle-orm";
|
|
15290
15647
|
|
|
15291
15648
|
// ../integration-commoncrawl/src/constants.ts
|
|
15292
15649
|
import os3 from "os";
|
|
@@ -15570,7 +15927,7 @@ async function queryBacklinks(opts) {
|
|
|
15570
15927
|
const reversed = opts.targets.map(reverseDomain);
|
|
15571
15928
|
const targetList = reversed.map(quote).join(", ");
|
|
15572
15929
|
const limitClause = opts.limitPerTarget ? `QUALIFY row_number() OVER (PARTITION BY t.target_rev_domain ORDER BY v.num_hosts DESC) <= ${Math.floor(opts.limitPerTarget)}` : "";
|
|
15573
|
-
const
|
|
15930
|
+
const sql14 = `
|
|
15574
15931
|
WITH vertices AS (
|
|
15575
15932
|
SELECT * FROM read_csv(
|
|
15576
15933
|
${quote(opts.vertexPath)},
|
|
@@ -15606,7 +15963,7 @@ async function queryBacklinks(opts) {
|
|
|
15606
15963
|
const conn = await instance.connect();
|
|
15607
15964
|
let rows;
|
|
15608
15965
|
try {
|
|
15609
|
-
const reader = await conn.runAndReadAll(
|
|
15966
|
+
const reader = await conn.runAndReadAll(sql14);
|
|
15610
15967
|
rows = reader.getRowObjects();
|
|
15611
15968
|
} finally {
|
|
15612
15969
|
conn.disconnectSync?.();
|
|
@@ -15683,7 +16040,7 @@ function pruneCachedRelease(release, opts = {}) {
|
|
|
15683
16040
|
}
|
|
15684
16041
|
|
|
15685
16042
|
// ../api-routes/src/backlinks-filter.ts
|
|
15686
|
-
import { and as and12, ne, notLike } from "drizzle-orm";
|
|
16043
|
+
import { and as and12, ne as ne2, notLike } from "drizzle-orm";
|
|
15687
16044
|
var BACKLINK_FILTER_PATTERNS = [
|
|
15688
16045
|
"*.google.com",
|
|
15689
16046
|
"*.googleusercontent.com",
|
|
@@ -15700,10 +16057,10 @@ function backlinkCrawlerExclusionClause() {
|
|
|
15700
16057
|
for (const pattern of BACKLINK_FILTER_PATTERNS) {
|
|
15701
16058
|
if (pattern.startsWith("*.")) {
|
|
15702
16059
|
const suffix = pattern.slice(2);
|
|
15703
|
-
conditions.push(
|
|
16060
|
+
conditions.push(ne2(backlinkDomains.linkingDomain, suffix));
|
|
15704
16061
|
conditions.push(notLike(backlinkDomains.linkingDomain, `%.${suffix}`));
|
|
15705
16062
|
} else {
|
|
15706
|
-
conditions.push(
|
|
16063
|
+
conditions.push(ne2(backlinkDomains.linkingDomain, pattern));
|
|
15707
16064
|
}
|
|
15708
16065
|
}
|
|
15709
16066
|
const combined = and12(...conditions);
|
|
@@ -15782,12 +16139,12 @@ function computeFilteredSummary(db, base) {
|
|
|
15782
16139
|
);
|
|
15783
16140
|
const filteredCondition = and13(baseDomainCondition, backlinkCrawlerExclusionClause());
|
|
15784
16141
|
const unfilteredAgg = db.select({
|
|
15785
|
-
count:
|
|
15786
|
-
total:
|
|
16142
|
+
count: sql7`count(*)`,
|
|
16143
|
+
total: sql7`coalesce(sum(${backlinkDomains.numHosts}), 0)`
|
|
15787
16144
|
}).from(backlinkDomains).where(baseDomainCondition).get();
|
|
15788
16145
|
const filteredAgg = db.select({
|
|
15789
|
-
count:
|
|
15790
|
-
total:
|
|
16146
|
+
count: sql7`count(*)`,
|
|
16147
|
+
total: sql7`coalesce(sum(${backlinkDomains.numHosts}), 0)`
|
|
15791
16148
|
}).from(backlinkDomains).where(filteredCondition).get();
|
|
15792
16149
|
const top10Rows = db.select({ numHosts: backlinkDomains.numHosts }).from(backlinkDomains).where(filteredCondition).orderBy(desc11(backlinkDomains.numHosts)).limit(10).all();
|
|
15793
16150
|
const totalLinkingDomains = Number(filteredAgg?.count ?? 0);
|
|
@@ -15961,7 +16318,7 @@ async function backlinksRoutes(app, opts) {
|
|
|
15961
16318
|
eq22(backlinkDomains.release, targetRelease)
|
|
15962
16319
|
);
|
|
15963
16320
|
const domainCondition = excludeCrawlers ? and13(baseDomainCondition, backlinkCrawlerExclusionClause()) : baseDomainCondition;
|
|
15964
|
-
const totalRow = app.db.select({ count:
|
|
16321
|
+
const totalRow = app.db.select({ count: sql7`count(*)` }).from(backlinkDomains).where(domainCondition).get();
|
|
15965
16322
|
const rows = app.db.select({
|
|
15966
16323
|
linkingDomain: backlinkDomains.linkingDomain,
|
|
15967
16324
|
numHosts: backlinkDomains.numHosts
|
|
@@ -15996,7 +16353,7 @@ async function backlinksRoutes(app, opts) {
|
|
|
15996
16353
|
|
|
15997
16354
|
// ../api-routes/src/traffic.ts
|
|
15998
16355
|
import crypto20 from "crypto";
|
|
15999
|
-
import { and as and14, desc as desc12, eq as eq23, gte, lte, sql as
|
|
16356
|
+
import { and as and14, desc as desc12, eq as eq23, gte as gte2, lte as lte2, sql as sql8 } from "drizzle-orm";
|
|
16000
16357
|
|
|
16001
16358
|
// ../integration-cloud-run/src/auth.ts
|
|
16002
16359
|
import crypto19 from "crypto";
|
|
@@ -16849,7 +17206,7 @@ async function trafficRoutes(app, opts) {
|
|
|
16849
17206
|
crawlerEventsHourly.status
|
|
16850
17207
|
],
|
|
16851
17208
|
set: {
|
|
16852
|
-
hits:
|
|
17209
|
+
hits: sql8`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
16853
17210
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
16854
17211
|
updatedAt: finishedAt
|
|
16855
17212
|
}
|
|
@@ -16884,7 +17241,7 @@ async function trafficRoutes(app, opts) {
|
|
|
16884
17241
|
aiReferralEventsHourly.status
|
|
16885
17242
|
],
|
|
16886
17243
|
set: {
|
|
16887
|
-
sessionsOrHits:
|
|
17244
|
+
sessionsOrHits: sql8`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
16888
17245
|
updatedAt: finishedAt
|
|
16889
17246
|
}
|
|
16890
17247
|
}).run();
|
|
@@ -16952,22 +17309,22 @@ async function trafficRoutes(app, opts) {
|
|
|
16952
17309
|
return response;
|
|
16953
17310
|
});
|
|
16954
17311
|
function buildSourceDetail(projectId, row, since) {
|
|
16955
|
-
const crawlerTotals = app.db.select({ total:
|
|
17312
|
+
const crawlerTotals = app.db.select({ total: sql8`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
16956
17313
|
and14(
|
|
16957
17314
|
eq23(crawlerEventsHourly.sourceId, row.id),
|
|
16958
|
-
|
|
17315
|
+
gte2(crawlerEventsHourly.tsHour, since)
|
|
16959
17316
|
)
|
|
16960
17317
|
).get();
|
|
16961
|
-
const aiTotals = app.db.select({ total:
|
|
17318
|
+
const aiTotals = app.db.select({ total: sql8`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
16962
17319
|
and14(
|
|
16963
17320
|
eq23(aiReferralEventsHourly.sourceId, row.id),
|
|
16964
|
-
|
|
17321
|
+
gte2(aiReferralEventsHourly.tsHour, since)
|
|
16965
17322
|
)
|
|
16966
17323
|
).get();
|
|
16967
|
-
const sampleTotals = app.db.select({ total:
|
|
17324
|
+
const sampleTotals = app.db.select({ total: sql8`COUNT(*)` }).from(rawEventSamples).where(
|
|
16968
17325
|
and14(
|
|
16969
17326
|
eq23(rawEventSamples.sourceId, row.id),
|
|
16970
|
-
|
|
17327
|
+
gte2(rawEventSamples.ts, since)
|
|
16971
17328
|
)
|
|
16972
17329
|
).get();
|
|
16973
17330
|
const latestRun = app.db.select().from(runs).where(
|
|
@@ -17061,12 +17418,12 @@ async function trafficRoutes(app, opts) {
|
|
|
17061
17418
|
if (kind === "all" || kind === TrafficEventKinds.crawler) {
|
|
17062
17419
|
const crawlerFilters = [
|
|
17063
17420
|
eq23(crawlerEventsHourly.projectId, project.id),
|
|
17064
|
-
|
|
17065
|
-
|
|
17421
|
+
gte2(crawlerEventsHourly.tsHour, sinceIso),
|
|
17422
|
+
lte2(crawlerEventsHourly.tsHour, untilIso)
|
|
17066
17423
|
];
|
|
17067
17424
|
if (sourceIdParam) crawlerFilters.push(eq23(crawlerEventsHourly.sourceId, sourceIdParam));
|
|
17068
17425
|
const crawlerWhere = and14(...crawlerFilters);
|
|
17069
|
-
const total = app.db.select({ total:
|
|
17426
|
+
const total = app.db.select({ total: sql8`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(crawlerWhere).get();
|
|
17070
17427
|
crawlerTotal = Number(total?.total ?? 0);
|
|
17071
17428
|
const rows = app.db.select().from(crawlerEventsHourly).where(crawlerWhere).orderBy(desc12(crawlerEventsHourly.tsHour)).limit(limit).all();
|
|
17072
17429
|
for (const r of rows) {
|
|
@@ -17086,12 +17443,12 @@ async function trafficRoutes(app, opts) {
|
|
|
17086
17443
|
if (kind === "all" || kind === TrafficEventKinds["ai-referral"]) {
|
|
17087
17444
|
const aiFilters = [
|
|
17088
17445
|
eq23(aiReferralEventsHourly.projectId, project.id),
|
|
17089
|
-
|
|
17090
|
-
|
|
17446
|
+
gte2(aiReferralEventsHourly.tsHour, sinceIso),
|
|
17447
|
+
lte2(aiReferralEventsHourly.tsHour, untilIso)
|
|
17091
17448
|
];
|
|
17092
17449
|
if (sourceIdParam) aiFilters.push(eq23(aiReferralEventsHourly.sourceId, sourceIdParam));
|
|
17093
17450
|
const aiWhere = and14(...aiFilters);
|
|
17094
|
-
const total = app.db.select({ total:
|
|
17451
|
+
const total = app.db.select({ total: sql8`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(aiWhere).get();
|
|
17095
17452
|
aiReferralTotal = Number(total?.total ?? 0);
|
|
17096
17453
|
const rows = app.db.select().from(aiReferralEventsHourly).where(aiWhere).orderBy(desc12(aiReferralEventsHourly.tsHour)).limit(limit).all();
|
|
17097
17454
|
for (const r of rows) {
|
|
@@ -17744,12 +18101,308 @@ var providersConfiguredCheck = {
|
|
|
17744
18101
|
};
|
|
17745
18102
|
var PROVIDERS_CHECKS = [providersConfiguredCheck];
|
|
17746
18103
|
|
|
18104
|
+
// ../api-routes/src/doctor/checks/traffic-source.ts
|
|
18105
|
+
import { and as and15, eq as eq24, gte as gte3, ne as ne3, sql as sql9 } from "drizzle-orm";
|
|
18106
|
+
var RECENT_DATA_WARN_DAYS = 7;
|
|
18107
|
+
var RECENT_DATA_FAIL_DAYS = 30;
|
|
18108
|
+
function skippedNoProject2() {
|
|
18109
|
+
return {
|
|
18110
|
+
status: CheckStatuses.skipped,
|
|
18111
|
+
code: "traffic.no-project",
|
|
18112
|
+
summary: "Project context required for traffic source checks.",
|
|
18113
|
+
remediation: "Run `canonry doctor --project <name>` to scope this check to a project."
|
|
18114
|
+
};
|
|
18115
|
+
}
|
|
18116
|
+
function loadProbes(ctx) {
|
|
18117
|
+
if (!ctx.project) return [];
|
|
18118
|
+
const rows = ctx.db.select().from(trafficSources).where(
|
|
18119
|
+
and15(
|
|
18120
|
+
eq24(trafficSources.projectId, ctx.project.id),
|
|
18121
|
+
ne3(trafficSources.status, TrafficSourceStatuses.archived)
|
|
18122
|
+
)
|
|
18123
|
+
).all();
|
|
18124
|
+
return rows.map((r) => ({
|
|
18125
|
+
id: r.id,
|
|
18126
|
+
projectId: r.projectId,
|
|
18127
|
+
projectName: ctx.project.name,
|
|
18128
|
+
sourceType: r.sourceType,
|
|
18129
|
+
displayName: r.displayName,
|
|
18130
|
+
status: r.status,
|
|
18131
|
+
lastSyncedAt: r.lastSyncedAt,
|
|
18132
|
+
lastError: r.lastError,
|
|
18133
|
+
configJson: r.configJson
|
|
18134
|
+
}));
|
|
18135
|
+
}
|
|
18136
|
+
var sourceConnectedCheck = {
|
|
18137
|
+
id: "traffic.source.connected",
|
|
18138
|
+
category: CheckCategories.integrations,
|
|
18139
|
+
scope: CheckScopes.project,
|
|
18140
|
+
title: "Traffic source connected",
|
|
18141
|
+
run: (ctx) => {
|
|
18142
|
+
if (!ctx.project) return skippedNoProject2();
|
|
18143
|
+
const sources = loadProbes(ctx);
|
|
18144
|
+
if (sources.length === 0) {
|
|
18145
|
+
return {
|
|
18146
|
+
status: CheckStatuses.skipped,
|
|
18147
|
+
code: "traffic.source.none",
|
|
18148
|
+
summary: "No server-side traffic source connected \u2014 server-log AI visibility data unavailable for this project.",
|
|
18149
|
+
remediation: "Connect a traffic source via `canonry traffic connect <type> <project>` to surface crawler hits and AI-referral arrivals from your server logs.",
|
|
18150
|
+
details: { sourceCount: 0 }
|
|
18151
|
+
};
|
|
18152
|
+
}
|
|
18153
|
+
const errored = sources.filter((s) => s.status === "error");
|
|
18154
|
+
if (errored.length > 0 && errored.length === sources.length) {
|
|
18155
|
+
return {
|
|
18156
|
+
status: CheckStatuses.fail,
|
|
18157
|
+
code: "traffic.source.all-errored",
|
|
18158
|
+
summary: `All ${sources.length} traffic source(s) are in error state. No data is being ingested.`,
|
|
18159
|
+
remediation: errored[0].lastError ? `Latest error: "${errored[0].lastError}". Re-connect the source or run \`canonry traffic sync <project> --source <id>\` to retry.` : "Run `canonry traffic sources <project>` to inspect the failing source(s) and re-connect.",
|
|
18160
|
+
details: { sourceCount: sources.length, erroredIds: errored.map((s) => s.id) }
|
|
18161
|
+
};
|
|
18162
|
+
}
|
|
18163
|
+
if (errored.length > 0) {
|
|
18164
|
+
return {
|
|
18165
|
+
status: CheckStatuses.warn,
|
|
18166
|
+
code: "traffic.source.partially-errored",
|
|
18167
|
+
summary: `${errored.length} of ${sources.length} traffic source(s) are in error state.`,
|
|
18168
|
+
remediation: "Run `canonry traffic sources <project>` to inspect the failing sources individually.",
|
|
18169
|
+
details: { sourceCount: sources.length, erroredIds: errored.map((s) => s.id) }
|
|
18170
|
+
};
|
|
18171
|
+
}
|
|
18172
|
+
return {
|
|
18173
|
+
status: CheckStatuses.ok,
|
|
18174
|
+
code: "traffic.source.connected",
|
|
18175
|
+
summary: `${sources.length} traffic source(s) connected: ${sources.map((s) => s.displayName).join(", ")}.`,
|
|
18176
|
+
details: { sourceCount: sources.length, sourceTypes: [...new Set(sources.map((s) => s.sourceType))] }
|
|
18177
|
+
};
|
|
18178
|
+
}
|
|
18179
|
+
};
|
|
18180
|
+
var recentDataCheck = {
|
|
18181
|
+
id: "traffic.source.recent-data",
|
|
18182
|
+
category: CheckCategories.integrations,
|
|
18183
|
+
scope: CheckScopes.project,
|
|
18184
|
+
title: "Traffic source recent data",
|
|
18185
|
+
run: (ctx) => {
|
|
18186
|
+
if (!ctx.project) return skippedNoProject2();
|
|
18187
|
+
const sources = loadProbes(ctx);
|
|
18188
|
+
if (sources.length === 0) {
|
|
18189
|
+
return {
|
|
18190
|
+
status: CheckStatuses.skipped,
|
|
18191
|
+
code: "traffic.recent-data.no-source",
|
|
18192
|
+
summary: "No traffic source connected \u2014 recent-data check skipped."
|
|
18193
|
+
};
|
|
18194
|
+
}
|
|
18195
|
+
const now = /* @__PURE__ */ new Date();
|
|
18196
|
+
const warnCutoff = new Date(now.getTime() - RECENT_DATA_WARN_DAYS * 24 * 60 * 6e4).toISOString();
|
|
18197
|
+
const failCutoff = new Date(now.getTime() - RECENT_DATA_FAIL_DAYS * 24 * 60 * 6e4).toISOString();
|
|
18198
|
+
const recentCrawlers = Number(
|
|
18199
|
+
ctx.db.select({ total: sql9`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
18200
|
+
and15(
|
|
18201
|
+
eq24(crawlerEventsHourly.projectId, ctx.project.id),
|
|
18202
|
+
gte3(crawlerEventsHourly.tsHour, warnCutoff)
|
|
18203
|
+
)
|
|
18204
|
+
).get()?.total ?? 0
|
|
18205
|
+
);
|
|
18206
|
+
const recentReferrals = Number(
|
|
18207
|
+
ctx.db.select({ total: sql9`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
18208
|
+
and15(
|
|
18209
|
+
eq24(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
18210
|
+
gte3(aiReferralEventsHourly.tsHour, warnCutoff)
|
|
18211
|
+
)
|
|
18212
|
+
).get()?.total ?? 0
|
|
18213
|
+
);
|
|
18214
|
+
if (recentCrawlers > 0 || recentReferrals > 0) {
|
|
18215
|
+
return {
|
|
18216
|
+
status: CheckStatuses.ok,
|
|
18217
|
+
code: "traffic.recent-data.fresh",
|
|
18218
|
+
summary: `${recentCrawlers} crawler hit(s) and ${recentReferrals} AI-referral arrival(s) in the last ${RECENT_DATA_WARN_DAYS} days.`,
|
|
18219
|
+
details: { crawlerHits: recentCrawlers, referralArrivals: recentReferrals, windowDays: RECENT_DATA_WARN_DAYS }
|
|
18220
|
+
};
|
|
18221
|
+
}
|
|
18222
|
+
const olderCrawlers = Number(
|
|
18223
|
+
ctx.db.select({ total: sql9`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
18224
|
+
and15(
|
|
18225
|
+
eq24(crawlerEventsHourly.projectId, ctx.project.id),
|
|
18226
|
+
gte3(crawlerEventsHourly.tsHour, failCutoff)
|
|
18227
|
+
)
|
|
18228
|
+
).get()?.total ?? 0
|
|
18229
|
+
);
|
|
18230
|
+
const lastSyncedAt = sources.map((s) => s.lastSyncedAt).filter(Boolean).sort().at(-1) ?? null;
|
|
18231
|
+
if (olderCrawlers > 0 || lastSyncedAt) {
|
|
18232
|
+
return {
|
|
18233
|
+
status: CheckStatuses.warn,
|
|
18234
|
+
code: "traffic.recent-data.stale",
|
|
18235
|
+
summary: `No crawler hits or AI-referral arrivals in the last ${RECENT_DATA_WARN_DAYS} days, though older data exists.`,
|
|
18236
|
+
remediation: lastSyncedAt ? `Last sync: ${lastSyncedAt}. Run \`canonry traffic sync <project>\` to refresh, or check the source connection.` : "Run `canonry traffic sync <project>` to pull recent events.",
|
|
18237
|
+
details: { lastSyncedAt, sourceCount: sources.length }
|
|
18238
|
+
};
|
|
18239
|
+
}
|
|
18240
|
+
return {
|
|
18241
|
+
status: CheckStatuses.fail,
|
|
18242
|
+
code: "traffic.recent-data.empty",
|
|
18243
|
+
summary: `No traffic data in the last ${RECENT_DATA_FAIL_DAYS} days. The source is connected but isn't ingesting.`,
|
|
18244
|
+
remediation: "Verify the source's configuration with `canonry traffic sources <project>` and run a manual sync to confirm credentials + scopes are still valid.",
|
|
18245
|
+
details: { sourceCount: sources.length }
|
|
18246
|
+
};
|
|
18247
|
+
}
|
|
18248
|
+
};
|
|
18249
|
+
async function runValidator(source, validator, fallbackId, fallbackLabel) {
|
|
18250
|
+
if (!validator) {
|
|
18251
|
+
return {
|
|
18252
|
+
source,
|
|
18253
|
+
output: {
|
|
18254
|
+
status: CheckStatuses.skipped,
|
|
18255
|
+
code: `traffic.${fallbackId}.no-validator`,
|
|
18256
|
+
summary: `No ${fallbackLabel} validator registered for source type "${source.sourceType}".`
|
|
18257
|
+
}
|
|
18258
|
+
};
|
|
18259
|
+
}
|
|
18260
|
+
try {
|
|
18261
|
+
const result = await validator(source);
|
|
18262
|
+
if (!result) {
|
|
18263
|
+
return {
|
|
18264
|
+
source,
|
|
18265
|
+
output: {
|
|
18266
|
+
status: CheckStatuses.skipped,
|
|
18267
|
+
code: `traffic.${fallbackId}.unsupported`,
|
|
18268
|
+
summary: `Validator for "${source.sourceType}" does not implement ${fallbackLabel} validation.`
|
|
18269
|
+
}
|
|
18270
|
+
};
|
|
18271
|
+
}
|
|
18272
|
+
return { source, output: result };
|
|
18273
|
+
} catch (e) {
|
|
18274
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
18275
|
+
return {
|
|
18276
|
+
source,
|
|
18277
|
+
output: {
|
|
18278
|
+
status: CheckStatuses.fail,
|
|
18279
|
+
code: `traffic.${fallbackId}.validator-error`,
|
|
18280
|
+
summary: `${fallbackLabel} validator threw: ${msg}.`,
|
|
18281
|
+
remediation: "Check the source configuration and credentials, then re-run the doctor."
|
|
18282
|
+
}
|
|
18283
|
+
};
|
|
18284
|
+
}
|
|
18285
|
+
}
|
|
18286
|
+
function summarizePerSourceResults(fallbackId, fallbackLabel, results) {
|
|
18287
|
+
const failed = results.filter((r) => r.output.status === CheckStatuses.fail);
|
|
18288
|
+
const warned = results.filter((r) => r.output.status === CheckStatuses.warn);
|
|
18289
|
+
const skipped = results.filter((r) => r.output.status === CheckStatuses.skipped);
|
|
18290
|
+
const ok = results.filter((r) => r.output.status === CheckStatuses.ok);
|
|
18291
|
+
const detail = {
|
|
18292
|
+
sources: results.map((r) => ({
|
|
18293
|
+
id: r.source.id,
|
|
18294
|
+
sourceType: r.source.sourceType,
|
|
18295
|
+
displayName: r.source.displayName,
|
|
18296
|
+
status: r.output.status,
|
|
18297
|
+
code: r.output.code,
|
|
18298
|
+
summary: r.output.summary
|
|
18299
|
+
}))
|
|
18300
|
+
};
|
|
18301
|
+
if (failed.length > 0) {
|
|
18302
|
+
return {
|
|
18303
|
+
status: CheckStatuses.fail,
|
|
18304
|
+
code: `traffic.${fallbackId}.failed`,
|
|
18305
|
+
summary: `${failed.length} of ${results.length} source(s) failed ${fallbackLabel} validation: ${failed.map((r) => `${r.source.displayName} (${r.output.summary})`).join("; ")}.`,
|
|
18306
|
+
remediation: failed[0].output.remediation ?? `Inspect the failing source(s) \u2014 see details.sources for per-source codes.`,
|
|
18307
|
+
details: detail
|
|
18308
|
+
};
|
|
18309
|
+
}
|
|
18310
|
+
if (warned.length > 0) {
|
|
18311
|
+
return {
|
|
18312
|
+
status: CheckStatuses.warn,
|
|
18313
|
+
code: `traffic.${fallbackId}.warned`,
|
|
18314
|
+
summary: `${warned.length} of ${results.length} source(s) raised warnings during ${fallbackLabel} validation.`,
|
|
18315
|
+
remediation: warned[0].output.remediation ?? `Review the warning(s) \u2014 see details.sources.`,
|
|
18316
|
+
details: detail
|
|
18317
|
+
};
|
|
18318
|
+
}
|
|
18319
|
+
if (ok.length > 0) {
|
|
18320
|
+
return {
|
|
18321
|
+
status: CheckStatuses.ok,
|
|
18322
|
+
code: `traffic.${fallbackId}.ok`,
|
|
18323
|
+
summary: `${ok.length} source(s) passed ${fallbackLabel} validation${skipped.length > 0 ? ` (${skipped.length} skipped)` : ""}.`,
|
|
18324
|
+
details: detail
|
|
18325
|
+
};
|
|
18326
|
+
}
|
|
18327
|
+
return {
|
|
18328
|
+
status: CheckStatuses.skipped,
|
|
18329
|
+
code: `traffic.${fallbackId}.all-skipped`,
|
|
18330
|
+
summary: `No source-type validator was available for any of the ${results.length} connected source(s).`,
|
|
18331
|
+
details: detail
|
|
18332
|
+
};
|
|
18333
|
+
}
|
|
18334
|
+
var credentialsCheck = {
|
|
18335
|
+
id: "traffic.source.credentials",
|
|
18336
|
+
category: CheckCategories.auth,
|
|
18337
|
+
scope: CheckScopes.project,
|
|
18338
|
+
title: "Traffic source credentials",
|
|
18339
|
+
run: async (ctx) => {
|
|
18340
|
+
if (!ctx.project) return skippedNoProject2();
|
|
18341
|
+
const sources = loadProbes(ctx);
|
|
18342
|
+
if (sources.length === 0) {
|
|
18343
|
+
return {
|
|
18344
|
+
status: CheckStatuses.skipped,
|
|
18345
|
+
code: "traffic.credentials.no-source",
|
|
18346
|
+
summary: "No traffic source connected \u2014 credentials check skipped."
|
|
18347
|
+
};
|
|
18348
|
+
}
|
|
18349
|
+
const validators = ctx.trafficSourceValidators ?? {};
|
|
18350
|
+
const results = await Promise.all(
|
|
18351
|
+
sources.map(
|
|
18352
|
+
(s) => runValidator(
|
|
18353
|
+
s,
|
|
18354
|
+
validators[s.sourceType]?.validateCredentials?.bind(validators[s.sourceType]),
|
|
18355
|
+
"credentials",
|
|
18356
|
+
"credentials"
|
|
18357
|
+
)
|
|
18358
|
+
)
|
|
18359
|
+
);
|
|
18360
|
+
return summarizePerSourceResults("credentials", "credentials", results);
|
|
18361
|
+
}
|
|
18362
|
+
};
|
|
18363
|
+
var scopesCheck2 = {
|
|
18364
|
+
id: "traffic.source.scopes",
|
|
18365
|
+
category: CheckCategories.auth,
|
|
18366
|
+
scope: CheckScopes.project,
|
|
18367
|
+
title: "Traffic source scopes",
|
|
18368
|
+
run: async (ctx) => {
|
|
18369
|
+
if (!ctx.project) return skippedNoProject2();
|
|
18370
|
+
const sources = loadProbes(ctx);
|
|
18371
|
+
if (sources.length === 0) {
|
|
18372
|
+
return {
|
|
18373
|
+
status: CheckStatuses.skipped,
|
|
18374
|
+
code: "traffic.scopes.no-source",
|
|
18375
|
+
summary: "No traffic source connected \u2014 scopes check skipped."
|
|
18376
|
+
};
|
|
18377
|
+
}
|
|
18378
|
+
const validators = ctx.trafficSourceValidators ?? {};
|
|
18379
|
+
const results = await Promise.all(
|
|
18380
|
+
sources.map(
|
|
18381
|
+
(s) => runValidator(
|
|
18382
|
+
s,
|
|
18383
|
+
validators[s.sourceType]?.validateScopes?.bind(validators[s.sourceType]),
|
|
18384
|
+
"scopes",
|
|
18385
|
+
"scopes"
|
|
18386
|
+
)
|
|
18387
|
+
)
|
|
18388
|
+
);
|
|
18389
|
+
return summarizePerSourceResults("scopes", "scopes", results);
|
|
18390
|
+
}
|
|
18391
|
+
};
|
|
18392
|
+
var TRAFFIC_SOURCE_CHECKS = [
|
|
18393
|
+
sourceConnectedCheck,
|
|
18394
|
+
recentDataCheck,
|
|
18395
|
+
credentialsCheck,
|
|
18396
|
+
scopesCheck2
|
|
18397
|
+
];
|
|
18398
|
+
|
|
17747
18399
|
// ../api-routes/src/doctor/registry.ts
|
|
17748
18400
|
var ALL_CHECKS = [
|
|
17749
18401
|
...GOOGLE_AUTH_CHECKS,
|
|
17750
18402
|
...BING_AUTH_CHECKS,
|
|
17751
18403
|
...GA_AUTH_CHECKS,
|
|
17752
|
-
...PROVIDERS_CHECKS
|
|
18404
|
+
...PROVIDERS_CHECKS,
|
|
18405
|
+
...TRAFFIC_SOURCE_CHECKS
|
|
17753
18406
|
];
|
|
17754
18407
|
var CHECK_BY_ID = Object.fromEntries(
|
|
17755
18408
|
ALL_CHECKS.map((check) => [check.id, check])
|
|
@@ -17836,7 +18489,8 @@ async function doctorRoutes(app, opts) {
|
|
|
17836
18489
|
ga4CredentialStore: opts.ga4CredentialStore,
|
|
17837
18490
|
getGoogleAuthConfig: opts.getGoogleAuthConfig,
|
|
17838
18491
|
redirectUri,
|
|
17839
|
-
providerSummary: opts.providerSummary
|
|
18492
|
+
providerSummary: opts.providerSummary,
|
|
18493
|
+
trafficSourceValidators: opts.trafficSourceValidators
|
|
17840
18494
|
};
|
|
17841
18495
|
return runChecks(ctx, ALL_CHECKS, { checkIds });
|
|
17842
18496
|
});
|
|
@@ -17856,7 +18510,8 @@ async function doctorRoutes(app, opts) {
|
|
|
17856
18510
|
ga4CredentialStore: opts.ga4CredentialStore,
|
|
17857
18511
|
getGoogleAuthConfig: opts.getGoogleAuthConfig,
|
|
17858
18512
|
redirectUri,
|
|
17859
|
-
providerSummary: opts.providerSummary
|
|
18513
|
+
providerSummary: opts.providerSummary,
|
|
18514
|
+
trafficSourceValidators: opts.trafficSourceValidators
|
|
17860
18515
|
};
|
|
17861
18516
|
return runChecks(ctx, ALL_CHECKS, { checkIds });
|
|
17862
18517
|
});
|
|
@@ -17995,13 +18650,57 @@ async function apiRoutes(app, opts) {
|
|
|
17995
18650
|
ga4CredentialStore: opts.ga4CredentialStore,
|
|
17996
18651
|
getGoogleAuthConfig: opts.getGoogleAuthConfig,
|
|
17997
18652
|
publicUrl: opts.publicUrl,
|
|
17998
|
-
providerSummary: opts.providerSummary
|
|
18653
|
+
providerSummary: opts.providerSummary,
|
|
18654
|
+
trafficSourceValidators: buildTrafficSourceValidators(opts)
|
|
17999
18655
|
});
|
|
18000
18656
|
if (opts.registerAuthenticatedRoutes) {
|
|
18001
18657
|
await opts.registerAuthenticatedRoutes(api);
|
|
18002
18658
|
}
|
|
18003
18659
|
}, { prefix: opts.routePrefix ?? "/api/v1" });
|
|
18004
18660
|
}
|
|
18661
|
+
function buildTrafficSourceValidators(opts) {
|
|
18662
|
+
const validators = {};
|
|
18663
|
+
if (opts.cloudRunCredentialStore) {
|
|
18664
|
+
const store = opts.cloudRunCredentialStore;
|
|
18665
|
+
const resolveToken = opts.resolveCloudRunAccessToken ?? defaultResolveAccessToken;
|
|
18666
|
+
validators["cloud-run"] = {
|
|
18667
|
+
validateCredentials: async (source) => {
|
|
18668
|
+
const record = store.getConnection(source.projectName);
|
|
18669
|
+
if (!record) {
|
|
18670
|
+
return {
|
|
18671
|
+
status: CheckStatuses.fail,
|
|
18672
|
+
code: "traffic.credentials.missing",
|
|
18673
|
+
summary: `No Cloud Run credential found in ~/.canonry/config.yaml for project "${source.projectName}".`,
|
|
18674
|
+
remediation: "Re-run `canonry traffic connect cloud-run <project> --gcp-project <id> --service-account-key <path>`."
|
|
18675
|
+
};
|
|
18676
|
+
}
|
|
18677
|
+
try {
|
|
18678
|
+
await resolveToken(record);
|
|
18679
|
+
return {
|
|
18680
|
+
status: CheckStatuses.ok,
|
|
18681
|
+
code: "traffic.credentials.resolved",
|
|
18682
|
+
summary: `Cloud Run access token resolves for "${source.displayName}" (project ${record.gcpProjectId}).`
|
|
18683
|
+
};
|
|
18684
|
+
} catch (e) {
|
|
18685
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
18686
|
+
return {
|
|
18687
|
+
status: CheckStatuses.fail,
|
|
18688
|
+
code: "traffic.credentials.resolve-failed",
|
|
18689
|
+
summary: `Failed to resolve Cloud Run access token: ${msg}.`,
|
|
18690
|
+
remediation: "Verify the service-account key in ~/.canonry/config.yaml is unexpired and well-formed. Re-connect the source if needed."
|
|
18691
|
+
};
|
|
18692
|
+
}
|
|
18693
|
+
},
|
|
18694
|
+
// Cloud Run scopes are implicit in the service-account key — Cloud
|
|
18695
|
+
// Logging viewer is the only required scope today, and it's enforced
|
|
18696
|
+
// at the IAM layer rather than baked into the token. We surface a
|
|
18697
|
+
// skipped result so the framework is uniform without producing a
|
|
18698
|
+
// false signal.
|
|
18699
|
+
validateScopes: () => null
|
|
18700
|
+
};
|
|
18701
|
+
}
|
|
18702
|
+
return Object.keys(validators).length > 0 ? validators : void 0;
|
|
18703
|
+
}
|
|
18005
18704
|
|
|
18006
18705
|
// src/server.ts
|
|
18007
18706
|
import os6 from "os";
|
|
@@ -20493,7 +21192,7 @@ import crypto22 from "crypto";
|
|
|
20493
21192
|
import fs7 from "fs";
|
|
20494
21193
|
import path9 from "path";
|
|
20495
21194
|
import os5 from "os";
|
|
20496
|
-
import { and as
|
|
21195
|
+
import { and as and16, eq as eq25, inArray as inArray7, sql as sql10 } from "drizzle-orm";
|
|
20497
21196
|
|
|
20498
21197
|
// src/run-telemetry.ts
|
|
20499
21198
|
import crypto21 from "crypto";
|
|
@@ -20838,7 +21537,7 @@ var JobRunner = class {
|
|
|
20838
21537
|
if (stale.length === 0) return;
|
|
20839
21538
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
20840
21539
|
for (const run of stale) {
|
|
20841
|
-
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(
|
|
21540
|
+
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(eq25(runs.id, run.id)).run();
|
|
20842
21541
|
log.warn("run.recovered-stale", { runId: run.id, previousStatus: run.status });
|
|
20843
21542
|
}
|
|
20844
21543
|
}
|
|
@@ -20872,10 +21571,10 @@ var JobRunner = class {
|
|
|
20872
21571
|
throw new Error(`Run ${runId} is not executable from status '${existingRun.status}'`);
|
|
20873
21572
|
}
|
|
20874
21573
|
if (existingRun.status === "queued") {
|
|
20875
|
-
this.db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
21574
|
+
this.db.update(runs).set({ status: "running", startedAt: now }).where(and16(eq25(runs.id, runId), eq25(runs.status, "queued"))).run();
|
|
20876
21575
|
}
|
|
20877
21576
|
this.throwIfRunCancelled(runId);
|
|
20878
|
-
const project = this.db.select().from(projects).where(
|
|
21577
|
+
const project = this.db.select().from(projects).where(eq25(projects.id, projectId)).get();
|
|
20879
21578
|
if (!project) {
|
|
20880
21579
|
throw new Error(`Project ${projectId} not found`);
|
|
20881
21580
|
}
|
|
@@ -20896,8 +21595,8 @@ var JobRunner = class {
|
|
|
20896
21595
|
throw new Error("No providers configured. Add at least one provider API key.");
|
|
20897
21596
|
}
|
|
20898
21597
|
log.info("run.dispatch", { runId, providerCount: activeProviders.length, providers: activeProviders.map((p) => p.adapter.name) });
|
|
20899
|
-
projectQueries = this.db.select().from(queries).where(
|
|
20900
|
-
const projectCompetitors = this.db.select().from(competitors).where(
|
|
21598
|
+
projectQueries = this.db.select().from(queries).where(eq25(queries.projectId, projectId)).all();
|
|
21599
|
+
const projectCompetitors = this.db.select().from(competitors).where(eq25(competitors.projectId, projectId)).all();
|
|
20901
21600
|
const competitorDomains = projectCompetitors.map((c) => c.domain);
|
|
20902
21601
|
const allDomains = effectiveDomains({
|
|
20903
21602
|
canonicalDomain: project.canonicalDomain,
|
|
@@ -20915,7 +21614,7 @@ var JobRunner = class {
|
|
|
20915
21614
|
const todayPeriod = getCurrentUsageDay();
|
|
20916
21615
|
for (const p of activeProviders) {
|
|
20917
21616
|
const providerScope = `${projectId}:${p.adapter.name}`;
|
|
20918
|
-
const providerUsage = this.db.select().from(usageCounters).where(
|
|
21617
|
+
const providerUsage = this.db.select().from(usageCounters).where(eq25(usageCounters.scope, providerScope)).all().filter((r) => r.period === todayPeriod && r.metric === "queries").reduce((sum, r) => sum + r.count, 0);
|
|
20919
21618
|
const limit = p.config.quotaPolicy.maxRequestsPerDay;
|
|
20920
21619
|
if (providerUsage + queriesPerProvider > limit) {
|
|
20921
21620
|
throw new Error(
|
|
@@ -21058,12 +21757,12 @@ var JobRunner = class {
|
|
|
21058
21757
|
const someFailed = providerErrors.size > 0;
|
|
21059
21758
|
if (allFailed) {
|
|
21060
21759
|
const errorDetail = serializeRunError(buildRunErrorFromMessages(providerErrors));
|
|
21061
|
-
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
21760
|
+
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq25(runs.id, runId)).run();
|
|
21062
21761
|
} else if (someFailed) {
|
|
21063
21762
|
const errorDetail = serializeRunError(buildRunErrorFromMessages(providerErrors));
|
|
21064
|
-
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
21763
|
+
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq25(runs.id, runId)).run();
|
|
21065
21764
|
} else {
|
|
21066
|
-
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
21765
|
+
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq25(runs.id, runId)).run();
|
|
21067
21766
|
}
|
|
21068
21767
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
21069
21768
|
const finalStatus = allFailed ? "failed" : someFailed ? "partial" : "completed";
|
|
@@ -21109,7 +21808,7 @@ var JobRunner = class {
|
|
|
21109
21808
|
status: "failed",
|
|
21110
21809
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21111
21810
|
error: errorMessage
|
|
21112
|
-
}).where(
|
|
21811
|
+
}).where(eq25(runs.id, runId)).run();
|
|
21113
21812
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
21114
21813
|
const abortReason = classifyRunAbortReason(errorMessage);
|
|
21115
21814
|
const phases = buildPhases({ startTime, providerCallStart, providerCallEnd });
|
|
@@ -21162,7 +21861,7 @@ var JobRunner = class {
|
|
|
21162
21861
|
updatedAt: now
|
|
21163
21862
|
}).onConflictDoUpdate({
|
|
21164
21863
|
target: [usageCounters.scope, usageCounters.period, usageCounters.metric],
|
|
21165
|
-
set: { count:
|
|
21864
|
+
set: { count: sql10`${usageCounters.count} + ${count}`, updatedAt: now }
|
|
21166
21865
|
}).run();
|
|
21167
21866
|
}
|
|
21168
21867
|
flushProviderUsage(projectId, providerDispatchCounts) {
|
|
@@ -21177,7 +21876,7 @@ var JobRunner = class {
|
|
|
21177
21876
|
finishedAt: runs.finishedAt,
|
|
21178
21877
|
error: runs.error,
|
|
21179
21878
|
trigger: runs.trigger
|
|
21180
|
-
}).from(runs).where(
|
|
21879
|
+
}).from(runs).where(eq25(runs.id, runId)).get();
|
|
21181
21880
|
}
|
|
21182
21881
|
isRunCancelled(runId) {
|
|
21183
21882
|
return this.getRunState(runId)?.status === "cancelled";
|
|
@@ -21193,7 +21892,7 @@ var JobRunner = class {
|
|
|
21193
21892
|
this.db.update(runs).set({
|
|
21194
21893
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21195
21894
|
error: currentRun.error ?? "Cancelled by user"
|
|
21196
|
-
}).where(
|
|
21895
|
+
}).where(eq25(runs.id, runId)).run();
|
|
21197
21896
|
}
|
|
21198
21897
|
trackEvent(
|
|
21199
21898
|
"run.completed",
|
|
@@ -21231,7 +21930,7 @@ function buildPhases(input) {
|
|
|
21231
21930
|
|
|
21232
21931
|
// src/gsc-sync.ts
|
|
21233
21932
|
import crypto23 from "crypto";
|
|
21234
|
-
import { eq as
|
|
21933
|
+
import { eq as eq26, and as and17, sql as sql11 } from "drizzle-orm";
|
|
21235
21934
|
var log2 = createLogger("GscSync");
|
|
21236
21935
|
function formatDate3(d) {
|
|
21237
21936
|
return d.toISOString().split("T")[0];
|
|
@@ -21243,13 +21942,13 @@ function daysAgo(n) {
|
|
|
21243
21942
|
}
|
|
21244
21943
|
async function executeGscSync(db, runId, projectId, opts) {
|
|
21245
21944
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
21246
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
21945
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq26(runs.id, runId)).run();
|
|
21247
21946
|
try {
|
|
21248
21947
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
21249
21948
|
if (!googleClientId || !googleClientSecret) {
|
|
21250
21949
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
21251
21950
|
}
|
|
21252
|
-
const project = db.select().from(projects).where(
|
|
21951
|
+
const project = db.select().from(projects).where(eq26(projects.id, projectId)).get();
|
|
21253
21952
|
if (!project) {
|
|
21254
21953
|
throw new Error(`Project not found: ${projectId}`);
|
|
21255
21954
|
}
|
|
@@ -21283,10 +21982,10 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
21283
21982
|
});
|
|
21284
21983
|
log2.info("fetch.complete", { runId, projectId, rowCount: rows.length });
|
|
21285
21984
|
db.delete(gscSearchData).where(
|
|
21286
|
-
|
|
21287
|
-
|
|
21288
|
-
|
|
21289
|
-
|
|
21985
|
+
and17(
|
|
21986
|
+
eq26(gscSearchData.projectId, projectId),
|
|
21987
|
+
sql11`${gscSearchData.date} >= ${startDate}`,
|
|
21988
|
+
sql11`${gscSearchData.date} <= ${endDate}`
|
|
21290
21989
|
)
|
|
21291
21990
|
).run();
|
|
21292
21991
|
const batchSize = 500;
|
|
@@ -21351,7 +22050,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
21351
22050
|
log2.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
|
|
21352
22051
|
}
|
|
21353
22052
|
}
|
|
21354
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
22053
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq26(gscUrlInspections.projectId, projectId)).all();
|
|
21355
22054
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
21356
22055
|
for (const row of allInspections) {
|
|
21357
22056
|
const existing = latestByUrl.get(row.url);
|
|
@@ -21372,7 +22071,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
21372
22071
|
}
|
|
21373
22072
|
}
|
|
21374
22073
|
const snapshotDate = formatDate3(/* @__PURE__ */ new Date());
|
|
21375
|
-
db.delete(gscCoverageSnapshots).where(
|
|
22074
|
+
db.delete(gscCoverageSnapshots).where(and17(eq26(gscCoverageSnapshots.projectId, projectId), eq26(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
21376
22075
|
db.insert(gscCoverageSnapshots).values({
|
|
21377
22076
|
id: crypto23.randomUUID(),
|
|
21378
22077
|
projectId,
|
|
@@ -21383,11 +22082,11 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
21383
22082
|
reasonBreakdown: JSON.stringify(reasonCounts),
|
|
21384
22083
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
21385
22084
|
}).run();
|
|
21386
|
-
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
22085
|
+
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq26(runs.id, runId)).run();
|
|
21387
22086
|
log2.info("sync.completed", { runId, projectId, searchDataRows: rows.length, urlInspections: topPages.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
21388
22087
|
} catch (err) {
|
|
21389
22088
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
21390
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
22089
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq26(runs.id, runId)).run();
|
|
21391
22090
|
log2.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
21392
22091
|
throw err;
|
|
21393
22092
|
}
|
|
@@ -21395,7 +22094,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
21395
22094
|
|
|
21396
22095
|
// src/gsc-inspect-sitemap.ts
|
|
21397
22096
|
import crypto24 from "crypto";
|
|
21398
|
-
import { eq as
|
|
22097
|
+
import { eq as eq27, and as and18 } from "drizzle-orm";
|
|
21399
22098
|
|
|
21400
22099
|
// src/sitemap-parser.ts
|
|
21401
22100
|
var log3 = createLogger("SitemapParser");
|
|
@@ -21516,13 +22215,13 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
21516
22215
|
var log4 = createLogger("InspectSitemap");
|
|
21517
22216
|
async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
21518
22217
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
21519
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
22218
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq27(runs.id, runId)).run();
|
|
21520
22219
|
try {
|
|
21521
22220
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
21522
22221
|
if (!googleClientId || !googleClientSecret) {
|
|
21523
22222
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
21524
22223
|
}
|
|
21525
|
-
const project = db.select().from(projects).where(
|
|
22224
|
+
const project = db.select().from(projects).where(eq27(projects.id, projectId)).get();
|
|
21526
22225
|
if (!project) {
|
|
21527
22226
|
throw new Error(`Project not found: ${projectId}`);
|
|
21528
22227
|
}
|
|
@@ -21590,7 +22289,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
21590
22289
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
21591
22290
|
}
|
|
21592
22291
|
}
|
|
21593
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
22292
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq27(gscUrlInspections.projectId, projectId)).all();
|
|
21594
22293
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
21595
22294
|
for (const row of allInspections) {
|
|
21596
22295
|
const existing = latestByUrl.get(row.url);
|
|
@@ -21611,7 +22310,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
21611
22310
|
}
|
|
21612
22311
|
}
|
|
21613
22312
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
21614
|
-
db.delete(gscCoverageSnapshots).where(
|
|
22313
|
+
db.delete(gscCoverageSnapshots).where(and18(eq27(gscCoverageSnapshots.projectId, projectId), eq27(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
21615
22314
|
db.insert(gscCoverageSnapshots).values({
|
|
21616
22315
|
id: crypto24.randomUUID(),
|
|
21617
22316
|
projectId,
|
|
@@ -21623,11 +22322,11 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
21623
22322
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
21624
22323
|
}).run();
|
|
21625
22324
|
const status = errors > 0 && inspected > 0 ? "partial" : errors === urls.length ? "failed" : "completed";
|
|
21626
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
22325
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq27(runs.id, runId)).run();
|
|
21627
22326
|
log4.info("inspect.completed", { runId, projectId, inspected, errors, total: urls.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
21628
22327
|
} catch (err) {
|
|
21629
22328
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
21630
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
22329
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq27(runs.id, runId)).run();
|
|
21631
22330
|
log4.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
21632
22331
|
throw err;
|
|
21633
22332
|
}
|
|
@@ -21635,7 +22334,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
21635
22334
|
|
|
21636
22335
|
// src/bing-inspect-sitemap.ts
|
|
21637
22336
|
import crypto25 from "crypto";
|
|
21638
|
-
import { eq as
|
|
22337
|
+
import { eq as eq28, desc as desc13 } from "drizzle-orm";
|
|
21639
22338
|
var log5 = createLogger("BingInspectSitemap");
|
|
21640
22339
|
function parseBingDate2(value) {
|
|
21641
22340
|
if (!value) return null;
|
|
@@ -21653,9 +22352,9 @@ function isBlockingIssueType2(issueType) {
|
|
|
21653
22352
|
}
|
|
21654
22353
|
async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
21655
22354
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
21656
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
22355
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq28(runs.id, runId)).run();
|
|
21657
22356
|
try {
|
|
21658
|
-
const project = db.select().from(projects).where(
|
|
22357
|
+
const project = db.select().from(projects).where(eq28(projects.id, projectId)).get();
|
|
21659
22358
|
if (!project) {
|
|
21660
22359
|
throw new Error(`Project not found: ${projectId}`);
|
|
21661
22360
|
}
|
|
@@ -21673,7 +22372,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
21673
22372
|
if (sitemapUrls.length === 0) {
|
|
21674
22373
|
throw new Error("No URLs found in sitemap");
|
|
21675
22374
|
}
|
|
21676
|
-
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(
|
|
22375
|
+
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(eq28(bingUrlInspections.projectId, projectId)).all();
|
|
21677
22376
|
const trackedUrls = new Set(trackedRows.map((r) => r.url));
|
|
21678
22377
|
const discovered = sitemapUrls.filter((u) => !trackedUrls.has(u));
|
|
21679
22378
|
log5.info("sitemap.diff", {
|
|
@@ -21756,7 +22455,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
21756
22455
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
21757
22456
|
}
|
|
21758
22457
|
}
|
|
21759
|
-
const allInspections = db.select().from(bingUrlInspections).where(
|
|
22458
|
+
const allInspections = db.select().from(bingUrlInspections).where(eq28(bingUrlInspections.projectId, projectId)).orderBy(desc13(bingUrlInspections.inspectedAt)).all();
|
|
21760
22459
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
21761
22460
|
const definitiveByUrl = /* @__PURE__ */ new Map();
|
|
21762
22461
|
for (const row of allInspections) {
|
|
@@ -21799,7 +22498,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
21799
22498
|
}
|
|
21800
22499
|
}).run();
|
|
21801
22500
|
const status = errors === sitemapUrls.length ? RunStatuses.failed : errors > 0 ? RunStatuses.partial : RunStatuses.completed;
|
|
21802
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
22501
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq28(runs.id, runId)).run();
|
|
21803
22502
|
log5.info("inspect.completed", {
|
|
21804
22503
|
runId,
|
|
21805
22504
|
projectId,
|
|
@@ -21813,7 +22512,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
21813
22512
|
});
|
|
21814
22513
|
} catch (err) {
|
|
21815
22514
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
21816
|
-
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
22515
|
+
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq28(runs.id, runId)).run();
|
|
21817
22516
|
log5.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
21818
22517
|
throw err;
|
|
21819
22518
|
}
|
|
@@ -21822,7 +22521,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
21822
22521
|
// src/commoncrawl-sync.ts
|
|
21823
22522
|
import crypto26 from "crypto";
|
|
21824
22523
|
import path10 from "path";
|
|
21825
|
-
import { and as
|
|
22524
|
+
import { and as and19, eq as eq29, sql as sql12 } from "drizzle-orm";
|
|
21826
22525
|
var log6 = createLogger("CommonCrawlSync");
|
|
21827
22526
|
var INSERT_CHUNK_SIZE = 1e4;
|
|
21828
22527
|
function defaultDeps() {
|
|
@@ -21848,7 +22547,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
21848
22547
|
phaseDetail: "downloading vertices + edges",
|
|
21849
22548
|
updatedAt: downloadStartedAt,
|
|
21850
22549
|
error: null
|
|
21851
|
-
}).where(
|
|
22550
|
+
}).where(eq29(ccReleaseSyncs.id, syncId)).run();
|
|
21852
22551
|
const paths = ccReleasePaths(release);
|
|
21853
22552
|
const releaseCacheDir = path10.join(deps.cacheDir, release);
|
|
21854
22553
|
const vertexPath = path10.join(releaseCacheDir, paths.vertexFilename);
|
|
@@ -21871,7 +22570,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
21871
22570
|
vertexSha256: vertex.sha256,
|
|
21872
22571
|
edgesSha256: edges.sha256,
|
|
21873
22572
|
updatedAt: downloadFinishedAt
|
|
21874
|
-
}).where(
|
|
22573
|
+
}).where(eq29(ccReleaseSyncs.id, syncId)).run();
|
|
21875
22574
|
const allProjects = db.select().from(projects).all();
|
|
21876
22575
|
const targets = Array.from(new Set(allProjects.map((p) => p.canonicalDomain)));
|
|
21877
22576
|
let rows = [];
|
|
@@ -21887,8 +22586,8 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
21887
22586
|
}
|
|
21888
22587
|
const queriedAt = deps.now().toISOString();
|
|
21889
22588
|
db.transaction((tx) => {
|
|
21890
|
-
tx.delete(backlinkDomains).where(
|
|
21891
|
-
tx.delete(backlinkSummaries).where(
|
|
22589
|
+
tx.delete(backlinkDomains).where(eq29(backlinkDomains.releaseSyncId, syncId)).run();
|
|
22590
|
+
tx.delete(backlinkSummaries).where(eq29(backlinkSummaries.releaseSyncId, syncId)).run();
|
|
21892
22591
|
const expanded = [];
|
|
21893
22592
|
for (const r of rows) {
|
|
21894
22593
|
const projectIds = projectsByDomain.get(r.targetDomain);
|
|
@@ -21947,7 +22646,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
21947
22646
|
domainsDiscovered: rows.length,
|
|
21948
22647
|
updatedAt: finishedAt,
|
|
21949
22648
|
error: null
|
|
21950
|
-
}).where(
|
|
22649
|
+
}).where(eq29(ccReleaseSyncs.id, syncId)).run();
|
|
21951
22650
|
log6.info("sync.completed", {
|
|
21952
22651
|
syncId,
|
|
21953
22652
|
release,
|
|
@@ -21977,7 +22676,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
21977
22676
|
error: errorMsg,
|
|
21978
22677
|
phaseDetail: null,
|
|
21979
22678
|
updatedAt: finishedAt
|
|
21980
|
-
}).where(
|
|
22679
|
+
}).where(eq29(ccReleaseSyncs.id, syncId)).run();
|
|
21981
22680
|
log6.error("sync.failed", { syncId, release, error: errorMsg });
|
|
21982
22681
|
throw err;
|
|
21983
22682
|
}
|
|
@@ -22013,7 +22712,7 @@ function computeSummary(rows) {
|
|
|
22013
22712
|
// src/backlink-extract.ts
|
|
22014
22713
|
import crypto27 from "crypto";
|
|
22015
22714
|
import fs8 from "fs";
|
|
22016
|
-
import { and as
|
|
22715
|
+
import { and as and20, desc as desc14, eq as eq30 } from "drizzle-orm";
|
|
22017
22716
|
var log7 = createLogger("BacklinkExtract");
|
|
22018
22717
|
function defaultDeps2() {
|
|
22019
22718
|
return {
|
|
@@ -22025,13 +22724,13 @@ function defaultDeps2() {
|
|
|
22025
22724
|
async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
22026
22725
|
const deps = { ...defaultDeps2(), ...opts.deps };
|
|
22027
22726
|
const startedAt = deps.now().toISOString();
|
|
22028
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
22727
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq30(runs.id, runId)).run();
|
|
22029
22728
|
try {
|
|
22030
|
-
const project = db.select().from(projects).where(
|
|
22729
|
+
const project = db.select().from(projects).where(eq30(projects.id, projectId)).get();
|
|
22031
22730
|
if (!project) {
|
|
22032
22731
|
throw new Error(`Project not found: ${projectId}`);
|
|
22033
22732
|
}
|
|
22034
|
-
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(
|
|
22733
|
+
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(eq30(ccReleaseSyncs.release, opts.release)).get() : db.select().from(ccReleaseSyncs).where(eq30(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).orderBy(desc14(ccReleaseSyncs.createdAt)).limit(1).get();
|
|
22035
22734
|
if (!sync) {
|
|
22036
22735
|
throw new Error("No ready release sync available \u2014 run `canonry backlinks sync` first");
|
|
22037
22736
|
}
|
|
@@ -22059,7 +22758,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
22059
22758
|
const targetDomain = project.canonicalDomain;
|
|
22060
22759
|
db.transaction((tx) => {
|
|
22061
22760
|
tx.delete(backlinkDomains).where(
|
|
22062
|
-
|
|
22761
|
+
and20(eq30(backlinkDomains.projectId, projectId), eq30(backlinkDomains.release, release))
|
|
22063
22762
|
).run();
|
|
22064
22763
|
if (rows.length > 0) {
|
|
22065
22764
|
const values = rows.map((r) => ({
|
|
@@ -22099,7 +22798,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
22099
22798
|
}).run();
|
|
22100
22799
|
});
|
|
22101
22800
|
const finishedAt = deps.now().toISOString();
|
|
22102
|
-
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
22801
|
+
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq30(runs.id, runId)).run();
|
|
22103
22802
|
log7.info("extract.completed", { runId, projectId, release, rows: rows.length });
|
|
22104
22803
|
} catch (err) {
|
|
22105
22804
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
@@ -22108,7 +22807,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
22108
22807
|
status: RunStatuses.failed,
|
|
22109
22808
|
error: errorMsg,
|
|
22110
22809
|
finishedAt
|
|
22111
|
-
}).where(
|
|
22810
|
+
}).where(eq30(runs.id, runId)).run();
|
|
22112
22811
|
log7.error("extract.failed", { runId, projectId, error: errorMsg });
|
|
22113
22812
|
throw err;
|
|
22114
22813
|
}
|
|
@@ -22181,7 +22880,7 @@ var ProviderRegistry = class {
|
|
|
22181
22880
|
|
|
22182
22881
|
// src/scheduler.ts
|
|
22183
22882
|
import cron from "node-cron";
|
|
22184
|
-
import { and as
|
|
22883
|
+
import { and as and21, eq as eq31 } from "drizzle-orm";
|
|
22185
22884
|
var log8 = createLogger("Scheduler");
|
|
22186
22885
|
function taskKey(projectId, kind) {
|
|
22187
22886
|
return `${projectId}::${kind}`;
|
|
@@ -22196,7 +22895,7 @@ var Scheduler = class {
|
|
|
22196
22895
|
}
|
|
22197
22896
|
/** Load all enabled schedules from DB and register cron jobs. */
|
|
22198
22897
|
start() {
|
|
22199
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
22898
|
+
const allSchedules = this.db.select().from(schedules).where(eq31(schedules.enabled, 1)).all();
|
|
22200
22899
|
for (const schedule of allSchedules) {
|
|
22201
22900
|
const missedRunAt = schedule.nextRunAt;
|
|
22202
22901
|
this.registerCronTask(schedule);
|
|
@@ -22226,7 +22925,7 @@ var Scheduler = class {
|
|
|
22226
22925
|
this.stopTask(key, existing, "Stopped");
|
|
22227
22926
|
this.tasks.delete(key);
|
|
22228
22927
|
}
|
|
22229
|
-
const schedule = this.db.select().from(schedules).where(
|
|
22928
|
+
const schedule = this.db.select().from(schedules).where(and21(eq31(schedules.projectId, projectId), eq31(schedules.kind, kind))).get();
|
|
22230
22929
|
if (schedule && schedule.enabled === 1) {
|
|
22231
22930
|
this.registerCronTask(schedule);
|
|
22232
22931
|
}
|
|
@@ -22267,14 +22966,14 @@ var Scheduler = class {
|
|
|
22267
22966
|
this.db.update(schedules).set({
|
|
22268
22967
|
nextRunAt: task.getNextRun()?.toISOString() ?? null,
|
|
22269
22968
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
22270
|
-
}).where(
|
|
22969
|
+
}).where(eq31(schedules.id, scheduleId)).run();
|
|
22271
22970
|
const label = schedule.preset ?? cronExpr;
|
|
22272
22971
|
log8.info("cron.registered", { projectId, kind, schedule: label, timezone });
|
|
22273
22972
|
}
|
|
22274
22973
|
triggerRun(scheduleId, projectId, kind) {
|
|
22275
22974
|
try {
|
|
22276
22975
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
22277
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
22976
|
+
const currentSchedule = this.db.select().from(schedules).where(eq31(schedules.id, scheduleId)).get();
|
|
22278
22977
|
if (!currentSchedule || currentSchedule.enabled !== 1) {
|
|
22279
22978
|
log8.warn("schedule.stale", { scheduleId, projectId, kind, msg: "schedule no longer exists or is disabled" });
|
|
22280
22979
|
this.remove(projectId, kind);
|
|
@@ -22282,7 +22981,7 @@ var Scheduler = class {
|
|
|
22282
22981
|
}
|
|
22283
22982
|
const task = this.tasks.get(taskKey(projectId, kind));
|
|
22284
22983
|
const nextRunAt = task?.getNextRun()?.toISOString() ?? null;
|
|
22285
|
-
const project = this.db.select().from(projects).where(
|
|
22984
|
+
const project = this.db.select().from(projects).where(eq31(projects.id, projectId)).get();
|
|
22286
22985
|
if (!project) {
|
|
22287
22986
|
log8.error("project.not-found", { projectId, kind, msg: "skipping scheduled run" });
|
|
22288
22987
|
this.remove(projectId, kind);
|
|
@@ -22302,7 +23001,7 @@ var Scheduler = class {
|
|
|
22302
23001
|
lastRunAt: now,
|
|
22303
23002
|
nextRunAt,
|
|
22304
23003
|
updatedAt: now
|
|
22305
|
-
}).where(
|
|
23004
|
+
}).where(eq31(schedules.id, currentSchedule.id)).run();
|
|
22306
23005
|
log8.info("traffic-sync.triggered", { projectName: project.name, sourceId });
|
|
22307
23006
|
this.callbacks.onTrafficSyncRequested(project.name, sourceId);
|
|
22308
23007
|
return;
|
|
@@ -22330,7 +23029,7 @@ var Scheduler = class {
|
|
|
22330
23029
|
this.db.update(schedules).set({
|
|
22331
23030
|
nextRunAt,
|
|
22332
23031
|
updatedAt: now
|
|
22333
|
-
}).where(
|
|
23032
|
+
}).where(eq31(schedules.id, currentSchedule.id)).run();
|
|
22334
23033
|
return;
|
|
22335
23034
|
}
|
|
22336
23035
|
const runId = queueResult.runId;
|
|
@@ -22338,7 +23037,7 @@ var Scheduler = class {
|
|
|
22338
23037
|
lastRunAt: now,
|
|
22339
23038
|
nextRunAt,
|
|
22340
23039
|
updatedAt: now
|
|
22341
|
-
}).where(
|
|
23040
|
+
}).where(eq31(schedules.id, currentSchedule.id)).run();
|
|
22342
23041
|
const scheduleProviders = parseJsonColumn(currentSchedule.providers, []);
|
|
22343
23042
|
const providers = scheduleProviders.length > 0 ? scheduleProviders : void 0;
|
|
22344
23043
|
log8.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
@@ -22350,7 +23049,7 @@ var Scheduler = class {
|
|
|
22350
23049
|
};
|
|
22351
23050
|
|
|
22352
23051
|
// src/notifier.ts
|
|
22353
|
-
import { eq as
|
|
23052
|
+
import { eq as eq32, desc as desc15, and as and22, or as or4 } from "drizzle-orm";
|
|
22354
23053
|
import crypto28 from "crypto";
|
|
22355
23054
|
var log9 = createLogger("Notifier");
|
|
22356
23055
|
var Notifier = class {
|
|
@@ -22363,18 +23062,18 @@ var Notifier = class {
|
|
|
22363
23062
|
/** Called after a run completes (success, partial, or failed). */
|
|
22364
23063
|
async onRunCompleted(runId, projectId) {
|
|
22365
23064
|
log9.info("run.completed", { runId, projectId });
|
|
22366
|
-
const notifs = this.db.select().from(notifications).where(
|
|
23065
|
+
const notifs = this.db.select().from(notifications).where(eq32(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
22367
23066
|
if (notifs.length === 0) {
|
|
22368
23067
|
log9.info("notifications.none-enabled", { projectId });
|
|
22369
23068
|
return;
|
|
22370
23069
|
}
|
|
22371
23070
|
log9.info("notifications.found", { projectId, count: notifs.length });
|
|
22372
|
-
const run = this.db.select().from(runs).where(
|
|
23071
|
+
const run = this.db.select().from(runs).where(eq32(runs.id, runId)).get();
|
|
22373
23072
|
if (!run) {
|
|
22374
23073
|
log9.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
22375
23074
|
return;
|
|
22376
23075
|
}
|
|
22377
|
-
const project = this.db.select().from(projects).where(
|
|
23076
|
+
const project = this.db.select().from(projects).where(eq32(projects.id, projectId)).get();
|
|
22378
23077
|
if (!project) {
|
|
22379
23078
|
log9.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
22380
23079
|
return;
|
|
@@ -22421,11 +23120,11 @@ var Notifier = class {
|
|
|
22421
23120
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
22422
23121
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
22423
23122
|
if (insightEvents.length === 0) return;
|
|
22424
|
-
const notifs = this.db.select().from(notifications).where(
|
|
23123
|
+
const notifs = this.db.select().from(notifications).where(eq32(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
22425
23124
|
if (notifs.length === 0) return;
|
|
22426
|
-
const run = this.db.select().from(runs).where(
|
|
23125
|
+
const run = this.db.select().from(runs).where(eq32(runs.id, runId)).get();
|
|
22427
23126
|
if (!run) return;
|
|
22428
|
-
const project = this.db.select().from(projects).where(
|
|
23127
|
+
const project = this.db.select().from(projects).where(eq32(projects.id, projectId)).get();
|
|
22429
23128
|
if (!project) return;
|
|
22430
23129
|
for (const notif of notifs) {
|
|
22431
23130
|
const config = parseJsonColumn(notif.config, { url: "", events: [] });
|
|
@@ -22456,9 +23155,9 @@ var Notifier = class {
|
|
|
22456
23155
|
}
|
|
22457
23156
|
computeTransitions(runId, projectId) {
|
|
22458
23157
|
const recentRuns = this.db.select().from(runs).where(
|
|
22459
|
-
|
|
22460
|
-
|
|
22461
|
-
or4(
|
|
23158
|
+
and22(
|
|
23159
|
+
eq32(runs.projectId, projectId),
|
|
23160
|
+
or4(eq32(runs.status, "completed"), eq32(runs.status, "partial"))
|
|
22462
23161
|
)
|
|
22463
23162
|
).orderBy(desc15(runs.createdAt)).limit(2).all();
|
|
22464
23163
|
if (recentRuns.length < 2) return [];
|
|
@@ -22470,12 +23169,12 @@ var Notifier = class {
|
|
|
22470
23169
|
query: queries.query,
|
|
22471
23170
|
provider: querySnapshots.provider,
|
|
22472
23171
|
citationState: querySnapshots.citationState
|
|
22473
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
23172
|
+
}).from(querySnapshots).leftJoin(queries, eq32(querySnapshots.queryId, queries.id)).where(eq32(querySnapshots.runId, currentRunId)).all();
|
|
22474
23173
|
const previousSnapshots = this.db.select({
|
|
22475
23174
|
queryId: querySnapshots.queryId,
|
|
22476
23175
|
provider: querySnapshots.provider,
|
|
22477
23176
|
citationState: querySnapshots.citationState
|
|
22478
|
-
}).from(querySnapshots).where(
|
|
23177
|
+
}).from(querySnapshots).where(eq32(querySnapshots.runId, previousRunId)).all();
|
|
22479
23178
|
const prevMap = /* @__PURE__ */ new Map();
|
|
22480
23179
|
for (const s of previousSnapshots) {
|
|
22481
23180
|
prevMap.set(`${s.queryId}:${s.provider}`, s.citationState);
|
|
@@ -22592,7 +23291,7 @@ var RunCoordinator = class {
|
|
|
22592
23291
|
|
|
22593
23292
|
// src/agent/session-registry.ts
|
|
22594
23293
|
import crypto30 from "crypto";
|
|
22595
|
-
import { eq as
|
|
23294
|
+
import { eq as eq34 } from "drizzle-orm";
|
|
22596
23295
|
|
|
22597
23296
|
// src/agent/session.ts
|
|
22598
23297
|
import fs11 from "fs";
|
|
@@ -22942,7 +23641,7 @@ function resolveSessionProviderAndModel(config, opts) {
|
|
|
22942
23641
|
|
|
22943
23642
|
// src/agent/memory-store.ts
|
|
22944
23643
|
import crypto29 from "crypto";
|
|
22945
|
-
import { and as
|
|
23644
|
+
import { and as and23, desc as desc16, eq as eq33, like as like2, sql as sql13 } from "drizzle-orm";
|
|
22946
23645
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
22947
23646
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
22948
23647
|
function rowToDto2(row) {
|
|
@@ -22956,7 +23655,7 @@ function rowToDto2(row) {
|
|
|
22956
23655
|
};
|
|
22957
23656
|
}
|
|
22958
23657
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
22959
|
-
const query = db.select().from(agentMemory).where(
|
|
23658
|
+
const query = db.select().from(agentMemory).where(eq33(agentMemory.projectId, projectId)).orderBy(desc16(agentMemory.updatedAt));
|
|
22960
23659
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
22961
23660
|
return rows.map(rowToDto2);
|
|
22962
23661
|
}
|
|
@@ -22987,12 +23686,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
22987
23686
|
updatedAt: now
|
|
22988
23687
|
}
|
|
22989
23688
|
}).run();
|
|
22990
|
-
const row = db.select().from(agentMemory).where(
|
|
23689
|
+
const row = db.select().from(agentMemory).where(and23(eq33(agentMemory.projectId, args.projectId), eq33(agentMemory.key, args.key))).get();
|
|
22991
23690
|
if (!row) throw new Error("memory upsert produced no row");
|
|
22992
23691
|
return rowToDto2(row);
|
|
22993
23692
|
}
|
|
22994
23693
|
function deleteMemoryEntry(db, projectId, key) {
|
|
22995
|
-
const result = db.delete(agentMemory).where(
|
|
23694
|
+
const result = db.delete(agentMemory).where(and23(eq33(agentMemory.projectId, projectId), eq33(agentMemory.key, key))).run();
|
|
22996
23695
|
const changes = result.changes ?? 0;
|
|
22997
23696
|
return changes > 0;
|
|
22998
23697
|
}
|
|
@@ -23021,16 +23720,16 @@ function writeCompactionNote(db, args) {
|
|
|
23021
23720
|
}).run();
|
|
23022
23721
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
23023
23722
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
23024
|
-
|
|
23025
|
-
|
|
23723
|
+
and23(
|
|
23724
|
+
eq33(agentMemory.projectId, args.projectId),
|
|
23026
23725
|
like2(agentMemory.key, `${sessionPrefix}%`)
|
|
23027
23726
|
)
|
|
23028
23727
|
).orderBy(desc16(agentMemory.updatedAt)).all();
|
|
23029
23728
|
const stale = existing.slice(COMPACTION_NOTES_PER_SESSION).map((r) => r.id);
|
|
23030
23729
|
if (stale.length > 0) {
|
|
23031
|
-
tx.delete(agentMemory).where(
|
|
23730
|
+
tx.delete(agentMemory).where(sql13`${agentMemory.id} IN (${sql13.join(stale.map((s) => sql13`${s}`), sql13`, `)})`).run();
|
|
23032
23731
|
}
|
|
23033
|
-
const row = tx.select().from(agentMemory).where(
|
|
23732
|
+
const row = tx.select().from(agentMemory).where(and23(eq33(agentMemory.projectId, args.projectId), eq33(agentMemory.key, key))).get();
|
|
23034
23733
|
if (row) inserted = rowToDto2(row);
|
|
23035
23734
|
});
|
|
23036
23735
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -23212,7 +23911,7 @@ var SessionRegistry = class {
|
|
|
23212
23911
|
modelProvider: effectiveProvider,
|
|
23213
23912
|
modelId: effectiveModelId,
|
|
23214
23913
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
23215
|
-
}).where(
|
|
23914
|
+
}).where(eq34(agentSessions.projectId, projectId)).run();
|
|
23216
23915
|
}
|
|
23217
23916
|
const agent2 = createAeroSession({
|
|
23218
23917
|
projectName,
|
|
@@ -23426,7 +24125,7 @@ ${lines.join("\n")}
|
|
|
23426
24125
|
modelProvider: nextProvider,
|
|
23427
24126
|
modelId: nextModelId,
|
|
23428
24127
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
23429
|
-
}).where(
|
|
24128
|
+
}).where(eq34(agentSessions.projectId, projectId)).run();
|
|
23430
24129
|
}
|
|
23431
24130
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
23432
24131
|
save(projectName) {
|
|
@@ -23588,11 +24287,11 @@ ${lines.join("\n")}
|
|
|
23588
24287
|
return id;
|
|
23589
24288
|
}
|
|
23590
24289
|
tryResolveProjectId(projectName) {
|
|
23591
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
24290
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq34(projects.name, projectName)).get();
|
|
23592
24291
|
return row?.id;
|
|
23593
24292
|
}
|
|
23594
24293
|
loadRow(projectId) {
|
|
23595
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
24294
|
+
const row = this.opts.db.select().from(agentSessions).where(eq34(agentSessions.projectId, projectId)).get();
|
|
23596
24295
|
return row ?? null;
|
|
23597
24296
|
}
|
|
23598
24297
|
insertRow(params) {
|
|
@@ -23611,14 +24310,14 @@ ${lines.join("\n")}
|
|
|
23611
24310
|
}
|
|
23612
24311
|
updateRow(projectId, patch) {
|
|
23613
24312
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
23614
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
24313
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq34(agentSessions.projectId, projectId)).run();
|
|
23615
24314
|
}
|
|
23616
24315
|
};
|
|
23617
24316
|
|
|
23618
24317
|
// src/agent/agent-routes.ts
|
|
23619
|
-
import { eq as
|
|
24318
|
+
import { eq as eq35 } from "drizzle-orm";
|
|
23620
24319
|
function resolveProject2(db, name) {
|
|
23621
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
24320
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq35(projects.name, name)).get();
|
|
23622
24321
|
if (!row) throw notFound("project", name);
|
|
23623
24322
|
return row;
|
|
23624
24323
|
}
|
|
@@ -23627,7 +24326,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
23627
24326
|
"/projects/:name/agent/transcript",
|
|
23628
24327
|
async (request) => {
|
|
23629
24328
|
const project = resolveProject2(opts.db, request.params.name);
|
|
23630
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
24329
|
+
const row = opts.db.select().from(agentSessions).where(eq35(agentSessions.projectId, project.id)).get();
|
|
23631
24330
|
if (!row) {
|
|
23632
24331
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
23633
24332
|
}
|
|
@@ -23651,7 +24350,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
23651
24350
|
async (request) => {
|
|
23652
24351
|
const project = resolveProject2(opts.db, request.params.name);
|
|
23653
24352
|
opts.sessionRegistry.reset(project.name);
|
|
23654
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
24353
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq35(agentSessions.projectId, project.id)).run();
|
|
23655
24354
|
return { status: "reset" };
|
|
23656
24355
|
}
|
|
23657
24356
|
);
|
|
@@ -24673,7 +25372,7 @@ async function createServer(opts) {
|
|
|
24673
25372
|
intelligenceService,
|
|
24674
25373
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
24675
25374
|
async ({ runId, projectId, insightCount, criticalOrHigh }) => {
|
|
24676
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
25375
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq36(projects.id, projectId)).get();
|
|
24677
25376
|
if (!project) return;
|
|
24678
25377
|
sessionRegistry.queueFollowUp(project.name, {
|
|
24679
25378
|
role: "user",
|
|
@@ -24833,7 +25532,7 @@ async function createServer(opts) {
|
|
|
24833
25532
|
const apiPrefix = basePath ? `${basePath}api/v1` : "/api/v1";
|
|
24834
25533
|
if (opts.config.apiKey) {
|
|
24835
25534
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
24836
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
25535
|
+
const existing = opts.db.select().from(apiKeys).where(eq36(apiKeys.keyHash, keyHash)).get();
|
|
24837
25536
|
if (!existing) {
|
|
24838
25537
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
24839
25538
|
opts.db.insert(apiKeys).values({
|
|
@@ -24885,7 +25584,7 @@ async function createServer(opts) {
|
|
|
24885
25584
|
};
|
|
24886
25585
|
const getDefaultApiKey = () => {
|
|
24887
25586
|
if (!opts.config.apiKey) return void 0;
|
|
24888
|
-
return opts.db.select().from(apiKeys).where(
|
|
25587
|
+
return opts.db.select().from(apiKeys).where(eq36(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
24889
25588
|
};
|
|
24890
25589
|
const createPasswordSession = (reply) => {
|
|
24891
25590
|
const key = getDefaultApiKey();
|
|
@@ -24942,12 +25641,12 @@ async function createServer(opts) {
|
|
|
24942
25641
|
return reply.send({ authenticated: true });
|
|
24943
25642
|
}
|
|
24944
25643
|
if (apiKey) {
|
|
24945
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
25644
|
+
const key = opts.db.select().from(apiKeys).where(eq36(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
24946
25645
|
if (!key || key.revokedAt) {
|
|
24947
25646
|
const err2 = authInvalid();
|
|
24948
25647
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
24949
25648
|
}
|
|
24950
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
25649
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq36(apiKeys.id, key.id)).run();
|
|
24951
25650
|
const sessionId = createSession(key.id);
|
|
24952
25651
|
reply.header("set-cookie", serializeSessionCookie({
|
|
24953
25652
|
name: SESSION_COOKIE_NAME,
|