@ainyc/canonry 4.11.1 → 4.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/assets/index-BofSsfDl.js +302 -0
- package/assets/assets/index-D0EPNRDs.css +1 -0
- package/assets/index.html +2 -2
- package/dist/{chunk-3SFDZPKU.js → chunk-DCE3B6KD.js} +177 -2
- package/dist/{chunk-5J5BVJF7.js → chunk-LNRDWAG3.js} +16 -1
- package/dist/{chunk-PBQ4Z4P4.js → chunk-RIGQFQJJ.js} +1705 -292
- package/dist/{chunk-565T7PMC.js → chunk-YDGT5CAY.js} +54 -2
- package/dist/cli.js +269 -103
- package/dist/index.d.ts +19 -0
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-XWOFLEHJ.js → intelligence-service-NT24OLLA.js} +2 -2
- package/dist/mcp.js +2 -2
- package/package.json +11 -9
- package/assets/assets/index-CGXCbiM_.css +0 -1
- package/assets/assets/index-DOcemxPD.js +0 -302
|
@@ -4,13 +4,14 @@ import {
|
|
|
4
4
|
configExists,
|
|
5
5
|
loadConfig,
|
|
6
6
|
saveConfigPatch
|
|
7
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-LNRDWAG3.js";
|
|
8
8
|
import {
|
|
9
9
|
DEFAULT_RUN_HISTORY_LIMIT,
|
|
10
10
|
IntelligenceService,
|
|
11
11
|
MIN_TREND_POINTS,
|
|
12
12
|
agentMemory,
|
|
13
13
|
agentSessions,
|
|
14
|
+
aiReferralEventsHourly,
|
|
14
15
|
apiKeys,
|
|
15
16
|
auditLog,
|
|
16
17
|
backlinkDomains,
|
|
@@ -36,6 +37,7 @@ import {
|
|
|
36
37
|
categorizeQueryByIntent,
|
|
37
38
|
ccReleaseSyncs,
|
|
38
39
|
competitors,
|
|
40
|
+
crawlerEventsHourly,
|
|
39
41
|
createLogger,
|
|
40
42
|
dropLegacyCredentialColumns,
|
|
41
43
|
extractLegacyCredentials,
|
|
@@ -58,10 +60,12 @@ import {
|
|
|
58
60
|
projects,
|
|
59
61
|
queries,
|
|
60
62
|
querySnapshots,
|
|
63
|
+
rawEventSamples,
|
|
61
64
|
runs,
|
|
62
65
|
schedules,
|
|
66
|
+
trafficSources,
|
|
63
67
|
usageCounters
|
|
64
|
-
} from "./chunk-
|
|
68
|
+
} from "./chunk-DCE3B6KD.js";
|
|
65
69
|
import {
|
|
66
70
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
67
71
|
AGENT_PROVIDER_IDS,
|
|
@@ -76,6 +80,11 @@ import {
|
|
|
76
80
|
RunKinds,
|
|
77
81
|
RunStatuses,
|
|
78
82
|
RunTriggers,
|
|
83
|
+
TrafficEventConfidences,
|
|
84
|
+
TrafficEvidenceKinds,
|
|
85
|
+
TrafficSourceAuthModes,
|
|
86
|
+
TrafficSourceStatuses,
|
|
87
|
+
TrafficSourceTypes,
|
|
79
88
|
absolutizeProjectUrl,
|
|
80
89
|
actionConfidenceLabel,
|
|
81
90
|
agentBusy,
|
|
@@ -137,7 +146,7 @@ import {
|
|
|
137
146
|
visibilityStateFromAnswerMentioned,
|
|
138
147
|
windowCutoff,
|
|
139
148
|
wordpressEnvSchema
|
|
140
|
-
} from "./chunk-
|
|
149
|
+
} from "./chunk-YDGT5CAY.js";
|
|
141
150
|
|
|
142
151
|
// src/telemetry.ts
|
|
143
152
|
import crypto from "crypto";
|
|
@@ -213,11 +222,11 @@ function trackEvent(event, properties) {
|
|
|
213
222
|
|
|
214
223
|
// src/server.ts
|
|
215
224
|
import { createRequire as createRequire3 } from "module";
|
|
216
|
-
import
|
|
225
|
+
import crypto30 from "crypto";
|
|
217
226
|
import fs12 from "fs";
|
|
218
227
|
import path14 from "path";
|
|
219
228
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
220
|
-
import { eq as
|
|
229
|
+
import { eq as eq35 } from "drizzle-orm";
|
|
221
230
|
import Fastify from "fastify";
|
|
222
231
|
|
|
223
232
|
// ../api-routes/src/auth.ts
|
|
@@ -2654,6 +2663,47 @@ function gscDateRange(report) {
|
|
|
2654
2663
|
function pluralize(count, singular, plural = `${singular}s`) {
|
|
2655
2664
|
return count === 1 ? singular : plural;
|
|
2656
2665
|
}
|
|
2666
|
+
var PROVIDER_DISPLAY_NAMES = {
|
|
2667
|
+
gemini: "Gemini",
|
|
2668
|
+
openai: "ChatGPT",
|
|
2669
|
+
claude: "Claude",
|
|
2670
|
+
perplexity: "Perplexity",
|
|
2671
|
+
local: "Local model",
|
|
2672
|
+
"cdp:chatgpt": "ChatGPT (browser)"
|
|
2673
|
+
};
|
|
2674
|
+
function providerDisplayName(name) {
|
|
2675
|
+
return PROVIDER_DISPLAY_NAMES[name] ?? name.charAt(0).toUpperCase() + name.slice(1);
|
|
2676
|
+
}
|
|
2677
|
+
function clientHorizonLabel(horizon) {
|
|
2678
|
+
switch (horizon) {
|
|
2679
|
+
case "immediate":
|
|
2680
|
+
return "Do now";
|
|
2681
|
+
case "short-term":
|
|
2682
|
+
return "This month";
|
|
2683
|
+
case "medium-term":
|
|
2684
|
+
return "Next quarter";
|
|
2685
|
+
}
|
|
2686
|
+
}
|
|
2687
|
+
function clientConfidenceLabel(confidence) {
|
|
2688
|
+
switch (confidence) {
|
|
2689
|
+
case "high":
|
|
2690
|
+
return "Strong evidence";
|
|
2691
|
+
case "medium":
|
|
2692
|
+
return "Some evidence";
|
|
2693
|
+
case "low":
|
|
2694
|
+
return "Worth trying";
|
|
2695
|
+
}
|
|
2696
|
+
}
|
|
2697
|
+
function clientTrendCopy(delta) {
|
|
2698
|
+
if (!delta) return null;
|
|
2699
|
+
if (delta.direction === "up") {
|
|
2700
|
+
return { text: `Up ${delta.deltaAbs.toFixed(1)} points since last check (was ${delta.prior}%)`, tone: "positive", arrow: "\u2191" };
|
|
2701
|
+
}
|
|
2702
|
+
if (delta.direction === "down") {
|
|
2703
|
+
return { text: `Down ${Math.abs(delta.deltaAbs).toFixed(1)} points since last check (was ${delta.prior}%)`, tone: "negative", arrow: "\u2193" };
|
|
2704
|
+
}
|
|
2705
|
+
return { text: `Holding steady since last check (was ${delta.prior}%)`, tone: "neutral", arrow: "\u2192" };
|
|
2706
|
+
}
|
|
2657
2707
|
function compactInlineList(items, limit = 3) {
|
|
2658
2708
|
const visible = items.slice(0, limit);
|
|
2659
2709
|
const more = items.length - visible.length;
|
|
@@ -3183,6 +3233,222 @@ table.report-table td .badge {
|
|
|
3183
3233
|
color: ${COLORS.textFaint};
|
|
3184
3234
|
font-size: 12px;
|
|
3185
3235
|
}
|
|
3236
|
+
.client-hero {
|
|
3237
|
+
background: ${COLORS.surface};
|
|
3238
|
+
border: 1px solid ${COLORS.border};
|
|
3239
|
+
border-radius: 16px;
|
|
3240
|
+
padding: 32px;
|
|
3241
|
+
margin-bottom: 24px;
|
|
3242
|
+
}
|
|
3243
|
+
.client-hero .client-hero-eyebrow {
|
|
3244
|
+
text-transform: uppercase;
|
|
3245
|
+
letter-spacing: 0.05em;
|
|
3246
|
+
font-size: 11px;
|
|
3247
|
+
font-weight: 600;
|
|
3248
|
+
color: ${COLORS.textFaint};
|
|
3249
|
+
}
|
|
3250
|
+
.client-hero .client-hero-number {
|
|
3251
|
+
font-size: 80px;
|
|
3252
|
+
line-height: 1;
|
|
3253
|
+
font-weight: 800;
|
|
3254
|
+
letter-spacing: -0.02em;
|
|
3255
|
+
color: ${COLORS.text};
|
|
3256
|
+
margin: 14px 0 18px;
|
|
3257
|
+
}
|
|
3258
|
+
.client-hero .client-hero-sentence {
|
|
3259
|
+
font-size: 17px;
|
|
3260
|
+
color: #d4d4d8;
|
|
3261
|
+
max-width: 720px;
|
|
3262
|
+
margin: 0;
|
|
3263
|
+
}
|
|
3264
|
+
.client-hero .client-hero-trend {
|
|
3265
|
+
margin-top: 14px;
|
|
3266
|
+
font-size: 14px;
|
|
3267
|
+
font-weight: 500;
|
|
3268
|
+
}
|
|
3269
|
+
.client-hero .client-hero-trend.tone-positive { color: ${COLORS.positive}; }
|
|
3270
|
+
.client-hero .client-hero-trend.tone-negative { color: ${COLORS.negative}; }
|
|
3271
|
+
.client-hero .client-hero-trend.tone-neutral { color: ${COLORS.textMuted}; }
|
|
3272
|
+
.client-metric-grid {
|
|
3273
|
+
display: grid;
|
|
3274
|
+
grid-template-columns: repeat(auto-fit, minmax(220px, 1fr));
|
|
3275
|
+
gap: 16px;
|
|
3276
|
+
margin-bottom: 24px;
|
|
3277
|
+
}
|
|
3278
|
+
.client-metric-tile {
|
|
3279
|
+
background: ${COLORS.surface};
|
|
3280
|
+
border: 1px solid ${COLORS.border};
|
|
3281
|
+
border-radius: 12px;
|
|
3282
|
+
padding: 22px 24px;
|
|
3283
|
+
}
|
|
3284
|
+
.client-metric-tile .label {
|
|
3285
|
+
text-transform: uppercase;
|
|
3286
|
+
letter-spacing: 0.05em;
|
|
3287
|
+
font-size: 11px;
|
|
3288
|
+
font-weight: 600;
|
|
3289
|
+
color: ${COLORS.textFaint};
|
|
3290
|
+
margin-bottom: 14px;
|
|
3291
|
+
}
|
|
3292
|
+
.client-metric-tile .value {
|
|
3293
|
+
font-size: 48px;
|
|
3294
|
+
line-height: 1;
|
|
3295
|
+
font-weight: 800;
|
|
3296
|
+
letter-spacing: -0.02em;
|
|
3297
|
+
color: ${COLORS.text};
|
|
3298
|
+
}
|
|
3299
|
+
.client-metric-tile .subtitle {
|
|
3300
|
+
margin-top: 10px;
|
|
3301
|
+
font-size: 12px;
|
|
3302
|
+
color: ${COLORS.textMuted};
|
|
3303
|
+
}
|
|
3304
|
+
.client-card {
|
|
3305
|
+
background: ${COLORS.surface};
|
|
3306
|
+
border: 1px solid ${COLORS.border};
|
|
3307
|
+
border-radius: 12px;
|
|
3308
|
+
padding: 22px 24px;
|
|
3309
|
+
margin-bottom: 16px;
|
|
3310
|
+
}
|
|
3311
|
+
.client-card h3 {
|
|
3312
|
+
font-size: 15px;
|
|
3313
|
+
font-weight: 600;
|
|
3314
|
+
margin: 0 0 4px;
|
|
3315
|
+
}
|
|
3316
|
+
.client-card .card-subtitle {
|
|
3317
|
+
font-size: 12px;
|
|
3318
|
+
color: ${COLORS.textMuted};
|
|
3319
|
+
margin: 0 0 18px;
|
|
3320
|
+
}
|
|
3321
|
+
.client-bar-list {
|
|
3322
|
+
display: flex;
|
|
3323
|
+
flex-direction: column;
|
|
3324
|
+
gap: 14px;
|
|
3325
|
+
}
|
|
3326
|
+
.client-bar-row {
|
|
3327
|
+
display: grid;
|
|
3328
|
+
grid-template-columns: 140px 1fr 130px;
|
|
3329
|
+
align-items: center;
|
|
3330
|
+
gap: 14px;
|
|
3331
|
+
font-size: 13px;
|
|
3332
|
+
}
|
|
3333
|
+
.client-bar-row .bar-label { color: #d4d4d8; }
|
|
3334
|
+
.client-bar-row .bar-track {
|
|
3335
|
+
height: 10px;
|
|
3336
|
+
background: ${COLORS.border};
|
|
3337
|
+
border-radius: 999px;
|
|
3338
|
+
overflow: hidden;
|
|
3339
|
+
}
|
|
3340
|
+
.client-bar-row .bar-fill {
|
|
3341
|
+
height: 100%;
|
|
3342
|
+
border-radius: 999px;
|
|
3343
|
+
background: ${COLORS.positive}b3;
|
|
3344
|
+
}
|
|
3345
|
+
.client-bar-row .bar-fill.bar-fill-neutral { background: #a1a1aaaa; }
|
|
3346
|
+
.client-bar-row .bar-fill.bar-fill-sky { background: #38bdf8b3; }
|
|
3347
|
+
.client-bar-row .bar-value {
|
|
3348
|
+
text-align: right;
|
|
3349
|
+
font-size: 13px;
|
|
3350
|
+
font-weight: 600;
|
|
3351
|
+
color: ${COLORS.text};
|
|
3352
|
+
font-variant-numeric: tabular-nums;
|
|
3353
|
+
}
|
|
3354
|
+
.client-bar-row .bar-value-sub { color: ${COLORS.textFaint}; font-weight: 400; }
|
|
3355
|
+
.client-progress-number {
|
|
3356
|
+
font-size: 56px;
|
|
3357
|
+
font-weight: 800;
|
|
3358
|
+
line-height: 1;
|
|
3359
|
+
letter-spacing: -0.02em;
|
|
3360
|
+
margin: 12px 0 4px;
|
|
3361
|
+
}
|
|
3362
|
+
.client-progress-number.tone-positive { color: ${COLORS.positive}; }
|
|
3363
|
+
.client-progress-number.tone-caution { color: ${COLORS.caution}; }
|
|
3364
|
+
.client-progress-number.tone-negative { color: ${COLORS.negative}; }
|
|
3365
|
+
.client-progress-bar {
|
|
3366
|
+
height: 12px;
|
|
3367
|
+
background: ${COLORS.border};
|
|
3368
|
+
border-radius: 999px;
|
|
3369
|
+
overflow: hidden;
|
|
3370
|
+
margin: 12px 0 14px;
|
|
3371
|
+
}
|
|
3372
|
+
.client-progress-fill { height: 100%; border-radius: 999px; }
|
|
3373
|
+
.client-progress-fill.tone-positive { background: ${COLORS.positive}b3; }
|
|
3374
|
+
.client-progress-fill.tone-caution { background: ${COLORS.caution}b3; }
|
|
3375
|
+
.client-progress-fill.tone-negative { background: ${COLORS.negative}b3; }
|
|
3376
|
+
.client-evidence-grid {
|
|
3377
|
+
display: grid;
|
|
3378
|
+
grid-template-columns: repeat(auto-fit, minmax(360px, 1fr));
|
|
3379
|
+
gap: 16px;
|
|
3380
|
+
}
|
|
3381
|
+
.client-opportunity-list {
|
|
3382
|
+
display: flex;
|
|
3383
|
+
flex-direction: column;
|
|
3384
|
+
gap: 8px;
|
|
3385
|
+
margin: 0;
|
|
3386
|
+
padding: 0;
|
|
3387
|
+
list-style: none;
|
|
3388
|
+
}
|
|
3389
|
+
.client-opportunity-list li {
|
|
3390
|
+
background: #09090b;
|
|
3391
|
+
border: 1px solid ${COLORS.border};
|
|
3392
|
+
border-radius: 8px;
|
|
3393
|
+
padding: 10px 14px;
|
|
3394
|
+
}
|
|
3395
|
+
.client-opportunity-list li .op-query {
|
|
3396
|
+
font-weight: 500;
|
|
3397
|
+
color: ${COLORS.text};
|
|
3398
|
+
font-size: 13px;
|
|
3399
|
+
}
|
|
3400
|
+
.client-opportunity-list li .op-action {
|
|
3401
|
+
margin-top: 2px;
|
|
3402
|
+
font-size: 11px;
|
|
3403
|
+
color: ${COLORS.textMuted};
|
|
3404
|
+
}
|
|
3405
|
+
.client-confidence-note {
|
|
3406
|
+
background: ${COLORS.surface};
|
|
3407
|
+
border: 1px solid ${COLORS.border};
|
|
3408
|
+
border-radius: 8px;
|
|
3409
|
+
padding: 10px 14px;
|
|
3410
|
+
font-size: 12px;
|
|
3411
|
+
color: ${COLORS.textMuted};
|
|
3412
|
+
margin-bottom: 6px;
|
|
3413
|
+
}
|
|
3414
|
+
.client-explainer {
|
|
3415
|
+
background: #09090b;
|
|
3416
|
+
border: 1px solid ${COLORS.border};
|
|
3417
|
+
border-radius: 12px;
|
|
3418
|
+
padding: 12px 16px;
|
|
3419
|
+
font-size: 12px;
|
|
3420
|
+
color: ${COLORS.textMuted};
|
|
3421
|
+
margin-bottom: 16px;
|
|
3422
|
+
line-height: 1.6;
|
|
3423
|
+
}
|
|
3424
|
+
.client-explainer strong { color: ${COLORS.text}; }
|
|
3425
|
+
.client-explainer .term { color: #d4d4d8; font-weight: 500; }
|
|
3426
|
+
.client-questions-list {
|
|
3427
|
+
display: grid;
|
|
3428
|
+
grid-template-columns: repeat(auto-fit, minmax(280px, 1fr));
|
|
3429
|
+
gap: 8px;
|
|
3430
|
+
margin: 0;
|
|
3431
|
+
padding: 0;
|
|
3432
|
+
list-style: none;
|
|
3433
|
+
}
|
|
3434
|
+
.client-questions-list li {
|
|
3435
|
+
display: flex;
|
|
3436
|
+
align-items: flex-start;
|
|
3437
|
+
gap: 12px;
|
|
3438
|
+
background: #09090b;
|
|
3439
|
+
border: 1px solid ${COLORS.border};
|
|
3440
|
+
border-radius: 8px;
|
|
3441
|
+
padding: 10px 14px;
|
|
3442
|
+
font-size: 13px;
|
|
3443
|
+
color: #d4d4d8;
|
|
3444
|
+
}
|
|
3445
|
+
.client-questions-list li .qnum {
|
|
3446
|
+
flex-shrink: 0;
|
|
3447
|
+
font-size: 11px;
|
|
3448
|
+
font-weight: 600;
|
|
3449
|
+
color: ${COLORS.textFaint};
|
|
3450
|
+
font-variant-numeric: tabular-nums;
|
|
3451
|
+
}
|
|
3186
3452
|
@media (max-width: 760px) {
|
|
3187
3453
|
.container { padding: 32px 16px 72px; }
|
|
3188
3454
|
.executive-hero { grid-template-columns: 1fr; }
|
|
@@ -3190,6 +3456,9 @@ table.report-table td .badge {
|
|
|
3190
3456
|
.source-bar-row { grid-template-columns: 1fr; gap: 6px; }
|
|
3191
3457
|
.source-bar-value { text-align: left; }
|
|
3192
3458
|
.chart-grid { grid-template-columns: 1fr; }
|
|
3459
|
+
.client-hero .client-hero-number { font-size: 56px; }
|
|
3460
|
+
.client-metric-tile .value { font-size: 36px; }
|
|
3461
|
+
.client-bar-row { grid-template-columns: 100px 1fr 100px; gap: 10px; }
|
|
3193
3462
|
}
|
|
3194
3463
|
@media print {
|
|
3195
3464
|
body { background: white; color: black; }
|
|
@@ -3380,68 +3649,82 @@ function renderTrafficDeltaTile(label, delta, countLabel) {
|
|
|
3380
3649
|
</div>`;
|
|
3381
3650
|
}
|
|
3382
3651
|
var WHATS_CHANGED_PERIOD_DAYS = 14;
|
|
3383
|
-
function renderProviderMovements(movements) {
|
|
3652
|
+
function renderProviderMovements(movements, audience) {
|
|
3384
3653
|
const meaningful = movements.filter((m) => m.direction !== "flat");
|
|
3385
3654
|
if (meaningful.length === 0) return "";
|
|
3655
|
+
const isClient = audience === "client";
|
|
3386
3656
|
const rows = meaningful.map((m) => {
|
|
3387
3657
|
const sign = m.deltaAbs > 0 ? "+" : "";
|
|
3388
3658
|
return `<tr>
|
|
3389
|
-
<td>${escapeHtml(m.provider)}</td>
|
|
3659
|
+
<td>${escapeHtml(isClient ? providerDisplayName(m.provider) : m.provider)}</td>
|
|
3390
3660
|
<td class="numeric">${m.prior}%</td>
|
|
3391
3661
|
<td class="numeric">${m.current}%</td>
|
|
3392
3662
|
<td class="numeric ${deltaToneClass(m.direction)}">${sign}${m.deltaAbs.toFixed(1)}% ${deltaArrow(m.direction)}</td>
|
|
3393
3663
|
</tr>`;
|
|
3394
3664
|
}).join("");
|
|
3395
|
-
|
|
3665
|
+
const heading = isClient ? "How each AI tool changed" : "AI engine movements";
|
|
3666
|
+
const colA = isClient ? "AI tool" : "Engine";
|
|
3667
|
+
const colB = isClient ? "Was" : "Prior";
|
|
3668
|
+
const colC = isClient ? "Now" : "Current";
|
|
3669
|
+
return `<div class="chart-card"><h3>${heading}</h3>
|
|
3396
3670
|
<table class="report-table">
|
|
3397
|
-
<thead><tr><th
|
|
3671
|
+
<thead><tr><th>${colA}</th><th class="numeric">${colB}</th><th class="numeric">${colC}</th><th class="numeric">Change</th></tr></thead>
|
|
3398
3672
|
<tbody>${rows}</tbody>
|
|
3399
3673
|
</table>
|
|
3400
3674
|
</div>`;
|
|
3401
3675
|
}
|
|
3402
|
-
function renderWinsLosses(insights2, heading, emptyMessage) {
|
|
3676
|
+
function renderWinsLosses(insights2, heading, emptyMessage, audience) {
|
|
3403
3677
|
if (insights2.length === 0) {
|
|
3404
3678
|
return `<div class="chart-card"><h3>${escapeHtml(heading)}</h3>
|
|
3405
3679
|
<p class="section-intro">${escapeHtml(emptyMessage)}</p>
|
|
3406
3680
|
</div>`;
|
|
3407
3681
|
}
|
|
3682
|
+
const isClient = audience === "client";
|
|
3408
3683
|
const rows = insights2.map((i) => {
|
|
3409
3684
|
const tone = severityTone(i.severity);
|
|
3410
3685
|
const countChip = i.instanceCount > 1 ? ` <span class="badge tone-neutral">\xD7 ${i.instanceCount}</span>` : "";
|
|
3686
|
+
const severityCell = isClient ? "" : `<td><span class="badge tone-${tone}">${escapeHtml(reportSeverityLabel(i.severity))}</span></td>`;
|
|
3411
3687
|
return `<tr>
|
|
3412
|
-
|
|
3688
|
+
${severityCell}
|
|
3413
3689
|
<td>${escapeHtml(i.title)}${countChip}</td>
|
|
3414
3690
|
<td>${escapeHtml(i.query)}</td>
|
|
3415
|
-
<td>${escapeHtml(i.provider)}</td>
|
|
3691
|
+
<td>${escapeHtml(isClient ? providerDisplayName(i.provider) : i.provider)}</td>
|
|
3416
3692
|
</tr>`;
|
|
3417
3693
|
}).join("");
|
|
3694
|
+
const headers = isClient ? `<tr><th>What changed</th><th>Customer question</th><th>AI tool</th></tr>` : `<tr><th>Severity</th><th>Title</th><th>Query</th><th>Provider</th></tr>`;
|
|
3418
3695
|
return `<div class="chart-card"><h3>${escapeHtml(heading)}</h3>
|
|
3419
3696
|
<table class="report-table">
|
|
3420
|
-
<thead
|
|
3697
|
+
<thead>${headers}</thead>
|
|
3421
3698
|
<tbody>${rows}</tbody>
|
|
3422
3699
|
</table>
|
|
3423
3700
|
</div>`;
|
|
3424
3701
|
}
|
|
3425
|
-
function renderWhatsChanged(report) {
|
|
3702
|
+
function renderWhatsChanged(report, audience) {
|
|
3426
3703
|
const w = report.whatsChanged;
|
|
3704
|
+
const isClient = audience === "client";
|
|
3705
|
+
const eyebrow = isClient ? "Since last check" : "Section 2";
|
|
3706
|
+
const title = isClient ? "What's different since last check" : "What's Changed";
|
|
3707
|
+
const intro = isClient ? "" : w.headline;
|
|
3427
3708
|
if (!w.enoughHistory && !w.gscClicksDelta && !w.aiReferralsDelta && w.wins.length === 0 && w.regressions.length === 0) {
|
|
3428
3709
|
return section(
|
|
3429
|
-
{ id: "whats-changed", eyebrow
|
|
3430
|
-
renderEmpty("Trends will appear after a few more checks.")
|
|
3710
|
+
{ id: "whats-changed", eyebrow, title, intro },
|
|
3711
|
+
renderEmpty(isClient ? "No comparison yet \u2014 trends will appear after a few more checks." : "Trends will appear after a few more checks.")
|
|
3431
3712
|
);
|
|
3432
3713
|
}
|
|
3433
3714
|
const rateTiles = `<div class="metric-grid">
|
|
3434
|
-
${renderRateDeltaTile("Citation rate", w.citationRate, "%")}
|
|
3435
|
-
${renderRateDeltaTile("Mention rate", w.mentionRate, "%")}
|
|
3436
|
-
${renderRateDeltaTile("Cited queries", w.citedQueryCount, "count")}
|
|
3437
|
-
${renderTrafficDeltaTile("GSC clicks", w.gscClicksDelta, "clicks")}
|
|
3438
|
-
${renderTrafficDeltaTile("AI referral sessions", w.aiReferralsDelta, "sessions")}
|
|
3715
|
+
${renderRateDeltaTile(isClient ? "AI links to your website" : "Citation rate", w.citationRate, "%")}
|
|
3716
|
+
${renderRateDeltaTile(isClient ? "AI mentions your name" : "Mention rate", w.mentionRate, "%")}
|
|
3717
|
+
${renderRateDeltaTile(isClient ? "Questions AI answered with you" : "Cited queries", w.citedQueryCount, "count")}
|
|
3718
|
+
${renderTrafficDeltaTile(isClient ? "Visitors from Google" : "GSC clicks", w.gscClicksDelta, isClient ? "visits" : "clicks")}
|
|
3719
|
+
${renderTrafficDeltaTile(isClient ? "Visitors from AI tools" : "AI referral sessions", w.aiReferralsDelta, isClient ? "visits" : "sessions")}
|
|
3439
3720
|
</div>`;
|
|
3440
|
-
const movements = renderProviderMovements(w.providerMovements);
|
|
3441
|
-
const
|
|
3442
|
-
const
|
|
3721
|
+
const movements = renderProviderMovements(w.providerMovements, audience);
|
|
3722
|
+
const winsHeading = isClient ? "What got better" : "Wins";
|
|
3723
|
+
const lossesHeading = isClient ? "What got worse" : "Regressions";
|
|
3724
|
+
const wins = renderWinsLosses(w.wins, winsHeading, isClient ? "No new wins this period." : "No new gains in the latest check.", audience);
|
|
3725
|
+
const regressions = renderWinsLosses(w.regressions, lossesHeading, isClient ? "Nothing got worse this period." : "No new regressions in the latest check.", audience);
|
|
3443
3726
|
return section(
|
|
3444
|
-
{ id: "whats-changed", eyebrow
|
|
3727
|
+
{ id: "whats-changed", eyebrow, title, intro },
|
|
3445
3728
|
`${rateTiles}${movements}${wins}${regressions}`
|
|
3446
3729
|
);
|
|
3447
3730
|
}
|
|
@@ -4095,8 +4378,9 @@ function renderRecommendedNextSteps(report) {
|
|
|
4095
4378
|
function actionAudienceMatches(action, audience) {
|
|
4096
4379
|
return action.audience === "both" || action.audience === audience;
|
|
4097
4380
|
}
|
|
4098
|
-
function renderActionCards(actions) {
|
|
4099
|
-
|
|
4381
|
+
function renderActionCards(actions, audience) {
|
|
4382
|
+
const isClient = audience === "client";
|
|
4383
|
+
if (actions.length === 0) return renderEmpty(isClient ? "No recommendations yet \u2014 run an AI check to populate this." : "No prioritized actions yet.");
|
|
4100
4384
|
return `<div class="action-card-grid">
|
|
4101
4385
|
${actions.map((action, idx) => {
|
|
4102
4386
|
const tone = reportActionTone(action);
|
|
@@ -4104,18 +4388,22 @@ function renderActionCards(actions) {
|
|
|
4104
4388
|
const evidence = action.evidence.length > 0 ? `<ul>${action.evidence.map((item) => `<li>${escapeHtml(item)}</li>`).join("")}</ul>` : "";
|
|
4105
4389
|
const proof = renderProofChips(action.evidence.length > 0 ? action.evidence : action.why, 3);
|
|
4106
4390
|
const details = why || evidence ? `<details class="action-details">
|
|
4107
|
-
<summary
|
|
4108
|
-
${why ? `<div><strong
|
|
4109
|
-
${evidence ? `<div><strong
|
|
4391
|
+
<summary>${isClient ? "See the data behind this" : "Evidence details"}</summary>
|
|
4392
|
+
${why ? `<div><strong>${isClient ? "Why this matters" : "Why"}</strong>${why}</div>` : ""}
|
|
4393
|
+
${evidence ? `<div><strong>${isClient ? "What we saw" : "Evidence"}</strong>${evidence}</div>` : ""}
|
|
4110
4394
|
</details>` : "";
|
|
4395
|
+
const horizonLabel = isClient ? clientHorizonLabel(action.horizon) : reportHorizonLabel(action.horizon);
|
|
4396
|
+
const confidenceLabel = isClient ? clientConfidenceLabel(action.confidence) : `${reportConfidenceLabel(action.confidence)} confidence`;
|
|
4397
|
+
const categoryBadge = isClient ? "" : `<span class="badge tone-neutral">${escapeHtml(reportActionCategoryLabel(action.category))}</span>`;
|
|
4398
|
+
const successLabel = isClient ? "What success looks like:" : "Win condition:";
|
|
4111
4399
|
return `<article class="action-card">
|
|
4112
4400
|
<div class="action-head">
|
|
4113
|
-
<div class="action-rank" title="Impact rank \u2014 1 is the highest-leverage action">${idx + 1}</div>
|
|
4401
|
+
<div class="action-rank" title="${isClient ? "Priority \u2014 1 will move the needle fastest" : "Impact rank \u2014 1 is the highest-leverage action"}">${idx + 1}</div>
|
|
4114
4402
|
<div>
|
|
4115
4403
|
<div class="action-meta">
|
|
4116
|
-
<span class="badge tone-${tone}">${escapeHtml(
|
|
4117
|
-
|
|
4118
|
-
<span class="badge tone-neutral">${escapeHtml(
|
|
4404
|
+
<span class="badge tone-${tone}">${escapeHtml(horizonLabel)}</span>
|
|
4405
|
+
${categoryBadge}
|
|
4406
|
+
<span class="badge tone-neutral">${escapeHtml(confidenceLabel)}</span>
|
|
4119
4407
|
</div>
|
|
4120
4408
|
<h3>${escapeHtml(action.title)}</h3>
|
|
4121
4409
|
</div>
|
|
@@ -4123,7 +4411,7 @@ function renderActionCards(actions) {
|
|
|
4123
4411
|
<p>${escapeHtml(action.action)}</p>
|
|
4124
4412
|
${proof}
|
|
4125
4413
|
${details}
|
|
4126
|
-
<div class="success-metric"><strong
|
|
4414
|
+
<div class="success-metric"><strong>${successLabel}</strong> ${escapeHtml(action.successMetric)}</div>
|
|
4127
4415
|
</article>`;
|
|
4128
4416
|
}).join("")}
|
|
4129
4417
|
</div>`;
|
|
@@ -4134,76 +4422,150 @@ function renderAudienceActionPlan(report, audience) {
|
|
|
4134
4422
|
return section(
|
|
4135
4423
|
{
|
|
4136
4424
|
id: audience === "client" ? "client-action-plan" : "agency-action-plan",
|
|
4137
|
-
eyebrow: audience === "client" ? "
|
|
4138
|
-
title: audience === "client" ? "What
|
|
4139
|
-
intro: audience === "client" ? "
|
|
4425
|
+
eyebrow: audience === "client" ? "Action plan" : "Agency actions",
|
|
4426
|
+
title: audience === "client" ? "What to do next" : "Agency Action Plan",
|
|
4427
|
+
intro: audience === "client" ? "Approve these in order. They are sorted by what will move the needle fastest." : "The highest-leverage work, sorted by urgency and evidence strength."
|
|
4140
4428
|
},
|
|
4141
|
-
renderActionCards(actions)
|
|
4429
|
+
renderActionCards(actions, audience)
|
|
4142
4430
|
);
|
|
4143
4431
|
}
|
|
4144
4432
|
function renderClientSummary(report) {
|
|
4145
4433
|
const s = report.executiveSummary;
|
|
4146
|
-
const
|
|
4147
|
-
|
|
4148
|
-
|
|
4149
|
-
|
|
4434
|
+
const sc = report.citationScorecard;
|
|
4435
|
+
const totalQ = s.totalQueryCount;
|
|
4436
|
+
const heroNumber = totalQ > 0 ? `${s.citationRate}%` : "\u2014";
|
|
4437
|
+
const heroSentence = totalQ > 0 ? `When customers asked AI ${totalQ} ${pluralize(totalQ, "question")} about your industry, AI linked to your website in ${s.citedQueryCount} of ${totalQ === 1 ? "them" : "those answers"}.` : "No AI check has been run yet. Run a check to see how AI tools answer customer questions about your business.";
|
|
4438
|
+
const trend = clientTrendCopy(report.whatsChanged.citationRate);
|
|
4439
|
+
const heroTrend = trend ? `<p class="client-hero-trend tone-${trend.tone}"><span style="margin-right:6px;">${trend.arrow}</span>${escapeHtml(trend.text)}</p>` : "";
|
|
4440
|
+
const hero = `<div class="client-hero">
|
|
4441
|
+
<div class="client-hero-eyebrow">Overview</div>
|
|
4442
|
+
<div class="client-hero-number">${heroNumber}</div>
|
|
4443
|
+
<p class="client-hero-sentence">${escapeHtml(heroSentence)}</p>
|
|
4444
|
+
${heroTrend}
|
|
4150
4445
|
</div>`;
|
|
4151
|
-
const
|
|
4152
|
-
|
|
4153
|
-
|
|
4154
|
-
|
|
4155
|
-
|
|
4156
|
-
|
|
4157
|
-
intro: report.clientSummary.overview
|
|
4158
|
-
},
|
|
4159
|
-
`<div class="chart-card">
|
|
4160
|
-
<h3>${escapeHtml(report.clientSummary.headline)}</h3>
|
|
4161
|
-
<p class="source-origin-headline">${escapeHtml(report.clientSummary.overview)}</p>
|
|
4446
|
+
const providerSubtitle = sc.providers.length > 0 ? sc.providers.map(providerDisplayName).join(", ") : `${formatNumber(s.queryCount)} ${pluralize(s.queryCount, "question")} tested`;
|
|
4447
|
+
const tiles = `<div class="client-metric-grid">
|
|
4448
|
+
<div class="client-metric-tile">
|
|
4449
|
+
<div class="label">AI mentions your name</div>
|
|
4450
|
+
<div class="value">${s.mentionRate}%</div>
|
|
4451
|
+
<div class="subtitle">${totalQ > 0 ? `Says your name in ${s.mentionedQueryCount} of ${totalQ} ${pluralize(totalQ, "answer")}` : "No data yet"}</div>
|
|
4162
4452
|
</div>
|
|
4163
|
-
|
|
4164
|
-
|
|
4165
|
-
|
|
4453
|
+
<div class="client-metric-tile">
|
|
4454
|
+
<div class="label">AI links to your website</div>
|
|
4455
|
+
<div class="value">${s.citationRate}%</div>
|
|
4456
|
+
<div class="subtitle">${totalQ > 0 ? `Cites your site as a source in ${s.citedQueryCount} of ${totalQ} ${pluralize(totalQ, "answer")}` : "No data yet"}</div>
|
|
4457
|
+
</div>
|
|
4458
|
+
<div class="client-metric-tile">
|
|
4459
|
+
<div class="label">AI tools tested</div>
|
|
4460
|
+
<div class="value">${formatNumber(s.providerCount)}</div>
|
|
4461
|
+
<div class="subtitle">${escapeHtml(providerSubtitle)}</div>
|
|
4462
|
+
</div>
|
|
4463
|
+
</div>`;
|
|
4464
|
+
const explainer = `<div class="client-explainer">
|
|
4465
|
+
<strong>Mentions and links are different.</strong>
|
|
4466
|
+
A <span class="term">mention</span> is when AI says your name out loud in its answer.
|
|
4467
|
+
A <span class="term">link</span> is when AI lists your website as a source it used.
|
|
4468
|
+
AI can do either, both, or neither \u2014 that's why we track both.
|
|
4469
|
+
</div>`;
|
|
4470
|
+
const questions = sc.queries.length > 0 ? `<div class="client-card">
|
|
4471
|
+
<h3>Customer questions we tested</h3>
|
|
4472
|
+
<p class="card-subtitle">These are the ${sc.queries.length} ${pluralize(sc.queries.length, "question we asked", "questions we asked")} every AI tool. The numbers above measure how often you came up.</p>
|
|
4473
|
+
<ol class="client-questions-list">
|
|
4474
|
+
${sc.queries.map((q, i) => `<li><span class="qnum">${String(i + 1).padStart(2, "0")}</span><span>"${escapeHtml(q)}"</span></li>`).join("")}
|
|
4475
|
+
</ol>
|
|
4476
|
+
</div>` : "";
|
|
4477
|
+
const providerBars = sc.providerRates.length > 0 ? `<div class="client-card">
|
|
4478
|
+
<h3>How often each AI tool links to your website</h3>
|
|
4479
|
+
<p class="card-subtitle">Higher is better. Each bar shows the share of customer questions where the AI cited your site.</p>
|
|
4480
|
+
<div class="client-bar-list">
|
|
4481
|
+
${sc.providerRates.map((r) => {
|
|
4482
|
+
const pct = Math.max(r.citationRate, 1.5);
|
|
4483
|
+
return `<div class="client-bar-row">
|
|
4484
|
+
<span class="bar-label">${escapeHtml(providerDisplayName(r.provider))}</span>
|
|
4485
|
+
<div class="bar-track"><div class="bar-fill" style="width:${pct}%"></div></div>
|
|
4486
|
+
<span class="bar-value">${r.citationRate}% <span class="bar-value-sub">(${r.citedCount}/${r.totalCount})</span></span>
|
|
4487
|
+
</div>`;
|
|
4488
|
+
}).join("")}
|
|
4489
|
+
</div>
|
|
4490
|
+
</div>` : "";
|
|
4491
|
+
const notes = report.clientSummary.confidenceNotes.length > 0 ? `<div>${report.clientSummary.confidenceNotes.map((note) => `<div class="client-confidence-note">${escapeHtml(note)}</div>`).join("")}</div>` : "";
|
|
4492
|
+
return `<section class="report-section" id="client-summary">${hero}${tiles}${explainer}${questions}${providerBars}${notes}</section>`;
|
|
4166
4493
|
}
|
|
4167
4494
|
function renderClientEvidenceSummary(report) {
|
|
4168
|
-
const
|
|
4169
|
-
|
|
4170
|
-
|
|
4171
|
-
|
|
4172
|
-
|
|
4173
|
-
|
|
4495
|
+
const ai = report.aiSourceOrigin.topDomains.slice(0, 5);
|
|
4496
|
+
const gsc = report.gsc;
|
|
4497
|
+
const indexing = report.indexingHealth;
|
|
4498
|
+
const opportunities = dedupeReportOpportunities(report).slice(0, 5);
|
|
4499
|
+
const aiMax = ai.length > 0 ? Math.max(...ai.map((d) => d.count)) : 0;
|
|
4500
|
+
const gscMax = gsc ? Math.max(...gsc.topQueries.slice(0, 5).map((q) => q.impressions), 1) : 0;
|
|
4501
|
+
const cards = [];
|
|
4502
|
+
if (ai.length > 0) {
|
|
4503
|
+
cards.push(`<div class="client-card">
|
|
4504
|
+
<h3>Where AI gets its answers</h3>
|
|
4505
|
+
<p class="card-subtitle">The websites AI tools cited most often when answering customer questions about your industry.</p>
|
|
4506
|
+
<div class="client-bar-list">
|
|
4507
|
+
${ai.map((d) => {
|
|
4508
|
+
const pct = aiMax > 0 ? Math.max(d.count / aiMax * 100, 1.5) : 0;
|
|
4509
|
+
const label = escapeHtml(d.domain) + (d.isCompetitor ? ' <span style="color:' + COLORS.textFaint + ';font-size:11px;">(competitor)</span>' : "");
|
|
4510
|
+
return `<div class="client-bar-row">
|
|
4511
|
+
<span class="bar-label">${label}</span>
|
|
4512
|
+
<div class="bar-track"><div class="bar-fill bar-fill-neutral" style="width:${pct}%"></div></div>
|
|
4513
|
+
<span class="bar-value">${formatNumber(d.count)}\xD7</span>
|
|
4514
|
+
</div>`;
|
|
4515
|
+
}).join("")}
|
|
4516
|
+
</div>
|
|
4174
4517
|
</div>`);
|
|
4175
4518
|
}
|
|
4176
|
-
if (
|
|
4177
|
-
|
|
4178
|
-
|
|
4179
|
-
|
|
4180
|
-
<
|
|
4519
|
+
if (indexing) {
|
|
4520
|
+
const tone = indexing.indexedPct >= 90 ? "positive" : indexing.indexedPct >= 70 ? "caution" : "negative";
|
|
4521
|
+
const fillPct = Math.max(indexing.indexedPct, 1.5);
|
|
4522
|
+
cards.push(`<div class="client-card">
|
|
4523
|
+
<h3>Pages Google can find on your site</h3>
|
|
4524
|
+
<p class="card-subtitle">Google indexing your site increases the chances of it appearing in AI search (especially Gemini).</p>
|
|
4525
|
+
<div class="client-progress-number tone-${tone}">${indexing.indexedPct}%</div>
|
|
4526
|
+
<div style="font-size:12px;color:${COLORS.textMuted};">${formatNumber(indexing.indexed)} of ${formatNumber(indexing.total)} pages indexed</div>
|
|
4527
|
+
<div class="client-progress-bar"><div class="client-progress-fill tone-${tone}" style="width:${fillPct}%"></div></div>
|
|
4528
|
+
<p style="margin:0;font-size:12px;color:${COLORS.textMuted};"><strong style="color:${COLORS.text};">${formatNumber(indexing.notIndexed)}</strong> ${pluralize(indexing.notIndexed, "page is", "pages are")} not indexed yet.</p>
|
|
4181
4529
|
</div>`);
|
|
4182
4530
|
}
|
|
4183
|
-
if (
|
|
4184
|
-
const
|
|
4185
|
-
|
|
4186
|
-
|
|
4187
|
-
|
|
4188
|
-
|
|
4531
|
+
if (gsc) {
|
|
4532
|
+
const queries2 = gsc.topQueries.slice(0, 5);
|
|
4533
|
+
const queryRows = queries2.length > 0 ? `<div class="client-bar-list">
|
|
4534
|
+
${queries2.map((q) => {
|
|
4535
|
+
const pct = gscMax > 0 ? Math.max(q.impressions / gscMax * 100, 1.5) : 0;
|
|
4536
|
+
return `<div class="client-bar-row">
|
|
4537
|
+
<span class="bar-label">${escapeHtml(q.query)}</span>
|
|
4538
|
+
<div class="bar-track"><div class="bar-fill bar-fill-sky" style="width:${pct}%"></div></div>
|
|
4539
|
+
<span class="bar-value">${formatNumber(q.impressions)} ${pluralize(q.impressions, "search", "searches")}</span>
|
|
4540
|
+
</div>`;
|
|
4541
|
+
}).join("")}
|
|
4542
|
+
</div>` : "";
|
|
4543
|
+
cards.push(`<div class="client-card">
|
|
4544
|
+
<h3>What people search Google for</h3>
|
|
4545
|
+
<p class="card-subtitle">You appeared in <strong style="color:${COLORS.text};">${formatNumber(gsc.totalImpressions)}</strong> Google searches and got <strong style="color:${COLORS.text};">${formatNumber(gsc.totalClicks)}</strong> ${pluralize(gsc.totalClicks, "click")} this period.</p>
|
|
4546
|
+
${queryRows}
|
|
4189
4547
|
</div>`);
|
|
4190
4548
|
}
|
|
4191
|
-
const opportunities = dedupeReportOpportunities(report);
|
|
4192
4549
|
if (opportunities.length > 0) {
|
|
4193
|
-
|
|
4194
|
-
<h3>
|
|
4195
|
-
<p>
|
|
4196
|
-
<ul
|
|
4550
|
+
cards.push(`<div class="client-card">
|
|
4551
|
+
<h3>Topics where you could improve</h3>
|
|
4552
|
+
<p class="card-subtitle">Customer questions where better content on your site would help AI cite you.</p>
|
|
4553
|
+
<ul class="client-opportunity-list">
|
|
4554
|
+
${opportunities.map((o) => `<li>
|
|
4555
|
+
<div class="op-query">${escapeHtml(o.query)}</div>
|
|
4556
|
+
<div class="op-action">${escapeHtml(contentActionLabel(o.action))}</div>
|
|
4557
|
+
</li>`).join("")}
|
|
4558
|
+
</ul>
|
|
4197
4559
|
</div>`);
|
|
4198
4560
|
}
|
|
4199
4561
|
return section(
|
|
4200
4562
|
{
|
|
4201
4563
|
id: "client-evidence-summary",
|
|
4202
|
-
eyebrow: "
|
|
4203
|
-
title: "
|
|
4204
|
-
intro: "
|
|
4564
|
+
eyebrow: "What we based this on",
|
|
4565
|
+
title: "The signals behind this plan",
|
|
4566
|
+
intro: "The data behind the recommendations above. Switch to Agency for the full breakdowns."
|
|
4205
4567
|
},
|
|
4206
|
-
|
|
4568
|
+
cards.length > 0 ? `<div class="client-evidence-grid">${cards.join("")}</div>` : renderEmpty("No supporting evidence yet \u2014 this fills in after the first AI check.")
|
|
4207
4569
|
);
|
|
4208
4570
|
}
|
|
4209
4571
|
function renderAgencyDiagnostics(report) {
|
|
@@ -4233,12 +4595,12 @@ function renderReportHtml(report, opts = {}) {
|
|
|
4233
4595
|
const title = opts.title ?? `Canonry ${audience} report \u2014 ${report.meta.project.displayName}`;
|
|
4234
4596
|
const sections = audience === "client" ? [
|
|
4235
4597
|
renderClientSummary(report),
|
|
4236
|
-
renderWhatsChanged(report),
|
|
4598
|
+
renderWhatsChanged(report, "client"),
|
|
4237
4599
|
renderAudienceActionPlan(report, "client"),
|
|
4238
4600
|
renderClientEvidenceSummary(report)
|
|
4239
4601
|
].join("\n") : [
|
|
4240
4602
|
renderExecutiveSummary(report),
|
|
4241
|
-
renderWhatsChanged(report),
|
|
4603
|
+
renderWhatsChanged(report, "agency"),
|
|
4242
4604
|
renderAudienceActionPlan(report, "agency"),
|
|
4243
4605
|
renderAgencyDiagnostics(report),
|
|
4244
4606
|
renderCitationScorecard(report),
|
|
@@ -4267,7 +4629,7 @@ function renderReportHtml(report, opts = {}) {
|
|
|
4267
4629
|
<body>
|
|
4268
4630
|
<div class="container">
|
|
4269
4631
|
<header class="header">
|
|
4270
|
-
<div class="eyebrow"
|
|
4632
|
+
<div class="eyebrow">AI Visibility Report</div>
|
|
4271
4633
|
<h1>${escapeHtml(report.meta.project.displayName)}</h1>
|
|
4272
4634
|
<div class="subtitle">${escapeHtml(report.meta.project.canonicalDomain)} \xB7 ${escapeHtml(report.meta.project.country)} / ${escapeHtml(report.meta.project.language.toUpperCase())}${renderHeaderLocationFragment(report.meta.location)} \xB7 Generated ${formatDate(report.meta.generatedAt)}</div>
|
|
4273
4635
|
</header>
|
|
@@ -9160,6 +9522,66 @@ var routeCatalog = [
|
|
|
9160
9522
|
200: { description: "History returned oldest-first by queriedAt." },
|
|
9161
9523
|
404: { description: "Project not found." }
|
|
9162
9524
|
}
|
|
9525
|
+
},
|
|
9526
|
+
{
|
|
9527
|
+
method: "post",
|
|
9528
|
+
path: "/api/v1/projects/{name}/traffic/connect/cloud-run",
|
|
9529
|
+
summary: "Connect a Cloud Run traffic source",
|
|
9530
|
+
description: "Stores the service-account JSON in `~/.canonry/config.yaml` and creates a `traffic_sources` row for the project. Reconnecting updates the existing active source rather than creating a duplicate.",
|
|
9531
|
+
tags: ["traffic"],
|
|
9532
|
+
parameters: [nameParameter],
|
|
9533
|
+
requestBody: {
|
|
9534
|
+
required: true,
|
|
9535
|
+
content: {
|
|
9536
|
+
"application/json": {
|
|
9537
|
+
schema: {
|
|
9538
|
+
type: "object",
|
|
9539
|
+
required: ["gcpProjectId", "keyJson"],
|
|
9540
|
+
properties: {
|
|
9541
|
+
gcpProjectId: stringSchema,
|
|
9542
|
+
serviceName: stringSchema,
|
|
9543
|
+
location: stringSchema,
|
|
9544
|
+
displayName: stringSchema,
|
|
9545
|
+
keyJson: { ...stringSchema, description: "Service-account JSON content." }
|
|
9546
|
+
}
|
|
9547
|
+
}
|
|
9548
|
+
}
|
|
9549
|
+
}
|
|
9550
|
+
},
|
|
9551
|
+
responses: {
|
|
9552
|
+
200: { description: "Traffic source DTO returned." },
|
|
9553
|
+
400: { description: "Invalid Cloud Run connection request." },
|
|
9554
|
+
404: { description: "Project not found." }
|
|
9555
|
+
}
|
|
9556
|
+
},
|
|
9557
|
+
{
|
|
9558
|
+
method: "post",
|
|
9559
|
+
path: "/api/v1/projects/{name}/traffic/sources/{id}/sync",
|
|
9560
|
+
summary: "Trigger a sync run for a traffic source",
|
|
9561
|
+
description: "Pulls request logs from the configured Cloud Run service for the lookback window, classifies crawler / AI-referral hits, and upserts hourly buckets and a bounded sample tail.",
|
|
9562
|
+
tags: ["traffic"],
|
|
9563
|
+
parameters: [
|
|
9564
|
+
nameParameter,
|
|
9565
|
+
{ name: "id", in: "path", required: true, description: "Traffic source ID.", schema: stringSchema }
|
|
9566
|
+
],
|
|
9567
|
+
requestBody: {
|
|
9568
|
+
required: false,
|
|
9569
|
+
content: {
|
|
9570
|
+
"application/json": {
|
|
9571
|
+
schema: {
|
|
9572
|
+
type: "object",
|
|
9573
|
+
properties: {
|
|
9574
|
+
sinceMinutes: { ...integerSchema, description: "Lookback window in minutes (default 60)." }
|
|
9575
|
+
}
|
|
9576
|
+
}
|
|
9577
|
+
}
|
|
9578
|
+
}
|
|
9579
|
+
},
|
|
9580
|
+
responses: {
|
|
9581
|
+
200: { description: "Sync summary returned." },
|
|
9582
|
+
400: { description: "Invalid sync request, missing credentials, or upstream pull error." },
|
|
9583
|
+
404: { description: "Project or traffic source not found." }
|
|
9584
|
+
}
|
|
9163
9585
|
}
|
|
9164
9586
|
];
|
|
9165
9587
|
var canonryLocalRouteCatalog = [
|
|
@@ -14908,7 +15330,7 @@ async function queryBacklinks(opts) {
|
|
|
14908
15330
|
const reversed = opts.targets.map(reverseDomain);
|
|
14909
15331
|
const targetList = reversed.map(quote).join(", ");
|
|
14910
15332
|
const limitClause = opts.limitPerTarget ? `QUALIFY row_number() OVER (PARTITION BY t.target_rev_domain ORDER BY v.num_hosts DESC) <= ${Math.floor(opts.limitPerTarget)}` : "";
|
|
14911
|
-
const
|
|
15333
|
+
const sql12 = `
|
|
14912
15334
|
WITH vertices AS (
|
|
14913
15335
|
SELECT * FROM read_csv(
|
|
14914
15336
|
${quote(opts.vertexPath)},
|
|
@@ -14944,7 +15366,7 @@ async function queryBacklinks(opts) {
|
|
|
14944
15366
|
const conn = await instance.connect();
|
|
14945
15367
|
let rows;
|
|
14946
15368
|
try {
|
|
14947
|
-
const reader = await conn.runAndReadAll(
|
|
15369
|
+
const reader = await conn.runAndReadAll(sql12);
|
|
14948
15370
|
rows = reader.getRowObjects();
|
|
14949
15371
|
} finally {
|
|
14950
15372
|
conn.disconnectSync?.();
|
|
@@ -15332,78 +15754,1016 @@ async function backlinksRoutes(app, opts) {
|
|
|
15332
15754
|
);
|
|
15333
15755
|
}
|
|
15334
15756
|
|
|
15335
|
-
// ../api-routes/src/
|
|
15336
|
-
|
|
15337
|
-
|
|
15338
|
-
|
|
15339
|
-
|
|
15340
|
-
|
|
15341
|
-
|
|
15342
|
-
|
|
15343
|
-
|
|
15344
|
-
|
|
15345
|
-
|
|
15346
|
-
|
|
15347
|
-
|
|
15348
|
-
|
|
15349
|
-
|
|
15350
|
-
|
|
15351
|
-
|
|
15352
|
-
|
|
15353
|
-
|
|
15354
|
-
|
|
15355
|
-
|
|
15356
|
-
|
|
15357
|
-
|
|
15358
|
-
|
|
15359
|
-
|
|
15360
|
-
|
|
15361
|
-
|
|
15362
|
-
|
|
15363
|
-
|
|
15364
|
-
|
|
15365
|
-
|
|
15366
|
-
|
|
15367
|
-
|
|
15368
|
-
|
|
15369
|
-
|
|
15370
|
-
|
|
15371
|
-
|
|
15372
|
-
|
|
15373
|
-
|
|
15374
|
-
|
|
15375
|
-
|
|
15376
|
-
|
|
15377
|
-
|
|
15378
|
-
|
|
15379
|
-
|
|
15380
|
-
|
|
15381
|
-
|
|
15382
|
-
|
|
15383
|
-
|
|
15384
|
-
|
|
15385
|
-
|
|
15386
|
-
|
|
15387
|
-
|
|
15388
|
-
|
|
15389
|
-
|
|
15390
|
-
|
|
15391
|
-
|
|
15392
|
-
|
|
15393
|
-
|
|
15394
|
-
|
|
15395
|
-
|
|
15396
|
-
}
|
|
15397
|
-
|
|
15398
|
-
|
|
15399
|
-
|
|
15400
|
-
|
|
15401
|
-
|
|
15402
|
-
|
|
15403
|
-
|
|
15404
|
-
|
|
15405
|
-
|
|
15406
|
-
|
|
15757
|
+
// ../api-routes/src/traffic.ts
|
|
15758
|
+
import crypto20 from "crypto";
|
|
15759
|
+
import { eq as eq23, sql as sql7 } from "drizzle-orm";
|
|
15760
|
+
|
|
15761
|
+
// ../integration-cloud-run/src/auth.ts
|
|
15762
|
+
import crypto19 from "crypto";
|
|
15763
|
+
var GOOGLE_TOKEN_URL3 = "https://oauth2.googleapis.com/token";
|
|
15764
|
+
var CLOUD_LOGGING_READ_SCOPE = "https://www.googleapis.com/auth/logging.read";
|
|
15765
|
+
var TOKEN_REQUEST_TIMEOUT_MS = 3e4;
|
|
15766
|
+
var CloudRunAuthError = class extends Error {
|
|
15767
|
+
constructor(message, httpStatus, body) {
|
|
15768
|
+
super(message);
|
|
15769
|
+
this.httpStatus = httpStatus;
|
|
15770
|
+
this.body = body;
|
|
15771
|
+
this.name = "CloudRunAuthError";
|
|
15772
|
+
}
|
|
15773
|
+
};
|
|
15774
|
+
function createServiceAccountJwt2(clientEmail, privateKey, scope) {
|
|
15775
|
+
if (!clientEmail) throw new CloudRunAuthError("clientEmail is required");
|
|
15776
|
+
if (!privateKey) throw new CloudRunAuthError("privateKey is required");
|
|
15777
|
+
if (!scope) throw new CloudRunAuthError("scope is required");
|
|
15778
|
+
const now = Math.floor(Date.now() / 1e3);
|
|
15779
|
+
const header = { alg: "RS256", typ: "JWT" };
|
|
15780
|
+
const payload = {
|
|
15781
|
+
iss: clientEmail,
|
|
15782
|
+
scope,
|
|
15783
|
+
aud: GOOGLE_TOKEN_URL3,
|
|
15784
|
+
iat: now,
|
|
15785
|
+
exp: now + 3600
|
|
15786
|
+
};
|
|
15787
|
+
const encode = (obj) => Buffer.from(JSON.stringify(obj)).toString("base64url");
|
|
15788
|
+
const headerB64 = encode(header);
|
|
15789
|
+
const payloadB64 = encode(payload);
|
|
15790
|
+
const signingInput = `${headerB64}.${payloadB64}`;
|
|
15791
|
+
const sign = crypto19.createSign("RSA-SHA256");
|
|
15792
|
+
sign.update(signingInput);
|
|
15793
|
+
const signature = sign.sign(privateKey, "base64url");
|
|
15794
|
+
return `${signingInput}.${signature}`;
|
|
15795
|
+
}
|
|
15796
|
+
async function getCloudLoggingAccessToken(clientEmail, privateKey) {
|
|
15797
|
+
const jwt = createServiceAccountJwt2(clientEmail, privateKey, CLOUD_LOGGING_READ_SCOPE);
|
|
15798
|
+
const res = await fetch(GOOGLE_TOKEN_URL3, {
|
|
15799
|
+
method: "POST",
|
|
15800
|
+
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
15801
|
+
body: new URLSearchParams({
|
|
15802
|
+
grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
|
|
15803
|
+
assertion: jwt
|
|
15804
|
+
}),
|
|
15805
|
+
signal: AbortSignal.timeout(TOKEN_REQUEST_TIMEOUT_MS)
|
|
15806
|
+
});
|
|
15807
|
+
if (!res.ok) {
|
|
15808
|
+
const body = await res.text().catch(() => "");
|
|
15809
|
+
throw new CloudRunAuthError(
|
|
15810
|
+
`Service-account token exchange failed (HTTP ${res.status})`,
|
|
15811
|
+
res.status,
|
|
15812
|
+
body.slice(0, 500)
|
|
15813
|
+
);
|
|
15814
|
+
}
|
|
15815
|
+
const data = await res.json();
|
|
15816
|
+
if (!data.access_token) {
|
|
15817
|
+
throw new CloudRunAuthError("Service-account token response missing access_token", res.status);
|
|
15818
|
+
}
|
|
15819
|
+
return data.access_token;
|
|
15820
|
+
}
|
|
15821
|
+
|
|
15822
|
+
// ../integration-cloud-run/src/filter.ts
|
|
15823
|
+
function assertNonEmpty(name, value) {
|
|
15824
|
+
if (!value.trim()) {
|
|
15825
|
+
throw new Error(`${name} must be a non-empty string`);
|
|
15826
|
+
}
|
|
15827
|
+
}
|
|
15828
|
+
function quoteLogFilterValue(value) {
|
|
15829
|
+
return JSON.stringify(value);
|
|
15830
|
+
}
|
|
15831
|
+
function normalizeTimestamp(value) {
|
|
15832
|
+
const date = value instanceof Date ? value : new Date(value);
|
|
15833
|
+
if (Number.isNaN(date.getTime())) {
|
|
15834
|
+
throw new Error(`Invalid timestamp: ${String(value)}`);
|
|
15835
|
+
}
|
|
15836
|
+
return date.toISOString();
|
|
15837
|
+
}
|
|
15838
|
+
function buildCloudRunLogFilter(options = {}) {
|
|
15839
|
+
const clauses = ['resource.type="cloud_run_revision"'];
|
|
15840
|
+
if (options.serviceName !== void 0) {
|
|
15841
|
+
assertNonEmpty("serviceName", options.serviceName);
|
|
15842
|
+
clauses.push(`resource.labels.service_name=${quoteLogFilterValue(options.serviceName)}`);
|
|
15843
|
+
}
|
|
15844
|
+
if (options.location !== void 0) {
|
|
15845
|
+
assertNonEmpty("location", options.location);
|
|
15846
|
+
clauses.push(`resource.labels.location=${quoteLogFilterValue(options.location)}`);
|
|
15847
|
+
}
|
|
15848
|
+
if (options.startTime !== void 0) {
|
|
15849
|
+
clauses.push(`timestamp >= ${quoteLogFilterValue(normalizeTimestamp(options.startTime))}`);
|
|
15850
|
+
}
|
|
15851
|
+
if (options.endTime !== void 0) {
|
|
15852
|
+
clauses.push(`timestamp < ${quoteLogFilterValue(normalizeTimestamp(options.endTime))}`);
|
|
15853
|
+
}
|
|
15854
|
+
const userAgentSubstrings = (options.userAgentSubstrings ?? []).map((pattern) => pattern.trim()).filter(Boolean);
|
|
15855
|
+
if (userAgentSubstrings.length > 0) {
|
|
15856
|
+
const uaClauses = userAgentSubstrings.map((pattern) => `httpRequest.userAgent:${quoteLogFilterValue(pattern)}`);
|
|
15857
|
+
clauses.push(`(${uaClauses.join(" OR ")})`);
|
|
15858
|
+
}
|
|
15859
|
+
const requestUrlSubstrings = (options.requestUrlSubstrings ?? []).map((pattern) => pattern.trim()).filter(Boolean);
|
|
15860
|
+
if (requestUrlSubstrings.length > 0) {
|
|
15861
|
+
const urlClauses = requestUrlSubstrings.map((pattern) => `httpRequest.requestUrl:${quoteLogFilterValue(pattern)}`);
|
|
15862
|
+
clauses.push(`(${urlClauses.join(" OR ")})`);
|
|
15863
|
+
}
|
|
15864
|
+
return clauses.join(" AND ");
|
|
15865
|
+
}
|
|
15866
|
+
|
|
15867
|
+
// ../integration-cloud-run/src/normalize.ts
|
|
15868
|
+
function numberOrNull(value) {
|
|
15869
|
+
if (value === void 0 || value === null) return null;
|
|
15870
|
+
const parsed = typeof value === "number" ? value : Number(value);
|
|
15871
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
15872
|
+
}
|
|
15873
|
+
function latencyToMs(value) {
|
|
15874
|
+
if (!value) return null;
|
|
15875
|
+
const secondsMatch = /^([0-9]+(?:\.[0-9]+)?)s$/.exec(value.trim());
|
|
15876
|
+
if (!secondsMatch) return null;
|
|
15877
|
+
const seconds = Number(secondsMatch[1]);
|
|
15878
|
+
return Number.isFinite(seconds) ? Math.round(seconds * 1e6) / 1e3 : null;
|
|
15879
|
+
}
|
|
15880
|
+
function normalizeLabels(labels) {
|
|
15881
|
+
if (!labels) return {};
|
|
15882
|
+
return Object.fromEntries(
|
|
15883
|
+
Object.entries(labels).filter((entry) => typeof entry[0] === "string" && typeof entry[1] === "string")
|
|
15884
|
+
);
|
|
15885
|
+
}
|
|
15886
|
+
function parseRequestUrl(requestUrl) {
|
|
15887
|
+
try {
|
|
15888
|
+
const url = requestUrl.startsWith("/") ? new URL(requestUrl, "https://canonry.local") : new URL(requestUrl);
|
|
15889
|
+
return {
|
|
15890
|
+
host: url.hostname === "canonry.local" ? null : url.hostname,
|
|
15891
|
+
path: url.pathname || "/",
|
|
15892
|
+
queryString: url.search ? url.search.slice(1) : null
|
|
15893
|
+
};
|
|
15894
|
+
} catch {
|
|
15895
|
+
return null;
|
|
15896
|
+
}
|
|
15897
|
+
}
|
|
15898
|
+
function buildEventId(entry, observedAt, requestUrl) {
|
|
15899
|
+
if (entry.insertId?.trim()) {
|
|
15900
|
+
return `cloud-run:${observedAt}:${entry.insertId}`;
|
|
15901
|
+
}
|
|
15902
|
+
return `cloud-run:${observedAt}:${requestUrl}`;
|
|
15903
|
+
}
|
|
15904
|
+
function normalizeCloudRunLogEntry(entry) {
|
|
15905
|
+
const request = entry.httpRequest;
|
|
15906
|
+
if (!request?.requestUrl) return null;
|
|
15907
|
+
const observedAt = entry.timestamp ?? entry.receiveTimestamp;
|
|
15908
|
+
if (!observedAt) return null;
|
|
15909
|
+
const urlParts = parseRequestUrl(request.requestUrl);
|
|
15910
|
+
if (!urlParts) return null;
|
|
15911
|
+
return {
|
|
15912
|
+
sourceType: TrafficSourceTypes["cloud-run"],
|
|
15913
|
+
evidenceKind: TrafficEvidenceKinds["raw-request"],
|
|
15914
|
+
confidence: TrafficEventConfidences.observed,
|
|
15915
|
+
eventId: buildEventId(entry, observedAt, request.requestUrl),
|
|
15916
|
+
observedAt,
|
|
15917
|
+
method: request.requestMethod ?? null,
|
|
15918
|
+
requestUrl: request.requestUrl,
|
|
15919
|
+
host: urlParts.host,
|
|
15920
|
+
path: urlParts.path,
|
|
15921
|
+
queryString: urlParts.queryString,
|
|
15922
|
+
status: numberOrNull(request.status),
|
|
15923
|
+
userAgent: request.userAgent ?? null,
|
|
15924
|
+
remoteIp: request.remoteIp ?? null,
|
|
15925
|
+
referer: request.referer ?? null,
|
|
15926
|
+
latencyMs: latencyToMs(request.latency),
|
|
15927
|
+
requestSizeBytes: numberOrNull(request.requestSize),
|
|
15928
|
+
responseSizeBytes: numberOrNull(request.responseSize),
|
|
15929
|
+
providerResource: {
|
|
15930
|
+
type: entry.resource?.type ?? null,
|
|
15931
|
+
labels: normalizeLabels(entry.resource?.labels)
|
|
15932
|
+
},
|
|
15933
|
+
providerLabels: normalizeLabels(entry.labels)
|
|
15934
|
+
};
|
|
15935
|
+
}
|
|
15936
|
+
|
|
15937
|
+
// ../integration-cloud-run/src/client.ts
|
|
15938
|
+
var CLOUD_LOGGING_ENTRIES_LIST_URL = "https://logging.googleapis.com/v2/entries:list";
|
|
15939
|
+
var DEFAULT_PAGE_SIZE = 1e3;
|
|
15940
|
+
var DEFAULT_MAX_PAGES = 1;
|
|
15941
|
+
var DEFAULT_TIMEOUT_MS = 3e4;
|
|
15942
|
+
var CloudRunLoggingApiError = class extends Error {
|
|
15943
|
+
constructor(message, status, body) {
|
|
15944
|
+
super(message);
|
|
15945
|
+
this.status = status;
|
|
15946
|
+
this.body = body;
|
|
15947
|
+
this.name = "CloudRunLoggingApiError";
|
|
15948
|
+
}
|
|
15949
|
+
};
|
|
15950
|
+
function validateAccessToken3(accessToken) {
|
|
15951
|
+
if (!accessToken.trim()) {
|
|
15952
|
+
throw new CloudRunLoggingApiError("Cloud Logging access token is required", 400);
|
|
15953
|
+
}
|
|
15954
|
+
}
|
|
15955
|
+
function validateProjectId(gcpProjectId) {
|
|
15956
|
+
if (!gcpProjectId.trim()) {
|
|
15957
|
+
throw new CloudRunLoggingApiError("GCP project ID is required", 400);
|
|
15958
|
+
}
|
|
15959
|
+
}
|
|
15960
|
+
function normalizePageSize(pageSize) {
|
|
15961
|
+
if (pageSize === void 0) return DEFAULT_PAGE_SIZE;
|
|
15962
|
+
if (!Number.isInteger(pageSize) || pageSize < 1) {
|
|
15963
|
+
throw new CloudRunLoggingApiError("pageSize must be a positive integer", 400);
|
|
15964
|
+
}
|
|
15965
|
+
return pageSize;
|
|
15966
|
+
}
|
|
15967
|
+
function normalizeMaxPages(maxPages) {
|
|
15968
|
+
if (maxPages === void 0) return DEFAULT_MAX_PAGES;
|
|
15969
|
+
if (!Number.isInteger(maxPages) || maxPages < 1) {
|
|
15970
|
+
throw new CloudRunLoggingApiError("maxPages must be a positive integer", 400);
|
|
15971
|
+
}
|
|
15972
|
+
return maxPages;
|
|
15973
|
+
}
|
|
15974
|
+
async function readErrorBody(response) {
|
|
15975
|
+
const text = await response.text().catch(() => "");
|
|
15976
|
+
if (!text) return void 0;
|
|
15977
|
+
return text.length <= 500 ? text : `${text.slice(0, 500)}... [truncated]`;
|
|
15978
|
+
}
|
|
15979
|
+
async function listCloudRunTrafficEvents(accessToken, options) {
|
|
15980
|
+
validateAccessToken3(accessToken);
|
|
15981
|
+
validateProjectId(options.gcpProjectId);
|
|
15982
|
+
const filter = buildCloudRunLogFilter(options);
|
|
15983
|
+
const pageSize = normalizePageSize(options.pageSize);
|
|
15984
|
+
const maxPages = normalizeMaxPages(options.maxPages);
|
|
15985
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
15986
|
+
let pageToken = options.pageToken;
|
|
15987
|
+
let rawEntryCount = 0;
|
|
15988
|
+
let skippedEntryCount = 0;
|
|
15989
|
+
const events = [];
|
|
15990
|
+
for (let page = 0; page < maxPages; page += 1) {
|
|
15991
|
+
const requestBody = {
|
|
15992
|
+
resourceNames: [`projects/${options.gcpProjectId}`],
|
|
15993
|
+
filter,
|
|
15994
|
+
orderBy: options.orderBy ?? "timestamp asc",
|
|
15995
|
+
pageSize
|
|
15996
|
+
};
|
|
15997
|
+
if (pageToken) {
|
|
15998
|
+
requestBody.pageToken = pageToken;
|
|
15999
|
+
}
|
|
16000
|
+
const response = await fetch(CLOUD_LOGGING_ENTRIES_LIST_URL, {
|
|
16001
|
+
method: "POST",
|
|
16002
|
+
headers: {
|
|
16003
|
+
Authorization: `Bearer ${accessToken}`,
|
|
16004
|
+
"Content-Type": "application/json"
|
|
16005
|
+
},
|
|
16006
|
+
body: JSON.stringify(requestBody),
|
|
16007
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
16008
|
+
});
|
|
16009
|
+
if (!response.ok) {
|
|
16010
|
+
const body2 = await readErrorBody(response);
|
|
16011
|
+
throw new CloudRunLoggingApiError(
|
|
16012
|
+
`Cloud Logging entries.list failed with HTTP ${response.status}`,
|
|
16013
|
+
response.status,
|
|
16014
|
+
body2
|
|
16015
|
+
);
|
|
16016
|
+
}
|
|
16017
|
+
const body = await response.json();
|
|
16018
|
+
const entries = body.entries ?? [];
|
|
16019
|
+
rawEntryCount += entries.length;
|
|
16020
|
+
for (const entry of entries) {
|
|
16021
|
+
const event = normalizeCloudRunLogEntry(entry);
|
|
16022
|
+
if (event) {
|
|
16023
|
+
events.push(event);
|
|
16024
|
+
} else {
|
|
16025
|
+
skippedEntryCount += 1;
|
|
16026
|
+
}
|
|
16027
|
+
}
|
|
16028
|
+
pageToken = body.nextPageToken;
|
|
16029
|
+
if (!pageToken) break;
|
|
16030
|
+
}
|
|
16031
|
+
return {
|
|
16032
|
+
events,
|
|
16033
|
+
rawEntryCount,
|
|
16034
|
+
skippedEntryCount,
|
|
16035
|
+
nextPageToken: pageToken,
|
|
16036
|
+
filter
|
|
16037
|
+
};
|
|
16038
|
+
}
|
|
16039
|
+
|
|
16040
|
+
// ../integration-traffic/src/rules.ts
|
|
16041
|
+
var DEFAULT_AI_CRAWLER_RULES = [
|
|
16042
|
+
{
|
|
16043
|
+
id: "openai-gptbot",
|
|
16044
|
+
operator: "OpenAI",
|
|
16045
|
+
product: "GPTBot",
|
|
16046
|
+
purpose: "training",
|
|
16047
|
+
userAgentPatterns: [/GPTBot\//i]
|
|
16048
|
+
},
|
|
16049
|
+
{
|
|
16050
|
+
id: "openai-searchbot",
|
|
16051
|
+
operator: "OpenAI",
|
|
16052
|
+
product: "OAI-SearchBot",
|
|
16053
|
+
purpose: "search",
|
|
16054
|
+
userAgentPatterns: [/OAI-SearchBot\//i]
|
|
16055
|
+
},
|
|
16056
|
+
{
|
|
16057
|
+
id: "openai-chatgpt-user",
|
|
16058
|
+
operator: "OpenAI",
|
|
16059
|
+
product: "ChatGPT-User",
|
|
16060
|
+
purpose: "user-agent",
|
|
16061
|
+
userAgentPatterns: [/ChatGPT-User\//i]
|
|
16062
|
+
},
|
|
16063
|
+
{
|
|
16064
|
+
id: "anthropic-claudebot",
|
|
16065
|
+
operator: "Anthropic",
|
|
16066
|
+
product: "ClaudeBot",
|
|
16067
|
+
purpose: "training",
|
|
16068
|
+
userAgentPatterns: [/ClaudeBot\//i, /Claude-Web\//i, /anthropic-ai/i]
|
|
16069
|
+
},
|
|
16070
|
+
{
|
|
16071
|
+
id: "perplexity-bot",
|
|
16072
|
+
operator: "Perplexity",
|
|
16073
|
+
product: "PerplexityBot",
|
|
16074
|
+
purpose: "search",
|
|
16075
|
+
userAgentPatterns: [/PerplexityBot\//i]
|
|
16076
|
+
},
|
|
16077
|
+
{
|
|
16078
|
+
id: "google-extended",
|
|
16079
|
+
operator: "Google",
|
|
16080
|
+
product: "Google-Extended",
|
|
16081
|
+
purpose: "training-control",
|
|
16082
|
+
userAgentPatterns: [/Google-Extended/i]
|
|
16083
|
+
},
|
|
16084
|
+
{
|
|
16085
|
+
id: "bytespider",
|
|
16086
|
+
operator: "ByteDance",
|
|
16087
|
+
product: "Bytespider",
|
|
16088
|
+
purpose: "training",
|
|
16089
|
+
userAgentPatterns: [/Bytespider/i]
|
|
16090
|
+
},
|
|
16091
|
+
{
|
|
16092
|
+
id: "applebot-extended",
|
|
16093
|
+
operator: "Apple",
|
|
16094
|
+
product: "Applebot-Extended",
|
|
16095
|
+
purpose: "training",
|
|
16096
|
+
userAgentPatterns: [/Applebot-Extended/i]
|
|
16097
|
+
},
|
|
16098
|
+
{
|
|
16099
|
+
id: "meta-externalagent",
|
|
16100
|
+
operator: "Meta",
|
|
16101
|
+
product: "meta-externalagent",
|
|
16102
|
+
purpose: "training",
|
|
16103
|
+
userAgentPatterns: [/meta-externalagent/i]
|
|
16104
|
+
},
|
|
16105
|
+
{
|
|
16106
|
+
id: "ccbot",
|
|
16107
|
+
operator: "Common Crawl",
|
|
16108
|
+
product: "CCBot",
|
|
16109
|
+
purpose: "crawl",
|
|
16110
|
+
userAgentPatterns: [/CCBot\//i]
|
|
16111
|
+
},
|
|
16112
|
+
{
|
|
16113
|
+
id: "cohere-ai",
|
|
16114
|
+
operator: "Cohere",
|
|
16115
|
+
product: "cohere-ai",
|
|
16116
|
+
purpose: "training",
|
|
16117
|
+
userAgentPatterns: [/cohere-ai/i]
|
|
16118
|
+
},
|
|
16119
|
+
{
|
|
16120
|
+
id: "diffbot",
|
|
16121
|
+
operator: "Diffbot",
|
|
16122
|
+
product: "Diffbot",
|
|
16123
|
+
purpose: "crawl",
|
|
16124
|
+
userAgentPatterns: [/Diffbot/i]
|
|
16125
|
+
},
|
|
16126
|
+
{
|
|
16127
|
+
id: "mistral-ai",
|
|
16128
|
+
operator: "Mistral AI",
|
|
16129
|
+
product: "MistralAI-User",
|
|
16130
|
+
purpose: "crawl",
|
|
16131
|
+
userAgentPatterns: [/MistralAI/i]
|
|
16132
|
+
}
|
|
16133
|
+
];
|
|
16134
|
+
var DEFAULT_AI_REFERRER_RULES = [
|
|
16135
|
+
{ domain: "chatgpt.com", operator: "OpenAI", product: "ChatGPT" },
|
|
16136
|
+
{ domain: "chat.openai.com", operator: "OpenAI", product: "ChatGPT" },
|
|
16137
|
+
{ domain: "perplexity.ai", operator: "Perplexity", product: "Perplexity" },
|
|
16138
|
+
{ domain: "claude.ai", operator: "Anthropic", product: "Claude" },
|
|
16139
|
+
{ domain: "gemini.google.com", operator: "Google", product: "Gemini" },
|
|
16140
|
+
{ domain: "copilot.microsoft.com", operator: "Microsoft", product: "Copilot" },
|
|
16141
|
+
{ domain: "phind.com", operator: "Phind", product: "Phind" },
|
|
16142
|
+
{ domain: "you.com", operator: "You.com", product: "You.com" },
|
|
16143
|
+
{ domain: "meta.ai", operator: "Meta", product: "Meta AI" }
|
|
16144
|
+
];
|
|
16145
|
+
|
|
16146
|
+
// ../integration-traffic/src/classifier.ts
|
|
16147
|
+
function normalizeHost(host) {
|
|
16148
|
+
return host.trim().toLowerCase().replace(/^www\./, "");
|
|
16149
|
+
}
|
|
16150
|
+
function hostMatches(host, domain) {
|
|
16151
|
+
const normalizedHost = normalizeHost(host);
|
|
16152
|
+
const normalizedDomain = normalizeHost(domain);
|
|
16153
|
+
return normalizedHost === normalizedDomain || normalizedHost.endsWith(`.${normalizedDomain}`);
|
|
16154
|
+
}
|
|
16155
|
+
function hostFromUrl(value) {
|
|
16156
|
+
if (!value) return null;
|
|
16157
|
+
try {
|
|
16158
|
+
return normalizeHost(new URL(value).hostname);
|
|
16159
|
+
} catch {
|
|
16160
|
+
return null;
|
|
16161
|
+
}
|
|
16162
|
+
}
|
|
16163
|
+
function utmSourceFromQuery(queryString) {
|
|
16164
|
+
if (!queryString) return null;
|
|
16165
|
+
const params = new URLSearchParams(queryString);
|
|
16166
|
+
const source = params.get("utm_source");
|
|
16167
|
+
return source ? normalizeHost(source) : null;
|
|
16168
|
+
}
|
|
16169
|
+
function classifyCrawler(event) {
|
|
16170
|
+
const userAgent = event.userAgent?.trim();
|
|
16171
|
+
if (!userAgent) return null;
|
|
16172
|
+
for (const rule of DEFAULT_AI_CRAWLER_RULES) {
|
|
16173
|
+
if (rule.userAgentPatterns.some((pattern) => pattern.test(userAgent))) {
|
|
16174
|
+
return {
|
|
16175
|
+
botId: rule.id,
|
|
16176
|
+
operator: rule.operator,
|
|
16177
|
+
product: rule.product,
|
|
16178
|
+
purpose: rule.purpose,
|
|
16179
|
+
verificationStatus: "claimed_unverified",
|
|
16180
|
+
matchedUserAgent: userAgent
|
|
16181
|
+
};
|
|
16182
|
+
}
|
|
16183
|
+
}
|
|
16184
|
+
return null;
|
|
16185
|
+
}
|
|
16186
|
+
function classifyAiReferral(event) {
|
|
16187
|
+
const refererHost = hostFromUrl(event.referer);
|
|
16188
|
+
if (refererHost) {
|
|
16189
|
+
const rule = DEFAULT_AI_REFERRER_RULES.find((candidate) => hostMatches(refererHost, candidate.domain));
|
|
16190
|
+
if (rule) {
|
|
16191
|
+
return {
|
|
16192
|
+
operator: rule.operator,
|
|
16193
|
+
product: rule.product,
|
|
16194
|
+
sourceDomain: refererHost,
|
|
16195
|
+
evidenceType: "referer"
|
|
16196
|
+
};
|
|
16197
|
+
}
|
|
16198
|
+
}
|
|
16199
|
+
const utmSource = utmSourceFromQuery(event.queryString);
|
|
16200
|
+
if (utmSource) {
|
|
16201
|
+
const rule = DEFAULT_AI_REFERRER_RULES.find((candidate) => hostMatches(utmSource, candidate.domain));
|
|
16202
|
+
if (rule) {
|
|
16203
|
+
return {
|
|
16204
|
+
operator: rule.operator,
|
|
16205
|
+
product: rule.product,
|
|
16206
|
+
sourceDomain: utmSource,
|
|
16207
|
+
evidenceType: "utm"
|
|
16208
|
+
};
|
|
16209
|
+
}
|
|
16210
|
+
}
|
|
16211
|
+
return null;
|
|
16212
|
+
}
|
|
16213
|
+
|
|
16214
|
+
// ../integration-traffic/src/rollup.ts
|
|
16215
|
+
var DEFAULT_SAMPLE_LIMIT = 25;
|
|
16216
|
+
var UUID_SEGMENT = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
16217
|
+
var LONG_HEX_SEGMENT = /^[0-9a-f]{16,}$/i;
|
|
16218
|
+
var NUMERIC_SEGMENT = /^\d+$/;
|
|
16219
|
+
function normalizeTrafficPathPattern(path15) {
|
|
16220
|
+
const cleanPath = path15.trim() || "/";
|
|
16221
|
+
const pathOnly = cleanPath.split("?")[0] || "/";
|
|
16222
|
+
const segments = pathOnly.split("/").map((segment) => {
|
|
16223
|
+
if (!segment) return segment;
|
|
16224
|
+
if (UUID_SEGMENT.test(segment) || LONG_HEX_SEGMENT.test(segment) || NUMERIC_SEGMENT.test(segment)) {
|
|
16225
|
+
return ":id";
|
|
16226
|
+
}
|
|
16227
|
+
return segment;
|
|
16228
|
+
});
|
|
16229
|
+
const normalized = segments.join("/");
|
|
16230
|
+
return normalized.startsWith("/") ? normalized : `/${normalized}`;
|
|
16231
|
+
}
|
|
16232
|
+
function hourBucket(value) {
|
|
16233
|
+
const date = new Date(value);
|
|
16234
|
+
if (Number.isNaN(date.getTime())) return value;
|
|
16235
|
+
date.setUTCMinutes(0, 0, 0);
|
|
16236
|
+
return date.toISOString();
|
|
16237
|
+
}
|
|
16238
|
+
function sortCrawlerBuckets(a, b) {
|
|
16239
|
+
return a.tsHour.localeCompare(b.tsHour) || a.botId.localeCompare(b.botId) || a.pathNormalized.localeCompare(b.pathNormalized) || String(a.status).localeCompare(String(b.status));
|
|
16240
|
+
}
|
|
16241
|
+
function sortReferralBuckets(a, b) {
|
|
16242
|
+
return a.tsHour.localeCompare(b.tsHour) || a.product.localeCompare(b.product) || a.sourceDomain.localeCompare(b.sourceDomain) || a.landingPathNormalized.localeCompare(b.landingPathNormalized) || String(a.status).localeCompare(String(b.status));
|
|
16243
|
+
}
|
|
16244
|
+
function topEntries(map, limit) {
|
|
16245
|
+
return [...map.values()].sort((a, b) => b.hits - a.hits || JSON.stringify(a.fields).localeCompare(JSON.stringify(b.fields))).slice(0, limit).map((entry) => ({ ...entry.fields, hits: entry.hits }));
|
|
16246
|
+
}
|
|
16247
|
+
function buildTrafficProbeReport(events, options = {}) {
|
|
16248
|
+
const sampleLimit = options.sampleLimit ?? DEFAULT_SAMPLE_LIMIT;
|
|
16249
|
+
const crawlerBuckets = /* @__PURE__ */ new Map();
|
|
16250
|
+
const aiReferralBuckets = /* @__PURE__ */ new Map();
|
|
16251
|
+
const topBots = /* @__PURE__ */ new Map();
|
|
16252
|
+
const topCrawlerPaths = /* @__PURE__ */ new Map();
|
|
16253
|
+
const topAiReferrers = /* @__PURE__ */ new Map();
|
|
16254
|
+
const topAiReferralLandingPaths = /* @__PURE__ */ new Map();
|
|
16255
|
+
let crawlerHits = 0;
|
|
16256
|
+
let aiReferralHits = 0;
|
|
16257
|
+
let unknownHits = 0;
|
|
16258
|
+
const samples = [];
|
|
16259
|
+
for (const event of events) {
|
|
16260
|
+
const tsHour = hourBucket(event.observedAt);
|
|
16261
|
+
const pathNormalized = normalizeTrafficPathPattern(event.path);
|
|
16262
|
+
const crawler = classifyCrawler(event);
|
|
16263
|
+
const aiReferral = classifyAiReferral(event);
|
|
16264
|
+
if (crawler) {
|
|
16265
|
+
crawlerHits += 1;
|
|
16266
|
+
const key = [
|
|
16267
|
+
tsHour,
|
|
16268
|
+
crawler.botId,
|
|
16269
|
+
crawler.verificationStatus,
|
|
16270
|
+
pathNormalized,
|
|
16271
|
+
event.status ?? "null"
|
|
16272
|
+
].join(" ");
|
|
16273
|
+
const existing = crawlerBuckets.get(key);
|
|
16274
|
+
if (existing) {
|
|
16275
|
+
existing.hits += 1;
|
|
16276
|
+
} else {
|
|
16277
|
+
crawlerBuckets.set(key, {
|
|
16278
|
+
tsHour,
|
|
16279
|
+
botId: crawler.botId,
|
|
16280
|
+
operator: crawler.operator,
|
|
16281
|
+
product: crawler.product,
|
|
16282
|
+
verificationStatus: crawler.verificationStatus,
|
|
16283
|
+
pathNormalized,
|
|
16284
|
+
status: event.status,
|
|
16285
|
+
hits: 1,
|
|
16286
|
+
sampledUserAgent: event.userAgent
|
|
16287
|
+
});
|
|
16288
|
+
}
|
|
16289
|
+
const botKey = `${crawler.botId} ${crawler.operator}`;
|
|
16290
|
+
const botEntry = topBots.get(botKey);
|
|
16291
|
+
if (botEntry) botEntry.hits += 1;
|
|
16292
|
+
else topBots.set(botKey, { fields: { botId: crawler.botId, operator: crawler.operator }, hits: 1 });
|
|
16293
|
+
incrementBucket(topCrawlerPaths, pathNormalized, { pathNormalized });
|
|
16294
|
+
}
|
|
16295
|
+
if (aiReferral) {
|
|
16296
|
+
aiReferralHits += 1;
|
|
16297
|
+
const key = [
|
|
16298
|
+
tsHour,
|
|
16299
|
+
aiReferral.product,
|
|
16300
|
+
aiReferral.sourceDomain,
|
|
16301
|
+
aiReferral.evidenceType,
|
|
16302
|
+
pathNormalized,
|
|
16303
|
+
event.status ?? "null"
|
|
16304
|
+
].join(" ");
|
|
16305
|
+
const existing = aiReferralBuckets.get(key);
|
|
16306
|
+
if (existing) {
|
|
16307
|
+
existing.hits += 1;
|
|
16308
|
+
} else {
|
|
16309
|
+
aiReferralBuckets.set(key, {
|
|
16310
|
+
tsHour,
|
|
16311
|
+
operator: aiReferral.operator,
|
|
16312
|
+
product: aiReferral.product,
|
|
16313
|
+
sourceDomain: aiReferral.sourceDomain,
|
|
16314
|
+
evidenceType: aiReferral.evidenceType,
|
|
16315
|
+
landingPathNormalized: pathNormalized,
|
|
16316
|
+
status: event.status,
|
|
16317
|
+
hits: 1
|
|
16318
|
+
});
|
|
16319
|
+
}
|
|
16320
|
+
incrementBucket(topAiReferrers, aiReferral.sourceDomain, {
|
|
16321
|
+
sourceDomain: aiReferral.sourceDomain,
|
|
16322
|
+
product: aiReferral.product
|
|
16323
|
+
});
|
|
16324
|
+
incrementBucket(topAiReferralLandingPaths, pathNormalized, { landingPathNormalized: pathNormalized });
|
|
16325
|
+
}
|
|
16326
|
+
if (!crawler && !aiReferral) unknownHits += 1;
|
|
16327
|
+
if (samples.length < sampleLimit) {
|
|
16328
|
+
samples.push({
|
|
16329
|
+
eventId: event.eventId,
|
|
16330
|
+
observedAt: event.observedAt,
|
|
16331
|
+
sourceType: event.sourceType,
|
|
16332
|
+
path: event.path,
|
|
16333
|
+
pathNormalized,
|
|
16334
|
+
status: event.status,
|
|
16335
|
+
userAgent: event.userAgent,
|
|
16336
|
+
referer: event.referer,
|
|
16337
|
+
crawler,
|
|
16338
|
+
aiReferral
|
|
16339
|
+
});
|
|
16340
|
+
}
|
|
16341
|
+
}
|
|
16342
|
+
return {
|
|
16343
|
+
generatedAt: options.generatedAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
16344
|
+
totals: {
|
|
16345
|
+
normalizedEvents: events.length,
|
|
16346
|
+
crawlerHits,
|
|
16347
|
+
aiReferralHits,
|
|
16348
|
+
unknownHits
|
|
16349
|
+
},
|
|
16350
|
+
crawlerEventsHourly: [...crawlerBuckets.values()].sort(sortCrawlerBuckets),
|
|
16351
|
+
aiReferralEventsHourly: [...aiReferralBuckets.values()].sort(sortReferralBuckets),
|
|
16352
|
+
topBots: topEntries(topBots, 10),
|
|
16353
|
+
topCrawlerPaths: topEntries(topCrawlerPaths, 10),
|
|
16354
|
+
topAiReferrers: topEntries(topAiReferrers, 10),
|
|
16355
|
+
topAiReferralLandingPaths: topEntries(topAiReferralLandingPaths, 10),
|
|
16356
|
+
samples
|
|
16357
|
+
};
|
|
16358
|
+
}
|
|
16359
|
+
function incrementBucket(map, key, fields) {
|
|
16360
|
+
const existing = map.get(key);
|
|
16361
|
+
if (existing) existing.hits += 1;
|
|
16362
|
+
else map.set(key, { fields, hits: 1 });
|
|
16363
|
+
}
|
|
16364
|
+
|
|
16365
|
+
// ../api-routes/src/traffic.ts
|
|
16366
|
+
var DEFAULT_SYNC_WINDOW_MINUTES = 60;
|
|
16367
|
+
var DEFAULT_PAGE_SIZE2 = 1e3;
|
|
16368
|
+
var DEFAULT_MAX_PAGES2 = 5;
|
|
16369
|
+
var DEFAULT_SAMPLE_LIMIT2 = 100;
|
|
16370
|
+
function parseSourceConfig(row) {
|
|
16371
|
+
return parseJsonColumn(row.configJson, {});
|
|
16372
|
+
}
|
|
16373
|
+
function rowToDto(row) {
|
|
16374
|
+
return {
|
|
16375
|
+
id: row.id,
|
|
16376
|
+
projectId: row.projectId,
|
|
16377
|
+
sourceType: row.sourceType,
|
|
16378
|
+
displayName: row.displayName,
|
|
16379
|
+
status: row.status,
|
|
16380
|
+
lastSyncedAt: row.lastSyncedAt ?? null,
|
|
16381
|
+
lastCursor: row.lastCursor ?? null,
|
|
16382
|
+
lastError: row.lastError ?? null,
|
|
16383
|
+
archivedAt: row.archivedAt ?? null,
|
|
16384
|
+
config: parseSourceConfig(row),
|
|
16385
|
+
createdAt: row.createdAt,
|
|
16386
|
+
updatedAt: row.updatedAt
|
|
16387
|
+
};
|
|
16388
|
+
}
|
|
16389
|
+
async function defaultResolveAccessToken(record) {
|
|
16390
|
+
if (record.authMode === TrafficSourceAuthModes["service-account"]) {
|
|
16391
|
+
if (!record.clientEmail || !record.privateKey) {
|
|
16392
|
+
throw validationError("Service-account credentials missing client_email or private_key");
|
|
16393
|
+
}
|
|
16394
|
+
return getCloudLoggingAccessToken(record.clientEmail, record.privateKey);
|
|
16395
|
+
}
|
|
16396
|
+
throw validationError(
|
|
16397
|
+
"OAuth-mode Cloud Run sync is not yet supported in v1. Provide a service-account key file."
|
|
16398
|
+
);
|
|
16399
|
+
}
|
|
16400
|
+
async function trafficRoutes(app, opts) {
|
|
16401
|
+
const pullEvents = opts.pullCloudRunEvents ?? listCloudRunTrafficEvents;
|
|
16402
|
+
const resolveAccessToken2 = opts.resolveCloudRunAccessToken ?? defaultResolveAccessToken;
|
|
16403
|
+
const syncWindowMinutes = opts.defaultSyncWindowMinutes ?? DEFAULT_SYNC_WINDOW_MINUTES;
|
|
16404
|
+
const pageSize = opts.defaultPageSize ?? DEFAULT_PAGE_SIZE2;
|
|
16405
|
+
const maxPages = opts.defaultMaxPages ?? DEFAULT_MAX_PAGES2;
|
|
16406
|
+
const sampleLimit = opts.defaultSampleLimit ?? DEFAULT_SAMPLE_LIMIT2;
|
|
16407
|
+
app.post("/projects/:name/traffic/connect/cloud-run", async (request) => {
|
|
16408
|
+
const project = resolveProject(app.db, request.params.name);
|
|
16409
|
+
const body = request.body ?? {};
|
|
16410
|
+
const { gcpProjectId, serviceName, location, displayName, keyJson } = body;
|
|
16411
|
+
if (!gcpProjectId || typeof gcpProjectId !== "string") {
|
|
16412
|
+
throw validationError("gcpProjectId is required");
|
|
16413
|
+
}
|
|
16414
|
+
if (!keyJson) {
|
|
16415
|
+
throw validationError(
|
|
16416
|
+
"keyJson is required for v1 (service-account JSON content). OAuth-mode Cloud Run is not yet supported."
|
|
16417
|
+
);
|
|
16418
|
+
}
|
|
16419
|
+
if (!opts.cloudRunCredentialStore) {
|
|
16420
|
+
throw validationError("Cloud Run credential storage is not configured for this deployment");
|
|
16421
|
+
}
|
|
16422
|
+
let parsed;
|
|
16423
|
+
try {
|
|
16424
|
+
parsed = JSON.parse(keyJson);
|
|
16425
|
+
} catch {
|
|
16426
|
+
throw validationError("Invalid JSON in keyJson");
|
|
16427
|
+
}
|
|
16428
|
+
if (!parsed.client_email || !parsed.private_key) {
|
|
16429
|
+
throw validationError("Service-account JSON must contain client_email and private_key");
|
|
16430
|
+
}
|
|
16431
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
16432
|
+
const existing = opts.cloudRunCredentialStore.getConnection(project.name);
|
|
16433
|
+
opts.cloudRunCredentialStore.upsertConnection({
|
|
16434
|
+
projectName: project.name,
|
|
16435
|
+
gcpProjectId,
|
|
16436
|
+
serviceName: serviceName ?? void 0,
|
|
16437
|
+
location: location ?? void 0,
|
|
16438
|
+
authMode: TrafficSourceAuthModes["service-account"],
|
|
16439
|
+
clientEmail: parsed.client_email,
|
|
16440
|
+
privateKey: parsed.private_key,
|
|
16441
|
+
createdAt: existing?.createdAt ?? now,
|
|
16442
|
+
updatedAt: now
|
|
16443
|
+
});
|
|
16444
|
+
const activeSource = app.db.select().from(trafficSources).where(eq23(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes["cloud-run"] && row.status !== TrafficSourceStatuses.archived);
|
|
16445
|
+
const config = {
|
|
16446
|
+
gcpProjectId,
|
|
16447
|
+
serviceName: serviceName ?? null,
|
|
16448
|
+
location: location ?? null,
|
|
16449
|
+
authMode: TrafficSourceAuthModes["service-account"]
|
|
16450
|
+
};
|
|
16451
|
+
const fallbackName = displayName ?? `Cloud Run \xB7 ${gcpProjectId}${serviceName ? ` / ${serviceName}` : ""}`;
|
|
16452
|
+
let sourceRow;
|
|
16453
|
+
if (activeSource) {
|
|
16454
|
+
app.db.update(trafficSources).set({
|
|
16455
|
+
displayName: fallbackName,
|
|
16456
|
+
status: TrafficSourceStatuses.connected,
|
|
16457
|
+
lastError: null,
|
|
16458
|
+
configJson: JSON.stringify(config),
|
|
16459
|
+
updatedAt: now
|
|
16460
|
+
}).where(eq23(trafficSources.id, activeSource.id)).run();
|
|
16461
|
+
sourceRow = app.db.select().from(trafficSources).where(eq23(trafficSources.id, activeSource.id)).get();
|
|
16462
|
+
} else {
|
|
16463
|
+
const newId = crypto20.randomUUID();
|
|
16464
|
+
app.db.insert(trafficSources).values({
|
|
16465
|
+
id: newId,
|
|
16466
|
+
projectId: project.id,
|
|
16467
|
+
sourceType: TrafficSourceTypes["cloud-run"],
|
|
16468
|
+
displayName: fallbackName,
|
|
16469
|
+
status: TrafficSourceStatuses.connected,
|
|
16470
|
+
lastSyncedAt: null,
|
|
16471
|
+
lastCursor: null,
|
|
16472
|
+
lastError: null,
|
|
16473
|
+
archivedAt: null,
|
|
16474
|
+
configJson: JSON.stringify(config),
|
|
16475
|
+
createdAt: now,
|
|
16476
|
+
updatedAt: now
|
|
16477
|
+
}).run();
|
|
16478
|
+
sourceRow = app.db.select().from(trafficSources).where(eq23(trafficSources.id, newId)).get();
|
|
16479
|
+
}
|
|
16480
|
+
writeAuditLog(app.db, {
|
|
16481
|
+
projectId: project.id,
|
|
16482
|
+
actor: "api",
|
|
16483
|
+
action: "traffic.cloud-run.connected",
|
|
16484
|
+
entityType: "traffic_source",
|
|
16485
|
+
entityId: sourceRow.id
|
|
16486
|
+
});
|
|
16487
|
+
return rowToDto(sourceRow);
|
|
16488
|
+
});
|
|
16489
|
+
app.post("/projects/:name/traffic/sources/:id/sync", async (request) => {
|
|
16490
|
+
const project = resolveProject(app.db, request.params.name);
|
|
16491
|
+
const sourceRow = app.db.select().from(trafficSources).where(eq23(trafficSources.id, request.params.id)).get();
|
|
16492
|
+
if (!sourceRow || sourceRow.projectId !== project.id) {
|
|
16493
|
+
throw notFound("Traffic source", request.params.id);
|
|
16494
|
+
}
|
|
16495
|
+
if (sourceRow.sourceType !== TrafficSourceTypes["cloud-run"]) {
|
|
16496
|
+
throw validationError(
|
|
16497
|
+
`Sync for source type "${sourceRow.sourceType}" is not implemented yet \u2014 only cloud-run is supported in v1.`
|
|
16498
|
+
);
|
|
16499
|
+
}
|
|
16500
|
+
const credentialStore = opts.cloudRunCredentialStore;
|
|
16501
|
+
if (!credentialStore) {
|
|
16502
|
+
throw validationError("Cloud Run credential storage is not configured for this deployment");
|
|
16503
|
+
}
|
|
16504
|
+
const credential = credentialStore.getConnection(project.name);
|
|
16505
|
+
if (!credential) {
|
|
16506
|
+
throw validationError(
|
|
16507
|
+
`No Cloud Run credential found for project "${project.name}". Run "canonry traffic connect cloud-run" first.`
|
|
16508
|
+
);
|
|
16509
|
+
}
|
|
16510
|
+
const config = parseSourceConfig(sourceRow);
|
|
16511
|
+
const gcpProjectId = config.gcpProjectId ?? credential.gcpProjectId;
|
|
16512
|
+
const serviceName = config.serviceName ?? credential.serviceName ?? void 0;
|
|
16513
|
+
const location = config.location ?? credential.location ?? void 0;
|
|
16514
|
+
const requestedMinutes = request.body?.sinceMinutes;
|
|
16515
|
+
const windowMinutes = Number.isFinite(requestedMinutes) && requestedMinutes && requestedMinutes > 0 ? Math.floor(requestedMinutes) : syncWindowMinutes;
|
|
16516
|
+
const windowEnd = /* @__PURE__ */ new Date();
|
|
16517
|
+
const requestedStartMs = windowEnd.getTime() - windowMinutes * 6e4;
|
|
16518
|
+
const lastSyncedMs = sourceRow.lastSyncedAt ? new Date(sourceRow.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY;
|
|
16519
|
+
const windowStart = new Date(
|
|
16520
|
+
Math.min(windowEnd.getTime(), Math.max(requestedStartMs, lastSyncedMs))
|
|
16521
|
+
);
|
|
16522
|
+
const startedAt = windowEnd.toISOString();
|
|
16523
|
+
const runId = crypto20.randomUUID();
|
|
16524
|
+
app.db.insert(runs).values({
|
|
16525
|
+
id: runId,
|
|
16526
|
+
projectId: project.id,
|
|
16527
|
+
kind: RunKinds["traffic-sync"],
|
|
16528
|
+
status: RunStatuses.running,
|
|
16529
|
+
trigger: RunTriggers.manual,
|
|
16530
|
+
startedAt,
|
|
16531
|
+
createdAt: startedAt
|
|
16532
|
+
}).run();
|
|
16533
|
+
let accessToken;
|
|
16534
|
+
try {
|
|
16535
|
+
accessToken = await resolveAccessToken2(credential);
|
|
16536
|
+
} catch (e) {
|
|
16537
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
16538
|
+
app.db.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq23(runs.id, runId)).run();
|
|
16539
|
+
app.db.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq23(trafficSources.id, sourceRow.id)).run();
|
|
16540
|
+
throw validationError(`Failed to resolve Cloud Run access token: ${msg}`);
|
|
16541
|
+
}
|
|
16542
|
+
let allEvents = [];
|
|
16543
|
+
try {
|
|
16544
|
+
const page = await pullEvents(accessToken, {
|
|
16545
|
+
gcpProjectId,
|
|
16546
|
+
serviceName,
|
|
16547
|
+
location,
|
|
16548
|
+
startTime: windowStart.toISOString(),
|
|
16549
|
+
endTime: windowEnd.toISOString(),
|
|
16550
|
+
pageSize,
|
|
16551
|
+
maxPages
|
|
16552
|
+
});
|
|
16553
|
+
allEvents = page.events;
|
|
16554
|
+
} catch (e) {
|
|
16555
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
16556
|
+
app.db.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq23(runs.id, runId)).run();
|
|
16557
|
+
app.db.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq23(trafficSources.id, sourceRow.id)).run();
|
|
16558
|
+
throw validationError(`Cloud Run pull failed: ${msg}`);
|
|
16559
|
+
}
|
|
16560
|
+
const report = buildTrafficProbeReport(allEvents, { sampleLimit });
|
|
16561
|
+
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
16562
|
+
let crawlerBucketRows = 0;
|
|
16563
|
+
let aiReferralBucketRows = 0;
|
|
16564
|
+
let sampleRows = 0;
|
|
16565
|
+
app.db.transaction((tx) => {
|
|
16566
|
+
for (const bucket of report.crawlerEventsHourly) {
|
|
16567
|
+
const status = bucket.status ?? 0;
|
|
16568
|
+
tx.insert(crawlerEventsHourly).values({
|
|
16569
|
+
projectId: project.id,
|
|
16570
|
+
sourceId: sourceRow.id,
|
|
16571
|
+
tsHour: bucket.tsHour,
|
|
16572
|
+
botId: bucket.botId,
|
|
16573
|
+
operator: bucket.operator,
|
|
16574
|
+
verificationStatus: bucket.verificationStatus,
|
|
16575
|
+
pathNormalized: bucket.pathNormalized,
|
|
16576
|
+
status,
|
|
16577
|
+
hits: bucket.hits,
|
|
16578
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
16579
|
+
createdAt: finishedAt,
|
|
16580
|
+
updatedAt: finishedAt
|
|
16581
|
+
}).onConflictDoUpdate({
|
|
16582
|
+
target: [
|
|
16583
|
+
crawlerEventsHourly.projectId,
|
|
16584
|
+
crawlerEventsHourly.sourceId,
|
|
16585
|
+
crawlerEventsHourly.tsHour,
|
|
16586
|
+
crawlerEventsHourly.botId,
|
|
16587
|
+
crawlerEventsHourly.verificationStatus,
|
|
16588
|
+
crawlerEventsHourly.pathNormalized,
|
|
16589
|
+
crawlerEventsHourly.status
|
|
16590
|
+
],
|
|
16591
|
+
set: {
|
|
16592
|
+
hits: sql7`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
16593
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
16594
|
+
updatedAt: finishedAt
|
|
16595
|
+
}
|
|
16596
|
+
}).run();
|
|
16597
|
+
crawlerBucketRows += 1;
|
|
16598
|
+
}
|
|
16599
|
+
for (const bucket of report.aiReferralEventsHourly) {
|
|
16600
|
+
const status = bucket.status ?? 0;
|
|
16601
|
+
tx.insert(aiReferralEventsHourly).values({
|
|
16602
|
+
projectId: project.id,
|
|
16603
|
+
sourceId: sourceRow.id,
|
|
16604
|
+
tsHour: bucket.tsHour,
|
|
16605
|
+
product: bucket.product,
|
|
16606
|
+
operator: bucket.operator,
|
|
16607
|
+
sourceDomain: bucket.sourceDomain,
|
|
16608
|
+
evidenceType: bucket.evidenceType,
|
|
16609
|
+
landingPathNormalized: bucket.landingPathNormalized,
|
|
16610
|
+
status,
|
|
16611
|
+
sessionsOrHits: bucket.hits,
|
|
16612
|
+
usersEstimated: null,
|
|
16613
|
+
createdAt: finishedAt,
|
|
16614
|
+
updatedAt: finishedAt
|
|
16615
|
+
}).onConflictDoUpdate({
|
|
16616
|
+
target: [
|
|
16617
|
+
aiReferralEventsHourly.projectId,
|
|
16618
|
+
aiReferralEventsHourly.sourceId,
|
|
16619
|
+
aiReferralEventsHourly.tsHour,
|
|
16620
|
+
aiReferralEventsHourly.product,
|
|
16621
|
+
aiReferralEventsHourly.sourceDomain,
|
|
16622
|
+
aiReferralEventsHourly.evidenceType,
|
|
16623
|
+
aiReferralEventsHourly.landingPathNormalized,
|
|
16624
|
+
aiReferralEventsHourly.status
|
|
16625
|
+
],
|
|
16626
|
+
set: {
|
|
16627
|
+
sessionsOrHits: sql7`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
16628
|
+
updatedAt: finishedAt
|
|
16629
|
+
}
|
|
16630
|
+
}).run();
|
|
16631
|
+
aiReferralBucketRows += 1;
|
|
16632
|
+
}
|
|
16633
|
+
for (const sample of report.samples) {
|
|
16634
|
+
const eventType = sample.crawler ? "crawler" : sample.aiReferral ? "ai_referral" : "unknown";
|
|
16635
|
+
const refererHost = (() => {
|
|
16636
|
+
if (!sample.referer) return null;
|
|
16637
|
+
try {
|
|
16638
|
+
return new URL(sample.referer).hostname;
|
|
16639
|
+
} catch {
|
|
16640
|
+
return null;
|
|
16641
|
+
}
|
|
16642
|
+
})();
|
|
16643
|
+
tx.insert(rawEventSamples).values({
|
|
16644
|
+
id: crypto20.randomUUID(),
|
|
16645
|
+
projectId: project.id,
|
|
16646
|
+
sourceId: sourceRow.id,
|
|
16647
|
+
ts: sample.observedAt,
|
|
16648
|
+
eventType,
|
|
16649
|
+
ipHash: null,
|
|
16650
|
+
userAgent: sample.userAgent,
|
|
16651
|
+
pathNormalized: sample.pathNormalized,
|
|
16652
|
+
status: sample.status,
|
|
16653
|
+
refererHost,
|
|
16654
|
+
classifierDetailsJson: JSON.stringify({
|
|
16655
|
+
crawler: sample.crawler,
|
|
16656
|
+
aiReferral: sample.aiReferral
|
|
16657
|
+
}),
|
|
16658
|
+
createdAt: finishedAt
|
|
16659
|
+
}).run();
|
|
16660
|
+
sampleRows += 1;
|
|
16661
|
+
}
|
|
16662
|
+
tx.update(trafficSources).set({
|
|
16663
|
+
status: TrafficSourceStatuses.connected,
|
|
16664
|
+
lastSyncedAt: finishedAt,
|
|
16665
|
+
lastError: null,
|
|
16666
|
+
updatedAt: finishedAt
|
|
16667
|
+
}).where(eq23(trafficSources.id, sourceRow.id)).run();
|
|
16668
|
+
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq23(runs.id, runId)).run();
|
|
16669
|
+
});
|
|
16670
|
+
writeAuditLog(app.db, {
|
|
16671
|
+
projectId: project.id,
|
|
16672
|
+
actor: "api",
|
|
16673
|
+
action: "traffic.cloud-run.synced",
|
|
16674
|
+
entityType: "traffic_source",
|
|
16675
|
+
entityId: sourceRow.id
|
|
16676
|
+
});
|
|
16677
|
+
const response = {
|
|
16678
|
+
sourceId: sourceRow.id,
|
|
16679
|
+
runId,
|
|
16680
|
+
syncedAt: finishedAt,
|
|
16681
|
+
pulledEvents: report.totals.normalizedEvents,
|
|
16682
|
+
crawlerHits: report.totals.crawlerHits,
|
|
16683
|
+
aiReferralHits: report.totals.aiReferralHits,
|
|
16684
|
+
unknownHits: report.totals.unknownHits,
|
|
16685
|
+
crawlerBucketRows,
|
|
16686
|
+
aiReferralBucketRows,
|
|
16687
|
+
sampleRows,
|
|
16688
|
+
windowStart: windowStart.toISOString(),
|
|
16689
|
+
windowEnd: windowEnd.toISOString()
|
|
16690
|
+
};
|
|
16691
|
+
return response;
|
|
16692
|
+
});
|
|
16693
|
+
}
|
|
16694
|
+
|
|
16695
|
+
// ../api-routes/src/doctor/checks/bing-auth.ts
|
|
16696
|
+
var BING_AUTH_CHECKS = [
|
|
16697
|
+
{
|
|
16698
|
+
id: "bing.auth.connection",
|
|
16699
|
+
category: CheckCategories.auth,
|
|
16700
|
+
scope: CheckScopes.project,
|
|
16701
|
+
title: "Bing WMT connection",
|
|
16702
|
+
run: async (ctx) => {
|
|
16703
|
+
if (!ctx.project) {
|
|
16704
|
+
return {
|
|
16705
|
+
status: CheckStatuses.skipped,
|
|
16706
|
+
code: "bing.auth.no-project",
|
|
16707
|
+
summary: "Project context required.",
|
|
16708
|
+
remediation: null
|
|
16709
|
+
};
|
|
16710
|
+
}
|
|
16711
|
+
const store = ctx.bingConnectionStore;
|
|
16712
|
+
if (!store) {
|
|
16713
|
+
return {
|
|
16714
|
+
status: CheckStatuses.skipped,
|
|
16715
|
+
code: "bing.auth.store-unavailable",
|
|
16716
|
+
summary: "Bing connection store is not configured for this deployment.",
|
|
16717
|
+
remediation: null
|
|
16718
|
+
};
|
|
16719
|
+
}
|
|
16720
|
+
const conn = store.getConnection(ctx.project.canonicalDomain);
|
|
16721
|
+
if (!conn) {
|
|
16722
|
+
return {
|
|
16723
|
+
status: CheckStatuses.fail,
|
|
16724
|
+
code: "bing.auth.no-connection",
|
|
16725
|
+
summary: `No Bing connection for ${ctx.project.canonicalDomain}.`,
|
|
16726
|
+
remediation: `Run \`canonry bing connect ${ctx.project.name} --api-key <key>\` to authorize.`
|
|
16727
|
+
};
|
|
16728
|
+
}
|
|
16729
|
+
if (!conn.apiKey) {
|
|
16730
|
+
return {
|
|
16731
|
+
status: CheckStatuses.fail,
|
|
16732
|
+
code: "bing.auth.no-api-key",
|
|
16733
|
+
summary: "Bing connection exists but has no API key stored.",
|
|
16734
|
+
remediation: `Run \`canonry bing connect ${ctx.project.name} --api-key <key>\` to re-authorize.`
|
|
16735
|
+
};
|
|
16736
|
+
}
|
|
16737
|
+
try {
|
|
16738
|
+
await getSites(conn.apiKey);
|
|
16739
|
+
return {
|
|
16740
|
+
status: CheckStatuses.ok,
|
|
16741
|
+
code: "bing.auth.connected",
|
|
16742
|
+
summary: "Bing API key is valid and can list sites.",
|
|
16743
|
+
remediation: null
|
|
16744
|
+
};
|
|
16745
|
+
} catch (err) {
|
|
16746
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
16747
|
+
return {
|
|
16748
|
+
status: CheckStatuses.fail,
|
|
16749
|
+
code: "bing.auth.verification-failed",
|
|
16750
|
+
summary: "Bing API key verification failed.",
|
|
16751
|
+
remediation: "Verify your Bing API key is correct and active in Bing Webmaster Tools.",
|
|
16752
|
+
details: { error: message }
|
|
16753
|
+
};
|
|
16754
|
+
}
|
|
16755
|
+
}
|
|
16756
|
+
},
|
|
16757
|
+
{
|
|
16758
|
+
id: "bing.auth.site-access",
|
|
16759
|
+
category: CheckCategories.auth,
|
|
16760
|
+
scope: CheckScopes.project,
|
|
16761
|
+
title: "Bing site access",
|
|
16762
|
+
run: async (ctx) => {
|
|
16763
|
+
if (!ctx.project) {
|
|
16764
|
+
return {
|
|
16765
|
+
status: CheckStatuses.skipped,
|
|
16766
|
+
code: "bing.auth.no-project",
|
|
15407
16767
|
summary: "Project context required.",
|
|
15408
16768
|
remediation: null
|
|
15409
16769
|
};
|
|
@@ -16183,6 +17543,11 @@ async function apiRoutes(app, opts) {
|
|
|
16183
17543
|
googleConnectionStore: opts.googleConnectionStore,
|
|
16184
17544
|
getGoogleAuthConfig: opts.getGoogleAuthConfig
|
|
16185
17545
|
});
|
|
17546
|
+
await api.register(trafficRoutes, {
|
|
17547
|
+
cloudRunCredentialStore: opts.cloudRunCredentialStore,
|
|
17548
|
+
pullCloudRunEvents: opts.pullCloudRunEvents,
|
|
17549
|
+
resolveCloudRunAccessToken: opts.resolveCloudRunAccessToken
|
|
17550
|
+
});
|
|
16186
17551
|
await api.register(backlinksRoutes, {
|
|
16187
17552
|
getBacklinksStatus: opts.getBacklinksStatus,
|
|
16188
17553
|
onInstallBacklinks: opts.onInstallBacklinks,
|
|
@@ -18608,8 +19973,40 @@ function removeGa4Connection(config, projectName) {
|
|
|
18608
19973
|
return true;
|
|
18609
19974
|
}
|
|
18610
19975
|
|
|
18611
|
-
// src/
|
|
19976
|
+
// src/cloud-run-config.ts
|
|
18612
19977
|
function ensureConnections3(config) {
|
|
19978
|
+
if (!config.cloudRun) config.cloudRun = {};
|
|
19979
|
+
if (!config.cloudRun.connections) config.cloudRun.connections = [];
|
|
19980
|
+
return config.cloudRun.connections;
|
|
19981
|
+
}
|
|
19982
|
+
function getCloudRunConnection(config, projectName) {
|
|
19983
|
+
return (config.cloudRun?.connections ?? []).find((c) => c.projectName === projectName);
|
|
19984
|
+
}
|
|
19985
|
+
function upsertCloudRunConnection(config, connection) {
|
|
19986
|
+
const connections = ensureConnections3(config);
|
|
19987
|
+
const index = connections.findIndex((c) => c.projectName === connection.projectName);
|
|
19988
|
+
if (index === -1) {
|
|
19989
|
+
connections.push(connection);
|
|
19990
|
+
return connection;
|
|
19991
|
+
}
|
|
19992
|
+
connections[index] = connection;
|
|
19993
|
+
return connection;
|
|
19994
|
+
}
|
|
19995
|
+
function removeCloudRunConnection(config, projectName) {
|
|
19996
|
+
const connections = config.cloudRun?.connections;
|
|
19997
|
+
if (!connections?.length) return false;
|
|
19998
|
+
const next = connections.filter((c) => c.projectName !== projectName);
|
|
19999
|
+
if (next.length === connections.length) return false;
|
|
20000
|
+
if (!config.cloudRun) return false;
|
|
20001
|
+
config.cloudRun.connections = next;
|
|
20002
|
+
if (next.length === 0) {
|
|
20003
|
+
delete config.cloudRun;
|
|
20004
|
+
}
|
|
20005
|
+
return true;
|
|
20006
|
+
}
|
|
20007
|
+
|
|
20008
|
+
// src/wordpress-config.ts
|
|
20009
|
+
function ensureConnections4(config) {
|
|
18613
20010
|
if (!config.wordpress) config.wordpress = {};
|
|
18614
20011
|
if (!config.wordpress.connections) config.wordpress.connections = [];
|
|
18615
20012
|
return config.wordpress.connections;
|
|
@@ -18626,7 +20023,7 @@ function getWordpressConnection(config, projectName) {
|
|
|
18626
20023
|
return (config.wordpress?.connections ?? []).find((connection) => connection.projectName === projectName);
|
|
18627
20024
|
}
|
|
18628
20025
|
function upsertWordpressConnection(config, connection) {
|
|
18629
|
-
const connections =
|
|
20026
|
+
const connections = ensureConnections4(config);
|
|
18630
20027
|
const normalized = normalizeConnection(connection);
|
|
18631
20028
|
const index = connections.findIndex((entry) => entry.projectName === connection.projectName);
|
|
18632
20029
|
if (index === -1) {
|
|
@@ -18660,11 +20057,11 @@ function removeWordpressConnection(config, projectName) {
|
|
|
18660
20057
|
}
|
|
18661
20058
|
|
|
18662
20059
|
// src/job-runner.ts
|
|
18663
|
-
import
|
|
20060
|
+
import crypto21 from "crypto";
|
|
18664
20061
|
import fs7 from "fs";
|
|
18665
20062
|
import path9 from "path";
|
|
18666
20063
|
import os4 from "os";
|
|
18667
|
-
import { and as and12, eq as
|
|
20064
|
+
import { and as and12, eq as eq24, inArray as inArray7, sql as sql8 } from "drizzle-orm";
|
|
18668
20065
|
|
|
18669
20066
|
// src/citation-utils.ts
|
|
18670
20067
|
function domainMatches(domain, canonicalDomain) {
|
|
@@ -18925,7 +20322,7 @@ var JobRunner = class {
|
|
|
18925
20322
|
if (stale.length === 0) return;
|
|
18926
20323
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
18927
20324
|
for (const run of stale) {
|
|
18928
|
-
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(
|
|
20325
|
+
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(eq24(runs.id, run.id)).run();
|
|
18929
20326
|
log.warn("run.recovered-stale", { runId: run.id, previousStatus: run.status });
|
|
18930
20327
|
}
|
|
18931
20328
|
}
|
|
@@ -18953,10 +20350,10 @@ var JobRunner = class {
|
|
|
18953
20350
|
throw new Error(`Run ${runId} is not executable from status '${existingRun.status}'`);
|
|
18954
20351
|
}
|
|
18955
20352
|
if (existingRun.status === "queued") {
|
|
18956
|
-
this.db.update(runs).set({ status: "running", startedAt: now }).where(and12(
|
|
20353
|
+
this.db.update(runs).set({ status: "running", startedAt: now }).where(and12(eq24(runs.id, runId), eq24(runs.status, "queued"))).run();
|
|
18957
20354
|
}
|
|
18958
20355
|
this.throwIfRunCancelled(runId);
|
|
18959
|
-
const project = this.db.select().from(projects).where(
|
|
20356
|
+
const project = this.db.select().from(projects).where(eq24(projects.id, projectId)).get();
|
|
18960
20357
|
if (!project) {
|
|
18961
20358
|
throw new Error(`Project ${projectId} not found`);
|
|
18962
20359
|
}
|
|
@@ -18976,8 +20373,8 @@ var JobRunner = class {
|
|
|
18976
20373
|
throw new Error("No providers configured. Add at least one provider API key.");
|
|
18977
20374
|
}
|
|
18978
20375
|
log.info("run.dispatch", { runId, providerCount: activeProviders.length, providers: activeProviders.map((p) => p.adapter.name) });
|
|
18979
|
-
projectQueries = this.db.select().from(queries).where(
|
|
18980
|
-
const projectCompetitors = this.db.select().from(competitors).where(
|
|
20376
|
+
projectQueries = this.db.select().from(queries).where(eq24(queries.projectId, projectId)).all();
|
|
20377
|
+
const projectCompetitors = this.db.select().from(competitors).where(eq24(competitors.projectId, projectId)).all();
|
|
18981
20378
|
const competitorDomains = projectCompetitors.map((c) => c.domain);
|
|
18982
20379
|
const allDomains = effectiveDomains({
|
|
18983
20380
|
canonicalDomain: project.canonicalDomain,
|
|
@@ -18993,7 +20390,7 @@ var JobRunner = class {
|
|
|
18993
20390
|
const todayPeriod = getCurrentUsageDay();
|
|
18994
20391
|
for (const p of activeProviders) {
|
|
18995
20392
|
const providerScope = `${projectId}:${p.adapter.name}`;
|
|
18996
|
-
const providerUsage = this.db.select().from(usageCounters).where(
|
|
20393
|
+
const providerUsage = this.db.select().from(usageCounters).where(eq24(usageCounters.scope, providerScope)).all().filter((r) => r.period === todayPeriod && r.metric === "queries").reduce((sum, r) => sum + r.count, 0);
|
|
18997
20394
|
const limit = p.config.quotaPolicy.maxRequestsPerDay;
|
|
18998
20395
|
if (providerUsage + queriesPerProvider > limit) {
|
|
18999
20396
|
throw new Error(
|
|
@@ -19053,7 +20450,7 @@ var JobRunner = class {
|
|
|
19053
20450
|
);
|
|
19054
20451
|
let screenshotRelPath = null;
|
|
19055
20452
|
if (raw.screenshotPath && fs7.existsSync(raw.screenshotPath)) {
|
|
19056
|
-
const snapshotId =
|
|
20453
|
+
const snapshotId = crypto21.randomUUID();
|
|
19057
20454
|
const screenshotDir = path9.join(os4.homedir(), ".canonry", "screenshots", runId);
|
|
19058
20455
|
if (!fs7.existsSync(screenshotDir)) fs7.mkdirSync(screenshotDir, { recursive: true });
|
|
19059
20456
|
const destPath = path9.join(screenshotDir, `${snapshotId}.png`);
|
|
@@ -19083,7 +20480,7 @@ var JobRunner = class {
|
|
|
19083
20480
|
}).run();
|
|
19084
20481
|
} else {
|
|
19085
20482
|
this.db.insert(querySnapshots).values({
|
|
19086
|
-
id:
|
|
20483
|
+
id: crypto21.randomUUID(),
|
|
19087
20484
|
runId,
|
|
19088
20485
|
queryId: q.id,
|
|
19089
20486
|
provider: providerName,
|
|
@@ -19134,12 +20531,12 @@ var JobRunner = class {
|
|
|
19134
20531
|
const someFailed = providerErrors.size > 0;
|
|
19135
20532
|
if (allFailed) {
|
|
19136
20533
|
const errorDetail = serializeRunError(buildRunErrorFromMessages(providerErrors));
|
|
19137
|
-
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
20534
|
+
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq24(runs.id, runId)).run();
|
|
19138
20535
|
} else if (someFailed) {
|
|
19139
20536
|
const errorDetail = serializeRunError(buildRunErrorFromMessages(providerErrors));
|
|
19140
|
-
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
20537
|
+
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq24(runs.id, runId)).run();
|
|
19141
20538
|
} else {
|
|
19142
|
-
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
20539
|
+
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq24(runs.id, runId)).run();
|
|
19143
20540
|
}
|
|
19144
20541
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
19145
20542
|
const finalStatus = allFailed ? "failed" : someFailed ? "partial" : "completed";
|
|
@@ -19174,7 +20571,7 @@ var JobRunner = class {
|
|
|
19174
20571
|
status: "failed",
|
|
19175
20572
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19176
20573
|
error: errorMessage
|
|
19177
|
-
}).where(
|
|
20574
|
+
}).where(eq24(runs.id, runId)).run();
|
|
19178
20575
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
19179
20576
|
trackEvent("run.completed", {
|
|
19180
20577
|
status: "failed",
|
|
@@ -19195,7 +20592,7 @@ var JobRunner = class {
|
|
|
19195
20592
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
19196
20593
|
const period = now.slice(0, 10);
|
|
19197
20594
|
this.db.insert(usageCounters).values({
|
|
19198
|
-
id:
|
|
20595
|
+
id: crypto21.randomUUID(),
|
|
19199
20596
|
scope,
|
|
19200
20597
|
period,
|
|
19201
20598
|
metric,
|
|
@@ -19203,7 +20600,7 @@ var JobRunner = class {
|
|
|
19203
20600
|
updatedAt: now
|
|
19204
20601
|
}).onConflictDoUpdate({
|
|
19205
20602
|
target: [usageCounters.scope, usageCounters.period, usageCounters.metric],
|
|
19206
|
-
set: { count:
|
|
20603
|
+
set: { count: sql8`${usageCounters.count} + ${count}`, updatedAt: now }
|
|
19207
20604
|
}).run();
|
|
19208
20605
|
}
|
|
19209
20606
|
flushProviderUsage(projectId, providerDispatchCounts) {
|
|
@@ -19217,7 +20614,7 @@ var JobRunner = class {
|
|
|
19217
20614
|
status: runs.status,
|
|
19218
20615
|
finishedAt: runs.finishedAt,
|
|
19219
20616
|
error: runs.error
|
|
19220
|
-
}).from(runs).where(
|
|
20617
|
+
}).from(runs).where(eq24(runs.id, runId)).get();
|
|
19221
20618
|
}
|
|
19222
20619
|
isRunCancelled(runId) {
|
|
19223
20620
|
return this.getRunState(runId)?.status === "cancelled";
|
|
@@ -19233,7 +20630,7 @@ var JobRunner = class {
|
|
|
19233
20630
|
this.db.update(runs).set({
|
|
19234
20631
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19235
20632
|
error: currentRun.error ?? "Cancelled by user"
|
|
19236
|
-
}).where(
|
|
20633
|
+
}).where(eq24(runs.id, runId)).run();
|
|
19237
20634
|
}
|
|
19238
20635
|
trackEvent("run.completed", {
|
|
19239
20636
|
status: "cancelled",
|
|
@@ -19255,8 +20652,8 @@ function getCurrentUsageDay() {
|
|
|
19255
20652
|
}
|
|
19256
20653
|
|
|
19257
20654
|
// src/gsc-sync.ts
|
|
19258
|
-
import
|
|
19259
|
-
import { eq as
|
|
20655
|
+
import crypto22 from "crypto";
|
|
20656
|
+
import { eq as eq25, and as and13, sql as sql9 } from "drizzle-orm";
|
|
19260
20657
|
var log2 = createLogger("GscSync");
|
|
19261
20658
|
function formatDate3(d) {
|
|
19262
20659
|
return d.toISOString().split("T")[0];
|
|
@@ -19268,13 +20665,13 @@ function daysAgo(n) {
|
|
|
19268
20665
|
}
|
|
19269
20666
|
async function executeGscSync(db, runId, projectId, opts) {
|
|
19270
20667
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
19271
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
20668
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq25(runs.id, runId)).run();
|
|
19272
20669
|
try {
|
|
19273
20670
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
19274
20671
|
if (!googleClientId || !googleClientSecret) {
|
|
19275
20672
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
19276
20673
|
}
|
|
19277
|
-
const project = db.select().from(projects).where(
|
|
20674
|
+
const project = db.select().from(projects).where(eq25(projects.id, projectId)).get();
|
|
19278
20675
|
if (!project) {
|
|
19279
20676
|
throw new Error(`Project not found: ${projectId}`);
|
|
19280
20677
|
}
|
|
@@ -19309,9 +20706,9 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
19309
20706
|
log2.info("fetch.complete", { runId, projectId, rowCount: rows.length });
|
|
19310
20707
|
db.delete(gscSearchData).where(
|
|
19311
20708
|
and13(
|
|
19312
|
-
|
|
19313
|
-
|
|
19314
|
-
|
|
20709
|
+
eq25(gscSearchData.projectId, projectId),
|
|
20710
|
+
sql9`${gscSearchData.date} >= ${startDate}`,
|
|
20711
|
+
sql9`${gscSearchData.date} <= ${endDate}`
|
|
19315
20712
|
)
|
|
19316
20713
|
).run();
|
|
19317
20714
|
const batchSize = 500;
|
|
@@ -19321,7 +20718,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
19321
20718
|
for (const row of batch) {
|
|
19322
20719
|
const [query, page, country, device, date] = row.keys;
|
|
19323
20720
|
db.insert(gscSearchData).values({
|
|
19324
|
-
id:
|
|
20721
|
+
id: crypto22.randomUUID(),
|
|
19325
20722
|
projectId,
|
|
19326
20723
|
syncRunId: runId,
|
|
19327
20724
|
date: date ?? "",
|
|
@@ -19355,7 +20752,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
19355
20752
|
const rich = ir.richResultsResult;
|
|
19356
20753
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
19357
20754
|
db.insert(gscUrlInspections).values({
|
|
19358
|
-
id:
|
|
20755
|
+
id: crypto22.randomUUID(),
|
|
19359
20756
|
projectId,
|
|
19360
20757
|
syncRunId: runId,
|
|
19361
20758
|
url: pageUrl,
|
|
@@ -19376,7 +20773,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
19376
20773
|
log2.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
|
|
19377
20774
|
}
|
|
19378
20775
|
}
|
|
19379
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
20776
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq25(gscUrlInspections.projectId, projectId)).all();
|
|
19380
20777
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
19381
20778
|
for (const row of allInspections) {
|
|
19382
20779
|
const existing = latestByUrl.get(row.url);
|
|
@@ -19397,9 +20794,9 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
19397
20794
|
}
|
|
19398
20795
|
}
|
|
19399
20796
|
const snapshotDate = formatDate3(/* @__PURE__ */ new Date());
|
|
19400
|
-
db.delete(gscCoverageSnapshots).where(and13(
|
|
20797
|
+
db.delete(gscCoverageSnapshots).where(and13(eq25(gscCoverageSnapshots.projectId, projectId), eq25(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
19401
20798
|
db.insert(gscCoverageSnapshots).values({
|
|
19402
|
-
id:
|
|
20799
|
+
id: crypto22.randomUUID(),
|
|
19403
20800
|
projectId,
|
|
19404
20801
|
syncRunId: runId,
|
|
19405
20802
|
date: snapshotDate,
|
|
@@ -19408,19 +20805,19 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
19408
20805
|
reasonBreakdown: JSON.stringify(reasonCounts),
|
|
19409
20806
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
19410
20807
|
}).run();
|
|
19411
|
-
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
20808
|
+
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq25(runs.id, runId)).run();
|
|
19412
20809
|
log2.info("sync.completed", { runId, projectId, searchDataRows: rows.length, urlInspections: topPages.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
19413
20810
|
} catch (err) {
|
|
19414
20811
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
19415
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
20812
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq25(runs.id, runId)).run();
|
|
19416
20813
|
log2.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
19417
20814
|
throw err;
|
|
19418
20815
|
}
|
|
19419
20816
|
}
|
|
19420
20817
|
|
|
19421
20818
|
// src/gsc-inspect-sitemap.ts
|
|
19422
|
-
import
|
|
19423
|
-
import { eq as
|
|
20819
|
+
import crypto23 from "crypto";
|
|
20820
|
+
import { eq as eq26, and as and14 } from "drizzle-orm";
|
|
19424
20821
|
|
|
19425
20822
|
// src/sitemap-parser.ts
|
|
19426
20823
|
var log3 = createLogger("SitemapParser");
|
|
@@ -19541,13 +20938,13 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
19541
20938
|
var log4 = createLogger("InspectSitemap");
|
|
19542
20939
|
async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
19543
20940
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
19544
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
20941
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq26(runs.id, runId)).run();
|
|
19545
20942
|
try {
|
|
19546
20943
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
19547
20944
|
if (!googleClientId || !googleClientSecret) {
|
|
19548
20945
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
19549
20946
|
}
|
|
19550
|
-
const project = db.select().from(projects).where(
|
|
20947
|
+
const project = db.select().from(projects).where(eq26(projects.id, projectId)).get();
|
|
19551
20948
|
if (!project) {
|
|
19552
20949
|
throw new Error(`Project not found: ${projectId}`);
|
|
19553
20950
|
}
|
|
@@ -19588,7 +20985,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
19588
20985
|
const rich = ir.richResultsResult;
|
|
19589
20986
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
19590
20987
|
db.insert(gscUrlInspections).values({
|
|
19591
|
-
id:
|
|
20988
|
+
id: crypto23.randomUUID(),
|
|
19592
20989
|
projectId,
|
|
19593
20990
|
syncRunId: runId,
|
|
19594
20991
|
url: pageUrl,
|
|
@@ -19615,7 +21012,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
19615
21012
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
19616
21013
|
}
|
|
19617
21014
|
}
|
|
19618
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
21015
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq26(gscUrlInspections.projectId, projectId)).all();
|
|
19619
21016
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
19620
21017
|
for (const row of allInspections) {
|
|
19621
21018
|
const existing = latestByUrl.get(row.url);
|
|
@@ -19636,9 +21033,9 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
19636
21033
|
}
|
|
19637
21034
|
}
|
|
19638
21035
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
19639
|
-
db.delete(gscCoverageSnapshots).where(and14(
|
|
21036
|
+
db.delete(gscCoverageSnapshots).where(and14(eq26(gscCoverageSnapshots.projectId, projectId), eq26(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
19640
21037
|
db.insert(gscCoverageSnapshots).values({
|
|
19641
|
-
id:
|
|
21038
|
+
id: crypto23.randomUUID(),
|
|
19642
21039
|
projectId,
|
|
19643
21040
|
syncRunId: runId,
|
|
19644
21041
|
date: snapshotDate,
|
|
@@ -19648,19 +21045,19 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
19648
21045
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
19649
21046
|
}).run();
|
|
19650
21047
|
const status = errors > 0 && inspected > 0 ? "partial" : errors === urls.length ? "failed" : "completed";
|
|
19651
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
21048
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq26(runs.id, runId)).run();
|
|
19652
21049
|
log4.info("inspect.completed", { runId, projectId, inspected, errors, total: urls.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
19653
21050
|
} catch (err) {
|
|
19654
21051
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
19655
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
21052
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq26(runs.id, runId)).run();
|
|
19656
21053
|
log4.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
19657
21054
|
throw err;
|
|
19658
21055
|
}
|
|
19659
21056
|
}
|
|
19660
21057
|
|
|
19661
21058
|
// src/bing-inspect-sitemap.ts
|
|
19662
|
-
import
|
|
19663
|
-
import { eq as
|
|
21059
|
+
import crypto24 from "crypto";
|
|
21060
|
+
import { eq as eq27, desc as desc12 } from "drizzle-orm";
|
|
19664
21061
|
var log5 = createLogger("BingInspectSitemap");
|
|
19665
21062
|
function parseBingDate2(value) {
|
|
19666
21063
|
if (!value) return null;
|
|
@@ -19678,9 +21075,9 @@ function isBlockingIssueType2(issueType) {
|
|
|
19678
21075
|
}
|
|
19679
21076
|
async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
19680
21077
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
19681
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
21078
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq27(runs.id, runId)).run();
|
|
19682
21079
|
try {
|
|
19683
|
-
const project = db.select().from(projects).where(
|
|
21080
|
+
const project = db.select().from(projects).where(eq27(projects.id, projectId)).get();
|
|
19684
21081
|
if (!project) {
|
|
19685
21082
|
throw new Error(`Project not found: ${projectId}`);
|
|
19686
21083
|
}
|
|
@@ -19698,7 +21095,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
19698
21095
|
if (sitemapUrls.length === 0) {
|
|
19699
21096
|
throw new Error("No URLs found in sitemap");
|
|
19700
21097
|
}
|
|
19701
|
-
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(
|
|
21098
|
+
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(eq27(bingUrlInspections.projectId, projectId)).all();
|
|
19702
21099
|
const trackedUrls = new Set(trackedRows.map((r) => r.url));
|
|
19703
21100
|
const discovered = sitemapUrls.filter((u) => !trackedUrls.has(u));
|
|
19704
21101
|
log5.info("sitemap.diff", {
|
|
@@ -19747,7 +21144,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
19747
21144
|
derivedInIndex = false;
|
|
19748
21145
|
}
|
|
19749
21146
|
db.insert(bingUrlInspections).values({
|
|
19750
|
-
id:
|
|
21147
|
+
id: crypto24.randomUUID(),
|
|
19751
21148
|
projectId,
|
|
19752
21149
|
url: pageUrl,
|
|
19753
21150
|
httpCode,
|
|
@@ -19781,7 +21178,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
19781
21178
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
19782
21179
|
}
|
|
19783
21180
|
}
|
|
19784
|
-
const allInspections = db.select().from(bingUrlInspections).where(
|
|
21181
|
+
const allInspections = db.select().from(bingUrlInspections).where(eq27(bingUrlInspections.projectId, projectId)).orderBy(desc12(bingUrlInspections.inspectedAt)).all();
|
|
19785
21182
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
19786
21183
|
const definitiveByUrl = /* @__PURE__ */ new Map();
|
|
19787
21184
|
for (const row of allInspections) {
|
|
@@ -19805,7 +21202,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
19805
21202
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
19806
21203
|
const snapNow = (/* @__PURE__ */ new Date()).toISOString();
|
|
19807
21204
|
db.insert(bingCoverageSnapshots).values({
|
|
19808
|
-
id:
|
|
21205
|
+
id: crypto24.randomUUID(),
|
|
19809
21206
|
projectId,
|
|
19810
21207
|
syncRunId: runId,
|
|
19811
21208
|
date: snapshotDate,
|
|
@@ -19824,7 +21221,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
19824
21221
|
}
|
|
19825
21222
|
}).run();
|
|
19826
21223
|
const status = errors === sitemapUrls.length ? RunStatuses.failed : errors > 0 ? RunStatuses.partial : RunStatuses.completed;
|
|
19827
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
21224
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq27(runs.id, runId)).run();
|
|
19828
21225
|
log5.info("inspect.completed", {
|
|
19829
21226
|
runId,
|
|
19830
21227
|
projectId,
|
|
@@ -19838,16 +21235,16 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
19838
21235
|
});
|
|
19839
21236
|
} catch (err) {
|
|
19840
21237
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
19841
|
-
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
21238
|
+
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq27(runs.id, runId)).run();
|
|
19842
21239
|
log5.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
19843
21240
|
throw err;
|
|
19844
21241
|
}
|
|
19845
21242
|
}
|
|
19846
21243
|
|
|
19847
21244
|
// src/commoncrawl-sync.ts
|
|
19848
|
-
import
|
|
21245
|
+
import crypto25 from "crypto";
|
|
19849
21246
|
import path10 from "path";
|
|
19850
|
-
import { and as and15, eq as
|
|
21247
|
+
import { and as and15, eq as eq28, sql as sql10 } from "drizzle-orm";
|
|
19851
21248
|
var log6 = createLogger("CommonCrawlSync");
|
|
19852
21249
|
var INSERT_CHUNK_SIZE = 1e4;
|
|
19853
21250
|
function defaultDeps() {
|
|
@@ -19873,7 +21270,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
19873
21270
|
phaseDetail: "downloading vertices + edges",
|
|
19874
21271
|
updatedAt: downloadStartedAt,
|
|
19875
21272
|
error: null
|
|
19876
|
-
}).where(
|
|
21273
|
+
}).where(eq28(ccReleaseSyncs.id, syncId)).run();
|
|
19877
21274
|
const paths = ccReleasePaths(release);
|
|
19878
21275
|
const releaseCacheDir = path10.join(deps.cacheDir, release);
|
|
19879
21276
|
const vertexPath = path10.join(releaseCacheDir, paths.vertexFilename);
|
|
@@ -19896,7 +21293,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
19896
21293
|
vertexSha256: vertex.sha256,
|
|
19897
21294
|
edgesSha256: edges.sha256,
|
|
19898
21295
|
updatedAt: downloadFinishedAt
|
|
19899
|
-
}).where(
|
|
21296
|
+
}).where(eq28(ccReleaseSyncs.id, syncId)).run();
|
|
19900
21297
|
const allProjects = db.select().from(projects).all();
|
|
19901
21298
|
const targets = Array.from(new Set(allProjects.map((p) => p.canonicalDomain)));
|
|
19902
21299
|
let rows = [];
|
|
@@ -19912,15 +21309,15 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
19912
21309
|
}
|
|
19913
21310
|
const queriedAt = deps.now().toISOString();
|
|
19914
21311
|
db.transaction((tx) => {
|
|
19915
|
-
tx.delete(backlinkDomains).where(
|
|
19916
|
-
tx.delete(backlinkSummaries).where(
|
|
21312
|
+
tx.delete(backlinkDomains).where(eq28(backlinkDomains.releaseSyncId, syncId)).run();
|
|
21313
|
+
tx.delete(backlinkSummaries).where(eq28(backlinkSummaries.releaseSyncId, syncId)).run();
|
|
19917
21314
|
const expanded = [];
|
|
19918
21315
|
for (const r of rows) {
|
|
19919
21316
|
const projectIds = projectsByDomain.get(r.targetDomain);
|
|
19920
21317
|
if (!projectIds) continue;
|
|
19921
21318
|
for (const projectId of projectIds) {
|
|
19922
21319
|
expanded.push({
|
|
19923
|
-
id:
|
|
21320
|
+
id: crypto25.randomUUID(),
|
|
19924
21321
|
projectId,
|
|
19925
21322
|
releaseSyncId: syncId,
|
|
19926
21323
|
release,
|
|
@@ -19940,7 +21337,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
19940
21337
|
const projectRows = rowsByProject.get(p.id) ?? [];
|
|
19941
21338
|
const summary = computeSummary(projectRows);
|
|
19942
21339
|
tx.insert(backlinkSummaries).values({
|
|
19943
|
-
id:
|
|
21340
|
+
id: crypto25.randomUUID(),
|
|
19944
21341
|
projectId: p.id,
|
|
19945
21342
|
releaseSyncId: syncId,
|
|
19946
21343
|
release,
|
|
@@ -19972,7 +21369,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
19972
21369
|
domainsDiscovered: rows.length,
|
|
19973
21370
|
updatedAt: finishedAt,
|
|
19974
21371
|
error: null
|
|
19975
|
-
}).where(
|
|
21372
|
+
}).where(eq28(ccReleaseSyncs.id, syncId)).run();
|
|
19976
21373
|
log6.info("sync.completed", {
|
|
19977
21374
|
syncId,
|
|
19978
21375
|
release,
|
|
@@ -20002,7 +21399,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
20002
21399
|
error: errorMsg,
|
|
20003
21400
|
phaseDetail: null,
|
|
20004
21401
|
updatedAt: finishedAt
|
|
20005
|
-
}).where(
|
|
21402
|
+
}).where(eq28(ccReleaseSyncs.id, syncId)).run();
|
|
20006
21403
|
log6.error("sync.failed", { syncId, release, error: errorMsg });
|
|
20007
21404
|
throw err;
|
|
20008
21405
|
}
|
|
@@ -20036,9 +21433,9 @@ function computeSummary(rows) {
|
|
|
20036
21433
|
}
|
|
20037
21434
|
|
|
20038
21435
|
// src/backlink-extract.ts
|
|
20039
|
-
import
|
|
21436
|
+
import crypto26 from "crypto";
|
|
20040
21437
|
import fs8 from "fs";
|
|
20041
|
-
import { and as and16, desc as desc13, eq as
|
|
21438
|
+
import { and as and16, desc as desc13, eq as eq29 } from "drizzle-orm";
|
|
20042
21439
|
var log7 = createLogger("BacklinkExtract");
|
|
20043
21440
|
function defaultDeps2() {
|
|
20044
21441
|
return {
|
|
@@ -20050,13 +21447,13 @@ function defaultDeps2() {
|
|
|
20050
21447
|
async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
20051
21448
|
const deps = { ...defaultDeps2(), ...opts.deps };
|
|
20052
21449
|
const startedAt = deps.now().toISOString();
|
|
20053
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
21450
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq29(runs.id, runId)).run();
|
|
20054
21451
|
try {
|
|
20055
|
-
const project = db.select().from(projects).where(
|
|
21452
|
+
const project = db.select().from(projects).where(eq29(projects.id, projectId)).get();
|
|
20056
21453
|
if (!project) {
|
|
20057
21454
|
throw new Error(`Project not found: ${projectId}`);
|
|
20058
21455
|
}
|
|
20059
|
-
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(
|
|
21456
|
+
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(eq29(ccReleaseSyncs.release, opts.release)).get() : db.select().from(ccReleaseSyncs).where(eq29(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).orderBy(desc13(ccReleaseSyncs.createdAt)).limit(1).get();
|
|
20060
21457
|
if (!sync) {
|
|
20061
21458
|
throw new Error("No ready release sync available \u2014 run `canonry backlinks sync` first");
|
|
20062
21459
|
}
|
|
@@ -20084,11 +21481,11 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
20084
21481
|
const targetDomain = project.canonicalDomain;
|
|
20085
21482
|
db.transaction((tx) => {
|
|
20086
21483
|
tx.delete(backlinkDomains).where(
|
|
20087
|
-
and16(
|
|
21484
|
+
and16(eq29(backlinkDomains.projectId, projectId), eq29(backlinkDomains.release, release))
|
|
20088
21485
|
).run();
|
|
20089
21486
|
if (rows.length > 0) {
|
|
20090
21487
|
const values = rows.map((r) => ({
|
|
20091
|
-
id:
|
|
21488
|
+
id: crypto26.randomUUID(),
|
|
20092
21489
|
projectId,
|
|
20093
21490
|
releaseSyncId: syncId,
|
|
20094
21491
|
release,
|
|
@@ -20101,7 +21498,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
20101
21498
|
}
|
|
20102
21499
|
const summary = computeSummary2(rows);
|
|
20103
21500
|
tx.insert(backlinkSummaries).values({
|
|
20104
|
-
id:
|
|
21501
|
+
id: crypto26.randomUUID(),
|
|
20105
21502
|
projectId,
|
|
20106
21503
|
releaseSyncId: syncId,
|
|
20107
21504
|
release,
|
|
@@ -20124,7 +21521,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
20124
21521
|
}).run();
|
|
20125
21522
|
});
|
|
20126
21523
|
const finishedAt = deps.now().toISOString();
|
|
20127
|
-
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
21524
|
+
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq29(runs.id, runId)).run();
|
|
20128
21525
|
log7.info("extract.completed", { runId, projectId, release, rows: rows.length });
|
|
20129
21526
|
} catch (err) {
|
|
20130
21527
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
@@ -20133,7 +21530,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
20133
21530
|
status: RunStatuses.failed,
|
|
20134
21531
|
error: errorMsg,
|
|
20135
21532
|
finishedAt
|
|
20136
|
-
}).where(
|
|
21533
|
+
}).where(eq29(runs.id, runId)).run();
|
|
20137
21534
|
log7.error("extract.failed", { runId, projectId, error: errorMsg });
|
|
20138
21535
|
throw err;
|
|
20139
21536
|
}
|
|
@@ -20206,7 +21603,7 @@ var ProviderRegistry = class {
|
|
|
20206
21603
|
|
|
20207
21604
|
// src/scheduler.ts
|
|
20208
21605
|
import cron from "node-cron";
|
|
20209
|
-
import { eq as
|
|
21606
|
+
import { eq as eq30 } from "drizzle-orm";
|
|
20210
21607
|
var log8 = createLogger("Scheduler");
|
|
20211
21608
|
var Scheduler = class {
|
|
20212
21609
|
db;
|
|
@@ -20218,7 +21615,7 @@ var Scheduler = class {
|
|
|
20218
21615
|
}
|
|
20219
21616
|
/** Load all enabled schedules from DB and register cron jobs. */
|
|
20220
21617
|
start() {
|
|
20221
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
21618
|
+
const allSchedules = this.db.select().from(schedules).where(eq30(schedules.enabled, 1)).all();
|
|
20222
21619
|
for (const schedule of allSchedules) {
|
|
20223
21620
|
const missedRunAt = schedule.nextRunAt;
|
|
20224
21621
|
this.registerCronTask(schedule);
|
|
@@ -20243,7 +21640,7 @@ var Scheduler = class {
|
|
|
20243
21640
|
this.stopTask(projectId, existing, "Stopped");
|
|
20244
21641
|
this.tasks.delete(projectId);
|
|
20245
21642
|
}
|
|
20246
|
-
const schedule = this.db.select().from(schedules).where(
|
|
21643
|
+
const schedule = this.db.select().from(schedules).where(eq30(schedules.projectId, projectId)).get();
|
|
20247
21644
|
if (schedule && schedule.enabled === 1) {
|
|
20248
21645
|
this.registerCronTask(schedule);
|
|
20249
21646
|
}
|
|
@@ -20276,14 +21673,14 @@ var Scheduler = class {
|
|
|
20276
21673
|
this.db.update(schedules).set({
|
|
20277
21674
|
nextRunAt: task.getNextRun()?.toISOString() ?? null,
|
|
20278
21675
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
20279
|
-
}).where(
|
|
21676
|
+
}).where(eq30(schedules.id, scheduleId)).run();
|
|
20280
21677
|
const label = schedule.preset ?? cronExpr;
|
|
20281
21678
|
log8.info("cron.registered", { projectId, schedule: label, timezone });
|
|
20282
21679
|
}
|
|
20283
21680
|
triggerRun(scheduleId, projectId) {
|
|
20284
21681
|
try {
|
|
20285
21682
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
20286
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
21683
|
+
const currentSchedule = this.db.select().from(schedules).where(eq30(schedules.id, scheduleId)).get();
|
|
20287
21684
|
if (!currentSchedule || currentSchedule.enabled !== 1) {
|
|
20288
21685
|
log8.warn("schedule.stale", { scheduleId, projectId, msg: "schedule no longer exists or is disabled" });
|
|
20289
21686
|
this.remove(projectId);
|
|
@@ -20291,7 +21688,7 @@ var Scheduler = class {
|
|
|
20291
21688
|
}
|
|
20292
21689
|
const task = this.tasks.get(projectId);
|
|
20293
21690
|
const nextRunAt = task?.getNextRun()?.toISOString() ?? null;
|
|
20294
|
-
const project = this.db.select().from(projects).where(
|
|
21691
|
+
const project = this.db.select().from(projects).where(eq30(projects.id, projectId)).get();
|
|
20295
21692
|
if (!project) {
|
|
20296
21693
|
log8.error("project.not-found", { projectId, msg: "skipping scheduled run" });
|
|
20297
21694
|
this.remove(projectId);
|
|
@@ -20320,7 +21717,7 @@ var Scheduler = class {
|
|
|
20320
21717
|
this.db.update(schedules).set({
|
|
20321
21718
|
nextRunAt,
|
|
20322
21719
|
updatedAt: now
|
|
20323
|
-
}).where(
|
|
21720
|
+
}).where(eq30(schedules.id, currentSchedule.id)).run();
|
|
20324
21721
|
return;
|
|
20325
21722
|
}
|
|
20326
21723
|
const runId = queueResult.runId;
|
|
@@ -20328,7 +21725,7 @@ var Scheduler = class {
|
|
|
20328
21725
|
lastRunAt: now,
|
|
20329
21726
|
nextRunAt,
|
|
20330
21727
|
updatedAt: now
|
|
20331
|
-
}).where(
|
|
21728
|
+
}).where(eq30(schedules.id, currentSchedule.id)).run();
|
|
20332
21729
|
const scheduleProviders = parseJsonColumn(currentSchedule.providers, []);
|
|
20333
21730
|
const providers = scheduleProviders.length > 0 ? scheduleProviders : void 0;
|
|
20334
21731
|
log8.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
@@ -20340,8 +21737,8 @@ var Scheduler = class {
|
|
|
20340
21737
|
};
|
|
20341
21738
|
|
|
20342
21739
|
// src/notifier.ts
|
|
20343
|
-
import { eq as
|
|
20344
|
-
import
|
|
21740
|
+
import { eq as eq31, desc as desc14, and as and17, or as or4 } from "drizzle-orm";
|
|
21741
|
+
import crypto27 from "crypto";
|
|
20345
21742
|
var log9 = createLogger("Notifier");
|
|
20346
21743
|
var Notifier = class {
|
|
20347
21744
|
db;
|
|
@@ -20353,18 +21750,18 @@ var Notifier = class {
|
|
|
20353
21750
|
/** Called after a run completes (success, partial, or failed). */
|
|
20354
21751
|
async onRunCompleted(runId, projectId) {
|
|
20355
21752
|
log9.info("run.completed", { runId, projectId });
|
|
20356
|
-
const notifs = this.db.select().from(notifications).where(
|
|
21753
|
+
const notifs = this.db.select().from(notifications).where(eq31(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
20357
21754
|
if (notifs.length === 0) {
|
|
20358
21755
|
log9.info("notifications.none-enabled", { projectId });
|
|
20359
21756
|
return;
|
|
20360
21757
|
}
|
|
20361
21758
|
log9.info("notifications.found", { projectId, count: notifs.length });
|
|
20362
|
-
const run = this.db.select().from(runs).where(
|
|
21759
|
+
const run = this.db.select().from(runs).where(eq31(runs.id, runId)).get();
|
|
20363
21760
|
if (!run) {
|
|
20364
21761
|
log9.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
20365
21762
|
return;
|
|
20366
21763
|
}
|
|
20367
|
-
const project = this.db.select().from(projects).where(
|
|
21764
|
+
const project = this.db.select().from(projects).where(eq31(projects.id, projectId)).get();
|
|
20368
21765
|
if (!project) {
|
|
20369
21766
|
log9.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
20370
21767
|
return;
|
|
@@ -20411,11 +21808,11 @@ var Notifier = class {
|
|
|
20411
21808
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
20412
21809
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
20413
21810
|
if (insightEvents.length === 0) return;
|
|
20414
|
-
const notifs = this.db.select().from(notifications).where(
|
|
21811
|
+
const notifs = this.db.select().from(notifications).where(eq31(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
20415
21812
|
if (notifs.length === 0) return;
|
|
20416
|
-
const run = this.db.select().from(runs).where(
|
|
21813
|
+
const run = this.db.select().from(runs).where(eq31(runs.id, runId)).get();
|
|
20417
21814
|
if (!run) return;
|
|
20418
|
-
const project = this.db.select().from(projects).where(
|
|
21815
|
+
const project = this.db.select().from(projects).where(eq31(projects.id, projectId)).get();
|
|
20419
21816
|
if (!project) return;
|
|
20420
21817
|
for (const notif of notifs) {
|
|
20421
21818
|
const config = parseJsonColumn(notif.config, { url: "", events: [] });
|
|
@@ -20447,8 +21844,8 @@ var Notifier = class {
|
|
|
20447
21844
|
computeTransitions(runId, projectId) {
|
|
20448
21845
|
const recentRuns = this.db.select().from(runs).where(
|
|
20449
21846
|
and17(
|
|
20450
|
-
|
|
20451
|
-
or4(
|
|
21847
|
+
eq31(runs.projectId, projectId),
|
|
21848
|
+
or4(eq31(runs.status, "completed"), eq31(runs.status, "partial"))
|
|
20452
21849
|
)
|
|
20453
21850
|
).orderBy(desc14(runs.createdAt)).limit(2).all();
|
|
20454
21851
|
if (recentRuns.length < 2) return [];
|
|
@@ -20460,12 +21857,12 @@ var Notifier = class {
|
|
|
20460
21857
|
query: queries.query,
|
|
20461
21858
|
provider: querySnapshots.provider,
|
|
20462
21859
|
citationState: querySnapshots.citationState
|
|
20463
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
21860
|
+
}).from(querySnapshots).leftJoin(queries, eq31(querySnapshots.queryId, queries.id)).where(eq31(querySnapshots.runId, currentRunId)).all();
|
|
20464
21861
|
const previousSnapshots = this.db.select({
|
|
20465
21862
|
queryId: querySnapshots.queryId,
|
|
20466
21863
|
provider: querySnapshots.provider,
|
|
20467
21864
|
citationState: querySnapshots.citationState
|
|
20468
|
-
}).from(querySnapshots).where(
|
|
21865
|
+
}).from(querySnapshots).where(eq31(querySnapshots.runId, previousRunId)).all();
|
|
20469
21866
|
const prevMap = /* @__PURE__ */ new Map();
|
|
20470
21867
|
for (const s of previousSnapshots) {
|
|
20471
21868
|
prevMap.set(`${s.queryId}:${s.provider}`, s.citationState);
|
|
@@ -20523,7 +21920,7 @@ var Notifier = class {
|
|
|
20523
21920
|
}
|
|
20524
21921
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
20525
21922
|
this.db.insert(auditLog).values({
|
|
20526
|
-
id:
|
|
21923
|
+
id: crypto27.randomUUID(),
|
|
20527
21924
|
projectId,
|
|
20528
21925
|
actor: "scheduler",
|
|
20529
21926
|
action: `notification.${status}`,
|
|
@@ -20581,8 +21978,8 @@ var RunCoordinator = class {
|
|
|
20581
21978
|
};
|
|
20582
21979
|
|
|
20583
21980
|
// src/agent/session-registry.ts
|
|
20584
|
-
import
|
|
20585
|
-
import { eq as
|
|
21981
|
+
import crypto29 from "crypto";
|
|
21982
|
+
import { eq as eq33 } from "drizzle-orm";
|
|
20586
21983
|
|
|
20587
21984
|
// src/agent/session.ts
|
|
20588
21985
|
import fs11 from "fs";
|
|
@@ -20931,11 +22328,11 @@ function resolveSessionProviderAndModel(config, opts) {
|
|
|
20931
22328
|
}
|
|
20932
22329
|
|
|
20933
22330
|
// src/agent/memory-store.ts
|
|
20934
|
-
import
|
|
20935
|
-
import { and as and18, desc as desc15, eq as
|
|
22331
|
+
import crypto28 from "crypto";
|
|
22332
|
+
import { and as and18, desc as desc15, eq as eq32, like as like2, sql as sql11 } from "drizzle-orm";
|
|
20936
22333
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
20937
22334
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
20938
|
-
function
|
|
22335
|
+
function rowToDto2(row) {
|
|
20939
22336
|
return {
|
|
20940
22337
|
id: row.id,
|
|
20941
22338
|
key: row.key,
|
|
@@ -20946,9 +22343,9 @@ function rowToDto(row) {
|
|
|
20946
22343
|
};
|
|
20947
22344
|
}
|
|
20948
22345
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
20949
|
-
const query = db.select().from(agentMemory).where(
|
|
22346
|
+
const query = db.select().from(agentMemory).where(eq32(agentMemory.projectId, projectId)).orderBy(desc15(agentMemory.updatedAt));
|
|
20950
22347
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
20951
|
-
return rows.map(
|
|
22348
|
+
return rows.map(rowToDto2);
|
|
20952
22349
|
}
|
|
20953
22350
|
function upsertMemoryEntry(db, args) {
|
|
20954
22351
|
if (Buffer.byteLength(args.value, "utf8") > AGENT_MEMORY_VALUE_MAX_BYTES) {
|
|
@@ -20960,7 +22357,7 @@ function upsertMemoryEntry(db, args) {
|
|
|
20960
22357
|
throw new Error(`memory key prefix "${COMPACTION_KEY_PREFIX}" is reserved for compaction notes`);
|
|
20961
22358
|
}
|
|
20962
22359
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
20963
|
-
const id =
|
|
22360
|
+
const id = crypto28.randomUUID();
|
|
20964
22361
|
db.insert(agentMemory).values({
|
|
20965
22362
|
id,
|
|
20966
22363
|
projectId: args.projectId,
|
|
@@ -20977,12 +22374,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
20977
22374
|
updatedAt: now
|
|
20978
22375
|
}
|
|
20979
22376
|
}).run();
|
|
20980
|
-
const row = db.select().from(agentMemory).where(and18(
|
|
22377
|
+
const row = db.select().from(agentMemory).where(and18(eq32(agentMemory.projectId, args.projectId), eq32(agentMemory.key, args.key))).get();
|
|
20981
22378
|
if (!row) throw new Error("memory upsert produced no row");
|
|
20982
|
-
return
|
|
22379
|
+
return rowToDto2(row);
|
|
20983
22380
|
}
|
|
20984
22381
|
function deleteMemoryEntry(db, projectId, key) {
|
|
20985
|
-
const result = db.delete(agentMemory).where(and18(
|
|
22382
|
+
const result = db.delete(agentMemory).where(and18(eq32(agentMemory.projectId, projectId), eq32(agentMemory.key, key))).run();
|
|
20986
22383
|
const changes = result.changes ?? 0;
|
|
20987
22384
|
return changes > 0;
|
|
20988
22385
|
}
|
|
@@ -20997,7 +22394,7 @@ function writeCompactionNote(db, args) {
|
|
|
20997
22394
|
}
|
|
20998
22395
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
20999
22396
|
const key = `${COMPACTION_KEY_PREFIX}${args.sessionId}:${now}`;
|
|
21000
|
-
const id =
|
|
22397
|
+
const id = crypto28.randomUUID();
|
|
21001
22398
|
let inserted;
|
|
21002
22399
|
db.transaction((tx) => {
|
|
21003
22400
|
tx.insert(agentMemory).values({
|
|
@@ -21012,16 +22409,16 @@ function writeCompactionNote(db, args) {
|
|
|
21012
22409
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
21013
22410
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
21014
22411
|
and18(
|
|
21015
|
-
|
|
22412
|
+
eq32(agentMemory.projectId, args.projectId),
|
|
21016
22413
|
like2(agentMemory.key, `${sessionPrefix}%`)
|
|
21017
22414
|
)
|
|
21018
22415
|
).orderBy(desc15(agentMemory.updatedAt)).all();
|
|
21019
22416
|
const stale = existing.slice(COMPACTION_NOTES_PER_SESSION).map((r) => r.id);
|
|
21020
22417
|
if (stale.length > 0) {
|
|
21021
|
-
tx.delete(agentMemory).where(
|
|
22418
|
+
tx.delete(agentMemory).where(sql11`${agentMemory.id} IN (${sql11.join(stale.map((s) => sql11`${s}`), sql11`, `)})`).run();
|
|
21022
22419
|
}
|
|
21023
|
-
const row = tx.select().from(agentMemory).where(and18(
|
|
21024
|
-
if (row) inserted =
|
|
22420
|
+
const row = tx.select().from(agentMemory).where(and18(eq32(agentMemory.projectId, args.projectId), eq32(agentMemory.key, key))).get();
|
|
22421
|
+
if (row) inserted = rowToDto2(row);
|
|
21025
22422
|
});
|
|
21026
22423
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
21027
22424
|
return inserted;
|
|
@@ -21202,7 +22599,7 @@ var SessionRegistry = class {
|
|
|
21202
22599
|
modelProvider: effectiveProvider,
|
|
21203
22600
|
modelId: effectiveModelId,
|
|
21204
22601
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
21205
|
-
}).where(
|
|
22602
|
+
}).where(eq33(agentSessions.projectId, projectId)).run();
|
|
21206
22603
|
}
|
|
21207
22604
|
const agent2 = createAeroSession({
|
|
21208
22605
|
projectName,
|
|
@@ -21416,7 +22813,7 @@ ${lines.join("\n")}
|
|
|
21416
22813
|
modelProvider: nextProvider,
|
|
21417
22814
|
modelId: nextModelId,
|
|
21418
22815
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
21419
|
-
}).where(
|
|
22816
|
+
}).where(eq33(agentSessions.projectId, projectId)).run();
|
|
21420
22817
|
}
|
|
21421
22818
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
21422
22819
|
save(projectName) {
|
|
@@ -21578,17 +22975,17 @@ ${lines.join("\n")}
|
|
|
21578
22975
|
return id;
|
|
21579
22976
|
}
|
|
21580
22977
|
tryResolveProjectId(projectName) {
|
|
21581
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
22978
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq33(projects.name, projectName)).get();
|
|
21582
22979
|
return row?.id;
|
|
21583
22980
|
}
|
|
21584
22981
|
loadRow(projectId) {
|
|
21585
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
22982
|
+
const row = this.opts.db.select().from(agentSessions).where(eq33(agentSessions.projectId, projectId)).get();
|
|
21586
22983
|
return row ?? null;
|
|
21587
22984
|
}
|
|
21588
22985
|
insertRow(params) {
|
|
21589
22986
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
21590
22987
|
this.opts.db.insert(agentSessions).values({
|
|
21591
|
-
id:
|
|
22988
|
+
id: crypto29.randomUUID(),
|
|
21592
22989
|
projectId: params.projectId,
|
|
21593
22990
|
systemPrompt: params.systemPrompt,
|
|
21594
22991
|
modelProvider: params.provider ?? params.modelProvider ?? AgentProviderIds.claude,
|
|
@@ -21601,14 +22998,14 @@ ${lines.join("\n")}
|
|
|
21601
22998
|
}
|
|
21602
22999
|
updateRow(projectId, patch) {
|
|
21603
23000
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
21604
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
23001
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq33(agentSessions.projectId, projectId)).run();
|
|
21605
23002
|
}
|
|
21606
23003
|
};
|
|
21607
23004
|
|
|
21608
23005
|
// src/agent/agent-routes.ts
|
|
21609
|
-
import { eq as
|
|
23006
|
+
import { eq as eq34 } from "drizzle-orm";
|
|
21610
23007
|
function resolveProject2(db, name) {
|
|
21611
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
23008
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq34(projects.name, name)).get();
|
|
21612
23009
|
if (!row) throw notFound("project", name);
|
|
21613
23010
|
return row;
|
|
21614
23011
|
}
|
|
@@ -21617,7 +23014,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
21617
23014
|
"/projects/:name/agent/transcript",
|
|
21618
23015
|
async (request) => {
|
|
21619
23016
|
const project = resolveProject2(opts.db, request.params.name);
|
|
21620
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
23017
|
+
const row = opts.db.select().from(agentSessions).where(eq34(agentSessions.projectId, project.id)).get();
|
|
21621
23018
|
if (!row) {
|
|
21622
23019
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
21623
23020
|
}
|
|
@@ -21641,7 +23038,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
21641
23038
|
async (request) => {
|
|
21642
23039
|
const project = resolveProject2(opts.db, request.params.name);
|
|
21643
23040
|
opts.sessionRegistry.reset(project.name);
|
|
21644
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
23041
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq34(agentSessions.projectId, project.id)).run();
|
|
21645
23042
|
return { status: "reset" };
|
|
21646
23043
|
}
|
|
21647
23044
|
);
|
|
@@ -22505,7 +23902,7 @@ function summarizeProviderConfig(provider, config) {
|
|
|
22505
23902
|
};
|
|
22506
23903
|
}
|
|
22507
23904
|
function hashApiKey(key) {
|
|
22508
|
-
return
|
|
23905
|
+
return crypto30.createHash("sha256").update(key).digest("hex");
|
|
22509
23906
|
}
|
|
22510
23907
|
function parseCookies2(header) {
|
|
22511
23908
|
if (!header) return {};
|
|
@@ -22663,7 +24060,7 @@ async function createServer(opts) {
|
|
|
22663
24060
|
intelligenceService,
|
|
22664
24061
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
22665
24062
|
async ({ runId, projectId, insightCount, criticalOrHigh }) => {
|
|
22666
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
24063
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq35(projects.id, projectId)).get();
|
|
22667
24064
|
if (!project) return;
|
|
22668
24065
|
sessionRegistry.queueFollowUp(project.name, {
|
|
22669
24066
|
role: "user",
|
|
@@ -22757,7 +24154,22 @@ async function createServer(opts) {
|
|
|
22757
24154
|
return removed;
|
|
22758
24155
|
}
|
|
22759
24156
|
};
|
|
22760
|
-
const
|
|
24157
|
+
const cloudRunCredentialStore = {
|
|
24158
|
+
getConnection: (projectName) => {
|
|
24159
|
+
return getCloudRunConnection(opts.config, projectName);
|
|
24160
|
+
},
|
|
24161
|
+
upsertConnection: (record) => {
|
|
24162
|
+
const updated = upsertCloudRunConnection(opts.config, record);
|
|
24163
|
+
saveConfigPatch(opts.config);
|
|
24164
|
+
return updated;
|
|
24165
|
+
},
|
|
24166
|
+
deleteConnection: (projectName) => {
|
|
24167
|
+
const removed = removeCloudRunConnection(opts.config, projectName);
|
|
24168
|
+
if (removed) saveConfigPatch(opts.config);
|
|
24169
|
+
return removed;
|
|
24170
|
+
}
|
|
24171
|
+
};
|
|
24172
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto30.randomBytes(32).toString("hex");
|
|
22761
24173
|
const googleConnectionStore = {
|
|
22762
24174
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
22763
24175
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -22803,11 +24215,11 @@ async function createServer(opts) {
|
|
|
22803
24215
|
const apiPrefix = basePath ? `${basePath}api/v1` : "/api/v1";
|
|
22804
24216
|
if (opts.config.apiKey) {
|
|
22805
24217
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
22806
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
24218
|
+
const existing = opts.db.select().from(apiKeys).where(eq35(apiKeys.keyHash, keyHash)).get();
|
|
22807
24219
|
if (!existing) {
|
|
22808
24220
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
22809
24221
|
opts.db.insert(apiKeys).values({
|
|
22810
|
-
id: `key_${
|
|
24222
|
+
id: `key_${crypto30.randomBytes(8).toString("hex")}`,
|
|
22811
24223
|
name: "default",
|
|
22812
24224
|
keyHash,
|
|
22813
24225
|
keyPrefix: prefix,
|
|
@@ -22831,7 +24243,7 @@ async function createServer(opts) {
|
|
|
22831
24243
|
};
|
|
22832
24244
|
const createSession = (apiKeyId) => {
|
|
22833
24245
|
pruneExpiredSessions();
|
|
22834
|
-
const sessionId =
|
|
24246
|
+
const sessionId = crypto30.randomBytes(32).toString("hex");
|
|
22835
24247
|
sessions.set(sessionId, {
|
|
22836
24248
|
apiKeyId,
|
|
22837
24249
|
expiresAt: Date.now() + SESSION_TTL_MS
|
|
@@ -22855,7 +24267,7 @@ async function createServer(opts) {
|
|
|
22855
24267
|
};
|
|
22856
24268
|
const getDefaultApiKey = () => {
|
|
22857
24269
|
if (!opts.config.apiKey) return void 0;
|
|
22858
|
-
return opts.db.select().from(apiKeys).where(
|
|
24270
|
+
return opts.db.select().from(apiKeys).where(eq35(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
22859
24271
|
};
|
|
22860
24272
|
const createPasswordSession = (reply) => {
|
|
22861
24273
|
const key = getDefaultApiKey();
|
|
@@ -22912,12 +24324,12 @@ async function createServer(opts) {
|
|
|
22912
24324
|
return reply.send({ authenticated: true });
|
|
22913
24325
|
}
|
|
22914
24326
|
if (apiKey) {
|
|
22915
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
24327
|
+
const key = opts.db.select().from(apiKeys).where(eq35(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
22916
24328
|
if (!key || key.revokedAt) {
|
|
22917
24329
|
const err2 = authInvalid();
|
|
22918
24330
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
22919
24331
|
}
|
|
22920
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
24332
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq35(apiKeys.id, key.id)).run();
|
|
22921
24333
|
const sessionId = createSession(key.id);
|
|
22922
24334
|
reply.header("set-cookie", serializeSessionCookie({
|
|
22923
24335
|
name: SESSION_COOKIE_NAME,
|
|
@@ -23027,7 +24439,7 @@ async function createServer(opts) {
|
|
|
23027
24439
|
deps: {
|
|
23028
24440
|
enqueueAutoExtract: ({ projectId, release: r }) => {
|
|
23029
24441
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
23030
|
-
const runId =
|
|
24442
|
+
const runId = crypto30.randomUUID();
|
|
23031
24443
|
opts.db.insert(runs).values({
|
|
23032
24444
|
id: runId,
|
|
23033
24445
|
projectId,
|
|
@@ -23100,6 +24512,7 @@ async function createServer(opts) {
|
|
|
23100
24512
|
},
|
|
23101
24513
|
wordpressConnectionStore,
|
|
23102
24514
|
ga4CredentialStore,
|
|
24515
|
+
cloudRunCredentialStore,
|
|
23103
24516
|
onRunCreated: (runId, projectId, providers2, location) => {
|
|
23104
24517
|
jobRunner.executeRun(runId, projectId, providers2, location).catch((err) => {
|
|
23105
24518
|
app.log.error({ runId, err }, "Job runner failed");
|
|
@@ -23162,7 +24575,7 @@ async function createServer(opts) {
|
|
|
23162
24575
|
const targetProjectIds = affectedProjectIds.length > 0 ? affectedProjectIds : [null];
|
|
23163
24576
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
23164
24577
|
opts.db.insert(auditLog).values(targetProjectIds.map((projectId) => ({
|
|
23165
|
-
id:
|
|
24578
|
+
id: crypto30.randomUUID(),
|
|
23166
24579
|
projectId,
|
|
23167
24580
|
actor: "api",
|
|
23168
24581
|
action: existing ? "provider.updated" : "provider.created",
|