@ainyc/canonry 4.137.0 → 4.139.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +9 -13
- package/assets/assets/{AuditHistoryPanel-B6GEmlVM.js → AuditHistoryPanel-DL9chF3F.js} +1 -1
- package/assets/assets/BacklinksPage-sA_tgQJv.js +1 -0
- package/assets/assets/ChartPrimitives-D7nLxIbe.js +1 -0
- package/assets/assets/HistoryPage-CLWbcu5m.js +1 -0
- package/assets/assets/ProjectPage-C_oXMCNU.js +8 -0
- package/assets/assets/{RunRow-MsV1RLPm.js → RunRow-DQ4icEiv.js} +1 -1
- package/assets/assets/RunsPage-D5sizm9T.js +1 -0
- package/assets/assets/SettingsPage-XdQjvWsg.js +1 -0
- package/assets/assets/TrafficPage-C12osXDa.js +1 -0
- package/assets/assets/TrafficSourceDetailPage-CXAiNK8Y.js +1 -0
- package/assets/assets/{arrow-left-CHU6s_Xn.js → arrow-left-LaI5a16g.js} +1 -1
- package/assets/assets/extract-error-message-CYWaTM3m.js +1 -0
- package/assets/assets/index-CguwySA5.js +213 -0
- package/assets/assets/index-CntuM6Th.css +1 -0
- package/assets/assets/trash-2-DoH00q7m.js +1 -0
- package/assets/index.html +2 -2
- package/dist/{chunk-XEX5CLQW.js → chunk-DLKUDNDA.js} +652 -436
- package/dist/{chunk-PBTB3FCN.js → chunk-IGU67QFM.js} +1677 -1152
- package/dist/{chunk-UVWI6VPP.js → chunk-IYTHIAAE.js} +5 -25
- package/dist/{chunk-7RF5EDUO.js → chunk-OA2EGCUS.js} +3094 -2959
- package/dist/cli.js +10 -38
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-JEFHUGVX.js → intelligence-service-SMYVN3KN.js} +2 -2
- package/dist/mcp.js +2 -2
- package/package.json +7 -7
- package/assets/assets/BacklinksPage-Cr3QegNo.js +0 -1
- package/assets/assets/ChartPrimitives-rhsGzU1A.js +0 -1
- package/assets/assets/HistoryPage-B3HrxyPP.js +0 -1
- package/assets/assets/ProjectPage-D3WiL3RI.js +0 -8
- package/assets/assets/RunsPage-BVlv0j3Y.js +0 -1
- package/assets/assets/SettingsPage-Dfd-T4o9.js +0 -1
- package/assets/assets/TrafficPage-Bl1jj73m.js +0 -1
- package/assets/assets/TrafficSourceDetailPage-CinQcKmY.js +0 -1
- package/assets/assets/extract-error-message-BDdyZgyf.js +0 -1
- package/assets/assets/index-BNulljlk.js +0 -213
- package/assets/assets/index-CglduHCN.css +0 -1
- package/assets/assets/trash-2-BE5oH1Tj.js +0 -1
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
loadConfig,
|
|
11
11
|
loadConfigRaw,
|
|
12
12
|
saveConfigPatch
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-IYTHIAAE.js";
|
|
14
14
|
import {
|
|
15
15
|
CC_CACHE_DIR,
|
|
16
16
|
DUCKDB_SPEC,
|
|
@@ -50,6 +50,7 @@ import {
|
|
|
50
50
|
deliverWebhook,
|
|
51
51
|
determineCitationState,
|
|
52
52
|
discoverySessions,
|
|
53
|
+
doctorHealthState,
|
|
53
54
|
downloadFile,
|
|
54
55
|
dropLegacyCredentialColumns,
|
|
55
56
|
executeDiscovery,
|
|
@@ -70,11 +71,9 @@ import {
|
|
|
70
71
|
gbpPlaceDetails,
|
|
71
72
|
getAttributes,
|
|
72
73
|
getCrawlIssues,
|
|
73
|
-
getLinkCounts,
|
|
74
74
|
getLodging,
|
|
75
75
|
getPlaceDetails,
|
|
76
76
|
getUrlInfo,
|
|
77
|
-
getUrlLinks,
|
|
78
77
|
groupRunsByCreatedAt,
|
|
79
78
|
gscCoverageSnapshots,
|
|
80
79
|
gscDailyTotals,
|
|
@@ -113,13 +112,16 @@ import {
|
|
|
113
112
|
refreshAccessToken,
|
|
114
113
|
researchRunQueries,
|
|
115
114
|
researchRuns,
|
|
115
|
+
resolveDestination,
|
|
116
|
+
resolveVercelSyncDeadlineMs,
|
|
116
117
|
resolveWebhookTarget,
|
|
117
118
|
runs,
|
|
118
119
|
schedules,
|
|
119
120
|
siteAuditPages,
|
|
120
121
|
siteAuditSnapshots,
|
|
122
|
+
toAlertView,
|
|
121
123
|
usageCounters
|
|
122
|
-
} from "./chunk-
|
|
124
|
+
} from "./chunk-IGU67QFM.js";
|
|
123
125
|
import {
|
|
124
126
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
125
127
|
AGENT_PROVIDER_IDS,
|
|
@@ -129,6 +131,7 @@ import {
|
|
|
129
131
|
AdsCampaignBiddingTypes,
|
|
130
132
|
AgentProviderIds,
|
|
131
133
|
BacklinkSources,
|
|
134
|
+
CITED_URL_CAPTURE_VERSION,
|
|
132
135
|
CcReleaseSyncStatuses,
|
|
133
136
|
CodingAgents,
|
|
134
137
|
DiscoveryCompetitorTypes,
|
|
@@ -137,6 +140,7 @@ import {
|
|
|
137
140
|
MemorySources,
|
|
138
141
|
ONBOARDING_FLOW_VERSION,
|
|
139
142
|
ProviderNames,
|
|
143
|
+
ROUTE_CAPABLE_CITED_URL_PROVIDERS,
|
|
140
144
|
ResearchQueryStatuses,
|
|
141
145
|
ResearchRunStatuses,
|
|
142
146
|
RunKinds,
|
|
@@ -167,12 +171,14 @@ import {
|
|
|
167
171
|
computeBacklinkSummaryMetrics,
|
|
168
172
|
contentActionLabel,
|
|
169
173
|
contentBriefDtoSchema,
|
|
174
|
+
deriveCitedUrlCandidates,
|
|
170
175
|
determineAnswerMentioned,
|
|
171
176
|
dollarsToMicros,
|
|
172
177
|
effectiveBrandNames,
|
|
173
178
|
effectiveDomains,
|
|
174
179
|
embedClientConfigForRequest,
|
|
175
180
|
factorStatusFromScore,
|
|
181
|
+
filterCapturedCitedUrls,
|
|
176
182
|
formatIsoDateInTimeZone,
|
|
177
183
|
frameAncestorsHeaderValue,
|
|
178
184
|
hostMatchesAnyDomain,
|
|
@@ -182,6 +188,7 @@ import {
|
|
|
182
188
|
isBrowserProvider,
|
|
183
189
|
isGhostTelemetryEvent,
|
|
184
190
|
isRetryableHttpError,
|
|
191
|
+
isVertexGroundingRedirect,
|
|
185
192
|
mapWithConcurrency,
|
|
186
193
|
normalizeIdTokens,
|
|
187
194
|
normalizeServedModel,
|
|
@@ -201,7 +208,7 @@ import {
|
|
|
201
208
|
validationError,
|
|
202
209
|
winnabilityClassLabel,
|
|
203
210
|
withRetry
|
|
204
|
-
} from "./chunk-
|
|
211
|
+
} from "./chunk-OA2EGCUS.js";
|
|
205
212
|
|
|
206
213
|
// src/telemetry.ts
|
|
207
214
|
import crypto from "crypto";
|
|
@@ -532,11 +539,11 @@ function checkLatestVersionForServer(opts) {
|
|
|
532
539
|
|
|
533
540
|
// src/server.ts
|
|
534
541
|
import { createRequire as createRequire3 } from "module";
|
|
535
|
-
import
|
|
542
|
+
import crypto22 from "crypto";
|
|
536
543
|
import fs8 from "fs";
|
|
537
544
|
import path9 from "path";
|
|
538
545
|
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
539
|
-
import { and as
|
|
546
|
+
import { and as and15, eq as eq21 } from "drizzle-orm";
|
|
540
547
|
import Fastify from "fastify";
|
|
541
548
|
import os5 from "os";
|
|
542
549
|
|
|
@@ -953,7 +960,14 @@ var geminiAdapter = {
|
|
|
953
960
|
model: raw.model,
|
|
954
961
|
servedModel: raw.servedModel,
|
|
955
962
|
groundingSources: raw.groundingSources,
|
|
956
|
-
searchQueries: raw.searchQueries
|
|
963
|
+
searchQueries: raw.searchQueries,
|
|
964
|
+
// Retrieval detection is not implemented for this provider. Its candidate
|
|
965
|
+
// marker is present on 100% of stored rows, so it has never been shown to
|
|
966
|
+
// discriminate a non-retrieving answer and wiring it up would hardcode
|
|
967
|
+
// `used`. `unknown` states what we actually know. The contract is a
|
|
968
|
+
// declaration about how we build the request, so it is always knowable.
|
|
969
|
+
retrievalStatus: "unknown",
|
|
970
|
+
retrievalContract: "native-auto-v1"
|
|
957
971
|
};
|
|
958
972
|
},
|
|
959
973
|
normalizeResult(raw) {
|
|
@@ -970,7 +984,8 @@ var geminiAdapter = {
|
|
|
970
984
|
answerText: normalized.answerText,
|
|
971
985
|
citedDomains: normalized.citedDomains,
|
|
972
986
|
groundingSources: normalized.groundingSources,
|
|
973
|
-
searchQueries: normalized.searchQueries
|
|
987
|
+
searchQueries: normalized.searchQueries,
|
|
988
|
+
retrievalStatus: "unknown"
|
|
974
989
|
};
|
|
975
990
|
},
|
|
976
991
|
async generateText(prompt, config) {
|
|
@@ -1295,7 +1310,14 @@ var openaiAdapter = {
|
|
|
1295
1310
|
model: raw.model,
|
|
1296
1311
|
servedModel: raw.servedModel,
|
|
1297
1312
|
groundingSources: raw.groundingSources,
|
|
1298
|
-
searchQueries: raw.searchQueries
|
|
1313
|
+
searchQueries: raw.searchQueries,
|
|
1314
|
+
// Retrieval detection is not implemented for this provider. Its candidate
|
|
1315
|
+
// marker is present on 100% of stored rows, so it has never been shown to
|
|
1316
|
+
// discriminate a non-retrieving answer and wiring it up would hardcode
|
|
1317
|
+
// `used`. `unknown` states what we actually know. The contract is a
|
|
1318
|
+
// declaration about how we build the request, so it is always knowable.
|
|
1319
|
+
retrievalStatus: "unknown",
|
|
1320
|
+
retrievalContract: "native-auto-v1"
|
|
1299
1321
|
};
|
|
1300
1322
|
},
|
|
1301
1323
|
normalizeResult(raw) {
|
|
@@ -1312,7 +1334,8 @@ var openaiAdapter = {
|
|
|
1312
1334
|
answerText: normalized.answerText,
|
|
1313
1335
|
citedDomains: normalized.citedDomains,
|
|
1314
1336
|
groundingSources: normalized.groundingSources,
|
|
1315
|
-
searchQueries: normalized.searchQueries
|
|
1337
|
+
searchQueries: normalized.searchQueries,
|
|
1338
|
+
retrievalStatus: "unknown"
|
|
1316
1339
|
};
|
|
1317
1340
|
},
|
|
1318
1341
|
async generateText(prompt, config) {
|
|
@@ -1345,6 +1368,7 @@ async function withRetry4(fn, options = {}) {
|
|
|
1345
1368
|
// ../provider-claude/src/normalize.ts
|
|
1346
1369
|
var DEFAULT_MODEL3 = "claude-sonnet-4-6";
|
|
1347
1370
|
var VALIDATION_PATTERN = /^claude-/;
|
|
1371
|
+
var CLAUDE_RETRIEVAL_CONTRACT = "search-required-v1";
|
|
1348
1372
|
function resolveModel2(config) {
|
|
1349
1373
|
const m = config.model;
|
|
1350
1374
|
if (!m) return DEFAULT_MODEL3;
|
|
@@ -1419,6 +1443,9 @@ async function executeTrackedQuery3(input) {
|
|
|
1419
1443
|
model,
|
|
1420
1444
|
max_tokens: 4096,
|
|
1421
1445
|
tools: [webSearchTool],
|
|
1446
|
+
// search-required-v1: retrieval is guaranteed by the API control, so the
|
|
1447
|
+
// query text and answer substance stay untouched. See the contract note.
|
|
1448
|
+
tool_choice: { type: "tool", name: "web_search" },
|
|
1422
1449
|
messages: [{ role: "user", content: input.query }]
|
|
1423
1450
|
})
|
|
1424
1451
|
);
|
|
@@ -1433,7 +1460,9 @@ async function executeTrackedQuery3(input) {
|
|
|
1433
1460
|
model,
|
|
1434
1461
|
servedModel: extractServedModel3(rawResponse),
|
|
1435
1462
|
groundingSources: parsed.groundingSources,
|
|
1436
|
-
searchQueries: parsed.searchQueries
|
|
1463
|
+
searchQueries: parsed.searchQueries,
|
|
1464
|
+
retrievalStatus: parsed.retrievalStatus,
|
|
1465
|
+
retrievalContract: CLAUDE_RETRIEVAL_CONTRACT
|
|
1437
1466
|
};
|
|
1438
1467
|
} catch (err) {
|
|
1439
1468
|
const msg = err instanceof Error ? err.message : String(err);
|
|
@@ -1451,7 +1480,8 @@ function normalizeResult3(raw) {
|
|
|
1451
1480
|
answerText: parsed.answerText,
|
|
1452
1481
|
citedDomains,
|
|
1453
1482
|
groundingSources,
|
|
1454
|
-
searchQueries
|
|
1483
|
+
searchQueries,
|
|
1484
|
+
retrievalStatus: useParsed ? parsed.retrievalStatus : raw.retrievalStatus
|
|
1455
1485
|
};
|
|
1456
1486
|
}
|
|
1457
1487
|
function hasParsedResponseContent3(rawResponse) {
|
|
@@ -1470,6 +1500,7 @@ function reparseStoredResult3(rawResponse) {
|
|
|
1470
1500
|
citedDomains: extractCitedDomainsFromSources3(groundingSources),
|
|
1471
1501
|
groundingSources,
|
|
1472
1502
|
searchQueries,
|
|
1503
|
+
retrievalStatus: extractRetrievalStatusFromRaw(rawResponse),
|
|
1473
1504
|
...providerErrors.length > 0 ? { providerError: `web_search tool error: ${providerErrors.join(", ")}` } : {}
|
|
1474
1505
|
};
|
|
1475
1506
|
}
|
|
@@ -1509,6 +1540,17 @@ function extractGroundingSourcesFromRaw2(rawResponse) {
|
|
|
1509
1540
|
}
|
|
1510
1541
|
return sources;
|
|
1511
1542
|
}
|
|
1543
|
+
function extractRetrievalStatusFromRaw(rawResponse) {
|
|
1544
|
+
try {
|
|
1545
|
+
const content = rawResponse.content;
|
|
1546
|
+
if (!Array.isArray(content) || content.length === 0) return "unknown";
|
|
1547
|
+
return content.some(
|
|
1548
|
+
(block) => block.type === "server_tool_use" && block.name === "web_search"
|
|
1549
|
+
) ? "used" : "not-used";
|
|
1550
|
+
} catch {
|
|
1551
|
+
return "unknown";
|
|
1552
|
+
}
|
|
1553
|
+
}
|
|
1512
1554
|
function extractSearchQueriesFromRaw3(rawResponse) {
|
|
1513
1555
|
const queries2 = /* @__PURE__ */ new Set();
|
|
1514
1556
|
try {
|
|
@@ -1656,7 +1698,9 @@ var claudeAdapter = {
|
|
|
1656
1698
|
model: raw.model,
|
|
1657
1699
|
servedModel: raw.servedModel,
|
|
1658
1700
|
groundingSources: raw.groundingSources,
|
|
1659
|
-
searchQueries: raw.searchQueries
|
|
1701
|
+
searchQueries: raw.searchQueries,
|
|
1702
|
+
retrievalStatus: raw.retrievalStatus,
|
|
1703
|
+
retrievalContract: raw.retrievalContract
|
|
1660
1704
|
};
|
|
1661
1705
|
},
|
|
1662
1706
|
normalizeResult(raw) {
|
|
@@ -1665,7 +1709,12 @@ var claudeAdapter = {
|
|
|
1665
1709
|
rawResponse: raw.rawResponse,
|
|
1666
1710
|
model: raw.model,
|
|
1667
1711
|
groundingSources: raw.groundingSources,
|
|
1668
|
-
searchQueries: raw.searchQueries
|
|
1712
|
+
searchQueries: raw.searchQueries,
|
|
1713
|
+
// The shared RawQueryResult now carries retrieval, so nothing is lost
|
|
1714
|
+
// across this boundary. A reconstruction that predates the field falls
|
|
1715
|
+
// back to `unknown` rather than asserting an absence.
|
|
1716
|
+
retrievalStatus: raw.retrievalStatus ?? "unknown",
|
|
1717
|
+
retrievalContract: raw.retrievalContract ?? CLAUDE_RETRIEVAL_CONTRACT
|
|
1669
1718
|
};
|
|
1670
1719
|
const normalized = normalizeResult3(claudeRaw);
|
|
1671
1720
|
return {
|
|
@@ -1673,7 +1722,8 @@ var claudeAdapter = {
|
|
|
1673
1722
|
answerText: normalized.answerText,
|
|
1674
1723
|
citedDomains: normalized.citedDomains,
|
|
1675
1724
|
groundingSources: normalized.groundingSources,
|
|
1676
|
-
searchQueries: normalized.searchQueries
|
|
1725
|
+
searchQueries: normalized.searchQueries,
|
|
1726
|
+
retrievalStatus: normalized.retrievalStatus
|
|
1677
1727
|
};
|
|
1678
1728
|
},
|
|
1679
1729
|
async generateText(prompt, config) {
|
|
@@ -1889,7 +1939,11 @@ var localAdapter = {
|
|
|
1889
1939
|
model: raw.model,
|
|
1890
1940
|
servedModel: raw.servedModel,
|
|
1891
1941
|
groundingSources: raw.groundingSources,
|
|
1892
|
-
searchQueries: raw.searchQueries
|
|
1942
|
+
searchQueries: raw.searchQueries,
|
|
1943
|
+
// A plain chat completion with no web access: there is no retrieval step
|
|
1944
|
+
// to report, which is what `not-applicable` means.
|
|
1945
|
+
retrievalStatus: "not-applicable",
|
|
1946
|
+
retrievalContract: "native-auto-v1"
|
|
1893
1947
|
};
|
|
1894
1948
|
},
|
|
1895
1949
|
normalizeResult(raw) {
|
|
@@ -1906,7 +1960,8 @@ var localAdapter = {
|
|
|
1906
1960
|
answerText: normalized.answerText,
|
|
1907
1961
|
citedDomains: normalized.citedDomains,
|
|
1908
1962
|
groundingSources: normalized.groundingSources,
|
|
1909
|
-
searchQueries: normalized.searchQueries
|
|
1963
|
+
searchQueries: normalized.searchQueries,
|
|
1964
|
+
retrievalStatus: "not-applicable"
|
|
1910
1965
|
};
|
|
1911
1966
|
},
|
|
1912
1967
|
async generateText(prompt, config) {
|
|
@@ -2355,7 +2410,8 @@ function normalizeResult5(raw) {
|
|
|
2355
2410
|
answerText,
|
|
2356
2411
|
citedDomains: extractCitedDomains(raw.groundingSources),
|
|
2357
2412
|
groundingSources: raw.groundingSources,
|
|
2358
|
-
searchQueries: raw.searchQueries
|
|
2413
|
+
searchQueries: raw.searchQueries,
|
|
2414
|
+
retrievalStatus: raw.retrievalStatus ?? "unknown"
|
|
2359
2415
|
};
|
|
2360
2416
|
}
|
|
2361
2417
|
|
|
@@ -2465,6 +2521,12 @@ var cdpChatgptAdapter = {
|
|
|
2465
2521
|
model: "chatgpt-web",
|
|
2466
2522
|
groundingSources,
|
|
2467
2523
|
searchQueries: [input.query],
|
|
2524
|
+
// This surface is scraped from the rendered web UI, which exposes no
|
|
2525
|
+
// reliable signal for whether a retrieval step ran behind the answer.
|
|
2526
|
+
// `searchQueries` here is the query we submitted, not evidence of a
|
|
2527
|
+
// search the product performed.
|
|
2528
|
+
retrievalStatus: "unknown",
|
|
2529
|
+
retrievalContract: "native-auto-v1",
|
|
2468
2530
|
screenshotPath: capturedScreenshotPath
|
|
2469
2531
|
};
|
|
2470
2532
|
} catch (err) {
|
|
@@ -2779,7 +2841,14 @@ var perplexityAdapter = {
|
|
|
2779
2841
|
model: raw.model,
|
|
2780
2842
|
servedModel: raw.servedModel,
|
|
2781
2843
|
groundingSources: raw.groundingSources,
|
|
2782
|
-
searchQueries: raw.searchQueries
|
|
2844
|
+
searchQueries: raw.searchQueries,
|
|
2845
|
+
// Retrieval detection is not implemented for this provider. Its candidate
|
|
2846
|
+
// marker is present on 100% of stored rows, so it has never been shown to
|
|
2847
|
+
// discriminate a non-retrieving answer and wiring it up would hardcode
|
|
2848
|
+
// `used`. `unknown` states what we actually know. The contract is a
|
|
2849
|
+
// declaration about how we build the request, so it is always knowable.
|
|
2850
|
+
retrievalStatus: "unknown",
|
|
2851
|
+
retrievalContract: "native-auto-v1"
|
|
2783
2852
|
};
|
|
2784
2853
|
},
|
|
2785
2854
|
normalizeResult(raw) {
|
|
@@ -2796,7 +2865,8 @@ var perplexityAdapter = {
|
|
|
2796
2865
|
answerText: normalized.answerText,
|
|
2797
2866
|
citedDomains: normalized.citedDomains,
|
|
2798
2867
|
groundingSources: normalized.groundingSources,
|
|
2799
|
-
searchQueries: normalized.searchQueries
|
|
2868
|
+
searchQueries: normalized.searchQueries,
|
|
2869
|
+
retrievalStatus: "unknown"
|
|
2800
2870
|
};
|
|
2801
2871
|
},
|
|
2802
2872
|
async generateText(prompt, config) {
|
|
@@ -3246,6 +3316,87 @@ function releaseDailyQueryQuota(db, input) {
|
|
|
3246
3316
|
db.update(usageCounters).set({ count: sql`MAX(0, ${usageCounters.count} - ${input.count})`, updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and(eq(usageCounters.scope, input.scope), eq(usageCounters.period, input.period), eq(usageCounters.metric, "queries"))).run();
|
|
3247
3317
|
}
|
|
3248
3318
|
|
|
3319
|
+
// src/cited-url-capture.ts
|
|
3320
|
+
var RESOLUTION_CONCURRENCY = 5;
|
|
3321
|
+
var RESOLUTION_TIMEOUT_MS = 5e3;
|
|
3322
|
+
var MAX_VERTEX_PROXY_HOPS = 3;
|
|
3323
|
+
var REDIRECT_STATUSES = /* @__PURE__ */ new Set([301, 302, 303, 307, 308]);
|
|
3324
|
+
function isRouteCapableProvider(provider) {
|
|
3325
|
+
return ROUTE_CAPABLE_CITED_URL_PROVIDERS.includes(provider);
|
|
3326
|
+
}
|
|
3327
|
+
function parseHttpUrl(value, base) {
|
|
3328
|
+
try {
|
|
3329
|
+
const url = new URL(value, base);
|
|
3330
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") return void 0;
|
|
3331
|
+
url.hash = "";
|
|
3332
|
+
return url;
|
|
3333
|
+
} catch {
|
|
3334
|
+
return void 0;
|
|
3335
|
+
}
|
|
3336
|
+
}
|
|
3337
|
+
async function resolveVertexGroundingRedirect(value, opts = {}) {
|
|
3338
|
+
const initial = parseHttpUrl(value);
|
|
3339
|
+
if (!initial || !isVertexGroundingRedirect(initial)) return void 0;
|
|
3340
|
+
const fetchImpl = opts.fetchImpl ?? globalThis.fetch;
|
|
3341
|
+
const timeoutMs = opts.timeoutMs ?? RESOLUTION_TIMEOUT_MS;
|
|
3342
|
+
let current = initial;
|
|
3343
|
+
for (let hop = 0; hop <= MAX_VERTEX_PROXY_HOPS; hop++) {
|
|
3344
|
+
const controller = new AbortController();
|
|
3345
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
3346
|
+
let response;
|
|
3347
|
+
try {
|
|
3348
|
+
response = await fetchImpl(current, { redirect: "manual", signal: controller.signal });
|
|
3349
|
+
} catch {
|
|
3350
|
+
return void 0;
|
|
3351
|
+
} finally {
|
|
3352
|
+
clearTimeout(timer);
|
|
3353
|
+
}
|
|
3354
|
+
if (!REDIRECT_STATUSES.has(response.status)) return void 0;
|
|
3355
|
+
const location = response.headers.get("location");
|
|
3356
|
+
if (!location) return void 0;
|
|
3357
|
+
const next = parseHttpUrl(location, current.toString());
|
|
3358
|
+
if (!next) return void 0;
|
|
3359
|
+
if (!isVertexGroundingRedirect(next)) return next.toString();
|
|
3360
|
+
current = next;
|
|
3361
|
+
}
|
|
3362
|
+
return void 0;
|
|
3363
|
+
}
|
|
3364
|
+
async function captureCitedUrls(provider, sources, opts = {}) {
|
|
3365
|
+
const sourceCount = sources.length;
|
|
3366
|
+
if (!isRouteCapableProvider(provider)) {
|
|
3367
|
+
return {
|
|
3368
|
+
citedUrls: null,
|
|
3369
|
+
captureStatus: "unsupported",
|
|
3370
|
+
sourceCount,
|
|
3371
|
+
resolvedCount: 0,
|
|
3372
|
+
captureVersion: CITED_URL_CAPTURE_VERSION
|
|
3373
|
+
};
|
|
3374
|
+
}
|
|
3375
|
+
const candidates = deriveCitedUrlCandidates(sources);
|
|
3376
|
+
const candidateCount = candidates.length;
|
|
3377
|
+
let resolved;
|
|
3378
|
+
try {
|
|
3379
|
+
resolved = await mapWithConcurrency(candidates, RESOLUTION_CONCURRENCY, async (candidate) => {
|
|
3380
|
+
const parsed = parseHttpUrl(candidate);
|
|
3381
|
+
if (!parsed) return void 0;
|
|
3382
|
+
if (!isVertexGroundingRedirect(parsed)) return candidate;
|
|
3383
|
+
return resolveVertexGroundingRedirect(candidate, opts);
|
|
3384
|
+
});
|
|
3385
|
+
} catch {
|
|
3386
|
+
resolved = candidates.map(() => void 0);
|
|
3387
|
+
}
|
|
3388
|
+
const acceptedPerSource = resolved.map((value) => filterCapturedCitedUrls([value]).at(0));
|
|
3389
|
+
const resolvedCount = acceptedPerSource.filter((value) => value !== void 0).length;
|
|
3390
|
+
const captureStatus = resolvedCount === candidateCount ? "complete" : resolvedCount > 0 ? "partial" : "failed";
|
|
3391
|
+
return {
|
|
3392
|
+
citedUrls: filterCapturedCitedUrls(acceptedPerSource),
|
|
3393
|
+
captureStatus,
|
|
3394
|
+
sourceCount,
|
|
3395
|
+
resolvedCount,
|
|
3396
|
+
captureVersion: CITED_URL_CAPTURE_VERSION
|
|
3397
|
+
};
|
|
3398
|
+
}
|
|
3399
|
+
|
|
3249
3400
|
// src/job-runner.ts
|
|
3250
3401
|
var log = createLogger("JobRunner");
|
|
3251
3402
|
var RunCancelledError = class extends Error {
|
|
@@ -3469,7 +3620,31 @@ var JobRunner = class {
|
|
|
3469
3620
|
config
|
|
3470
3621
|
);
|
|
3471
3622
|
this.throwIfRunCancelled(runId);
|
|
3472
|
-
const
|
|
3623
|
+
const providerResult = adapter.normalizeResult(raw);
|
|
3624
|
+
const rawGroundingSources = providerResult.groundingSources;
|
|
3625
|
+
const normalized = {
|
|
3626
|
+
...providerResult,
|
|
3627
|
+
groundingSources: Array.isArray(rawGroundingSources) ? rawGroundingSources : []
|
|
3628
|
+
};
|
|
3629
|
+
let citedUrlCapture;
|
|
3630
|
+
try {
|
|
3631
|
+
citedUrlCapture = await captureCitedUrls(providerName, rawGroundingSources);
|
|
3632
|
+
} catch (err) {
|
|
3633
|
+
citedUrlCapture = {
|
|
3634
|
+
citedUrls: [],
|
|
3635
|
+
captureStatus: "failed",
|
|
3636
|
+
sourceCount: normalized.groundingSources.length,
|
|
3637
|
+
resolvedCount: 0,
|
|
3638
|
+
captureVersion: CITED_URL_CAPTURE_VERSION
|
|
3639
|
+
};
|
|
3640
|
+
log.warn("query.cited-url-capture-failed", {
|
|
3641
|
+
runId,
|
|
3642
|
+
provider: providerName,
|
|
3643
|
+
query: q.query,
|
|
3644
|
+
error: err instanceof Error ? err.message : String(err)
|
|
3645
|
+
});
|
|
3646
|
+
}
|
|
3647
|
+
this.throwIfRunCancelled(runId);
|
|
3473
3648
|
log.info("query.result", { runId, provider: providerName, query: q.query, citedDomains: normalized.citedDomains, groundingSources: normalized.groundingSources.map((s) => s.uri), matchDomains: allDomains });
|
|
3474
3649
|
const citationState = determineCitationState(normalized, allDomains);
|
|
3475
3650
|
const answerMentioned = determineAnswerMentioned(
|
|
@@ -3505,6 +3680,16 @@ var JobRunner = class {
|
|
|
3505
3680
|
answerMentioned,
|
|
3506
3681
|
answerText: normalized.answerText,
|
|
3507
3682
|
citedDomains: normalized.citedDomains,
|
|
3683
|
+
citedUrls: citedUrlCapture.citedUrls,
|
|
3684
|
+
captureStatus: citedUrlCapture.captureStatus,
|
|
3685
|
+
sourceCount: citedUrlCapture.sourceCount,
|
|
3686
|
+
resolvedCount: citedUrlCapture.resolvedCount,
|
|
3687
|
+
captureVersion: citedUrlCapture.captureVersion,
|
|
3688
|
+
// Retrieval is the adapter's own observation, never inferred
|
|
3689
|
+
// from citation counts. Both branches record it so no snapshot
|
|
3690
|
+
// can be written unmarked.
|
|
3691
|
+
retrievalStatus: normalized.retrievalStatus,
|
|
3692
|
+
retrievalContract: raw.retrievalContract,
|
|
3508
3693
|
competitorOverlap: overlap,
|
|
3509
3694
|
recommendedCompetitors: extractedCompetitors,
|
|
3510
3695
|
location: runLocation?.label ?? null,
|
|
@@ -3531,6 +3716,16 @@ var JobRunner = class {
|
|
|
3531
3716
|
answerMentioned,
|
|
3532
3717
|
answerText: normalized.answerText,
|
|
3533
3718
|
citedDomains: normalized.citedDomains,
|
|
3719
|
+
citedUrls: citedUrlCapture.citedUrls,
|
|
3720
|
+
captureStatus: citedUrlCapture.captureStatus,
|
|
3721
|
+
sourceCount: citedUrlCapture.sourceCount,
|
|
3722
|
+
resolvedCount: citedUrlCapture.resolvedCount,
|
|
3723
|
+
captureVersion: citedUrlCapture.captureVersion,
|
|
3724
|
+
// Retrieval is the adapter's own observation, never inferred
|
|
3725
|
+
// from citation counts. Both branches record it so no snapshot
|
|
3726
|
+
// can be written unmarked.
|
|
3727
|
+
retrievalStatus: normalized.retrievalStatus,
|
|
3728
|
+
retrievalContract: raw.retrievalContract,
|
|
3534
3729
|
competitorOverlap: overlap,
|
|
3535
3730
|
recommendedCompetitors: extractedCompetitors,
|
|
3536
3731
|
location: runLocation?.label ?? null,
|
|
@@ -5967,155 +6162,10 @@ function computeSummary2(rows) {
|
|
|
5967
6162
|
return computeBacklinkSummaryMetrics(rows);
|
|
5968
6163
|
}
|
|
5969
6164
|
|
|
5970
|
-
// src/
|
|
6165
|
+
// src/discovery-run.ts
|
|
5971
6166
|
import crypto13 from "crypto";
|
|
5972
6167
|
import { and as and9, eq as eq11 } from "drizzle-orm";
|
|
5973
|
-
var log11 = createLogger("
|
|
5974
|
-
var DEFAULT_MAX_PAGES = 200;
|
|
5975
|
-
var LINK_COUNTS_MAX_PAGES = 50;
|
|
5976
|
-
var URL_LINKS_MAX_PAGES = 20;
|
|
5977
|
-
function bingReleaseId(date) {
|
|
5978
|
-
return `bing-${date.toISOString().slice(0, 10)}`;
|
|
5979
|
-
}
|
|
5980
|
-
function aggregateInboundLinksByDomain(links, targetDomain) {
|
|
5981
|
-
const target = hostOf(targetDomain);
|
|
5982
|
-
const byHost = /* @__PURE__ */ new Map();
|
|
5983
|
-
for (const link of links) {
|
|
5984
|
-
const host = hostOf(link.Url);
|
|
5985
|
-
if (!host) continue;
|
|
5986
|
-
if (target && hostMatchesDomain(host, target)) continue;
|
|
5987
|
-
let urls = byHost.get(host);
|
|
5988
|
-
if (!urls) {
|
|
5989
|
-
urls = /* @__PURE__ */ new Set();
|
|
5990
|
-
byHost.set(host, urls);
|
|
5991
|
-
}
|
|
5992
|
-
urls.add(link.Url);
|
|
5993
|
-
}
|
|
5994
|
-
return [...byHost.entries()].map(([linkingDomain, urls]) => ({ linkingDomain, numHosts: urls.size })).sort((a, b) => b.numHosts - a.numHosts || a.linkingDomain.localeCompare(b.linkingDomain));
|
|
5995
|
-
}
|
|
5996
|
-
function computeBingSummary(rows) {
|
|
5997
|
-
return computeBacklinkSummaryMetrics(rows);
|
|
5998
|
-
}
|
|
5999
|
-
function defaultDeps4() {
|
|
6000
|
-
return { getLinkCounts, getUrlLinks, now: () => /* @__PURE__ */ new Date() };
|
|
6001
|
-
}
|
|
6002
|
-
async function executeBingBacklinkSync(db, runId, projectId, opts) {
|
|
6003
|
-
const deps = { ...defaultDeps4(), ...opts.deps };
|
|
6004
|
-
const startedAt = deps.now().toISOString();
|
|
6005
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq11(runs.id, runId)).run();
|
|
6006
|
-
try {
|
|
6007
|
-
const project = db.select().from(projects).where(eq11(projects.id, projectId)).get();
|
|
6008
|
-
if (!project) throw new Error(`Project not found: ${projectId}`);
|
|
6009
|
-
const conn = opts.resolveConnection(project.canonicalDomain);
|
|
6010
|
-
if (!conn) throw new Error(`No Bing Webmaster connection for ${project.canonicalDomain}`);
|
|
6011
|
-
const siteUrl = conn.siteUrl;
|
|
6012
|
-
if (!siteUrl) {
|
|
6013
|
-
throw new Error(`Bing connection for ${project.canonicalDomain} has no verified site selected`);
|
|
6014
|
-
}
|
|
6015
|
-
const pages = await deps.getLinkCounts(conn.apiKey, siteUrl, { maxPages: LINK_COUNTS_MAX_PAGES });
|
|
6016
|
-
const maxPages = Math.max(1, opts.maxPages ?? DEFAULT_MAX_PAGES);
|
|
6017
|
-
const targetPages = [...pages].sort((a, b) => b.Count - a.Count).slice(0, maxPages);
|
|
6018
|
-
if (pages.length > targetPages.length) {
|
|
6019
|
-
log11.info("bing-sync.pages-capped", {
|
|
6020
|
-
runId,
|
|
6021
|
-
projectId,
|
|
6022
|
-
sitePagesFound: pages.length,
|
|
6023
|
-
sitePagesPulled: targetPages.length
|
|
6024
|
-
});
|
|
6025
|
-
}
|
|
6026
|
-
const allLinks = [];
|
|
6027
|
-
let pageFailures = 0;
|
|
6028
|
-
for (const page of targetPages) {
|
|
6029
|
-
try {
|
|
6030
|
-
const links = await deps.getUrlLinks(conn.apiKey, siteUrl, page.Url, { maxPages: URL_LINKS_MAX_PAGES });
|
|
6031
|
-
allLinks.push(...links);
|
|
6032
|
-
} catch (err) {
|
|
6033
|
-
pageFailures++;
|
|
6034
|
-
log11.warn("bing-sync.page-failed", {
|
|
6035
|
-
runId,
|
|
6036
|
-
projectId,
|
|
6037
|
-
page: page.Url,
|
|
6038
|
-
error: err instanceof Error ? err.message : String(err)
|
|
6039
|
-
});
|
|
6040
|
-
}
|
|
6041
|
-
}
|
|
6042
|
-
if (targetPages.length > 0 && pageFailures === targetPages.length) {
|
|
6043
|
-
throw new Error(`All ${targetPages.length} Bing inbound-link page fetch(es) failed`);
|
|
6044
|
-
}
|
|
6045
|
-
const rows = aggregateInboundLinksByDomain(allLinks, project.canonicalDomain);
|
|
6046
|
-
const summary = computeBingSummary(rows);
|
|
6047
|
-
const queriedAt = deps.now().toISOString();
|
|
6048
|
-
const release = bingReleaseId(deps.now());
|
|
6049
|
-
const targetDomain = project.canonicalDomain;
|
|
6050
|
-
const source = BacklinkSources["bing-webmaster"];
|
|
6051
|
-
const existing = db.select({ id: backlinkSummaries.id }).from(backlinkSummaries).where(and9(
|
|
6052
|
-
eq11(backlinkSummaries.projectId, projectId),
|
|
6053
|
-
eq11(backlinkSummaries.source, source),
|
|
6054
|
-
eq11(backlinkSummaries.release, release)
|
|
6055
|
-
)).get();
|
|
6056
|
-
const preserveExisting = pageFailures > 0 && !!existing;
|
|
6057
|
-
if (!preserveExisting) {
|
|
6058
|
-
db.transaction((tx) => {
|
|
6059
|
-
tx.delete(backlinkDomains).where(and9(
|
|
6060
|
-
eq11(backlinkDomains.projectId, projectId),
|
|
6061
|
-
eq11(backlinkDomains.source, source),
|
|
6062
|
-
eq11(backlinkDomains.release, release)
|
|
6063
|
-
)).run();
|
|
6064
|
-
if (rows.length > 0) {
|
|
6065
|
-
tx.insert(backlinkDomains).values(rows.map((r) => ({
|
|
6066
|
-
id: crypto13.randomUUID(),
|
|
6067
|
-
projectId,
|
|
6068
|
-
releaseSyncId: null,
|
|
6069
|
-
source,
|
|
6070
|
-
release,
|
|
6071
|
-
targetDomain,
|
|
6072
|
-
linkingDomain: r.linkingDomain,
|
|
6073
|
-
numHosts: r.numHosts,
|
|
6074
|
-
createdAt: queriedAt
|
|
6075
|
-
}))).run();
|
|
6076
|
-
}
|
|
6077
|
-
tx.insert(backlinkSummaries).values({
|
|
6078
|
-
id: crypto13.randomUUID(),
|
|
6079
|
-
projectId,
|
|
6080
|
-
releaseSyncId: null,
|
|
6081
|
-
source,
|
|
6082
|
-
release,
|
|
6083
|
-
targetDomain,
|
|
6084
|
-
totalLinkingDomains: summary.totalLinkingDomains,
|
|
6085
|
-
totalHosts: summary.totalHosts,
|
|
6086
|
-
top10HostsShare: summary.top10HostsShare,
|
|
6087
|
-
queriedAt,
|
|
6088
|
-
createdAt: queriedAt
|
|
6089
|
-
}).onConflictDoUpdate({
|
|
6090
|
-
target: [backlinkSummaries.projectId, backlinkSummaries.source, backlinkSummaries.release],
|
|
6091
|
-
set: {
|
|
6092
|
-
targetDomain,
|
|
6093
|
-
totalLinkingDomains: summary.totalLinkingDomains,
|
|
6094
|
-
totalHosts: summary.totalHosts,
|
|
6095
|
-
top10HostsShare: summary.top10HostsShare,
|
|
6096
|
-
queriedAt
|
|
6097
|
-
}
|
|
6098
|
-
}).run();
|
|
6099
|
-
});
|
|
6100
|
-
}
|
|
6101
|
-
const finishedAt = deps.now().toISOString();
|
|
6102
|
-
const status = pageFailures > 0 ? RunStatuses.partial : RunStatuses.completed;
|
|
6103
|
-
const error = preserveExisting ? `Kept existing ${release} snapshot; ${pageFailures} of ${targetPages.length} inbound-link page fetches failed` : pageFailures > 0 ? `${pageFailures} of ${targetPages.length} inbound-link page fetches failed` : null;
|
|
6104
|
-
db.update(runs).set({ status, error, finishedAt }).where(eq11(runs.id, runId)).run();
|
|
6105
|
-
log11.info("bing-sync.completed", { runId, projectId, release, rows: rows.length, status, pageFailures, preserveExisting });
|
|
6106
|
-
} catch (err) {
|
|
6107
|
-
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
6108
|
-
const finishedAt = deps.now().toISOString();
|
|
6109
|
-
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt }).where(eq11(runs.id, runId)).run();
|
|
6110
|
-
log11.error("bing-sync.failed", { runId, projectId, error: errorMsg });
|
|
6111
|
-
throw err;
|
|
6112
|
-
}
|
|
6113
|
-
}
|
|
6114
|
-
|
|
6115
|
-
// src/discovery-run.ts
|
|
6116
|
-
import crypto14 from "crypto";
|
|
6117
|
-
import { and as and10, eq as eq12 } from "drizzle-orm";
|
|
6118
|
-
var log12 = createLogger("DiscoveryRun");
|
|
6168
|
+
var log11 = createLogger("DiscoveryRun");
|
|
6119
6169
|
var DEFAULT_SEED_COUNT = 30;
|
|
6120
6170
|
var EMBED_RETRY_MAX_RETRIES = 3;
|
|
6121
6171
|
var EMBED_RETRY_MAX_DELAY_MS = 1e4;
|
|
@@ -6148,7 +6198,7 @@ async function embedWithRetry(fn, opts = {}) {
|
|
|
6148
6198
|
maxDelayMs: EMBED_RETRY_MAX_DELAY_MS,
|
|
6149
6199
|
sleep: opts.sleep,
|
|
6150
6200
|
onRetry: opts.onRetry ?? (({ attempt, delayMs }) => {
|
|
6151
|
-
|
|
6201
|
+
log11.warn("discovery.embed.retry", { attempt, delayMs });
|
|
6152
6202
|
})
|
|
6153
6203
|
}
|
|
6154
6204
|
);
|
|
@@ -6156,11 +6206,11 @@ async function embedWithRetry(fn, opts = {}) {
|
|
|
6156
6206
|
var QUERIES_PER_INTENT_BUCKET = 6;
|
|
6157
6207
|
async function executeDiscoveryRun(opts) {
|
|
6158
6208
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
6159
|
-
opts.db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
6209
|
+
opts.db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq11(runs.id, opts.runId)).run();
|
|
6160
6210
|
try {
|
|
6161
|
-
const projectRow = opts.db.select().from(projects).where(
|
|
6211
|
+
const projectRow = opts.db.select().from(projects).where(eq11(projects.id, opts.projectId)).get();
|
|
6162
6212
|
if (!projectRow) throw new Error(`Project ${opts.projectId} not found`);
|
|
6163
|
-
const projectCompetitors = opts.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
6213
|
+
const projectCompetitors = opts.db.select({ domain: competitors.domain }).from(competitors).where(eq11(competitors.projectId, opts.projectId)).all().map((r) => r.domain.toLowerCase());
|
|
6164
6214
|
const canonicalDomains = effectiveDomains({
|
|
6165
6215
|
canonicalDomain: projectRow.canonicalDomain,
|
|
6166
6216
|
ownedDomains: projectRow.ownedDomains
|
|
@@ -6198,8 +6248,8 @@ async function executeDiscoveryRun(opts) {
|
|
|
6198
6248
|
seedProvider: result.seedProvider,
|
|
6199
6249
|
result
|
|
6200
6250
|
});
|
|
6201
|
-
opts.db.update(runs).set({ status: RunStatuses.completed, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
6202
|
-
|
|
6251
|
+
opts.db.update(runs).set({ status: RunStatuses.completed, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq11(runs.id, opts.runId)).run();
|
|
6252
|
+
log11.info("discovery.completed", {
|
|
6203
6253
|
runId: opts.runId,
|
|
6204
6254
|
sessionId: opts.sessionId,
|
|
6205
6255
|
buckets: result.buckets,
|
|
@@ -6207,13 +6257,13 @@ async function executeDiscoveryRun(opts) {
|
|
|
6207
6257
|
});
|
|
6208
6258
|
} catch (err) {
|
|
6209
6259
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
6210
|
-
|
|
6260
|
+
log11.error("discovery.failed", { runId: opts.runId, sessionId: opts.sessionId, error: errorMsg });
|
|
6211
6261
|
markSessionFailed(opts.db, opts.sessionId, errorMsg);
|
|
6212
6262
|
opts.db.update(runs).set({
|
|
6213
6263
|
status: RunStatuses.failed,
|
|
6214
6264
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
6215
6265
|
error: errorMsg
|
|
6216
|
-
}).where(
|
|
6266
|
+
}).where(eq11(runs.id, opts.runId)).run();
|
|
6217
6267
|
}
|
|
6218
6268
|
}
|
|
6219
6269
|
function buildDefaultDeps(registry) {
|
|
@@ -6462,13 +6512,13 @@ function writeDiscoveryInsight(db, input) {
|
|
|
6462
6512
|
totalProbes
|
|
6463
6513
|
});
|
|
6464
6514
|
db.transaction((tx) => {
|
|
6465
|
-
tx.update(insights).set({ dismissed: true }).where(
|
|
6466
|
-
|
|
6467
|
-
|
|
6468
|
-
|
|
6515
|
+
tx.update(insights).set({ dismissed: true }).where(and9(
|
|
6516
|
+
eq11(insights.projectId, input.projectId),
|
|
6517
|
+
eq11(insights.type, "discovery.basket-divergence"),
|
|
6518
|
+
eq11(insights.dismissed, false)
|
|
6469
6519
|
)).run();
|
|
6470
6520
|
tx.insert(insights).values({
|
|
6471
|
-
id:
|
|
6521
|
+
id: crypto13.randomUUID(),
|
|
6472
6522
|
projectId: input.projectId,
|
|
6473
6523
|
runId: input.runId,
|
|
6474
6524
|
type: "discovery.basket-divergence",
|
|
@@ -6504,10 +6554,10 @@ function buildDiscoveryInsightTitle(input) {
|
|
|
6504
6554
|
}
|
|
6505
6555
|
|
|
6506
6556
|
// src/execute-site-audit.ts
|
|
6507
|
-
import
|
|
6508
|
-
import { eq as
|
|
6557
|
+
import crypto14 from "crypto";
|
|
6558
|
+
import { eq as eq12 } from "drizzle-orm";
|
|
6509
6559
|
import { runSitemapAudit } from "@ainyc/aeo-audit";
|
|
6510
|
-
var
|
|
6560
|
+
var log12 = createLogger("SiteAudit");
|
|
6511
6561
|
var SITE_AUDIT_DEFAULT_PAGE_LIMIT = 500;
|
|
6512
6562
|
var SITE_AUDIT_MAX_PAGE_LIMIT = 2e3;
|
|
6513
6563
|
function toHomepageUrl(canonicalDomain) {
|
|
@@ -6568,15 +6618,15 @@ function computeFactorAverages(pages) {
|
|
|
6568
6618
|
}
|
|
6569
6619
|
async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
6570
6620
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
6571
|
-
db.update(runs).set({ status: "running", startedAt }).where(
|
|
6621
|
+
db.update(runs).set({ status: "running", startedAt }).where(eq12(runs.id, runId)).run();
|
|
6572
6622
|
try {
|
|
6573
|
-
const project = db.select().from(projects).where(
|
|
6623
|
+
const project = db.select().from(projects).where(eq12(projects.id, projectId)).get();
|
|
6574
6624
|
if (!project) {
|
|
6575
6625
|
throw new Error(`Project not found: ${projectId}`);
|
|
6576
6626
|
}
|
|
6577
6627
|
const homepageUrl = toHomepageUrl(project.canonicalDomain);
|
|
6578
6628
|
const limit = clampSiteAuditLimit(opts.limit);
|
|
6579
|
-
|
|
6629
|
+
log12.info("start", { runId, projectId, homepageUrl, sitemapUrl: opts.sitemapUrl ?? null, limit });
|
|
6580
6630
|
await assertSiteAuditUrlAllowed(homepageUrl, "canonicalDomain");
|
|
6581
6631
|
if (opts.sitemapUrl) await assertSiteAuditUrlAllowed(opts.sitemapUrl, "sitemapUrl");
|
|
6582
6632
|
const report = await runSitemapAudit(homepageUrl, { sitemapUrl: opts.sitemapUrl, limit });
|
|
@@ -6584,7 +6634,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
6584
6634
|
const pagesErrored = report.pages.filter((page) => page.status === "error").length;
|
|
6585
6635
|
const auditable = report.pagesDiscovered - report.pagesSkipped;
|
|
6586
6636
|
if (auditable > report.pagesAudited) {
|
|
6587
|
-
|
|
6637
|
+
log12.info("truncated", {
|
|
6588
6638
|
runId,
|
|
6589
6639
|
projectId,
|
|
6590
6640
|
auditable,
|
|
@@ -6603,7 +6653,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
6603
6653
|
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
6604
6654
|
db.transaction((tx) => {
|
|
6605
6655
|
tx.insert(siteAuditSnapshots).values({
|
|
6606
|
-
id:
|
|
6656
|
+
id: crypto14.randomUUID(),
|
|
6607
6657
|
projectId,
|
|
6608
6658
|
runId,
|
|
6609
6659
|
sitemapUrl: report.sitemapUrl,
|
|
@@ -6632,7 +6682,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
6632
6682
|
}).run();
|
|
6633
6683
|
for (const page of report.pages) {
|
|
6634
6684
|
tx.insert(siteAuditPages).values({
|
|
6635
|
-
id:
|
|
6685
|
+
id: crypto14.randomUUID(),
|
|
6636
6686
|
projectId,
|
|
6637
6687
|
runId,
|
|
6638
6688
|
url: page.url,
|
|
@@ -6643,9 +6693,9 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
6643
6693
|
createdAt: finishedAt
|
|
6644
6694
|
}).run();
|
|
6645
6695
|
}
|
|
6646
|
-
tx.update(runs).set({ status, finishedAt }).where(
|
|
6696
|
+
tx.update(runs).set({ status, finishedAt }).where(eq12(runs.id, runId)).run();
|
|
6647
6697
|
});
|
|
6648
|
-
|
|
6698
|
+
log12.info("completed", {
|
|
6649
6699
|
runId,
|
|
6650
6700
|
projectId,
|
|
6651
6701
|
status,
|
|
@@ -6655,14 +6705,14 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
6655
6705
|
});
|
|
6656
6706
|
} catch (err) {
|
|
6657
6707
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
6658
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
6659
|
-
|
|
6708
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq12(runs.id, runId)).run();
|
|
6709
|
+
log12.error("failed", { runId, projectId, error: errorMsg });
|
|
6660
6710
|
throw err;
|
|
6661
6711
|
}
|
|
6662
6712
|
}
|
|
6663
6713
|
|
|
6664
6714
|
// src/commands/backfill.ts
|
|
6665
|
-
import { and as
|
|
6715
|
+
import { and as and10, eq as eq13, inArray as inArray4, isNull, sql as sql5 } from "drizzle-orm";
|
|
6666
6716
|
var SNAPSHOT_BATCH_SIZE = 500;
|
|
6667
6717
|
async function backfillAnswerVisibilityCommand(opts) {
|
|
6668
6718
|
const config = loadConfig();
|
|
@@ -6670,7 +6720,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
6670
6720
|
migrate(db);
|
|
6671
6721
|
const projectFilter = opts?.project?.trim();
|
|
6672
6722
|
const isDryRun = opts?.dryRun === true;
|
|
6673
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
6723
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq13(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
6674
6724
|
let examined = 0;
|
|
6675
6725
|
let updated = 0;
|
|
6676
6726
|
let wouldUpdate = 0;
|
|
@@ -6678,10 +6728,10 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
6678
6728
|
let reparsed = 0;
|
|
6679
6729
|
let providerErrors = 0;
|
|
6680
6730
|
if (scopedProjects.length > 0) {
|
|
6681
|
-
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
6682
|
-
|
|
6731
|
+
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(and10(
|
|
6732
|
+
eq13(runs.kind, RunKinds["answer-visibility"]),
|
|
6683
6733
|
inArray4(runs.projectId, scopedProjects.map((project) => project.id))
|
|
6684
|
-
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
6734
|
+
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(eq13(runs.kind, RunKinds["answer-visibility"])).all();
|
|
6685
6735
|
const runIdsByProject = /* @__PURE__ */ new Map();
|
|
6686
6736
|
for (const run of runRows) {
|
|
6687
6737
|
const existing = runIdsByProject.get(run.projectId);
|
|
@@ -6689,7 +6739,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
6689
6739
|
else runIdsByProject.set(run.projectId, [run.id]);
|
|
6690
6740
|
}
|
|
6691
6741
|
for (const project of scopedProjects) {
|
|
6692
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
6742
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq13(competitors.projectId, project.id)).all().map((row) => row.domain);
|
|
6693
6743
|
const runIds = runIdsByProject.get(project.id) ?? [];
|
|
6694
6744
|
if (runIds.length === 0) continue;
|
|
6695
6745
|
const projectDomains = effectiveDomains({
|
|
@@ -6735,7 +6785,12 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
6735
6785
|
answerText,
|
|
6736
6786
|
citedDomains: reparsedResult.citedDomains,
|
|
6737
6787
|
groundingSources: reparsedResult.groundingSources,
|
|
6738
|
-
searchQueries: reparsedResult.searchQueries
|
|
6788
|
+
searchQueries: reparsedResult.searchQueries,
|
|
6789
|
+
// Backfill recomputes mention and citation state from stored
|
|
6790
|
+
// rows; it never observes retrieval and never writes it back
|
|
6791
|
+
// (nextPatch carries no retrieval field). `unknown` keeps this
|
|
6792
|
+
// reconstruction from asserting anything about a search.
|
|
6793
|
+
retrievalStatus: "unknown"
|
|
6739
6794
|
};
|
|
6740
6795
|
const nextCitationState = determineCitationState(normalized, projectDomains);
|
|
6741
6796
|
const nextCitedDomains = reparsedResult.citedDomains;
|
|
@@ -6777,7 +6832,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
6777
6832
|
} else {
|
|
6778
6833
|
db.transaction((tx) => {
|
|
6779
6834
|
for (const update of pendingUpdates) {
|
|
6780
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
6835
|
+
tx.update(querySnapshots).set(update.patch).where(eq13(querySnapshots.id, update.id)).run();
|
|
6781
6836
|
}
|
|
6782
6837
|
});
|
|
6783
6838
|
updated += pendingUpdates.length;
|
|
@@ -6826,13 +6881,13 @@ No DB writes performed. Re-run without --dry-run to apply.`);
|
|
|
6826
6881
|
function backfillNormalizedPaths(db, opts) {
|
|
6827
6882
|
const baseConditions = [];
|
|
6828
6883
|
if (opts?.projectId) {
|
|
6829
|
-
baseConditions.push(
|
|
6884
|
+
baseConditions.push(eq13(gaTrafficSnapshots.projectId, opts.projectId));
|
|
6830
6885
|
}
|
|
6831
6886
|
const rows = db.select({
|
|
6832
6887
|
id: gaTrafficSnapshots.id,
|
|
6833
6888
|
landingPage: gaTrafficSnapshots.landingPage,
|
|
6834
6889
|
landingPageNormalized: gaTrafficSnapshots.landingPageNormalized
|
|
6835
|
-
}).from(gaTrafficSnapshots).where(baseConditions.length > 0 ?
|
|
6890
|
+
}).from(gaTrafficSnapshots).where(baseConditions.length > 0 ? and10(...baseConditions) : void 0).all();
|
|
6836
6891
|
let updated = 0;
|
|
6837
6892
|
let unchanged = 0;
|
|
6838
6893
|
if (rows.length > 0) {
|
|
@@ -6847,7 +6902,7 @@ function backfillNormalizedPaths(db, opts) {
|
|
|
6847
6902
|
unchanged++;
|
|
6848
6903
|
continue;
|
|
6849
6904
|
}
|
|
6850
|
-
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(
|
|
6905
|
+
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(eq13(gaTrafficSnapshots.id, row.id)).run();
|
|
6851
6906
|
updated++;
|
|
6852
6907
|
}
|
|
6853
6908
|
});
|
|
@@ -6861,7 +6916,7 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
6861
6916
|
const projectFilter = opts?.project?.trim();
|
|
6862
6917
|
let projectId;
|
|
6863
6918
|
if (projectFilter) {
|
|
6864
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
6919
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq13(projects.name, projectFilter)).get();
|
|
6865
6920
|
if (!project) {
|
|
6866
6921
|
const result2 = {
|
|
6867
6922
|
project: projectFilter,
|
|
@@ -6898,13 +6953,13 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
6898
6953
|
function backfillAiReferralPaths(db, opts) {
|
|
6899
6954
|
const baseConditions = [];
|
|
6900
6955
|
if (opts?.projectId) {
|
|
6901
|
-
baseConditions.push(
|
|
6956
|
+
baseConditions.push(eq13(gaAiReferrals.projectId, opts.projectId));
|
|
6902
6957
|
}
|
|
6903
6958
|
const rows = db.select({
|
|
6904
6959
|
id: gaAiReferrals.id,
|
|
6905
6960
|
landingPage: gaAiReferrals.landingPage,
|
|
6906
6961
|
landingPageNormalized: gaAiReferrals.landingPageNormalized
|
|
6907
|
-
}).from(gaAiReferrals).where(baseConditions.length > 0 ?
|
|
6962
|
+
}).from(gaAiReferrals).where(baseConditions.length > 0 ? and10(...baseConditions) : void 0).all();
|
|
6908
6963
|
let updated = 0;
|
|
6909
6964
|
let unchanged = 0;
|
|
6910
6965
|
if (rows.length > 0) {
|
|
@@ -6919,7 +6974,7 @@ function backfillAiReferralPaths(db, opts) {
|
|
|
6919
6974
|
unchanged++;
|
|
6920
6975
|
continue;
|
|
6921
6976
|
}
|
|
6922
|
-
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(
|
|
6977
|
+
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(eq13(gaAiReferrals.id, row.id)).run();
|
|
6923
6978
|
updated++;
|
|
6924
6979
|
}
|
|
6925
6980
|
});
|
|
@@ -6933,7 +6988,7 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
6933
6988
|
const projectFilter = opts?.project?.trim();
|
|
6934
6989
|
let projectId;
|
|
6935
6990
|
if (projectFilter) {
|
|
6936
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
6991
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq13(projects.name, projectFilter)).get();
|
|
6937
6992
|
if (!project) {
|
|
6938
6993
|
const result2 = {
|
|
6939
6994
|
project: projectFilter,
|
|
@@ -6969,10 +7024,10 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
6969
7024
|
}
|
|
6970
7025
|
function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
6971
7026
|
const isDryRun = opts?.dryRun === true;
|
|
6972
|
-
const project = db.select().from(projects).where(
|
|
7027
|
+
const project = db.select().from(projects).where(eq13(projects.id, projectId)).get();
|
|
6973
7028
|
if (!project) return { examined: 0, updated: 0, mentioned: 0 };
|
|
6974
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
6975
|
-
const runRows = db.select({ id: runs.id }).from(runs).where(
|
|
7029
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq13(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
7030
|
+
const runRows = db.select({ id: runs.id }).from(runs).where(and10(eq13(runs.kind, RunKinds["answer-visibility"]), eq13(runs.projectId, projectId))).all();
|
|
6976
7031
|
const runIds = runRows.map((r) => r.id);
|
|
6977
7032
|
let examined = 0;
|
|
6978
7033
|
let updated = 0;
|
|
@@ -7014,6 +7069,8 @@ function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
|
7014
7069
|
answerText,
|
|
7015
7070
|
citedDomains,
|
|
7016
7071
|
groundingSources,
|
|
7072
|
+
// See the note above: retrieval is not observed or written here.
|
|
7073
|
+
retrievalStatus: "unknown",
|
|
7017
7074
|
searchQueries: []
|
|
7018
7075
|
};
|
|
7019
7076
|
const nextCompetitorOverlap = computeCompetitorOverlap(normalized, competitorDomains);
|
|
@@ -7044,7 +7101,7 @@ function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
|
7044
7101
|
} else {
|
|
7045
7102
|
db.transaction((tx) => {
|
|
7046
7103
|
for (const update of pendingUpdates) {
|
|
7047
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
7104
|
+
tx.update(querySnapshots).set(update.patch).where(eq13(querySnapshots.id, update.id)).run();
|
|
7048
7105
|
}
|
|
7049
7106
|
});
|
|
7050
7107
|
updated += pendingUpdates.length;
|
|
@@ -7059,7 +7116,7 @@ async function backfillAnswerMentionsCommand(opts) {
|
|
|
7059
7116
|
migrate(db);
|
|
7060
7117
|
const projectFilter = opts?.project?.trim();
|
|
7061
7118
|
const isDryRun = opts?.dryRun === true;
|
|
7062
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
7119
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq13(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
7063
7120
|
let examined = 0;
|
|
7064
7121
|
let updated = 0;
|
|
7065
7122
|
let wouldUpdate = 0;
|
|
@@ -7119,7 +7176,7 @@ function readStoredGroundingSources(rawResponse) {
|
|
|
7119
7176
|
return result;
|
|
7120
7177
|
}
|
|
7121
7178
|
async function backfillInsightsCommand(project, opts) {
|
|
7122
|
-
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-
|
|
7179
|
+
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-SMYVN3KN.js");
|
|
7123
7180
|
const config = loadConfig();
|
|
7124
7181
|
const db = createClient(config.database);
|
|
7125
7182
|
migrate(db);
|
|
@@ -7278,7 +7335,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
7278
7335
|
const config = loadConfig();
|
|
7279
7336
|
const db = createClient(config.database);
|
|
7280
7337
|
migrate(db);
|
|
7281
|
-
const project = db.select().from(projects).where(
|
|
7338
|
+
const project = db.select().from(projects).where(eq13(projects.name, opts.project)).get();
|
|
7282
7339
|
if (!project) {
|
|
7283
7340
|
throw new Error(`Project "${opts.project}" not found`);
|
|
7284
7341
|
}
|
|
@@ -7289,8 +7346,8 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
7289
7346
|
process.stderr.write(`Recovering orphan snapshot attribution for "${project.name}"${mode}...
|
|
7290
7347
|
`);
|
|
7291
7348
|
}
|
|
7292
|
-
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(
|
|
7293
|
-
|
|
7349
|
+
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(and10(
|
|
7350
|
+
eq13(auditLog.projectId, project.id),
|
|
7294
7351
|
inArray4(auditLog.action, ["keywords.appended", "keywords.deleted", "queries.appended", "queries.deleted", "queries.replaced"])
|
|
7295
7352
|
)).orderBy(auditLog.createdAt).all();
|
|
7296
7353
|
const history = replayQueryAuditLog(events);
|
|
@@ -7298,8 +7355,8 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
7298
7355
|
runId: runs.id,
|
|
7299
7356
|
createdAt: runs.createdAt,
|
|
7300
7357
|
location: runs.location
|
|
7301
|
-
}).from(runs).innerJoin(querySnapshots,
|
|
7302
|
-
|
|
7358
|
+
}).from(runs).innerJoin(querySnapshots, eq13(querySnapshots.runId, runs.id)).where(and10(
|
|
7359
|
+
eq13(runs.projectId, project.id),
|
|
7303
7360
|
isNull(querySnapshots.queryId),
|
|
7304
7361
|
isNull(querySnapshots.queryText)
|
|
7305
7362
|
)).groupBy(runs.id).orderBy(runs.createdAt).all();
|
|
@@ -7321,8 +7378,8 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
7321
7378
|
provider: querySnapshots.provider,
|
|
7322
7379
|
createdAt: querySnapshots.createdAt,
|
|
7323
7380
|
answerText: querySnapshots.answerText
|
|
7324
|
-
}).from(querySnapshots).where(
|
|
7325
|
-
|
|
7381
|
+
}).from(querySnapshots).where(and10(
|
|
7382
|
+
eq13(querySnapshots.runId, run.runId),
|
|
7326
7383
|
isNull(querySnapshots.queryId),
|
|
7327
7384
|
isNull(querySnapshots.queryText)
|
|
7328
7385
|
)).orderBy(querySnapshots.provider, querySnapshots.createdAt).all();
|
|
@@ -7388,7 +7445,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
7388
7445
|
if (!isDryRun && updates.length > 0) {
|
|
7389
7446
|
db.transaction((tx) => {
|
|
7390
7447
|
for (const u of updates) {
|
|
7391
|
-
tx.update(querySnapshots).set({ queryText: u.queryText }).where(
|
|
7448
|
+
tx.update(querySnapshots).set({ queryText: u.queryText }).where(eq13(querySnapshots.id, u.id)).run();
|
|
7392
7449
|
}
|
|
7393
7450
|
});
|
|
7394
7451
|
}
|
|
@@ -7462,7 +7519,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7462
7519
|
const projectFilter = opts?.project?.trim();
|
|
7463
7520
|
const isDryRun = opts?.dryRun === true;
|
|
7464
7521
|
const isJson = isMachineFormat(opts?.format);
|
|
7465
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
7522
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq13(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
7466
7523
|
if (scopedProjects.length === 0) {
|
|
7467
7524
|
if (projectFilter && !isJson) {
|
|
7468
7525
|
process.stderr.write(`No project named "${projectFilter}".
|
|
@@ -7487,8 +7544,8 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7487
7544
|
dryRun: isDryRun,
|
|
7488
7545
|
byBot: {}
|
|
7489
7546
|
};
|
|
7490
|
-
const unknownCountRow = db.select({ n: sql5`count(*)` }).from(rawEventSamples).where(
|
|
7491
|
-
|
|
7547
|
+
const unknownCountRow = db.select({ n: sql5`count(*)` }).from(rawEventSamples).where(and10(
|
|
7548
|
+
eq13(rawEventSamples.eventType, "unknown"),
|
|
7492
7549
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
7493
7550
|
)).get();
|
|
7494
7551
|
result.unknownBefore = Number(unknownCountRow?.n ?? 0);
|
|
@@ -7500,8 +7557,8 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7500
7557
|
userAgent: rawEventSamples.userAgent,
|
|
7501
7558
|
pathNormalized: rawEventSamples.pathNormalized,
|
|
7502
7559
|
status: rawEventSamples.status
|
|
7503
|
-
}).from(rawEventSamples).where(
|
|
7504
|
-
|
|
7560
|
+
}).from(rawEventSamples).where(and10(
|
|
7561
|
+
eq13(rawEventSamples.eventType, "unknown"),
|
|
7505
7562
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
7506
7563
|
)).all();
|
|
7507
7564
|
result.examined = unknownSamples.length;
|
|
@@ -7540,7 +7597,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7540
7597
|
result.reclassified++;
|
|
7541
7598
|
result.byBot[classified.botId] = (result.byBot[classified.botId] ?? 0) + 1;
|
|
7542
7599
|
if (isDryRun) continue;
|
|
7543
|
-
db.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(
|
|
7600
|
+
db.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(eq13(rawEventSamples.id, snap.id)).run();
|
|
7544
7601
|
const tsHour = new Date(snap.ts);
|
|
7545
7602
|
tsHour.setUTCMinutes(0, 0, 0);
|
|
7546
7603
|
if (userFetch) {
|
|
@@ -7604,8 +7661,8 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7604
7661
|
}
|
|
7605
7662
|
}
|
|
7606
7663
|
if (!isDryRun) {
|
|
7607
|
-
const afterRow = db.select({ n: sql5`count(*)` }).from(rawEventSamples).where(
|
|
7608
|
-
|
|
7664
|
+
const afterRow = db.select({ n: sql5`count(*)` }).from(rawEventSamples).where(and10(
|
|
7665
|
+
eq13(rawEventSamples.eventType, "unknown"),
|
|
7609
7666
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
7610
7667
|
)).get();
|
|
7611
7668
|
result.unknownAfter = Number(afterRow?.n ?? 0);
|
|
@@ -7640,7 +7697,7 @@ No DB writes performed. Re-run without --dry-run to apply.`);
|
|
|
7640
7697
|
}
|
|
7641
7698
|
|
|
7642
7699
|
// src/commands/skills.ts
|
|
7643
|
-
import
|
|
7700
|
+
import crypto15 from "crypto";
|
|
7644
7701
|
import fs4 from "fs";
|
|
7645
7702
|
import os4 from "os";
|
|
7646
7703
|
import path5 from "path";
|
|
@@ -7695,7 +7752,7 @@ function walkRelative(dir, prefix = "") {
|
|
|
7695
7752
|
return out.sort();
|
|
7696
7753
|
}
|
|
7697
7754
|
function sha256File(filePath) {
|
|
7698
|
-
return
|
|
7755
|
+
return crypto15.createHash("sha256").update(fs4.readFileSync(filePath)).digest("hex");
|
|
7699
7756
|
}
|
|
7700
7757
|
function readSkillManifest(skillDir) {
|
|
7701
7758
|
try {
|
|
@@ -8037,10 +8094,11 @@ var ProviderRegistry = class {
|
|
|
8037
8094
|
};
|
|
8038
8095
|
|
|
8039
8096
|
// src/scheduler.ts
|
|
8040
|
-
import
|
|
8097
|
+
import crypto16 from "crypto";
|
|
8041
8098
|
import cron from "node-cron";
|
|
8042
|
-
import { and as
|
|
8043
|
-
var
|
|
8099
|
+
import { and as and11, eq as eq14, inArray as inArray5, notExists, sql as sql6 } from "drizzle-orm";
|
|
8100
|
+
var log13 = createLogger("Scheduler");
|
|
8101
|
+
var DEFAULT_HEALTH_CRON = "0 */6 * * *";
|
|
8044
8102
|
function taskKey(projectId, kind) {
|
|
8045
8103
|
return `${projectId}::${kind}`;
|
|
8046
8104
|
}
|
|
@@ -8052,18 +8110,60 @@ var Scheduler = class {
|
|
|
8052
8110
|
this.db = db;
|
|
8053
8111
|
this.callbacks = callbacks;
|
|
8054
8112
|
}
|
|
8113
|
+
/**
|
|
8114
|
+
* Give every project a health schedule if it has never had one.
|
|
8115
|
+
*
|
|
8116
|
+
* Ships the schedule with the feature. The six checks that existed before
|
|
8117
|
+
* this had never run on a live instance because nothing scheduled them, and
|
|
8118
|
+
* a health check nobody enables is indistinguishable from no health check.
|
|
8119
|
+
*
|
|
8120
|
+
* Keyed on the row EXISTING, not on it being enabled, so an operator who
|
|
8121
|
+
* deliberately turns this off keeps it off across restarts. Seeded here
|
|
8122
|
+
* rather than in a migration because migrations must stay additive to remain
|
|
8123
|
+
* downgrade-safe, and inserting rows is a data mutation.
|
|
8124
|
+
*
|
|
8125
|
+
* Every 6h, not daily: the cadence has to be shorter than the time-to-damage
|
|
8126
|
+
* it guards. A Vercel source begins discarding traffic once it is 24h behind,
|
|
8127
|
+
* so a daily pass could only ever observe the loss after it started.
|
|
8128
|
+
*/
|
|
8129
|
+
ensureHealthSchedules() {
|
|
8130
|
+
const projectsWithoutHealth = this.db.select({ id: projects.id }).from(projects).where(notExists(
|
|
8131
|
+
this.db.select({ one: sql6`1` }).from(schedules).where(and11(
|
|
8132
|
+
eq14(schedules.projectId, projects.id),
|
|
8133
|
+
eq14(schedules.kind, SchedulableRunKinds.doctor)
|
|
8134
|
+
))
|
|
8135
|
+
)).all();
|
|
8136
|
+
if (projectsWithoutHealth.length === 0) return;
|
|
8137
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8138
|
+
for (const project of projectsWithoutHealth) {
|
|
8139
|
+
this.db.insert(schedules).values({
|
|
8140
|
+
id: crypto16.randomUUID(),
|
|
8141
|
+
projectId: project.id,
|
|
8142
|
+
kind: SchedulableRunKinds.doctor,
|
|
8143
|
+
cronExpr: DEFAULT_HEALTH_CRON,
|
|
8144
|
+
timezone: "UTC",
|
|
8145
|
+
enabled: true,
|
|
8146
|
+
providers: [],
|
|
8147
|
+
nextRunAt: nextRunFromCron(DEFAULT_HEALTH_CRON, "UTC"),
|
|
8148
|
+
createdAt: now,
|
|
8149
|
+
updatedAt: now
|
|
8150
|
+
}).run();
|
|
8151
|
+
}
|
|
8152
|
+
log13.info("health-schedule.seeded", { projectCount: projectsWithoutHealth.length, cron: DEFAULT_HEALTH_CRON });
|
|
8153
|
+
}
|
|
8055
8154
|
/** Load all enabled schedules from DB and register cron jobs. */
|
|
8056
8155
|
start() {
|
|
8057
|
-
|
|
8156
|
+
this.ensureHealthSchedules();
|
|
8157
|
+
const allSchedules = this.db.select().from(schedules).where(eq14(schedules.enabled, true)).all();
|
|
8058
8158
|
for (const schedule of allSchedules) {
|
|
8059
8159
|
const missedRunAt = schedule.nextRunAt;
|
|
8060
8160
|
this.registerCronTask(schedule);
|
|
8061
8161
|
if (missedRunAt && new Date(missedRunAt) < /* @__PURE__ */ new Date()) {
|
|
8062
|
-
|
|
8162
|
+
log13.info("run.catch-up", { projectId: schedule.projectId, kind: schedule.kind, missedRunAt });
|
|
8063
8163
|
this.triggerRun(schedule.id, schedule.projectId, schedule.kind);
|
|
8064
8164
|
}
|
|
8065
8165
|
}
|
|
8066
|
-
|
|
8166
|
+
log13.info("started", { scheduleCount: allSchedules.length });
|
|
8067
8167
|
}
|
|
8068
8168
|
/** Stop all cron tasks for graceful shutdown. */
|
|
8069
8169
|
stop() {
|
|
@@ -8084,7 +8184,7 @@ var Scheduler = class {
|
|
|
8084
8184
|
this.stopTask(key, existing, "Stopped");
|
|
8085
8185
|
this.tasks.delete(key);
|
|
8086
8186
|
}
|
|
8087
|
-
const schedule = this.db.select().from(schedules).where(
|
|
8187
|
+
const schedule = this.db.select().from(schedules).where(and11(eq14(schedules.projectId, projectId), eq14(schedules.kind, kind))).get();
|
|
8088
8188
|
if (schedule && schedule.enabled) {
|
|
8089
8189
|
this.registerCronTask(schedule);
|
|
8090
8190
|
}
|
|
@@ -8107,13 +8207,13 @@ var Scheduler = class {
|
|
|
8107
8207
|
stopTask(key, task, verb) {
|
|
8108
8208
|
void task.stop();
|
|
8109
8209
|
void task.destroy();
|
|
8110
|
-
|
|
8210
|
+
log13.info(`task.${verb.toLowerCase()}`, { key });
|
|
8111
8211
|
}
|
|
8112
8212
|
registerCronTask(schedule) {
|
|
8113
8213
|
const { id: scheduleId, projectId, cronExpr, timezone } = schedule;
|
|
8114
8214
|
const kind = schedule.kind;
|
|
8115
8215
|
if (!cron.validate(cronExpr)) {
|
|
8116
|
-
|
|
8216
|
+
log13.error("cron.invalid", { projectId, kind, cronExpr });
|
|
8117
8217
|
return;
|
|
8118
8218
|
}
|
|
8119
8219
|
const task = cron.schedule(cronExpr, () => {
|
|
@@ -8125,51 +8225,51 @@ var Scheduler = class {
|
|
|
8125
8225
|
this.db.update(schedules).set({
|
|
8126
8226
|
nextRunAt: nextRunFromCron(cronExpr, timezone),
|
|
8127
8227
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
8128
|
-
}).where(
|
|
8228
|
+
}).where(eq14(schedules.id, scheduleId)).run();
|
|
8129
8229
|
const label = schedule.preset ?? cronExpr;
|
|
8130
|
-
|
|
8230
|
+
log13.info("cron.registered", { projectId, kind, schedule: label, timezone });
|
|
8131
8231
|
}
|
|
8132
8232
|
triggerRun(scheduleId, projectId, kind) {
|
|
8133
8233
|
try {
|
|
8134
8234
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8135
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
8235
|
+
const currentSchedule = this.db.select().from(schedules).where(eq14(schedules.id, scheduleId)).get();
|
|
8136
8236
|
if (!currentSchedule || !currentSchedule.enabled) {
|
|
8137
|
-
|
|
8237
|
+
log13.warn("schedule.stale", { scheduleId, projectId, kind, msg: "schedule no longer exists or is disabled" });
|
|
8138
8238
|
this.remove(projectId, kind);
|
|
8139
8239
|
return;
|
|
8140
8240
|
}
|
|
8141
8241
|
const nextRunAt = nextRunFromCron(currentSchedule.cronExpr, currentSchedule.timezone);
|
|
8142
|
-
const project = this.db.select().from(projects).where(
|
|
8242
|
+
const project = this.db.select().from(projects).where(eq14(projects.id, projectId)).get();
|
|
8143
8243
|
if (!project) {
|
|
8144
|
-
|
|
8244
|
+
log13.error("project.not-found", { projectId, kind, msg: "skipping scheduled run" });
|
|
8145
8245
|
this.remove(projectId, kind);
|
|
8146
8246
|
return;
|
|
8147
8247
|
}
|
|
8148
8248
|
if (kind === SchedulableRunKinds["traffic-sync"]) {
|
|
8149
8249
|
const sourceId = currentSchedule.sourceId;
|
|
8150
8250
|
if (!sourceId) {
|
|
8151
|
-
|
|
8251
|
+
log13.warn("traffic-sync.missing-source", { scheduleId, projectId });
|
|
8152
8252
|
return;
|
|
8153
8253
|
}
|
|
8154
8254
|
if (!this.callbacks.onTrafficSyncRequested) {
|
|
8155
|
-
|
|
8255
|
+
log13.warn("traffic-sync.no-callback", { scheduleId, projectId, msg: "host did not register onTrafficSyncRequested" });
|
|
8156
8256
|
return;
|
|
8157
8257
|
}
|
|
8158
8258
|
this.db.update(schedules).set({
|
|
8159
8259
|
lastRunAt: now,
|
|
8160
8260
|
nextRunAt,
|
|
8161
8261
|
updatedAt: now
|
|
8162
|
-
}).where(
|
|
8163
|
-
|
|
8262
|
+
}).where(eq14(schedules.id, currentSchedule.id)).run();
|
|
8263
|
+
log13.info("traffic-sync.triggered", { projectName: project.name, sourceId });
|
|
8164
8264
|
this.callbacks.onTrafficSyncRequested(project.name, sourceId);
|
|
8165
8265
|
return;
|
|
8166
8266
|
}
|
|
8167
8267
|
if (kind === SchedulableRunKinds["gbp-sync"]) {
|
|
8168
8268
|
if (!this.callbacks.onGbpSyncRequested) {
|
|
8169
|
-
|
|
8269
|
+
log13.warn("gbp-sync.no-callback", { scheduleId, projectId, msg: "host did not register onGbpSyncRequested" });
|
|
8170
8270
|
return;
|
|
8171
8271
|
}
|
|
8172
|
-
const runId2 =
|
|
8272
|
+
const runId2 = crypto16.randomUUID();
|
|
8173
8273
|
this.db.insert(runs).values({
|
|
8174
8274
|
id: runId2,
|
|
8175
8275
|
projectId,
|
|
@@ -8182,27 +8282,27 @@ var Scheduler = class {
|
|
|
8182
8282
|
lastRunAt: now,
|
|
8183
8283
|
nextRunAt,
|
|
8184
8284
|
updatedAt: now
|
|
8185
|
-
}).where(
|
|
8186
|
-
|
|
8285
|
+
}).where(eq14(schedules.id, currentSchedule.id)).run();
|
|
8286
|
+
log13.info("gbp-sync.triggered", { runId: runId2, projectName: project.name });
|
|
8187
8287
|
this.callbacks.onGbpSyncRequested(runId2, projectId);
|
|
8188
8288
|
return;
|
|
8189
8289
|
}
|
|
8190
8290
|
if (kind === SchedulableRunKinds["ads-sync"]) {
|
|
8191
8291
|
if (!this.callbacks.onAdsSyncRequested) {
|
|
8192
|
-
|
|
8292
|
+
log13.warn("ads-sync.no-callback", { scheduleId, projectId, msg: "host did not register onAdsSyncRequested" });
|
|
8193
8293
|
return;
|
|
8194
8294
|
}
|
|
8195
|
-
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(
|
|
8196
|
-
|
|
8197
|
-
|
|
8295
|
+
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(and11(
|
|
8296
|
+
eq14(runs.projectId, projectId),
|
|
8297
|
+
eq14(runs.kind, RunKinds["ads-sync"]),
|
|
8198
8298
|
inArray5(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
8199
8299
|
)).get();
|
|
8200
8300
|
if (activeAdsRun) {
|
|
8201
|
-
|
|
8202
|
-
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(
|
|
8301
|
+
log13.info("ads-sync.skipped-active", { projectName: project.name, activeRunId: activeAdsRun.id });
|
|
8302
|
+
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq14(schedules.id, currentSchedule.id)).run();
|
|
8203
8303
|
return;
|
|
8204
8304
|
}
|
|
8205
|
-
const runId2 =
|
|
8305
|
+
const runId2 = crypto16.randomUUID();
|
|
8206
8306
|
this.db.insert(runs).values({
|
|
8207
8307
|
id: runId2,
|
|
8208
8308
|
projectId,
|
|
@@ -8215,55 +8315,69 @@ var Scheduler = class {
|
|
|
8215
8315
|
lastRunAt: now,
|
|
8216
8316
|
nextRunAt,
|
|
8217
8317
|
updatedAt: now
|
|
8218
|
-
}).where(
|
|
8219
|
-
|
|
8318
|
+
}).where(eq14(schedules.id, currentSchedule.id)).run();
|
|
8319
|
+
log13.info("ads-sync.triggered", { runId: runId2, projectName: project.name });
|
|
8220
8320
|
this.callbacks.onAdsSyncRequested(runId2, projectId);
|
|
8221
8321
|
return;
|
|
8222
8322
|
}
|
|
8223
8323
|
if (kind === SchedulableRunKinds["data-refresh"]) {
|
|
8224
8324
|
if (!this.callbacks.onDataRefreshRequested) {
|
|
8225
|
-
|
|
8325
|
+
log13.warn("data-refresh.no-callback", { scheduleId, projectId, msg: "host did not register onDataRefreshRequested" });
|
|
8226
8326
|
return;
|
|
8227
8327
|
}
|
|
8228
8328
|
this.db.update(schedules).set({
|
|
8229
8329
|
lastRunAt: now,
|
|
8230
8330
|
nextRunAt,
|
|
8231
8331
|
updatedAt: now
|
|
8232
|
-
}).where(
|
|
8233
|
-
|
|
8332
|
+
}).where(eq14(schedules.id, currentSchedule.id)).run();
|
|
8333
|
+
log13.info("data-refresh.triggered", { projectName: project.name });
|
|
8234
8334
|
this.callbacks.onDataRefreshRequested(project.name);
|
|
8235
8335
|
return;
|
|
8236
8336
|
}
|
|
8337
|
+
if (kind === SchedulableRunKinds.doctor) {
|
|
8338
|
+
if (!this.callbacks.onDoctorRequested) {
|
|
8339
|
+
log13.warn("doctor.no-callback", { scheduleId, projectId, msg: "host did not register onDoctorRequested" });
|
|
8340
|
+
return;
|
|
8341
|
+
}
|
|
8342
|
+
this.db.update(schedules).set({
|
|
8343
|
+
lastRunAt: now,
|
|
8344
|
+
nextRunAt,
|
|
8345
|
+
updatedAt: now
|
|
8346
|
+
}).where(eq14(schedules.id, currentSchedule.id)).run();
|
|
8347
|
+
log13.info("doctor.triggered", { projectName: project.name });
|
|
8348
|
+
this.callbacks.onDoctorRequested(project.name);
|
|
8349
|
+
return;
|
|
8350
|
+
}
|
|
8237
8351
|
if (kind === SchedulableRunKinds["backlinks-sync"]) {
|
|
8238
8352
|
if (!this.callbacks.onBacklinksSyncRequested) {
|
|
8239
|
-
|
|
8353
|
+
log13.warn("backlinks-sync.no-callback", { scheduleId, projectId, msg: "host did not register onBacklinksSyncRequested" });
|
|
8240
8354
|
return;
|
|
8241
8355
|
}
|
|
8242
8356
|
this.db.update(schedules).set({
|
|
8243
8357
|
lastRunAt: now,
|
|
8244
8358
|
nextRunAt,
|
|
8245
8359
|
updatedAt: now
|
|
8246
|
-
}).where(
|
|
8247
|
-
|
|
8360
|
+
}).where(eq14(schedules.id, currentSchedule.id)).run();
|
|
8361
|
+
log13.info("backlinks-sync.triggered", { projectName: project.name });
|
|
8248
8362
|
this.callbacks.onBacklinksSyncRequested(project.name);
|
|
8249
8363
|
return;
|
|
8250
8364
|
}
|
|
8251
8365
|
if (kind === SchedulableRunKinds["site-audit"]) {
|
|
8252
8366
|
if (!this.callbacks.onSiteAuditRequested) {
|
|
8253
|
-
|
|
8367
|
+
log13.warn("site-audit.no-callback", { scheduleId, projectId, msg: "host did not register onSiteAuditRequested" });
|
|
8254
8368
|
return;
|
|
8255
8369
|
}
|
|
8256
|
-
const active = this.db.select({ id: runs.id }).from(runs).where(
|
|
8257
|
-
|
|
8258
|
-
|
|
8370
|
+
const active = this.db.select({ id: runs.id }).from(runs).where(and11(
|
|
8371
|
+
eq14(runs.projectId, projectId),
|
|
8372
|
+
eq14(runs.kind, RunKinds["site-audit"]),
|
|
8259
8373
|
inArray5(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
8260
8374
|
)).get();
|
|
8261
8375
|
if (active) {
|
|
8262
|
-
|
|
8263
|
-
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(
|
|
8376
|
+
log13.info("site-audit.skipped-active", { projectName: project.name, activeRunId: active.id });
|
|
8377
|
+
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq14(schedules.id, currentSchedule.id)).run();
|
|
8264
8378
|
return;
|
|
8265
8379
|
}
|
|
8266
|
-
const runId2 =
|
|
8380
|
+
const runId2 = crypto16.randomUUID();
|
|
8267
8381
|
this.db.insert(runs).values({
|
|
8268
8382
|
id: runId2,
|
|
8269
8383
|
projectId,
|
|
@@ -8276,8 +8390,8 @@ var Scheduler = class {
|
|
|
8276
8390
|
lastRunAt: now,
|
|
8277
8391
|
nextRunAt,
|
|
8278
8392
|
updatedAt: now
|
|
8279
|
-
}).where(
|
|
8280
|
-
|
|
8393
|
+
}).where(eq14(schedules.id, currentSchedule.id)).run();
|
|
8394
|
+
log13.info("site-audit.triggered", { runId: runId2, projectName: project.name });
|
|
8281
8395
|
this.callbacks.onSiteAuditRequested(runId2, projectId);
|
|
8282
8396
|
return;
|
|
8283
8397
|
}
|
|
@@ -8286,7 +8400,7 @@ var Scheduler = class {
|
|
|
8286
8400
|
if (project.defaultLocation) {
|
|
8287
8401
|
const loc = projectLocations.find((l) => l.label === project.defaultLocation);
|
|
8288
8402
|
if (!loc) {
|
|
8289
|
-
|
|
8403
|
+
log13.warn("default-location.stale", { scheduleId, projectId, label: project.defaultLocation });
|
|
8290
8404
|
return;
|
|
8291
8405
|
}
|
|
8292
8406
|
resolvedLocation = loc;
|
|
@@ -8300,11 +8414,11 @@ var Scheduler = class {
|
|
|
8300
8414
|
location: locationLabel
|
|
8301
8415
|
});
|
|
8302
8416
|
if (queueResult.conflict) {
|
|
8303
|
-
|
|
8417
|
+
log13.info("run.skipped-active", { projectName: project.name, activeRunId: queueResult.activeRunId });
|
|
8304
8418
|
this.db.update(schedules).set({
|
|
8305
8419
|
nextRunAt,
|
|
8306
8420
|
updatedAt: now
|
|
8307
|
-
}).where(
|
|
8421
|
+
}).where(eq14(schedules.id, currentSchedule.id)).run();
|
|
8308
8422
|
return;
|
|
8309
8423
|
}
|
|
8310
8424
|
const runId = queueResult.runId;
|
|
@@ -8312,19 +8426,19 @@ var Scheduler = class {
|
|
|
8312
8426
|
lastRunAt: now,
|
|
8313
8427
|
nextRunAt,
|
|
8314
8428
|
updatedAt: now
|
|
8315
|
-
}).where(
|
|
8429
|
+
}).where(eq14(schedules.id, currentSchedule.id)).run();
|
|
8316
8430
|
const scheduleProviders = currentSchedule.providers;
|
|
8317
8431
|
const providers = scheduleProviders.length > 0 ? scheduleProviders : void 0;
|
|
8318
|
-
|
|
8432
|
+
log13.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
8319
8433
|
this.callbacks.onRunCreated(runId, projectId, providers, resolvedLocation);
|
|
8320
8434
|
} catch (err) {
|
|
8321
|
-
|
|
8435
|
+
log13.error("trigger.error", { scheduleId, projectId, kind, error: err instanceof Error ? err.message : String(err) });
|
|
8322
8436
|
}
|
|
8323
8437
|
}
|
|
8324
8438
|
};
|
|
8325
8439
|
|
|
8326
8440
|
// src/data-refresh.ts
|
|
8327
|
-
var
|
|
8441
|
+
var log14 = createLogger("DataRefresh");
|
|
8328
8442
|
async function refreshAllIntegrations(client, projectName) {
|
|
8329
8443
|
const integrations = [
|
|
8330
8444
|
{ name: "gsc", run: () => client.gscSync(projectName, {}) },
|
|
@@ -8337,19 +8451,19 @@ async function refreshAllIntegrations(client, projectName) {
|
|
|
8337
8451
|
results.forEach((result, idx) => {
|
|
8338
8452
|
const integration = integrations[idx].name;
|
|
8339
8453
|
if (result.status === "fulfilled") {
|
|
8340
|
-
|
|
8454
|
+
log14.info("integration.refreshed", { projectName, integration });
|
|
8341
8455
|
} else {
|
|
8342
8456
|
const reason = result.reason;
|
|
8343
8457
|
const message = reason instanceof Error ? reason.message : String(reason);
|
|
8344
|
-
|
|
8458
|
+
log14.warn("integration.refresh-failed", { projectName, integration, error: message });
|
|
8345
8459
|
}
|
|
8346
8460
|
});
|
|
8347
8461
|
}
|
|
8348
8462
|
|
|
8349
8463
|
// src/notifier.ts
|
|
8350
|
-
import { eq as
|
|
8351
|
-
import
|
|
8352
|
-
var
|
|
8464
|
+
import { eq as eq15, desc as desc5, and as and12, inArray as inArray6, or } from "drizzle-orm";
|
|
8465
|
+
import crypto17 from "crypto";
|
|
8466
|
+
var log15 = createLogger("Notifier");
|
|
8353
8467
|
var Notifier = class {
|
|
8354
8468
|
db;
|
|
8355
8469
|
serverUrl;
|
|
@@ -8359,26 +8473,26 @@ var Notifier = class {
|
|
|
8359
8473
|
}
|
|
8360
8474
|
/** Called after a run completes (success, partial, or failed). */
|
|
8361
8475
|
async onRunCompleted(runId, projectId) {
|
|
8362
|
-
|
|
8363
|
-
const notifs = this.db.select().from(notifications).where(
|
|
8476
|
+
log15.info("run.completed", { runId, projectId });
|
|
8477
|
+
const notifs = this.db.select().from(notifications).where(eq15(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
8364
8478
|
if (notifs.length === 0) {
|
|
8365
|
-
|
|
8479
|
+
log15.info("notifications.none-enabled", { projectId });
|
|
8366
8480
|
return;
|
|
8367
8481
|
}
|
|
8368
|
-
|
|
8369
|
-
const run = this.db.select().from(runs).where(
|
|
8482
|
+
log15.info("notifications.found", { projectId, count: notifs.length });
|
|
8483
|
+
const run = this.db.select().from(runs).where(eq15(runs.id, runId)).get();
|
|
8370
8484
|
if (!run) {
|
|
8371
|
-
|
|
8485
|
+
log15.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
8372
8486
|
return;
|
|
8373
8487
|
}
|
|
8374
|
-
const project = this.db.select().from(projects).where(
|
|
8488
|
+
const project = this.db.select().from(projects).where(eq15(projects.id, projectId)).get();
|
|
8375
8489
|
if (!project) {
|
|
8376
|
-
|
|
8490
|
+
log15.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
8377
8491
|
return;
|
|
8378
8492
|
}
|
|
8379
8493
|
const transitions = this.computeTransitions(runId, projectId);
|
|
8380
8494
|
const events = [];
|
|
8381
|
-
|
|
8495
|
+
log15.info("run.status", { runId: run.id, status: run.status, projectId });
|
|
8382
8496
|
if (run.status === "completed" || run.status === "partial") {
|
|
8383
8497
|
events.push("run.completed");
|
|
8384
8498
|
}
|
|
@@ -8394,7 +8508,7 @@ var Notifier = class {
|
|
|
8394
8508
|
if (!config.url) continue;
|
|
8395
8509
|
const subscribedEvents = config.events;
|
|
8396
8510
|
const matchingEvents = events.filter((e) => subscribedEvents.includes(e));
|
|
8397
|
-
|
|
8511
|
+
log15.info("notification.match", { notificationId: notif.id, subscribedEvents, matchedEvents: matchingEvents });
|
|
8398
8512
|
if (matchingEvents.length === 0) continue;
|
|
8399
8513
|
for (const event of matchingEvents) {
|
|
8400
8514
|
const relevantTransitions = event === "citation.lost" ? lostTransitions : event === "citation.gained" ? gainedTransitions : transitions;
|
|
@@ -8410,6 +8524,101 @@ var Notifier = class {
|
|
|
8410
8524
|
}
|
|
8411
8525
|
}
|
|
8412
8526
|
}
|
|
8527
|
+
/**
|
|
8528
|
+
* Record the outcome of a scheduled health pass and notify on transitions.
|
|
8529
|
+
*
|
|
8530
|
+
* Edge-triggered on `(status, code)`. A pass that finds the same problem as
|
|
8531
|
+
* last time is stored but not re-sent: an operator who receives the same
|
|
8532
|
+
* warning every day stops reading the channel, which is how a real
|
|
8533
|
+
* degradation goes unnoticed. A pass whose worst code CHANGES does notify,
|
|
8534
|
+
* even at the same severity, because a different cause is different news.
|
|
8535
|
+
*
|
|
8536
|
+
* Returns the event emitted, or null when nothing changed — the return value
|
|
8537
|
+
* is what tests assert against, so "stayed quiet" is observable rather than
|
|
8538
|
+
* inferred from the absence of a webhook.
|
|
8539
|
+
*/
|
|
8540
|
+
async onHealthChecked(projectId, report) {
|
|
8541
|
+
const project = this.db.select().from(projects).where(eq15(projects.id, projectId)).get();
|
|
8542
|
+
if (!project) {
|
|
8543
|
+
log15.error("project.not-found", { projectId, msg: "skipping health notification" });
|
|
8544
|
+
return null;
|
|
8545
|
+
}
|
|
8546
|
+
const rank = (status2) => status2 === "fail" ? 2 : status2 === "warn" ? 1 : 0;
|
|
8547
|
+
const categoryRank = (category) => {
|
|
8548
|
+
switch (category) {
|
|
8549
|
+
case "database":
|
|
8550
|
+
return 6;
|
|
8551
|
+
case "integrations":
|
|
8552
|
+
return 5;
|
|
8553
|
+
case "providers":
|
|
8554
|
+
return 4;
|
|
8555
|
+
case "auth":
|
|
8556
|
+
return 3;
|
|
8557
|
+
case "schedules":
|
|
8558
|
+
return 2;
|
|
8559
|
+
case "config":
|
|
8560
|
+
return 1;
|
|
8561
|
+
default:
|
|
8562
|
+
return 0;
|
|
8563
|
+
}
|
|
8564
|
+
};
|
|
8565
|
+
const graded = report.checks.filter((c) => c.status === "fail" || c.status === "warn" || c.status === "ok");
|
|
8566
|
+
const failing = graded.filter((c) => c.status !== "ok").sort((a, b) => rank(b.status) - rank(a.status) || categoryRank(b.category) - categoryRank(a.category) || a.id.localeCompare(b.id));
|
|
8567
|
+
const worst = failing.at(0) ?? null;
|
|
8568
|
+
const noSignal = graded.length === 0;
|
|
8569
|
+
const status = noSignal ? "warn" : worst ? worst.status : "ok";
|
|
8570
|
+
const code = noSignal ? "health.no-signal" : worst?.code ?? "health.ok";
|
|
8571
|
+
const summary = noSignal ? `No health check produced a result (${report.checks.length} skipped) \u2014 health is unknown, not confirmed.` : worst?.summary ?? `All ${graded.length} health check(s) passing.`;
|
|
8572
|
+
const previous = this.db.select().from(doctorHealthState).where(eq15(doctorHealthState.projectId, projectId)).get();
|
|
8573
|
+
const previousStatus = previous?.status ?? null;
|
|
8574
|
+
let event = null;
|
|
8575
|
+
if (previous === void 0) {
|
|
8576
|
+
if (status !== "ok") event = "health.degraded";
|
|
8577
|
+
} else if (status === "ok" && previousStatus !== "ok") {
|
|
8578
|
+
event = "health.recovered";
|
|
8579
|
+
} else if (status !== "ok" && (previousStatus === "ok" || previous.code !== code)) {
|
|
8580
|
+
event = "health.degraded";
|
|
8581
|
+
}
|
|
8582
|
+
const now = report.checkedAt;
|
|
8583
|
+
const observation = { projectId, status, code, summary, checkedAt: now };
|
|
8584
|
+
if (previous === void 0) {
|
|
8585
|
+
this.db.insert(doctorHealthState).values({ ...observation, notifiedAt: null }).run();
|
|
8586
|
+
} else {
|
|
8587
|
+
this.db.update(doctorHealthState).set(observation).where(eq15(doctorHealthState.projectId, projectId)).run();
|
|
8588
|
+
}
|
|
8589
|
+
if (!event) {
|
|
8590
|
+
log15.info("health.unchanged", { projectId, status, code });
|
|
8591
|
+
return null;
|
|
8592
|
+
}
|
|
8593
|
+
const notifs = this.db.select().from(notifications).where(eq15(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
8594
|
+
const payload = {
|
|
8595
|
+
source: "canonry",
|
|
8596
|
+
event,
|
|
8597
|
+
project: { name: project.name, canonicalDomain: project.canonicalDomain },
|
|
8598
|
+
health: {
|
|
8599
|
+
status,
|
|
8600
|
+
code,
|
|
8601
|
+
summary,
|
|
8602
|
+
remediation: worst?.remediation ?? null,
|
|
8603
|
+
checkedAt: now,
|
|
8604
|
+
previousStatus,
|
|
8605
|
+
failing: failing.map((c) => ({ id: c.id, status: c.status, code: c.code, summary: c.summary }))
|
|
8606
|
+
},
|
|
8607
|
+
dashboardUrl: `${this.serverUrl}/projects/${project.name}`
|
|
8608
|
+
};
|
|
8609
|
+
let delivered = 0;
|
|
8610
|
+
for (const notif of notifs) {
|
|
8611
|
+
const config = notif.config;
|
|
8612
|
+
if (!config.url) continue;
|
|
8613
|
+
await this.sendWebhook(config.url, payload, notif.id, projectId, notif.webhookSecret ?? null);
|
|
8614
|
+
delivered += 1;
|
|
8615
|
+
}
|
|
8616
|
+
if (delivered > 0) {
|
|
8617
|
+
this.db.update(doctorHealthState).set({ notifiedAt: now }).where(eq15(doctorHealthState.projectId, projectId)).run();
|
|
8618
|
+
}
|
|
8619
|
+
log15.info("health.notified", { projectId, event, status, code, subscribers: notifs.length, delivered });
|
|
8620
|
+
return event;
|
|
8621
|
+
}
|
|
8413
8622
|
/** Dispatch insight webhooks for critical/high severity insights after a run. */
|
|
8414
8623
|
async dispatchInsightWebhooks(runId, projectId, result) {
|
|
8415
8624
|
const insightEvents = [];
|
|
@@ -8418,11 +8627,11 @@ var Notifier = class {
|
|
|
8418
8627
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
8419
8628
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
8420
8629
|
if (insightEvents.length === 0) return;
|
|
8421
|
-
const notifs = this.db.select().from(notifications).where(
|
|
8630
|
+
const notifs = this.db.select().from(notifications).where(eq15(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
8422
8631
|
if (notifs.length === 0) return;
|
|
8423
|
-
const run = this.db.select().from(runs).where(
|
|
8632
|
+
const run = this.db.select().from(runs).where(eq15(runs.id, runId)).get();
|
|
8424
8633
|
if (!run) return;
|
|
8425
|
-
const project = this.db.select().from(projects).where(
|
|
8634
|
+
const project = this.db.select().from(projects).where(eq15(projects.id, projectId)).get();
|
|
8426
8635
|
if (!project) return;
|
|
8427
8636
|
for (const notif of notifs) {
|
|
8428
8637
|
const config = notif.config;
|
|
@@ -8452,12 +8661,12 @@ var Notifier = class {
|
|
|
8452
8661
|
}
|
|
8453
8662
|
}
|
|
8454
8663
|
computeTransitions(runId, projectId) {
|
|
8455
|
-
const thisRun = this.db.select().from(runs).where(
|
|
8664
|
+
const thisRun = this.db.select().from(runs).where(eq15(runs.id, runId)).get();
|
|
8456
8665
|
if (!thisRun) return [];
|
|
8457
|
-
const groupSiblings = this.db.select().from(runs).where(
|
|
8458
|
-
|
|
8459
|
-
|
|
8460
|
-
|
|
8666
|
+
const groupSiblings = this.db.select().from(runs).where(and12(
|
|
8667
|
+
eq15(runs.projectId, projectId),
|
|
8668
|
+
eq15(runs.kind, thisRun.kind),
|
|
8669
|
+
eq15(runs.createdAt, thisRun.createdAt)
|
|
8461
8670
|
)).all();
|
|
8462
8671
|
const stillPending = groupSiblings.some((r) => r.status === "queued" || r.status === "running");
|
|
8463
8672
|
if (stillPending) return [];
|
|
@@ -8473,17 +8682,17 @@ var Notifier = class {
|
|
|
8473
8682
|
return candidate.id > best.id ? candidate : best;
|
|
8474
8683
|
});
|
|
8475
8684
|
if (winner.id !== runId) return [];
|
|
8476
|
-
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
8685
|
+
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(eq15(projects.id, projectId)).get();
|
|
8477
8686
|
const locationCount = Math.max(
|
|
8478
8687
|
1,
|
|
8479
8688
|
(projectLocations?.locations ?? []).length
|
|
8480
8689
|
);
|
|
8481
8690
|
const RECENT_FETCH_LIMIT = Math.max(8, locationCount * 4);
|
|
8482
8691
|
const recentRuns = this.db.select().from(runs).where(
|
|
8483
|
-
|
|
8484
|
-
|
|
8485
|
-
|
|
8486
|
-
or(
|
|
8692
|
+
and12(
|
|
8693
|
+
eq15(runs.projectId, projectId),
|
|
8694
|
+
eq15(runs.kind, thisRun.kind),
|
|
8695
|
+
or(eq15(runs.status, "completed"), eq15(runs.status, "partial"))
|
|
8487
8696
|
)
|
|
8488
8697
|
).orderBy(desc5(runs.createdAt), desc5(runs.id)).limit(RECENT_FETCH_LIMIT).all();
|
|
8489
8698
|
const groups = groupRunsByCreatedAt(recentRuns);
|
|
@@ -8500,7 +8709,7 @@ var Notifier = class {
|
|
|
8500
8709
|
provider: querySnapshots.provider,
|
|
8501
8710
|
location: querySnapshots.location,
|
|
8502
8711
|
citationState: querySnapshots.citationState
|
|
8503
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
8712
|
+
}).from(querySnapshots).leftJoin(queries, eq15(querySnapshots.queryId, queries.id)).where(inArray6(querySnapshots.runId, currentRunIds)).all();
|
|
8504
8713
|
const previousSnapshots = this.db.select({
|
|
8505
8714
|
queryId: querySnapshots.queryId,
|
|
8506
8715
|
provider: querySnapshots.provider,
|
|
@@ -8530,26 +8739,29 @@ var Notifier = class {
|
|
|
8530
8739
|
return transitions;
|
|
8531
8740
|
}
|
|
8532
8741
|
async sendWebhook(url, payload, notificationId, projectId, webhookSecret) {
|
|
8742
|
+
const destination = resolveDestination(url);
|
|
8743
|
+
const body = destination.render ? destination.render(toAlertView(payload)) : payload;
|
|
8744
|
+
const signingSecret = destination.signed ? webhookSecret : null;
|
|
8533
8745
|
const targetLabel = redactNotificationUrl(url).urlDisplay;
|
|
8534
8746
|
const targetCheck = await resolveWebhookTarget(url);
|
|
8535
8747
|
if (!targetCheck.ok) {
|
|
8536
|
-
|
|
8748
|
+
log15.error("webhook.ssrf-blocked", { url: targetLabel, reason: targetCheck.message });
|
|
8537
8749
|
this.logDelivery(projectId, notificationId, payload.event, "failed", `SSRF: ${targetCheck.message}`);
|
|
8538
8750
|
return;
|
|
8539
8751
|
}
|
|
8540
|
-
|
|
8752
|
+
log15.info("webhook.send", { event: payload.event, url: targetLabel });
|
|
8541
8753
|
const maxRetries = 3;
|
|
8542
8754
|
const delays = [1e3, 4e3, 16e3];
|
|
8543
8755
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
8544
8756
|
try {
|
|
8545
|
-
const response = await deliverWebhook(targetCheck.target,
|
|
8757
|
+
const response = await deliverWebhook(targetCheck.target, body, signingSecret);
|
|
8546
8758
|
if (response.status >= 200 && response.status < 300) {
|
|
8547
|
-
|
|
8759
|
+
log15.info("webhook.delivered", { event: payload.event, url: targetLabel, httpStatus: response.status });
|
|
8548
8760
|
this.logDelivery(projectId, notificationId, payload.event, "sent", null);
|
|
8549
8761
|
return;
|
|
8550
8762
|
}
|
|
8551
8763
|
const errorDetail = response.error ?? `HTTP ${response.status}`;
|
|
8552
|
-
|
|
8764
|
+
log15.warn("webhook.attempt-failed", { event: payload.event, url: targetLabel, attempt: attempt + 1, maxRetries, httpStatus: response.status, error: errorDetail });
|
|
8553
8765
|
if (attempt === maxRetries - 1) {
|
|
8554
8766
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
8555
8767
|
}
|
|
@@ -8557,7 +8769,7 @@ var Notifier = class {
|
|
|
8557
8769
|
const errorDetail = err instanceof Error ? err.message : String(err);
|
|
8558
8770
|
if (attempt === maxRetries - 1) {
|
|
8559
8771
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
8560
|
-
|
|
8772
|
+
log15.error("webhook.exhausted", { event: payload.event, url: targetLabel, maxRetries, error: errorDetail });
|
|
8561
8773
|
}
|
|
8562
8774
|
}
|
|
8563
8775
|
if (attempt < maxRetries - 1) {
|
|
@@ -8567,7 +8779,7 @@ var Notifier = class {
|
|
|
8567
8779
|
}
|
|
8568
8780
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
8569
8781
|
this.db.insert(auditLog).values({
|
|
8570
|
-
id:
|
|
8782
|
+
id: crypto17.randomUUID(),
|
|
8571
8783
|
projectId,
|
|
8572
8784
|
actor: "scheduler",
|
|
8573
8785
|
action: `notification.${status}`,
|
|
@@ -8580,8 +8792,8 @@ var Notifier = class {
|
|
|
8580
8792
|
};
|
|
8581
8793
|
|
|
8582
8794
|
// src/run-coordinator.ts
|
|
8583
|
-
import { eq as
|
|
8584
|
-
var
|
|
8795
|
+
import { eq as eq16 } from "drizzle-orm";
|
|
8796
|
+
var log16 = createLogger("RunCoordinator");
|
|
8585
8797
|
var RunCoordinator = class {
|
|
8586
8798
|
constructor(db, notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
8587
8799
|
this.db = db;
|
|
@@ -8596,10 +8808,10 @@ var RunCoordinator = class {
|
|
|
8596
8808
|
onInsightsGenerated;
|
|
8597
8809
|
onAeroEvent;
|
|
8598
8810
|
async onRunCompleted(runId, projectId) {
|
|
8599
|
-
const runRow = this.db.select().from(runs).where(
|
|
8811
|
+
const runRow = this.db.select().from(runs).where(eq16(runs.id, runId)).get();
|
|
8600
8812
|
const kind = runRow?.kind ?? RunKinds["answer-visibility"];
|
|
8601
8813
|
if (runRow?.trigger === RunTriggers.probe) {
|
|
8602
|
-
|
|
8814
|
+
log16.info("probe.skip-side-effects", { runId, projectId, kind });
|
|
8603
8815
|
return;
|
|
8604
8816
|
}
|
|
8605
8817
|
let insightCount = 0;
|
|
@@ -8616,12 +8828,12 @@ var RunCoordinator = class {
|
|
|
8616
8828
|
try {
|
|
8617
8829
|
await this.onInsightsGenerated(runId, projectId, result);
|
|
8618
8830
|
} catch (err) {
|
|
8619
|
-
|
|
8831
|
+
log16.error("insight-webhook.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
8620
8832
|
}
|
|
8621
8833
|
}
|
|
8622
8834
|
}
|
|
8623
8835
|
} catch (err) {
|
|
8624
|
-
|
|
8836
|
+
log16.error("intelligence.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
8625
8837
|
}
|
|
8626
8838
|
} else if (kind === RunKinds["gbp-sync"]) {
|
|
8627
8839
|
try {
|
|
@@ -8634,17 +8846,17 @@ var RunCoordinator = class {
|
|
|
8634
8846
|
try {
|
|
8635
8847
|
await this.onInsightsGenerated(runId, projectId, analysisResultFromInsights(gbpInsights));
|
|
8636
8848
|
} catch (err) {
|
|
8637
|
-
|
|
8849
|
+
log16.error("gbp-insight-webhook.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
8638
8850
|
}
|
|
8639
8851
|
}
|
|
8640
8852
|
} catch (err) {
|
|
8641
|
-
|
|
8853
|
+
log16.error("gbp-intelligence.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
8642
8854
|
}
|
|
8643
8855
|
}
|
|
8644
8856
|
try {
|
|
8645
8857
|
await this.notifier.onRunCompleted(runId, projectId);
|
|
8646
8858
|
} catch (err) {
|
|
8647
|
-
|
|
8859
|
+
log16.error("notifier.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
8648
8860
|
}
|
|
8649
8861
|
if (this.onAeroEvent) {
|
|
8650
8862
|
try {
|
|
@@ -8657,7 +8869,7 @@ var RunCoordinator = class {
|
|
|
8657
8869
|
};
|
|
8658
8870
|
await this.onAeroEvent(ctx);
|
|
8659
8871
|
} catch (err) {
|
|
8660
|
-
|
|
8872
|
+
log16.error("aero.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
8661
8873
|
}
|
|
8662
8874
|
}
|
|
8663
8875
|
}
|
|
@@ -8672,7 +8884,7 @@ var RunCoordinator = class {
|
|
|
8672
8884
|
* so the Aero queue is never starved of a follow-up.
|
|
8673
8885
|
*/
|
|
8674
8886
|
buildDiscoveryAeroContext(runId, projectId, status, error) {
|
|
8675
|
-
const session = this.db.select().from(discoverySessions).where(
|
|
8887
|
+
const session = this.db.select().from(discoverySessions).where(eq16(discoverySessions.runId, runId)).get();
|
|
8676
8888
|
const competitorMap = session ? session.competitorMap : [];
|
|
8677
8889
|
return {
|
|
8678
8890
|
kind: RunKinds["aeo-discover-probe"],
|
|
@@ -8714,8 +8926,8 @@ function analysisResultFromInsights(insights2) {
|
|
|
8714
8926
|
}
|
|
8715
8927
|
|
|
8716
8928
|
// src/agent/session-registry.ts
|
|
8717
|
-
import
|
|
8718
|
-
import { eq as
|
|
8929
|
+
import crypto21 from "crypto";
|
|
8930
|
+
import { eq as eq18 } from "drizzle-orm";
|
|
8719
8931
|
|
|
8720
8932
|
// src/agent/session.ts
|
|
8721
8933
|
import fs7 from "fs";
|
|
@@ -9458,7 +9670,7 @@ function buildAeroStateTools(ctx, opts = {}) {
|
|
|
9458
9670
|
}
|
|
9459
9671
|
|
|
9460
9672
|
// src/agent/llm-usage.ts
|
|
9461
|
-
import
|
|
9673
|
+
import crypto18 from "crypto";
|
|
9462
9674
|
var AeroLlmUsageFeatures = {
|
|
9463
9675
|
turn: "aero.turn"
|
|
9464
9676
|
};
|
|
@@ -9477,7 +9689,7 @@ function recordLlmUsageEvent(args) {
|
|
|
9477
9689
|
const usage = args.message.usage;
|
|
9478
9690
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9479
9691
|
args.db.insert(llmUsageEvents).values({
|
|
9480
|
-
id:
|
|
9692
|
+
id: crypto18.randomUUID(),
|
|
9481
9693
|
projectId: args.projectId,
|
|
9482
9694
|
runId: args.runId,
|
|
9483
9695
|
agentSessionId: args.agentSessionId,
|
|
@@ -9553,7 +9765,7 @@ function splitAeroAnthropicSystemCachePayload(payload, model) {
|
|
|
9553
9765
|
}
|
|
9554
9766
|
|
|
9555
9767
|
// src/agent/tool-usage.ts
|
|
9556
|
-
import
|
|
9768
|
+
import crypto19 from "crypto";
|
|
9557
9769
|
var AeroToolEventStatuses = {
|
|
9558
9770
|
success: "success",
|
|
9559
9771
|
error: "error"
|
|
@@ -9616,7 +9828,7 @@ function createAeroToolUsageHooks(args) {
|
|
|
9616
9828
|
function recordAgentToolEvent(args) {
|
|
9617
9829
|
try {
|
|
9618
9830
|
args.db.insert(agentToolEvents).values({
|
|
9619
|
-
id:
|
|
9831
|
+
id: crypto19.randomUUID(),
|
|
9620
9832
|
projectId: args.projectId,
|
|
9621
9833
|
agentSessionId: args.agentSessionId,
|
|
9622
9834
|
toolCallId: args.toolCallId,
|
|
@@ -9759,7 +9971,7 @@ function resolveSessionProviderAndModel(config, opts) {
|
|
|
9759
9971
|
import { Type as Type4 } from "@sinclair/typebox";
|
|
9760
9972
|
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
9761
9973
|
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
|
|
9762
|
-
var
|
|
9974
|
+
var log17 = createLogger("RemoteMcp");
|
|
9763
9975
|
var AERO_EXCLUDED_MCP_TOOLS2 = /* @__PURE__ */ new Set([]);
|
|
9764
9976
|
var MAX_TOOL_RESULT_CHARS2 = 2e4;
|
|
9765
9977
|
function truncate(text) {
|
|
@@ -9812,7 +10024,7 @@ async function loadExternalMcpTools(servers, opts = {}) {
|
|
|
9812
10024
|
try {
|
|
9813
10025
|
client = await connect(server);
|
|
9814
10026
|
} catch (err) {
|
|
9815
|
-
|
|
10027
|
+
log17.error("external-mcp.connect-failed", {
|
|
9816
10028
|
label,
|
|
9817
10029
|
error: err instanceof Error ? err.message : String(err)
|
|
9818
10030
|
});
|
|
@@ -9822,7 +10034,7 @@ async function loadExternalMcpTools(servers, opts = {}) {
|
|
|
9822
10034
|
try {
|
|
9823
10035
|
listed = await client.listTools();
|
|
9824
10036
|
} catch (err) {
|
|
9825
|
-
|
|
10037
|
+
log17.error("external-mcp.list-failed", {
|
|
9826
10038
|
label,
|
|
9827
10039
|
error: err instanceof Error ? err.message : String(err)
|
|
9828
10040
|
});
|
|
@@ -9833,7 +10045,7 @@ async function loadExternalMcpTools(servers, opts = {}) {
|
|
|
9833
10045
|
if (!isReadOnly(tool)) continue;
|
|
9834
10046
|
tools.push(adaptRemoteTool(client, tool));
|
|
9835
10047
|
}
|
|
9836
|
-
|
|
10048
|
+
log17.info("external-mcp.loaded", {
|
|
9837
10049
|
label,
|
|
9838
10050
|
discovered: listed.tools.length,
|
|
9839
10051
|
adopted: tools.length
|
|
@@ -9843,8 +10055,8 @@ async function loadExternalMcpTools(servers, opts = {}) {
|
|
|
9843
10055
|
}
|
|
9844
10056
|
|
|
9845
10057
|
// src/agent/memory-store.ts
|
|
9846
|
-
import
|
|
9847
|
-
import { and as
|
|
10058
|
+
import crypto20 from "crypto";
|
|
10059
|
+
import { and as and13, desc as desc6, eq as eq17, like, sql as sql7 } from "drizzle-orm";
|
|
9848
10060
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
9849
10061
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
9850
10062
|
function rowToDto(row) {
|
|
@@ -9858,7 +10070,7 @@ function rowToDto(row) {
|
|
|
9858
10070
|
};
|
|
9859
10071
|
}
|
|
9860
10072
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
9861
|
-
const query = db.select().from(agentMemory).where(
|
|
10073
|
+
const query = db.select().from(agentMemory).where(eq17(agentMemory.projectId, projectId)).orderBy(desc6(agentMemory.updatedAt));
|
|
9862
10074
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
9863
10075
|
return rows.map(rowToDto);
|
|
9864
10076
|
}
|
|
@@ -9872,7 +10084,7 @@ function upsertMemoryEntry(db, args) {
|
|
|
9872
10084
|
throw new Error(`memory key prefix "${COMPACTION_KEY_PREFIX}" is reserved for compaction notes`);
|
|
9873
10085
|
}
|
|
9874
10086
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9875
|
-
const id =
|
|
10087
|
+
const id = crypto20.randomUUID();
|
|
9876
10088
|
db.insert(agentMemory).values({
|
|
9877
10089
|
id,
|
|
9878
10090
|
projectId: args.projectId,
|
|
@@ -9889,12 +10101,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
9889
10101
|
updatedAt: now
|
|
9890
10102
|
}
|
|
9891
10103
|
}).run();
|
|
9892
|
-
const row = db.select().from(agentMemory).where(
|
|
10104
|
+
const row = db.select().from(agentMemory).where(and13(eq17(agentMemory.projectId, args.projectId), eq17(agentMemory.key, args.key))).get();
|
|
9893
10105
|
if (!row) throw new Error("memory upsert produced no row");
|
|
9894
10106
|
return rowToDto(row);
|
|
9895
10107
|
}
|
|
9896
10108
|
function deleteMemoryEntry(db, projectId, key) {
|
|
9897
|
-
const result = db.delete(agentMemory).where(
|
|
10109
|
+
const result = db.delete(agentMemory).where(and13(eq17(agentMemory.projectId, projectId), eq17(agentMemory.key, key))).run();
|
|
9898
10110
|
const changes = result.changes ?? 0;
|
|
9899
10111
|
return changes > 0;
|
|
9900
10112
|
}
|
|
@@ -9909,7 +10121,7 @@ function writeCompactionNote(db, args) {
|
|
|
9909
10121
|
}
|
|
9910
10122
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9911
10123
|
const key = `${COMPACTION_KEY_PREFIX}${args.sessionId}:${now}`;
|
|
9912
|
-
const id =
|
|
10124
|
+
const id = crypto20.randomUUID();
|
|
9913
10125
|
let inserted;
|
|
9914
10126
|
db.transaction((tx) => {
|
|
9915
10127
|
tx.insert(agentMemory).values({
|
|
@@ -9923,16 +10135,16 @@ function writeCompactionNote(db, args) {
|
|
|
9923
10135
|
}).run();
|
|
9924
10136
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
9925
10137
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
9926
|
-
|
|
9927
|
-
|
|
10138
|
+
and13(
|
|
10139
|
+
eq17(agentMemory.projectId, args.projectId),
|
|
9928
10140
|
like(agentMemory.key, `${sessionPrefix}%`)
|
|
9929
10141
|
)
|
|
9930
10142
|
).orderBy(desc6(agentMemory.updatedAt)).all();
|
|
9931
10143
|
const stale = existing.slice(COMPACTION_NOTES_PER_SESSION).map((r) => r.id);
|
|
9932
10144
|
if (stale.length > 0) {
|
|
9933
|
-
tx.delete(agentMemory).where(
|
|
10145
|
+
tx.delete(agentMemory).where(sql7`${agentMemory.id} IN (${sql7.join(stale.map((s) => sql7`${s}`), sql7`, `)})`).run();
|
|
9934
10146
|
}
|
|
9935
|
-
const row = tx.select().from(agentMemory).where(
|
|
10147
|
+
const row = tx.select().from(agentMemory).where(and13(eq17(agentMemory.projectId, args.projectId), eq17(agentMemory.key, key))).get();
|
|
9936
10148
|
if (row) inserted = rowToDto(row);
|
|
9937
10149
|
});
|
|
9938
10150
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -10065,7 +10277,7 @@ async function compactMessages(args) {
|
|
|
10065
10277
|
}
|
|
10066
10278
|
|
|
10067
10279
|
// src/agent/session-registry.ts
|
|
10068
|
-
var
|
|
10280
|
+
var log18 = createLogger("SessionRegistry");
|
|
10069
10281
|
var MAX_HYDRATE_NOTES = 20;
|
|
10070
10282
|
var MAX_HYDRATE_BYTES = 32 * 1024;
|
|
10071
10283
|
function escapeMemoryFragment(value) {
|
|
@@ -10109,7 +10321,7 @@ var SessionRegistry = class {
|
|
|
10109
10321
|
if (!this.externalToolsPromise) {
|
|
10110
10322
|
this.externalToolsPromise = loadExternalMcpTools(this.opts.config.externalMcpServers).catch(
|
|
10111
10323
|
(err) => {
|
|
10112
|
-
|
|
10324
|
+
log18.error("external-mcp.load-failed", {
|
|
10113
10325
|
error: err instanceof Error ? err.message : String(err)
|
|
10114
10326
|
});
|
|
10115
10327
|
return [];
|
|
@@ -10159,7 +10371,7 @@ var SessionRegistry = class {
|
|
|
10159
10371
|
modelProvider: effectiveProvider,
|
|
10160
10372
|
modelId: effectiveModelId,
|
|
10161
10373
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
10162
|
-
}).where(
|
|
10374
|
+
}).where(eq18(agentSessions.projectId, projectId)).run();
|
|
10163
10375
|
}
|
|
10164
10376
|
const agent2 = createAeroSession({
|
|
10165
10377
|
projectName,
|
|
@@ -10188,7 +10400,7 @@ var SessionRegistry = class {
|
|
|
10188
10400
|
}
|
|
10189
10401
|
const { provider, modelId } = resolveSessionProviderAndModel(this.opts.config, preferences);
|
|
10190
10402
|
const systemPrompt = loadAeroSystemPrompt();
|
|
10191
|
-
const sessionId =
|
|
10403
|
+
const sessionId = crypto21.randomUUID();
|
|
10192
10404
|
const agent = createAeroSession({
|
|
10193
10405
|
projectName,
|
|
10194
10406
|
client: this.opts.client,
|
|
@@ -10354,13 +10566,13 @@ ${lines.join("\n")}
|
|
|
10354
10566
|
agent.state.messages = result.messages;
|
|
10355
10567
|
agent.state.systemPrompt = this.buildHydratedSystemPrompt(projectId, row.systemPrompt);
|
|
10356
10568
|
this.save(projectName);
|
|
10357
|
-
|
|
10569
|
+
log18.info("compaction.completed", {
|
|
10358
10570
|
projectName,
|
|
10359
10571
|
removedCount: result.removedCount,
|
|
10360
10572
|
summaryBytes: Buffer.byteLength(result.summary, "utf8")
|
|
10361
10573
|
});
|
|
10362
10574
|
} catch (err) {
|
|
10363
|
-
|
|
10575
|
+
log18.error("compaction.failed", {
|
|
10364
10576
|
projectName,
|
|
10365
10577
|
error: err instanceof Error ? err.message : String(err)
|
|
10366
10578
|
});
|
|
@@ -10393,7 +10605,7 @@ ${lines.join("\n")}
|
|
|
10393
10605
|
modelProvider: nextProvider,
|
|
10394
10606
|
modelId: nextModelId,
|
|
10395
10607
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
10396
|
-
}).where(
|
|
10608
|
+
}).where(eq18(agentSessions.projectId, projectId)).run();
|
|
10397
10609
|
}
|
|
10398
10610
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
10399
10611
|
save(projectName) {
|
|
@@ -10461,7 +10673,7 @@ ${lines.join("\n")}
|
|
|
10461
10673
|
await agent.prompt(msgs);
|
|
10462
10674
|
this.save(projectName);
|
|
10463
10675
|
} catch (err) {
|
|
10464
|
-
|
|
10676
|
+
log18.error("drain.failed", {
|
|
10465
10677
|
projectName,
|
|
10466
10678
|
error: err instanceof Error ? err.message : String(err)
|
|
10467
10679
|
});
|
|
@@ -10557,17 +10769,17 @@ ${lines.join("\n")}
|
|
|
10557
10769
|
return id;
|
|
10558
10770
|
}
|
|
10559
10771
|
tryResolveProjectId(projectName) {
|
|
10560
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
10772
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq18(projects.name, projectName)).get();
|
|
10561
10773
|
return row?.id;
|
|
10562
10774
|
}
|
|
10563
10775
|
loadRow(projectId) {
|
|
10564
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
10776
|
+
const row = this.opts.db.select().from(agentSessions).where(eq18(agentSessions.projectId, projectId)).get();
|
|
10565
10777
|
return row ?? null;
|
|
10566
10778
|
}
|
|
10567
10779
|
insertRow(params) {
|
|
10568
10780
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
10569
10781
|
this.opts.db.insert(agentSessions).values({
|
|
10570
|
-
id: params.id ??
|
|
10782
|
+
id: params.id ?? crypto21.randomUUID(),
|
|
10571
10783
|
projectId: params.projectId,
|
|
10572
10784
|
systemPrompt: params.systemPrompt,
|
|
10573
10785
|
modelProvider: params.provider ?? params.modelProvider ?? AgentProviderIds.claude,
|
|
@@ -10580,14 +10792,14 @@ ${lines.join("\n")}
|
|
|
10580
10792
|
}
|
|
10581
10793
|
updateRow(projectId, patch) {
|
|
10582
10794
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
10583
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
10795
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq18(agentSessions.projectId, projectId)).run();
|
|
10584
10796
|
}
|
|
10585
10797
|
};
|
|
10586
10798
|
|
|
10587
10799
|
// src/agent/agent-routes.ts
|
|
10588
|
-
import { eq as
|
|
10800
|
+
import { eq as eq19 } from "drizzle-orm";
|
|
10589
10801
|
function resolveProject(db, name) {
|
|
10590
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
10802
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq19(projects.name, name)).get();
|
|
10591
10803
|
if (!row) throw notFound("project", name);
|
|
10592
10804
|
return row;
|
|
10593
10805
|
}
|
|
@@ -10596,7 +10808,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
10596
10808
|
"/projects/:name/agent/transcript",
|
|
10597
10809
|
async (request) => {
|
|
10598
10810
|
const project = resolveProject(opts.db, request.params.name);
|
|
10599
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
10811
|
+
const row = opts.db.select().from(agentSessions).where(eq19(agentSessions.projectId, project.id)).get();
|
|
10600
10812
|
if (!row) {
|
|
10601
10813
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
10602
10814
|
}
|
|
@@ -10620,7 +10832,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
10620
10832
|
async (request) => {
|
|
10621
10833
|
const project = resolveProject(opts.db, request.params.name);
|
|
10622
10834
|
opts.sessionRegistry.reset(project.name);
|
|
10623
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
10835
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq19(agentSessions.projectId, project.id)).run();
|
|
10624
10836
|
return { status: "reset" };
|
|
10625
10837
|
}
|
|
10626
10838
|
);
|
|
@@ -11054,7 +11266,7 @@ function formatAuditFactorScore(factor) {
|
|
|
11054
11266
|
}
|
|
11055
11267
|
|
|
11056
11268
|
// src/snapshot-service.ts
|
|
11057
|
-
var
|
|
11269
|
+
var log19 = createLogger("Snapshot");
|
|
11058
11270
|
var ANALYSIS_PROVIDER_PRIORITY = ["openai", "claude", "gemini", "perplexity", "local"];
|
|
11059
11271
|
var SNAPSHOT_QUERY_COUNT = 6;
|
|
11060
11272
|
var ProviderExecutionGate2 = class {
|
|
@@ -11200,7 +11412,7 @@ var SnapshotService = class {
|
|
|
11200
11412
|
return mapAuditReport(report);
|
|
11201
11413
|
} catch (err) {
|
|
11202
11414
|
const message = err instanceof Error ? err.message : String(err);
|
|
11203
|
-
|
|
11415
|
+
log19.warn("audit.failed", { homepageUrl, error: message });
|
|
11204
11416
|
return {
|
|
11205
11417
|
url: homepageUrl,
|
|
11206
11418
|
finalUrl: homepageUrl,
|
|
@@ -11229,7 +11441,7 @@ var SnapshotService = class {
|
|
|
11229
11441
|
queries: parsedQueries
|
|
11230
11442
|
};
|
|
11231
11443
|
} catch (err) {
|
|
11232
|
-
|
|
11444
|
+
log19.warn("profile.generation-failed", {
|
|
11233
11445
|
domain: ctx.domain,
|
|
11234
11446
|
provider: ctx.analysisProvider.adapter.name,
|
|
11235
11447
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -11371,7 +11583,7 @@ var SnapshotService = class {
|
|
|
11371
11583
|
recommendedActions: uniqueStrings(parsed.recommendedActions ?? []).slice(0, 4)
|
|
11372
11584
|
};
|
|
11373
11585
|
} catch (err) {
|
|
11374
|
-
|
|
11586
|
+
log19.warn("response.analysis-failed", {
|
|
11375
11587
|
provider: ctx.analysisProvider.adapter.name,
|
|
11376
11588
|
error: err instanceof Error ? err.message : String(err)
|
|
11377
11589
|
});
|
|
@@ -11626,20 +11838,20 @@ function clipText(value, length) {
|
|
|
11626
11838
|
}
|
|
11627
11839
|
|
|
11628
11840
|
// src/research-runner.ts
|
|
11629
|
-
import { and as
|
|
11841
|
+
import { and as and14, eq as eq20, inArray as inArray7, sql as sql8 } from "drizzle-orm";
|
|
11630
11842
|
var unfinishedResearchQueryStatuses = [ResearchQueryStatuses.queued, ResearchQueryStatuses.running];
|
|
11631
11843
|
function finalResearchRunStatus(completed, failed) {
|
|
11632
11844
|
if (failed === 0) return ResearchRunStatuses.completed;
|
|
11633
11845
|
return completed > 0 ? ResearchRunStatuses.partial : ResearchRunStatuses.failed;
|
|
11634
11846
|
}
|
|
11635
11847
|
async function executeResearchRun(db, registry, runId, projectId) {
|
|
11636
|
-
const run = db.select().from(researchRuns).where(
|
|
11848
|
+
const run = db.select().from(researchRuns).where(and14(eq20(researchRuns.id, runId), eq20(researchRuns.projectId, projectId))).get();
|
|
11637
11849
|
if (!run || run.status !== ResearchRunStatuses.queued) return;
|
|
11638
|
-
const project = db.select().from(projects).where(
|
|
11850
|
+
const project = db.select().from(projects).where(eq20(projects.id, projectId)).get();
|
|
11639
11851
|
if (!project) return;
|
|
11640
11852
|
const provider = registry.get(run.provider);
|
|
11641
11853
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
11642
|
-
const claim = db.update(researchRuns).set({ status: ResearchRunStatuses.running, startedAt: now }).where(
|
|
11854
|
+
const claim = db.update(researchRuns).set({ status: ResearchRunStatuses.running, startedAt: now }).where(and14(eq20(researchRuns.id, runId), eq20(researchRuns.status, ResearchRunStatuses.queued))).run();
|
|
11643
11855
|
if (claim.changes !== 1) return;
|
|
11644
11856
|
let reserved = 0;
|
|
11645
11857
|
let dispatched = 0;
|
|
@@ -11663,8 +11875,8 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
11663
11875
|
}
|
|
11664
11876
|
reserved = run.totalQueries;
|
|
11665
11877
|
reservation = { scope, period };
|
|
11666
|
-
const rows = db.select().from(researchRunQueries).where(
|
|
11667
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
11878
|
+
const rows = db.select().from(researchRunQueries).where(eq20(researchRunQueries.researchRunId, runId)).orderBy(researchRunQueries.position).all();
|
|
11879
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq20(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
11668
11880
|
const domains = effectiveDomains(project);
|
|
11669
11881
|
const brands = effectiveBrandNames(project);
|
|
11670
11882
|
const config = { ...provider.config, model: run.resolvedModel };
|
|
@@ -11672,7 +11884,7 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
11672
11884
|
await mapWithConcurrency(rows, Math.max(1, provider.config.quotaPolicy.maxConcurrency), async (row) => {
|
|
11673
11885
|
try {
|
|
11674
11886
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
11675
|
-
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.running, startedAt }).where(
|
|
11887
|
+
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.running, startedAt }).where(and14(eq20(researchRunQueries.id, row.id), eq20(researchRunQueries.status, ResearchQueryStatuses.queued))).run();
|
|
11676
11888
|
const raw = await gate.run(async () => {
|
|
11677
11889
|
dispatched++;
|
|
11678
11890
|
return provider.adapter.executeTrackedQuery({ query: row.queryText, canonicalDomains: domains, competitorDomains, ...run.location ? { location: run.location } : {} }, config);
|
|
@@ -11699,7 +11911,7 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
11699
11911
|
citationState: determineCitationState(normalized, domains),
|
|
11700
11912
|
rawResponse: raw.rawResponse,
|
|
11701
11913
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
11702
|
-
}).where(
|
|
11914
|
+
}).where(and14(eq20(researchRunQueries.id, row.id), eq20(researchRunQueries.status, ResearchQueryStatuses.running))).run();
|
|
11703
11915
|
if (completed.changes === 1) incrementResearchProgress(db, runId, "completedQueries");
|
|
11704
11916
|
} catch (error) {
|
|
11705
11917
|
try {
|
|
@@ -11723,18 +11935,18 @@ async function executeResearchRun(db, registry, runId, projectId) {
|
|
|
11723
11935
|
}
|
|
11724
11936
|
}
|
|
11725
11937
|
function incrementResearchProgress(db, runId, column) {
|
|
11726
|
-
db.update(researchRuns).set({ [column]:
|
|
11938
|
+
db.update(researchRuns).set({ [column]: sql8`${researchRuns[column]} + 1` }).where(eq20(researchRuns.id, runId)).run();
|
|
11727
11939
|
}
|
|
11728
11940
|
function markResearchQueryFailed(db, runId, queryId, error) {
|
|
11729
11941
|
const message = error instanceof Error ? error.message : String(error);
|
|
11730
|
-
const failed = db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error: message, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
11942
|
+
const failed = db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error: message, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and14(eq20(researchRunQueries.id, queryId), inArray7(researchRunQueries.status, unfinishedResearchQueryStatuses))).run();
|
|
11731
11943
|
if (failed.changes === 1) incrementResearchProgress(db, runId, "failedQueries");
|
|
11732
11944
|
}
|
|
11733
11945
|
function markUnfinishedResearchQueriesFailed(db, runId, error) {
|
|
11734
|
-
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
11946
|
+
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and14(eq20(researchRunQueries.researchRunId, runId), inArray7(researchRunQueries.status, unfinishedResearchQueryStatuses))).run();
|
|
11735
11947
|
}
|
|
11736
11948
|
function finalizeResearchRun(db, runId, fatalError) {
|
|
11737
|
-
const rows = db.select({ status: researchRunQueries.status }).from(researchRunQueries).where(
|
|
11949
|
+
const rows = db.select({ status: researchRunQueries.status }).from(researchRunQueries).where(eq20(researchRunQueries.researchRunId, runId)).all();
|
|
11738
11950
|
const completed = rows.filter((row) => row.status === ResearchQueryStatuses.completed).length;
|
|
11739
11951
|
const failed = rows.filter((row) => row.status === ResearchQueryStatuses.failed).length;
|
|
11740
11952
|
db.update(researchRuns).set({
|
|
@@ -11743,13 +11955,13 @@ function finalizeResearchRun(db, runId, fatalError) {
|
|
|
11743
11955
|
failedQueries: failed,
|
|
11744
11956
|
error: fatalError ?? (failed === rows.length ? "Every research query failed." : null),
|
|
11745
11957
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
11746
|
-
}).where(
|
|
11958
|
+
}).where(eq20(researchRuns.id, runId)).run();
|
|
11747
11959
|
}
|
|
11748
11960
|
|
|
11749
11961
|
// src/server.ts
|
|
11750
11962
|
var _require3 = createRequire3(import.meta.url);
|
|
11751
11963
|
var { version: PKG_VERSION2 } = _require3("../package.json");
|
|
11752
|
-
var
|
|
11964
|
+
var log20 = createLogger("Server");
|
|
11753
11965
|
var DEFAULT_QUOTA = {
|
|
11754
11966
|
maxConcurrency: 2,
|
|
11755
11967
|
maxRequestsPerMinute: 10,
|
|
@@ -11792,14 +12004,14 @@ function summarizeProviderConfig(config) {
|
|
|
11792
12004
|
};
|
|
11793
12005
|
}
|
|
11794
12006
|
function hashApiKey(key) {
|
|
11795
|
-
return
|
|
12007
|
+
return crypto22.createHash("sha256").update(key).digest("hex");
|
|
11796
12008
|
}
|
|
11797
12009
|
var DASHBOARD_SCRYPT_KEYLEN = 64;
|
|
11798
12010
|
var DASHBOARD_SCRYPT_COST = 1 << 15;
|
|
11799
12011
|
var DASHBOARD_SCRYPT_MAXMEM = 64 * 1024 * 1024;
|
|
11800
12012
|
function hashDashboardPassword(password) {
|
|
11801
|
-
const salt =
|
|
11802
|
-
const derived =
|
|
12013
|
+
const salt = crypto22.randomBytes(16);
|
|
12014
|
+
const derived = crypto22.scryptSync(password, salt, DASHBOARD_SCRYPT_KEYLEN, {
|
|
11803
12015
|
N: DASHBOARD_SCRYPT_COST,
|
|
11804
12016
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
11805
12017
|
});
|
|
@@ -11820,14 +12032,14 @@ function verifyDashboardPassword(password, storedHash) {
|
|
|
11820
12032
|
} catch {
|
|
11821
12033
|
return { ok: false, needsRehash: false };
|
|
11822
12034
|
}
|
|
11823
|
-
const derived =
|
|
12035
|
+
const derived = crypto22.scryptSync(password, salt, expected.length, {
|
|
11824
12036
|
N: DASHBOARD_SCRYPT_COST,
|
|
11825
12037
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
11826
12038
|
});
|
|
11827
12039
|
if (derived.length !== expected.length)
|
|
11828
12040
|
return { ok: false, needsRehash: false };
|
|
11829
12041
|
return {
|
|
11830
|
-
ok:
|
|
12042
|
+
ok: crypto22.timingSafeEqual(derived, expected),
|
|
11831
12043
|
needsRehash: false
|
|
11832
12044
|
};
|
|
11833
12045
|
}
|
|
@@ -11836,7 +12048,7 @@ function verifyDashboardPassword(password, storedHash) {
|
|
|
11836
12048
|
const expected = Buffer.from(storedHash, "hex");
|
|
11837
12049
|
if (candidate.length !== expected.length)
|
|
11838
12050
|
return { ok: false, needsRehash: false };
|
|
11839
|
-
const ok =
|
|
12051
|
+
const ok = crypto22.timingSafeEqual(candidate, expected);
|
|
11840
12052
|
return { ok, needsRehash: ok };
|
|
11841
12053
|
}
|
|
11842
12054
|
return { ok: false, needsRehash: false };
|
|
@@ -11899,7 +12111,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
11899
12111
|
}
|
|
11900
12112
|
if (migratedGoogle > 0) {
|
|
11901
12113
|
saveConfigPatch({ google: config.google });
|
|
11902
|
-
|
|
12114
|
+
log20.info("credentials.migrated", { type: "google", count: migratedGoogle });
|
|
11903
12115
|
}
|
|
11904
12116
|
let migratedGa4 = 0;
|
|
11905
12117
|
for (const row of rows.ga4) {
|
|
@@ -11917,7 +12129,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
11917
12129
|
}
|
|
11918
12130
|
if (migratedGa4 > 0) {
|
|
11919
12131
|
saveConfigPatch({ ga4: config.ga4 });
|
|
11920
|
-
|
|
12132
|
+
log20.info("credentials.migrated", { type: "ga4", count: migratedGa4 });
|
|
11921
12133
|
}
|
|
11922
12134
|
}
|
|
11923
12135
|
function isLoopbackBindHost(host) {
|
|
@@ -11974,11 +12186,11 @@ async function createServer(opts) {
|
|
|
11974
12186
|
applyLegacyCredentials(legacyRows, opts.config);
|
|
11975
12187
|
dropLegacyCredentialColumns(opts.db);
|
|
11976
12188
|
} catch (err) {
|
|
11977
|
-
|
|
12189
|
+
log20.warn("credentials.migration.failed", {
|
|
11978
12190
|
error: err instanceof Error ? err.message : String(err)
|
|
11979
12191
|
});
|
|
11980
12192
|
}
|
|
11981
|
-
|
|
12193
|
+
log20.info("providers.configured", {
|
|
11982
12194
|
providers: Object.keys(providers).filter((k) => {
|
|
11983
12195
|
const p = providers[k];
|
|
11984
12196
|
return p?.apiKey || p?.baseUrl || p?.vertexProject;
|
|
@@ -12042,7 +12254,7 @@ async function createServer(opts) {
|
|
|
12042
12254
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
12043
12255
|
async (ctx) => {
|
|
12044
12256
|
if (!sessionRegistry) return;
|
|
12045
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
12257
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq21(projects.id, ctx.projectId)).get();
|
|
12046
12258
|
if (!project) return;
|
|
12047
12259
|
let content;
|
|
12048
12260
|
if (ctx.kind === RunKinds["aeo-discover-probe"]) {
|
|
@@ -12377,6 +12589,24 @@ async function createServer(opts) {
|
|
|
12377
12589
|
onAdsSyncRequested: (runId, projectId) => {
|
|
12378
12590
|
runAdsSync(runId, projectId);
|
|
12379
12591
|
},
|
|
12592
|
+
onDoctorRequested: (projectName) => {
|
|
12593
|
+
void (async () => {
|
|
12594
|
+
try {
|
|
12595
|
+
const report = await aeroClient.runDoctor({ project: projectName });
|
|
12596
|
+
const project = opts.db.select().from(projects).where(eq21(projects.name, projectName)).get();
|
|
12597
|
+
if (!project) {
|
|
12598
|
+
app.log.warn({ projectName }, "doctor schedule fired for an unknown project");
|
|
12599
|
+
return;
|
|
12600
|
+
}
|
|
12601
|
+
await notifier.onHealthChecked(project.id, {
|
|
12602
|
+
checks: report.checks,
|
|
12603
|
+
checkedAt: report.generatedAt
|
|
12604
|
+
});
|
|
12605
|
+
} catch (err) {
|
|
12606
|
+
app.log.warn({ projectName, err }, "scheduled doctor pass failed");
|
|
12607
|
+
}
|
|
12608
|
+
})();
|
|
12609
|
+
},
|
|
12380
12610
|
onDataRefreshRequested: (projectName) => {
|
|
12381
12611
|
void refreshAllIntegrations(aeroClient, projectName);
|
|
12382
12612
|
},
|
|
@@ -12391,9 +12621,9 @@ async function createServer(opts) {
|
|
|
12391
12621
|
});
|
|
12392
12622
|
if (!probed) return;
|
|
12393
12623
|
const alreadySynced = opts.db.select().from(ccReleaseSyncs).where(
|
|
12394
|
-
|
|
12395
|
-
|
|
12396
|
-
|
|
12624
|
+
and15(
|
|
12625
|
+
eq21(ccReleaseSyncs.release, probed.release),
|
|
12626
|
+
eq21(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)
|
|
12397
12627
|
)
|
|
12398
12628
|
).limit(1).get();
|
|
12399
12629
|
if (alreadySynced) {
|
|
@@ -12414,18 +12644,6 @@ async function createServer(opts) {
|
|
|
12414
12644
|
);
|
|
12415
12645
|
});
|
|
12416
12646
|
})();
|
|
12417
|
-
const project = opts.db.select().from(projects).where(eq22(projects.name, projectName)).get();
|
|
12418
|
-
if (project && bingConnectionStore.getConnection(project.canonicalDomain)) {
|
|
12419
|
-
aeroClient.backlinksBingSync(projectName).catch((err) => {
|
|
12420
|
-
app.log.error(
|
|
12421
|
-
{
|
|
12422
|
-
projectName,
|
|
12423
|
-
err: err instanceof Error ? err.message : String(err)
|
|
12424
|
-
},
|
|
12425
|
-
"Scheduled Bing backlinks sync failed"
|
|
12426
|
-
);
|
|
12427
|
-
});
|
|
12428
|
-
}
|
|
12429
12647
|
},
|
|
12430
12648
|
onSiteAuditRequested: (runId, projectId) => {
|
|
12431
12649
|
runSiteAudit(runId, projectId);
|
|
@@ -12560,7 +12778,7 @@ async function createServer(opts) {
|
|
|
12560
12778
|
return removed;
|
|
12561
12779
|
}
|
|
12562
12780
|
};
|
|
12563
|
-
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ??
|
|
12781
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto22.randomBytes(32).toString("hex");
|
|
12564
12782
|
const googleConnectionStore = {
|
|
12565
12783
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
12566
12784
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -12633,11 +12851,11 @@ async function createServer(opts) {
|
|
|
12633
12851
|
const googlePublicUrl = resolveGooglePublicUrl(opts.config, basePath);
|
|
12634
12852
|
if (opts.config.apiKey) {
|
|
12635
12853
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
12636
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
12854
|
+
const existing = opts.db.select().from(apiKeys).where(eq21(apiKeys.keyHash, keyHash)).get();
|
|
12637
12855
|
if (!existing) {
|
|
12638
12856
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
12639
12857
|
opts.db.insert(apiKeys).values({
|
|
12640
|
-
id: `key_${
|
|
12858
|
+
id: `key_${crypto22.randomBytes(8).toString("hex")}`,
|
|
12641
12859
|
name: "default",
|
|
12642
12860
|
keyHash,
|
|
12643
12861
|
keyPrefix: prefix,
|
|
@@ -12661,7 +12879,7 @@ async function createServer(opts) {
|
|
|
12661
12879
|
};
|
|
12662
12880
|
const createSession = (apiKeyId) => {
|
|
12663
12881
|
pruneExpiredSessions();
|
|
12664
|
-
const sessionId =
|
|
12882
|
+
const sessionId = crypto22.randomBytes(32).toString("hex");
|
|
12665
12883
|
sessions.set(sessionId, {
|
|
12666
12884
|
apiKeyId,
|
|
12667
12885
|
expiresAt: Date.now() + SESSION_TTL_MS
|
|
@@ -12685,7 +12903,7 @@ async function createServer(opts) {
|
|
|
12685
12903
|
};
|
|
12686
12904
|
const getDefaultApiKey = () => {
|
|
12687
12905
|
if (!opts.config.apiKey) return void 0;
|
|
12688
|
-
return opts.db.select().from(apiKeys).where(
|
|
12906
|
+
return opts.db.select().from(apiKeys).where(eq21(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
12689
12907
|
};
|
|
12690
12908
|
const createPasswordSession = (reply) => {
|
|
12691
12909
|
const key = getDefaultApiKey();
|
|
@@ -12709,7 +12927,7 @@ async function createServer(opts) {
|
|
|
12709
12927
|
if (!header) return false;
|
|
12710
12928
|
const parts = header.split(" ");
|
|
12711
12929
|
if (parts.length !== 2 || parts[0] !== "Bearer") return false;
|
|
12712
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
12930
|
+
const key = opts.db.select().from(apiKeys).where(eq21(apiKeys.keyHash, hashApiKey(parts[1]))).get();
|
|
12713
12931
|
return Boolean(key && !key.revokedAt);
|
|
12714
12932
|
};
|
|
12715
12933
|
app.get(apiPrefix + "/session", async (request, reply) => {
|
|
@@ -12786,12 +13004,12 @@ async function createServer(opts) {
|
|
|
12786
13004
|
return reply.send({ authenticated: true });
|
|
12787
13005
|
}
|
|
12788
13006
|
if (apiKey) {
|
|
12789
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
13007
|
+
const key = opts.db.select().from(apiKeys).where(eq21(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
12790
13008
|
if (!key || key.revokedAt) {
|
|
12791
13009
|
const err2 = authInvalid();
|
|
12792
13010
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
12793
13011
|
}
|
|
12794
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
13012
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq21(apiKeys.id, key.id)).run();
|
|
12795
13013
|
const sessionId = createSession(key.id);
|
|
12796
13014
|
reply.header(
|
|
12797
13015
|
"set-cookie",
|
|
@@ -12902,6 +13120,11 @@ async function createServer(opts) {
|
|
|
12902
13120
|
// installer; cloud deployments inherit the secure default of `false` by
|
|
12903
13121
|
// not passing this option. Override with CANONRY_ALLOW_LOOPBACK_WEBHOOKS=0.
|
|
12904
13122
|
allowLoopbackWebhooks: process.env.CANONRY_ALLOW_LOOPBACK_WEBHOOKS !== "0",
|
|
13123
|
+
// Wall-clock budget for one incremental Vercel drain. This is the only lever
|
|
13124
|
+
// that decides whether a source catches up or falls further behind, and it
|
|
13125
|
+
// was previously reachable only from tests, so a source losing ground could
|
|
13126
|
+
// not be rescued without shipping code. Unset keeps the built-in default.
|
|
13127
|
+
vercelSyncDeadlineMs: resolveVercelSyncDeadlineMs(process.env),
|
|
12905
13128
|
// Local-only Aero agent routes. Registered here so they inherit api-routes'
|
|
12906
13129
|
// auth plugin — bare `registerAgentRoutes(app, ...)` would skip auth.
|
|
12907
13130
|
registerAuthenticatedRoutes: async (scope) => {
|
|
@@ -12984,7 +13207,7 @@ async function createServer(opts) {
|
|
|
12984
13207
|
deps: {
|
|
12985
13208
|
enqueueAutoExtract: ({ projectId, release: r }) => {
|
|
12986
13209
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
12987
|
-
const runId =
|
|
13210
|
+
const runId = crypto22.randomUUID();
|
|
12988
13211
|
opts.db.insert(runs).values({
|
|
12989
13212
|
id: runId,
|
|
12990
13213
|
projectId,
|
|
@@ -13014,13 +13237,6 @@ async function createServer(opts) {
|
|
|
13014
13237
|
}
|
|
13015
13238
|
);
|
|
13016
13239
|
},
|
|
13017
|
-
onBingBacklinkSyncRequested: (runId, projectId) => {
|
|
13018
|
-
executeBingBacklinkSync(opts.db, runId, projectId, {
|
|
13019
|
-
resolveConnection: (domain) => bingConnectionStore.getConnection(domain)
|
|
13020
|
-
}).catch((err) => {
|
|
13021
|
-
app.log.error({ runId, err }, "Bing backlink sync failed");
|
|
13022
|
-
});
|
|
13023
|
-
},
|
|
13024
13240
|
onDiscoveryRunRequested: (input) => {
|
|
13025
13241
|
executeDiscoveryRun({
|
|
13026
13242
|
db: opts.db,
|
|
@@ -13116,7 +13332,7 @@ async function createServer(opts) {
|
|
|
13116
13332
|
...inspectOpts,
|
|
13117
13333
|
config: opts.config
|
|
13118
13334
|
}).then(() => {
|
|
13119
|
-
const finished = opts.db.select({ status: runs.status }).from(runs).where(
|
|
13335
|
+
const finished = opts.db.select({ status: runs.status }).from(runs).where(eq21(runs.id, runId)).get();
|
|
13120
13336
|
if (finished?.status === RunStatuses.completed || finished?.status === RunStatuses.partial) {
|
|
13121
13337
|
return maybeRefreshGscCoverage(opts.db, opts.config, projectId);
|
|
13122
13338
|
}
|
|
@@ -13210,7 +13426,7 @@ async function createServer(opts) {
|
|
|
13210
13426
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
13211
13427
|
opts.db.insert(auditLog).values(
|
|
13212
13428
|
targetProjectIds.map((projectId) => ({
|
|
13213
|
-
id:
|
|
13429
|
+
id: crypto22.randomUUID(),
|
|
13214
13430
|
projectId,
|
|
13215
13431
|
actor: "api",
|
|
13216
13432
|
action: existing ? "provider.updated" : "provider.created",
|
|
@@ -13465,7 +13681,7 @@ async function createServer(opts) {
|
|
|
13465
13681
|
}
|
|
13466
13682
|
checkLatestVersionForServer();
|
|
13467
13683
|
scheduler.start();
|
|
13468
|
-
for (const run of opts.db.select({ id: researchRuns.id, projectId: researchRuns.projectId }).from(researchRuns).where(
|
|
13684
|
+
for (const run of opts.db.select({ id: researchRuns.id, projectId: researchRuns.projectId }).from(researchRuns).where(eq21(researchRuns.status, ResearchRunStatuses.queued)).all()) {
|
|
13469
13685
|
dispatchResearchRun(run.id, run.projectId);
|
|
13470
13686
|
}
|
|
13471
13687
|
app.addHook("onClose", async () => {
|