@canonry/canonry 7.87.2 → 7.90.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +43 -5
- package/assets/assets/{AgentSetupPage-_ZP3F_zA.js → AgentSetupPage-oSoM5flN.js} +6 -6
- package/assets/assets/AuditHistoryPanel-YrEggsyA.js +1 -0
- package/assets/assets/{BacklinksPage-CsvVgXTZ.js → BacklinksPage-vXUxxplH.js} +1 -1
- package/assets/assets/{ChartPrimitives-4D5wgAn5.js → ChartPrimitives-Cs2iavxU.js} +1 -1
- package/assets/assets/{Hint-CfeZmZig.js → Hint-uoVNfr0U.js} +1 -1
- package/assets/assets/{HistoryPage-BCxpCul0.js → HistoryPage-VJ_JQ7R3.js} +1 -1
- package/assets/assets/{InfoTooltip-DrBIPuSU.js → InfoTooltip-CNLlIWeP.js} +1 -1
- package/assets/assets/MeasurementPropertyPage-BZKiS1V0.js +4 -0
- package/assets/assets/ProjectPage-p-rDlzSq.js +11 -0
- package/assets/assets/{RunsPage-CfQ_kqlu.js → RunsPage-CjkPCc9Z.js} +1 -1
- package/assets/assets/{SettingsPage-PIgRvqJc.js → SettingsPage-BNHMlx6C.js} +1 -1
- package/assets/assets/SiteHealthSection-BZhulwSy.js +2 -0
- package/assets/assets/ToneBadge-B3rD6bDF.js +60 -0
- package/assets/assets/TrafficPage-Dr8NTQpA.js +40 -0
- package/assets/assets/TrafficSourceDetailPage-C79eQilP.js +1 -0
- package/assets/assets/{account-context-CVjb3ZSn.js → account-context-GstYbqpX.js} +1 -1
- package/assets/assets/agent-trust-copy-Ca4ys8J2.js +1 -0
- package/assets/assets/{card-BnVez9nG.js → card-CgdGerYK.js} +1 -1
- package/assets/assets/{format-helpers-DVXZD864.js → format-helpers-ByAY4dv0.js} +2 -2
- package/assets/assets/{index-BOUrrC9J.css → index-Bhpq6fUo.css} +1 -1
- package/assets/assets/index-BrHemYsG.js +60 -0
- package/assets/assets/query-client-DjRrRIsq.js +1 -0
- package/assets/assets/react-query-B8JD_K7g.js +1 -0
- package/assets/assets/{react-sigma_core.esm.min-CVfGIEIE.js → react-sigma_core.esm.min-OHqBWE8n.js} +1 -1
- package/assets/assets/server-traffic-Bt9bexaP.js +1 -0
- package/assets/assets/{service-BGqeyqhZ.js → service-SgtD5EOS.js} +1 -1
- package/assets/assets/{vendor-lucide-CoTHJwai.js → vendor-lucide-Bo69m9KC.js} +1 -1
- package/assets/assets/{vendor-markdown-cQgSk2i0.js → vendor-markdown-BiRP51B0.js} +1 -1
- package/assets/assets/{vendor-radix-wS8sWIWE.js → vendor-radix-PQeN8gOe.js} +1 -1
- package/assets/assets/{vendor-recharts-AnNqECZW.js → vendor-recharts-CY6tH-r6.js} +1 -1
- package/assets/assets/{vendor-tanstack-q5SvZtcv.js → vendor-tanstack-tBjrfzKu.js} +1 -1
- package/assets/index.html +15 -15
- package/dist/{ads-ZUZWSIBV.js → ads-6BAITIBY.js} +3 -3
- package/dist/{agent-YDS3WY7B.js → agent-BE3D65U4.js} +7 -7
- package/dist/{agent-ask-7VUPXTXL.js → agent-ask-TT6BAV2T.js} +3 -3
- package/dist/{agent-conversations-WPEBK3BS.js → agent-conversations-GJDAWU36.js} +3 -3
- package/dist/{agent-journeys-H5CCYLNB.js → agent-journeys-UEL27GNY.js} +3 -3
- package/dist/{agent-memory-CNPKPIWD.js → agent-memory-PB74OP46.js} +3 -3
- package/dist/{agent-providers-U7FLJO5I.js → agent-providers-5WFMNRRK.js} +3 -3
- package/dist/{agent-transcript-VY3BQNPU.js → agent-transcript-IAGB3JXG.js} +3 -3
- package/dist/{analytics-PYUNO7C2.js → analytics-5BVE7KUN.js} +3 -3
- package/dist/{apply-YNBEYC2G.js → apply-YJJWQ4JX.js} +3 -3
- package/dist/{backfill-6AYHWEQX.js → backfill-6OLGZIEV.js} +10 -10
- package/dist/{backlinks-LVHZ2MQ7.js → backlinks-4JK26OIO.js} +3 -3
- package/dist/bing-NZYYPB2M.js +794 -0
- package/dist/{bootstrap-27NOBWTP.js → bootstrap-6KHK63W3.js} +10 -10
- package/dist/{cdp-HCYY4SXR.js → cdp-PTXWVX6T.js} +7 -7
- package/dist/{chunk-JDTKG7KJ.js → chunk-2UULABPI.js} +7 -3
- package/dist/{chunk-GXTCDNX7.js → chunk-7ASKZHS4.js} +1 -1
- package/dist/{chunk-ICS2WJ7H.js → chunk-7VJJXLRP.js} +3 -3
- package/dist/{chunk-DOPUEXMC.js → chunk-AFYKUX2X.js} +2 -2
- package/dist/{chunk-LYTBHEID.js → chunk-C2XWWIJZ.js} +571 -346
- package/dist/{chunk-TFB2S6TQ.js → chunk-DF4B6OGX.js} +4 -4
- package/dist/{chunk-6F5ULQDY.js → chunk-DZO6CATO.js} +4 -4
- package/dist/{chunk-F643WELJ.js → chunk-ETIIPSWL.js} +412 -9
- package/dist/{chunk-BJHHD7VE.js → chunk-F442MSX2.js} +2 -2
- package/dist/{chunk-TEQF4ZJA.js → chunk-F6WNKKGG.js} +188 -2
- package/dist/{chunk-JAYUWQAV.js → chunk-GKX5F7NX.js} +1 -1
- package/dist/{chunk-IE4D47JE.js → chunk-HUQDD26S.js} +7 -7
- package/dist/{chunk-BVJA3TV2.js → chunk-I7DAEE3L.js} +5 -5
- package/dist/{chunk-4LNZTL7M.js → chunk-L2MDMZFT.js} +3 -3
- package/dist/{chunk-3KK6FWDF.js → chunk-LQWMNDZR.js} +1 -1
- package/dist/{chunk-INH773FB.js → chunk-LTZNUDSI.js} +3 -3
- package/dist/{chunk-ZJVQ72VC.js → chunk-N7LRPMWR.js} +2 -2
- package/dist/{chunk-T2MGGW6Y.js → chunk-NKGQCCUS.js} +6 -6
- package/dist/{chunk-POMLL2OZ.js → chunk-NPX5SL54.js} +1 -1
- package/dist/{chunk-2FJB3CSS.js → chunk-OOVWMWMT.js} +1 -1
- package/dist/{chunk-IVGVPPQT.js → chunk-OVUFIG22.js} +1156 -646
- package/dist/{chunk-MNLAIB67.js → chunk-PID3C4B7.js} +1 -1
- package/dist/{chunk-FAMCDCVK.js → chunk-Q6EZEVXE.js} +1 -1
- package/dist/{chunk-GDU6IMGB.js → chunk-QLVTMPFW.js} +2 -2
- package/dist/{chunk-2AXFKYJU.js → chunk-QWNDNZJ3.js} +2 -2
- package/dist/{chunk-AI75KS7G.js → chunk-SDXMG4VX.js} +2792 -1612
- package/dist/{chunk-IK7QEKKD.js → chunk-TA4L7U2M.js} +1 -1
- package/dist/{chunk-4FWDITQG.js → chunk-UG7TZJIG.js} +1 -1
- package/dist/{chunk-AYYZ4HKJ.js → chunk-USHY3SH6.js} +291 -1
- package/dist/{chunk-TN53B5AB.js → chunk-WNDBACO7.js} +240 -3
- package/dist/{chunk-CBWGUGME.js → chunk-WYMBWCG5.js} +20 -2
- package/dist/{chunk-IBGZ6D6R.js → chunk-XCEA5OMI.js} +2 -2
- package/dist/{chunk-ZCNKGBIV.js → chunk-YB2GAK32.js} +5 -5
- package/dist/{chunk-3FHBPJ5R.js → chunk-YPJCZD5H.js} +4 -4
- package/dist/{chunk-GBWCEOZR.js → chunk-ZV6PMFF4.js} +1 -1
- package/dist/{citations-3EZTHA65.js → citations-WX4VO6ZI.js} +3 -3
- package/dist/{cli-catalog-CBTM3JKP.js → cli-catalog-BTRNR2UP.js} +3 -3
- package/dist/{cli-commands-H7BFVUGC.js → cli-commands-5ISZA43S.js} +7 -7
- package/dist/cli.js +13 -13
- package/dist/{competitor-VH4TXJLT.js → competitor-HZRZWCEN.js} +3 -3
- package/dist/{content-TSMNHXJZ.js → content-NOZDHI3M.js} +3 -3
- package/dist/{dashboard-KT2ACPI5.js → dashboard-TZLWHXP4.js} +6 -6
- package/dist/{demo-LWSLULEZ.js → demo-2PYBQD47.js} +1 -1
- package/dist/{demo-server-UENMDVMG.js → demo-server-JEQNGMMV.js} +12 -12
- package/dist/{discover-XTJXTKCI.js → discover-A2XFXHOC.js} +3 -3
- package/dist/{discover-eval-IVQI5YEW.js → discover-eval-A6SBUYXF.js} +3 -3
- package/dist/{doctor-SJ4SHSQO.js → doctor-GZACELPF.js} +3 -3
- package/dist/{evidence-C6T556YH.js → evidence-RVFMUWIL.js} +3 -3
- package/dist/{export-cmd-JQZ5ZRSI.js → export-cmd-X2TFZGNJ.js} +3 -3
- package/dist/{ga-W2WHVRHN.js → ga-TOT3V67A.js} +3 -3
- package/dist/{gbp-3AZIV5WF.js → gbp-KP4YZU6S.js} +3 -3
- package/dist/{google-3CKWPZT7.js → google-GCFSTV5N.js} +4 -4
- package/dist/{google-marketing-PJBGQYYW.js → google-marketing-ANS65KQN.js} +1 -1
- package/dist/{health-cmd-4JTQMCXM.js → health-cmd-UKMNLU6D.js} +3 -3
- package/dist/{history-TXUJFEO2.js → history-TWGYSAZZ.js} +3 -3
- package/dist/index.d.ts +1861 -11
- package/dist/index.js +25 -25
- package/dist/{init-TF3OPCEC.js → init-K4623VOY.js} +12 -12
- package/dist/{insights-XZQO4B45.js → insights-7IOC7WIR.js} +3 -3
- package/dist/intelligence-service-42Q3OVI5.js +16 -0
- package/dist/{journeys-VFIV5JYV.js → journeys-NWKT45PT.js} +3 -3
- package/dist/{keys-PV23G4LW.js → keys-ZZYZUHUK.js} +3 -3
- package/dist/{keyword-D76KBIV5.js → keyword-F6EIX5KM.js} +3 -3
- package/dist/{logs-DYDZ6IEJ.js → logs-B45FXS6E.js} +3 -3
- package/dist/{mcp-GAR5LDQD.js → mcp-WPNCNWA5.js} +4 -4
- package/dist/mcp.js +9 -9
- package/dist/{measurement-plan-K24EOGMI.js → measurement-plan-QT5HJ3KC.js} +5 -5
- package/dist/{notify-XQHMPDQQ.js → notify-Z3LIF3SO.js} +3 -3
- package/dist/{organic-evidence-UHHE7AO4.js → organic-evidence-R5TM7XUB.js} +3 -3
- package/dist/{overview-AUZ3XRS5.js → overview-6TZHJLQN.js} +4 -4
- package/dist/{project-FUCPU2BD.js → project-VMPVOQ2A.js} +3 -3
- package/dist/{query-HRMSXDKL.js → query-EXSH5QP7.js} +3 -3
- package/dist/{research-R23XLZJ3.js → research-6R6BEL4X.js} +3 -3
- package/dist/{results-export-N6IV3KG4.js → results-export-ZJRL3ECU.js} +3 -3
- package/dist/{run-CJLG5EXH.js → run-C4VXAJE3.js} +4 -4
- package/dist/{schedule-IPDCQ2I5.js → schedule-KC6ETWMO.js} +3 -3
- package/dist/{search-ZU4AKJ76.js → search-SUR6BEQD.js} +3 -3
- package/dist/{serve-LLIUILD2.js → serve-TVODHNFU.js} +26 -26
- package/dist/{session-2NQKSF2N.js → session-D42KUKCZ.js} +13 -13
- package/dist/{settings-4FS5NRKP.js → settings-LTCG3D5B.js} +3 -3
- package/dist/sitemaps-YMTZLF5S.js +46 -0
- package/dist/{skills-GSFHACGN.js → skills-THMG3L64.js} +2 -2
- package/dist/{snapshot-4NUVPDK3.js → snapshot-ZEVB3EE4.js} +3 -3
- package/dist/{src-JQYZ2TP6.js → src-HPVSTXVF.js} +20 -2
- package/dist/{status-ASBLMFP2.js → status-LPCPTWKO.js} +4 -4
- package/dist/{technical-aeo-QXHRBGYY.js → technical-aeo-J6LNT45B.js} +3 -3
- package/dist/{telemetry-J2I577I3.js → telemetry-BPY34GDV.js} +4 -4
- package/dist/{tools-UPJRW4DN.js → tools-K3G43ESI.js} +11 -11
- package/dist/{traffic-GWBZKYR5.js → traffic-BOIFAPAR.js} +5 -5
- package/dist/{trends-YOLCJBNM.js → trends-A2MDRSQQ.js} +3 -3
- package/dist/{users-LPSCZV3M.js → users-VJOQODJK.js} +3 -3
- package/dist/{visibility-stats-JJQTZSSK.js → visibility-stats-YNMJYASA.js} +3 -3
- package/dist/{wordpress-4IKDT5KT.js → wordpress-BWG6D7VH.js} +3 -3
- package/package.json +1 -1
- package/assets/assets/AuditHistoryPanel-9B0XpYsN.js +0 -1
- package/assets/assets/MeasurementPropertyPage-BObvWjVx.js +0 -4
- package/assets/assets/ProjectPage-y378tOaT.js +0 -11
- package/assets/assets/SiteHealthSection-DuUE1cB0.js +0 -2
- package/assets/assets/ToneBadge-BgTLSTKp.js +0 -60
- package/assets/assets/TrafficPage-B_YunwjH.js +0 -40
- package/assets/assets/TrafficSourceDetailPage-C51W87fU.js +0 -1
- package/assets/assets/agent-trust-copy-BlpC8jo5.js +0 -1
- package/assets/assets/extract-error-message-Dqirbrl0.js +0 -1
- package/assets/assets/index-CsvEqhK4.js +0 -60
- package/assets/assets/query-client-CVc7wMKJ.js +0 -1
- package/assets/assets/react-query-CSlTSOjz.js +0 -1
- package/dist/bing-A2MUG6AV.js +0 -418
- package/dist/intelligence-service-LEH5UVIC.js +0 -16
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
answerServiceAdapters,
|
|
3
3
|
answerServiceDescriptors
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-TA4L7U2M.js";
|
|
5
5
|
import {
|
|
6
6
|
AeroLlmUsageFeatures,
|
|
7
7
|
aeroTurnStatus,
|
|
@@ -27,14 +27,14 @@ import {
|
|
|
27
27
|
resolveConfiguredAgentProvider,
|
|
28
28
|
resolveSessionProviderAndModel,
|
|
29
29
|
setAeroSystemPrompt
|
|
30
|
-
} from "./chunk-
|
|
30
|
+
} from "./chunk-NKGQCCUS.js";
|
|
31
31
|
import {
|
|
32
32
|
AeroToolProfiles,
|
|
33
33
|
AeroToolScopes,
|
|
34
34
|
buildAeroStateTools,
|
|
35
35
|
isAeroToolProfile,
|
|
36
36
|
truncateToolResult
|
|
37
|
-
} from "./chunk-
|
|
37
|
+
} from "./chunk-DF4B6OGX.js";
|
|
38
38
|
import {
|
|
39
39
|
getGoogleAuthConfig,
|
|
40
40
|
getGoogleConnection,
|
|
@@ -53,7 +53,7 @@ import {
|
|
|
53
53
|
removeCloudflareTrafficConnection,
|
|
54
54
|
removeCloudflareTrafficConnectionBySourceId,
|
|
55
55
|
upsertCloudflareTrafficConnection
|
|
56
|
-
} from "./chunk-
|
|
56
|
+
} from "./chunk-UG7TZJIG.js";
|
|
57
57
|
import {
|
|
58
58
|
claudeAdapter,
|
|
59
59
|
embedQueries,
|
|
@@ -65,7 +65,7 @@ import {
|
|
|
65
65
|
openrouterAdapter,
|
|
66
66
|
perplexityAdapter,
|
|
67
67
|
startTrafficReclassification
|
|
68
|
-
} from "./chunk-
|
|
68
|
+
} from "./chunk-I7DAEE3L.js";
|
|
69
69
|
import {
|
|
70
70
|
classifyUsageSurface,
|
|
71
71
|
createApiUsageTelemetry,
|
|
@@ -77,7 +77,7 @@ import {
|
|
|
77
77
|
startOutcomeTimer,
|
|
78
78
|
trackFeatureCompleted,
|
|
79
79
|
trackInstallState
|
|
80
|
-
} from "./chunk-
|
|
80
|
+
} from "./chunk-DZO6CATO.js";
|
|
81
81
|
import {
|
|
82
82
|
isLoopbackBindHost
|
|
83
83
|
} from "./chunk-TSENU4GG.js";
|
|
@@ -85,7 +85,7 @@ import {
|
|
|
85
85
|
isApiProviderRegistrable,
|
|
86
86
|
isCdpProviderRegistrable,
|
|
87
87
|
registeredProviderNames
|
|
88
|
-
} from "./chunk-
|
|
88
|
+
} from "./chunk-LQWMNDZR.js";
|
|
89
89
|
import {
|
|
90
90
|
checkLatestVersionForServer,
|
|
91
91
|
getOrCreateAnonymousId,
|
|
@@ -97,17 +97,17 @@ import {
|
|
|
97
97
|
recordDashboardEvent,
|
|
98
98
|
setTelemetryPreference,
|
|
99
99
|
trackEvent
|
|
100
|
-
} from "./chunk-
|
|
100
|
+
} from "./chunk-AFYKUX2X.js";
|
|
101
101
|
import {
|
|
102
102
|
createCanonryMcpServer
|
|
103
|
-
} from "./chunk-
|
|
103
|
+
} from "./chunk-7VJJXLRP.js";
|
|
104
104
|
import {
|
|
105
105
|
CANONRY_MCP_TIERS,
|
|
106
106
|
CANONRY_MCP_TOOLKIT_NAMES
|
|
107
107
|
} from "./chunk-MTW37GXF.js";
|
|
108
108
|
import {
|
|
109
109
|
ApiClient
|
|
110
|
-
} from "./chunk-
|
|
110
|
+
} from "./chunk-F6WNKKGG.js";
|
|
111
111
|
import {
|
|
112
112
|
AGENT_PROVIDERS,
|
|
113
113
|
adapterSupportsBatch,
|
|
@@ -139,21 +139,21 @@ import {
|
|
|
139
139
|
saveConfigPatch,
|
|
140
140
|
sentimentInstallReadiness,
|
|
141
141
|
updateSharedProviderExecutionGate
|
|
142
|
-
} from "./chunk-
|
|
142
|
+
} from "./chunk-OOVWMWMT.js";
|
|
143
143
|
import {
|
|
144
144
|
getBundledSkillSnapshots
|
|
145
|
-
} from "./chunk-
|
|
145
|
+
} from "./chunk-Q6EZEVXE.js";
|
|
146
146
|
import {
|
|
147
147
|
computeFactorAverages,
|
|
148
148
|
deleteSiteCrawlGraphLayout,
|
|
149
149
|
persistSiteCrawlGraphLayout,
|
|
150
150
|
prepareSiteCrawlGraphLayout,
|
|
151
151
|
recordedSharePct
|
|
152
|
-
} from "./chunk-
|
|
152
|
+
} from "./chunk-N7LRPMWR.js";
|
|
153
153
|
import {
|
|
154
154
|
CanonryMcpToolNames,
|
|
155
155
|
canonryMcpTools
|
|
156
|
-
} from "./chunk-
|
|
156
|
+
} from "./chunk-WNDBACO7.js";
|
|
157
157
|
import {
|
|
158
158
|
PACKAGE_VERSION
|
|
159
159
|
} from "./chunk-AOJU5LKX.js";
|
|
@@ -162,16 +162,16 @@ import {
|
|
|
162
162
|
} from "./chunk-HGMYEDAQ.js";
|
|
163
163
|
import {
|
|
164
164
|
IntelligenceService
|
|
165
|
-
} from "./chunk-
|
|
165
|
+
} from "./chunk-YB2GAK32.js";
|
|
166
166
|
import {
|
|
167
167
|
computeCitedCompetitorDomains,
|
|
168
168
|
determineCitationState
|
|
169
|
-
} from "./chunk-
|
|
169
|
+
} from "./chunk-GKX5F7NX.js";
|
|
170
170
|
import {
|
|
171
171
|
addLogListener,
|
|
172
172
|
createFastifyLogger,
|
|
173
173
|
createLogger
|
|
174
|
-
} from "./chunk-
|
|
174
|
+
} from "./chunk-XCEA5OMI.js";
|
|
175
175
|
import {
|
|
176
176
|
BingApiError,
|
|
177
177
|
CC_CACHE_DIR,
|
|
@@ -199,6 +199,9 @@ import {
|
|
|
199
199
|
auditFromRequest,
|
|
200
200
|
backfillProjectAnswerMentions,
|
|
201
201
|
backfillProjectAnswerMentionsInChunks,
|
|
202
|
+
bingDateToDay,
|
|
203
|
+
bingKeywordMarket,
|
|
204
|
+
bingPositionOrNull,
|
|
202
205
|
blockedAddressReason,
|
|
203
206
|
buildLocationProfileFields,
|
|
204
207
|
captureSimpleMeasurementDefinition,
|
|
@@ -222,13 +225,31 @@ import {
|
|
|
222
225
|
fetchDailyMetrics,
|
|
223
226
|
fetchSearchAnalytics,
|
|
224
227
|
formatStorefrontAddress,
|
|
228
|
+
getActivePagePreviewBlocks,
|
|
225
229
|
getAttributes,
|
|
226
230
|
getAuthUrl,
|
|
231
|
+
getBlockedUrls,
|
|
232
|
+
getConnectedPages,
|
|
233
|
+
getContentSubmissionQuota,
|
|
234
|
+
getCountryRegionSettings,
|
|
227
235
|
getCrawlIssues,
|
|
236
|
+
getCrawlSettings,
|
|
237
|
+
getCrawlStats,
|
|
238
|
+
getDeepLinkBlocks,
|
|
239
|
+
getFeedDetails,
|
|
240
|
+
getFeeds,
|
|
241
|
+
getFetchedUrls,
|
|
242
|
+
getKeywordHistory,
|
|
228
243
|
getLodging,
|
|
244
|
+
getPageStats,
|
|
229
245
|
getPlaceDetails,
|
|
230
246
|
getPlaceReviewSignals,
|
|
247
|
+
getQueryParameters,
|
|
248
|
+
getQueryStats,
|
|
249
|
+
getRankAndTrafficStats,
|
|
250
|
+
getSiteRoles,
|
|
231
251
|
getUrlInfo,
|
|
252
|
+
getUrlSubmissionQuota,
|
|
232
253
|
hasOutstandingProviderBatch,
|
|
233
254
|
hasProxyHeaders,
|
|
234
255
|
hashApiKey,
|
|
@@ -238,6 +259,7 @@ import {
|
|
|
238
259
|
hashSessionToken,
|
|
239
260
|
inspectUrl,
|
|
240
261
|
installDuckdb,
|
|
262
|
+
isBlockingCrawlIssue,
|
|
241
263
|
isDuckdbInstalled,
|
|
242
264
|
isValidReleaseId,
|
|
243
265
|
latestQueryBasketRevision,
|
|
@@ -246,6 +268,7 @@ import {
|
|
|
246
268
|
listMonthlyKeywords,
|
|
247
269
|
listPlaceActionLinks,
|
|
248
270
|
listReviews,
|
|
271
|
+
listSitemaps,
|
|
249
272
|
loadDuckdb,
|
|
250
273
|
markSessionFailed,
|
|
251
274
|
measurementRunCompleteness,
|
|
@@ -255,6 +278,7 @@ import {
|
|
|
255
278
|
nextRunFromCron,
|
|
256
279
|
nextRunFromSchedule,
|
|
257
280
|
notProbeRun,
|
|
281
|
+
parseBingDate,
|
|
258
282
|
parseCookieHeader,
|
|
259
283
|
previewCompetitorAutoAliases,
|
|
260
284
|
probeLatestRelease,
|
|
@@ -292,7 +316,7 @@ import {
|
|
|
292
316
|
webhookResponseReason,
|
|
293
317
|
webhookTargetRefusalReason,
|
|
294
318
|
writeAuditLog
|
|
295
|
-
} from "./chunk-
|
|
319
|
+
} from "./chunk-SDXMG4VX.js";
|
|
296
320
|
import {
|
|
297
321
|
OperationalLogStore,
|
|
298
322
|
SentimentIdempotencyConflict,
|
|
@@ -311,6 +335,13 @@ import {
|
|
|
311
335
|
backlinkSummaries,
|
|
312
336
|
bingConnections,
|
|
313
337
|
bingCoverageSnapshots,
|
|
338
|
+
bingCrawlStats,
|
|
339
|
+
bingKeywordDemand,
|
|
340
|
+
bingPageStats,
|
|
341
|
+
bingQueryStats,
|
|
342
|
+
bingSiteSnapshots,
|
|
343
|
+
bingSitemaps,
|
|
344
|
+
bingTrafficDaily,
|
|
314
345
|
bingUrlInspections,
|
|
315
346
|
ccReleaseSyncs,
|
|
316
347
|
competitors,
|
|
@@ -339,6 +370,7 @@ import {
|
|
|
339
370
|
gscDataWatermarks,
|
|
340
371
|
gscQueryDailyTotals,
|
|
341
372
|
gscSearchData,
|
|
373
|
+
gscSitemaps,
|
|
342
374
|
gscUrlInspections,
|
|
343
375
|
gtmConnections,
|
|
344
376
|
gtmRawSnapshots,
|
|
@@ -376,11 +408,12 @@ import {
|
|
|
376
408
|
siteCrawlRunRequests,
|
|
377
409
|
siteCrawlSnapshots,
|
|
378
410
|
siteLivenessState,
|
|
411
|
+
sitemapListSyncs,
|
|
379
412
|
trafficSources,
|
|
380
413
|
unrecognizedAgentEventsHourly,
|
|
381
414
|
unrecognizedAgentNotifyState,
|
|
382
415
|
usageCounters
|
|
383
|
-
} from "./chunk-
|
|
416
|
+
} from "./chunk-USHY3SH6.js";
|
|
384
417
|
import {
|
|
385
418
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
386
419
|
AI_ENGINE_DOMAINS,
|
|
@@ -390,6 +423,7 @@ import {
|
|
|
390
423
|
AgentProviderIds,
|
|
391
424
|
AppError,
|
|
392
425
|
BacklinkSources,
|
|
426
|
+
BingSiteSnapshotKinds,
|
|
393
427
|
CHECK_RUNTIME_ERROR_SUFFIX,
|
|
394
428
|
CITED_URL_CAPTURE_VERSION,
|
|
395
429
|
CcReleaseSyncStatuses,
|
|
@@ -636,7 +670,7 @@ import {
|
|
|
636
670
|
winnabilityClassLabel,
|
|
637
671
|
withRetry,
|
|
638
672
|
withSkippedProviders
|
|
639
|
-
} from "./chunk-
|
|
673
|
+
} from "./chunk-ETIIPSWL.js";
|
|
640
674
|
|
|
641
675
|
// src/google-marketing-runtime.ts
|
|
642
676
|
import crypto2 from "crypto";
|
|
@@ -3320,13 +3354,13 @@ function createGoogleMarketingRuntime(options) {
|
|
|
3320
3354
|
|
|
3321
3355
|
// src/server.ts
|
|
3322
3356
|
import { createRequire } from "module";
|
|
3323
|
-
import
|
|
3357
|
+
import crypto27 from "crypto";
|
|
3324
3358
|
import fs5 from "fs";
|
|
3325
3359
|
import path5 from "path";
|
|
3326
3360
|
import { isIP as isIP2 } from "net";
|
|
3327
3361
|
import { fileURLToPath } from "url";
|
|
3328
3362
|
import { setImmediate as yieldToEventLoop4 } from "timers/promises";
|
|
3329
|
-
import { and as
|
|
3363
|
+
import { and as and29, eq as eq35, isNotNull as isNotNull5, isNull as isNull8, lte as lte3, max, ne as ne4, or as or4, sql as sql15 } from "drizzle-orm";
|
|
3330
3364
|
|
|
3331
3365
|
// src/seo-runtime.ts
|
|
3332
3366
|
import { eq, inArray } from "drizzle-orm";
|
|
@@ -5940,7 +5974,7 @@ function nonBlank(value) {
|
|
|
5940
5974
|
return trimmed ? trimmed : void 0;
|
|
5941
5975
|
}
|
|
5942
5976
|
function resolveBuildCommit(env = process.env) {
|
|
5943
|
-
const embedded = true ? "
|
|
5977
|
+
const embedded = true ? "c97db3b634ae1944197825d1b93afaee2b7f41ad" : void 0;
|
|
5944
5978
|
return nonBlank(embedded) ?? nonBlank(env.CANONRY_COMMIT);
|
|
5945
5979
|
}
|
|
5946
5980
|
function resolveInstanceIdentity(env = process.env) {
|
|
@@ -7916,15 +7950,15 @@ var JobRunner = class {
|
|
|
7916
7950
|
const dispatchable = [];
|
|
7917
7951
|
for (const registered of activeProviders) {
|
|
7918
7952
|
const name = registered.adapter.name;
|
|
7919
|
-
const
|
|
7953
|
+
const count6 = unitsByProvider.get(name)?.length ?? 0;
|
|
7920
7954
|
const scope = `${projectId}:${adapterConnectionName(registered.adapter)}`;
|
|
7921
7955
|
const limit = registered.config.quotaPolicy.maxRequestsPerDay;
|
|
7922
|
-
const quota = reserveDailyQueryQuota(this.db, { scope, period: todayPeriod, count:
|
|
7956
|
+
const quota = reserveDailyQueryQuota(this.db, { scope, period: todayPeriod, count: count6, limit });
|
|
7923
7957
|
if (!quota.reserved) {
|
|
7924
|
-
providerErrors.set(name, `Daily quota exceeded for ${name}: ${quota.used} queries used today, limit is ${limit}. This fill needs ${
|
|
7958
|
+
providerErrors.set(name, `Daily quota exceeded for ${name}: ${quota.used} queries used today, limit is ${limit}. This fill needs ${count6} more.`);
|
|
7925
7959
|
continue;
|
|
7926
7960
|
}
|
|
7927
|
-
providerReservations.set(name, { scope, period: todayPeriod, reserved:
|
|
7961
|
+
providerReservations.set(name, { scope, period: todayPeriod, reserved: count6 });
|
|
7928
7962
|
dispatchable.push(registered);
|
|
7929
7963
|
}
|
|
7930
7964
|
const executionGates = /* @__PURE__ */ new Map();
|
|
@@ -8036,8 +8070,8 @@ var JobRunner = class {
|
|
|
8036
8070
|
const remaining = /* @__PURE__ */ new Map();
|
|
8037
8071
|
for (const slot of state.missing) remaining.set(slot.provider, (remaining.get(slot.provider) ?? 0) + 1);
|
|
8038
8072
|
const providers = {};
|
|
8039
|
-
for (const [provider,
|
|
8040
|
-
providers[provider] = fresh[provider] ?? previous[provider] ?? { message: `Expected measurements not run yet: ${
|
|
8073
|
+
for (const [provider, count6] of remaining) {
|
|
8074
|
+
providers[provider] = fresh[provider] ?? previous[provider] ?? { message: `Expected measurements not run yet: ${count6}.` };
|
|
8041
8075
|
}
|
|
8042
8076
|
txDb.update(runs).set({ error: serializeRunError({ providers }) }).where(and3(eq4(runs.id, input.runId), eq4(runs.status, "partial"))).run();
|
|
8043
8077
|
}
|
|
@@ -8294,7 +8328,7 @@ var JobRunner = class {
|
|
|
8294
8328
|
log2.error("run.answer-fields-rescore-failed", { runId: recording.runId, projectId, error: describeError(err) });
|
|
8295
8329
|
}
|
|
8296
8330
|
}
|
|
8297
|
-
incrementUsage(scope, metric,
|
|
8331
|
+
incrementUsage(scope, metric, count6) {
|
|
8298
8332
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8299
8333
|
const period = now.slice(0, 10);
|
|
8300
8334
|
this.db.insert(usageCounters).values({
|
|
@@ -8302,11 +8336,11 @@ var JobRunner = class {
|
|
|
8302
8336
|
scope,
|
|
8303
8337
|
period,
|
|
8304
8338
|
metric,
|
|
8305
|
-
count:
|
|
8339
|
+
count: count6,
|
|
8306
8340
|
updatedAt: now
|
|
8307
8341
|
}).onConflictDoUpdate({
|
|
8308
8342
|
target: [usageCounters.scope, usageCounters.period, usageCounters.metric],
|
|
8309
|
-
set: { count: sql2`${usageCounters.count} + ${
|
|
8343
|
+
set: { count: sql2`${usageCounters.count} + ${count6}`, updatedAt: now }
|
|
8310
8344
|
}).run();
|
|
8311
8345
|
}
|
|
8312
8346
|
flushProviderUsage(providerDispatchCounts, providerReservations) {
|
|
@@ -8589,6 +8623,64 @@ function googleRunFailure(err) {
|
|
|
8589
8623
|
|
|
8590
8624
|
// src/gsc-sync.ts
|
|
8591
8625
|
var log3 = createLogger("GscSync");
|
|
8626
|
+
var GSC_SITEMAP_INDEX_MAX_PER_SYNC = 20;
|
|
8627
|
+
function countOrNull(value) {
|
|
8628
|
+
if (value === void 0 || value === "") return null;
|
|
8629
|
+
const n = Number(value);
|
|
8630
|
+
return Number.isFinite(n) && n >= 0 ? n : null;
|
|
8631
|
+
}
|
|
8632
|
+
function gscSitemapUrlCount(sitemap) {
|
|
8633
|
+
if (!sitemap.contents || sitemap.contents.length === 0) return null;
|
|
8634
|
+
return sitemap.contents.reduce((sum, content) => sum + (countOrNull(content.submitted) ?? 0), 0);
|
|
8635
|
+
}
|
|
8636
|
+
async function syncGscSitemaps(db, projectId, runId, accessToken, propertyId) {
|
|
8637
|
+
const top = await listSitemaps(accessToken, propertyId);
|
|
8638
|
+
const indexes = top.filter((sitemap) => sitemap.isSitemapsIndex);
|
|
8639
|
+
const read = indexes.slice(0, GSC_SITEMAP_INDEX_MAX_PER_SYNC);
|
|
8640
|
+
const files = /* @__PURE__ */ new Map();
|
|
8641
|
+
for (const index of read) files.set(index.path, await listSitemaps(accessToken, propertyId, index.path));
|
|
8642
|
+
const syncedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
8643
|
+
const toRow = (sitemap, parentUrl) => ({
|
|
8644
|
+
projectId,
|
|
8645
|
+
url: sitemap.path,
|
|
8646
|
+
parentUrl,
|
|
8647
|
+
isIndex: Boolean(sitemap.isSitemapsIndex),
|
|
8648
|
+
isPending: Boolean(sitemap.isPending),
|
|
8649
|
+
type: sitemap.type ?? null,
|
|
8650
|
+
urlCount: gscSitemapUrlCount(sitemap),
|
|
8651
|
+
warnings: countOrNull(sitemap.warnings),
|
|
8652
|
+
errors: countOrNull(sitemap.errors),
|
|
8653
|
+
submittedAt: sitemap.lastSubmitted ?? null,
|
|
8654
|
+
lastDownloadedAt: sitemap.lastDownloaded ?? null,
|
|
8655
|
+
syncRunId: runId,
|
|
8656
|
+
syncedAt
|
|
8657
|
+
});
|
|
8658
|
+
const rows = [
|
|
8659
|
+
...top.map((sitemap) => toRow(sitemap, null)),
|
|
8660
|
+
...[...files.entries()].flatMap(([parent, children]) => children.map((child) => toRow({ ...child, isSitemapsIndex: false }, parent)))
|
|
8661
|
+
];
|
|
8662
|
+
const byUrl = /* @__PURE__ */ new Map();
|
|
8663
|
+
for (const row of rows) if (!byUrl.has(row.url)) byUrl.set(row.url, row);
|
|
8664
|
+
const unread = new Set(indexes.slice(GSC_SITEMAP_INDEX_MAX_PER_SYNC).map((index) => index.path));
|
|
8665
|
+
db.transaction((tx) => {
|
|
8666
|
+
const stored = tx.select({ url: gscSitemaps.url, parentUrl: gscSitemaps.parentUrl }).from(gscSitemaps).where(eq7(gscSitemaps.projectId, projectId)).all();
|
|
8667
|
+
for (const row of stored) {
|
|
8668
|
+
if (byUrl.has(row.url) || row.parentUrl !== null && unread.has(row.parentUrl)) continue;
|
|
8669
|
+
tx.delete(gscSitemaps).where(and5(eq7(gscSitemaps.projectId, projectId), eq7(gscSitemaps.url, row.url))).run();
|
|
8670
|
+
}
|
|
8671
|
+
for (const row of byUrl.values()) {
|
|
8672
|
+
tx.insert(gscSitemaps).values({ id: crypto8.randomUUID(), ...row }).onConflictDoUpdate({
|
|
8673
|
+
target: [gscSitemaps.projectId, gscSitemaps.url],
|
|
8674
|
+
set: { ...row }
|
|
8675
|
+
}).run();
|
|
8676
|
+
}
|
|
8677
|
+
tx.insert(sitemapListSyncs).values({ projectId, engine: "google", syncedAt, syncRunId: runId }).onConflictDoUpdate({
|
|
8678
|
+
target: [sitemapListSyncs.projectId, sitemapListSyncs.engine],
|
|
8679
|
+
set: { syncedAt, syncRunId: runId }
|
|
8680
|
+
}).run();
|
|
8681
|
+
});
|
|
8682
|
+
return { sitemaps: byUrl.size, indexesRead: read.length };
|
|
8683
|
+
}
|
|
8592
8684
|
async function executeGscSync(db, runId, projectId, opts) {
|
|
8593
8685
|
const reportOutcome = startRunOutcome(db, runId, FeatureNames.search_console, "sync");
|
|
8594
8686
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
@@ -8744,6 +8836,12 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
8744
8836
|
}).run();
|
|
8745
8837
|
}
|
|
8746
8838
|
log3.info("query-totals.complete", { runId, projectId, rowCount: queryTotalRows.length });
|
|
8839
|
+
try {
|
|
8840
|
+
const stored = await syncGscSitemaps(db, projectId, runId, accessToken, propertyId);
|
|
8841
|
+
log3.info("sitemaps.complete", { runId, projectId, ...stored });
|
|
8842
|
+
} catch (err) {
|
|
8843
|
+
log3.warn("sitemaps.failed", { runId, projectId, error: describeError(err) });
|
|
8844
|
+
}
|
|
8747
8845
|
const coverage = writeCoverageSnapshot(db, projectId, runId);
|
|
8748
8846
|
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq7(runs.id, runId)).run();
|
|
8749
8847
|
log3.info("sync.completed", { runId, projectId, searchDataRows: rows.length, indexed: coverage.indexed, notIndexed: coverage.notIndexed, unknown: coverage.unknown, verifiedByInspection: coverage.verifiedByInspection });
|
|
@@ -10595,20 +10693,6 @@ function bingRunFailure(err) {
|
|
|
10595
10693
|
return runFailure(err, err instanceof BingApiError && err.isThrottle ? OutcomeReasonCodes.RATE_LIMITED : void 0);
|
|
10596
10694
|
}
|
|
10597
10695
|
var BING_INSPECT_SPACING_MS = 2e3;
|
|
10598
|
-
function parseBingDate(value) {
|
|
10599
|
-
if (!value) return null;
|
|
10600
|
-
const match = /\/Date\((-?\d+)(?:[-+]\d+)?\)\//.exec(value);
|
|
10601
|
-
if (!match) return null;
|
|
10602
|
-
const ms = parseInt(match[1], 10);
|
|
10603
|
-
if (ms <= 0) return null;
|
|
10604
|
-
return new Date(ms).toISOString();
|
|
10605
|
-
}
|
|
10606
|
-
function isBlockingIssueType(issueType) {
|
|
10607
|
-
if (!issueType) return true;
|
|
10608
|
-
const trimmed = issueType.trim();
|
|
10609
|
-
if (!trimmed) return true;
|
|
10610
|
-
return trimmed.split(/\s+/).some((flag) => !/^(?:None|Seo(?:Issues|Concerns))$/i.test(flag));
|
|
10611
|
-
}
|
|
10612
10696
|
async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
10613
10697
|
const reportOutcome = startRunOutcome(db, runId, FeatureNames.bing, "sync");
|
|
10614
10698
|
let abortCause;
|
|
@@ -10653,7 +10737,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
10653
10737
|
try {
|
|
10654
10738
|
const issues = await getCrawlIssues(conn.apiKey, conn.siteUrl);
|
|
10655
10739
|
for (const issue of issues) {
|
|
10656
|
-
if (issue.Url &&
|
|
10740
|
+
if (issue.Url && isBlockingCrawlIssue(issue.IssueType ?? null)) {
|
|
10657
10741
|
blockedUrls.add(issue.Url);
|
|
10658
10742
|
}
|
|
10659
10743
|
}
|
|
@@ -10846,10 +10930,428 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
10846
10930
|
}
|
|
10847
10931
|
}
|
|
10848
10932
|
|
|
10849
|
-
// src/
|
|
10933
|
+
// src/bing-sync.ts
|
|
10850
10934
|
import crypto15 from "crypto";
|
|
10851
|
-
import { and as and9,
|
|
10852
|
-
var log10 = createLogger("
|
|
10935
|
+
import { and as and9, eq as eq13, notInArray as notInArray2, sql as sql5 } from "drizzle-orm";
|
|
10936
|
+
var log10 = createLogger("BingSync");
|
|
10937
|
+
var BING_SYNC_SPACING_MS = 2e3;
|
|
10938
|
+
var BING_KEYWORD_DEMAND_REFRESH_DAYS = 7;
|
|
10939
|
+
var BING_KEYWORD_DEMAND_MAX_PER_SYNC = 30;
|
|
10940
|
+
var BING_SITEMAP_INDEX_MAX_PER_SYNC = 20;
|
|
10941
|
+
var BING_NOT_AUTHORIZED = 14;
|
|
10942
|
+
var BingSyncThrottled = class extends Error {
|
|
10943
|
+
constructor(cause) {
|
|
10944
|
+
super(describeError(cause), { cause });
|
|
10945
|
+
this.name = "BingSyncThrottled";
|
|
10946
|
+
}
|
|
10947
|
+
};
|
|
10948
|
+
function isThrottle(err) {
|
|
10949
|
+
return err instanceof BingApiError && (err.isThrottle || err.status === 429);
|
|
10950
|
+
}
|
|
10951
|
+
function count3(value) {
|
|
10952
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.round(value) : 0;
|
|
10953
|
+
}
|
|
10954
|
+
var BLOCKED_ENTITY_TYPES = ["page", "directory"];
|
|
10955
|
+
var BLOCKED_REQUEST_TYPES = ["cache-only", "full-removal"];
|
|
10956
|
+
var COUNTRY_REGION_TYPES = ["page", "directory", "domain", "subdomain"];
|
|
10957
|
+
var SITE_ROLES = ["administrator", "read-only", "read-write"];
|
|
10958
|
+
function enumWord(words, value) {
|
|
10959
|
+
return typeof value === "number" && Number.isInteger(value) ? words[value] ?? null : null;
|
|
10960
|
+
}
|
|
10961
|
+
function convertDates(row) {
|
|
10962
|
+
const out = {};
|
|
10963
|
+
for (const [key, value] of Object.entries(row)) {
|
|
10964
|
+
if (key === "__type") continue;
|
|
10965
|
+
out[key] = typeof value === "string" && value.startsWith("/Date(") ? parseBingDate(value) : value;
|
|
10966
|
+
}
|
|
10967
|
+
return out;
|
|
10968
|
+
}
|
|
10969
|
+
function mergeWeekly(rows) {
|
|
10970
|
+
const merged = /* @__PURE__ */ new Map();
|
|
10971
|
+
for (const row of rows) {
|
|
10972
|
+
const date = bingDateToDay(row.Date);
|
|
10973
|
+
const key = typeof row.Query === "string" ? row.Query.trim() : "";
|
|
10974
|
+
if (!date || !key) continue;
|
|
10975
|
+
const id = `${date}\0${key}`;
|
|
10976
|
+
const entry = merged.get(id) ?? { date, key, clicks: 0, impressions: 0, clickPosWeight: 0, clickPosSum: 0, imprPosWeight: 0, imprPosSum: 0 };
|
|
10977
|
+
const clicks = count3(row.Clicks);
|
|
10978
|
+
const impressions = count3(row.Impressions);
|
|
10979
|
+
entry.clicks += clicks;
|
|
10980
|
+
entry.impressions += impressions;
|
|
10981
|
+
const clickPos = bingPositionOrNull(row.AvgClickPosition);
|
|
10982
|
+
if (clickPos !== null) {
|
|
10983
|
+
const weight = Math.max(clicks, 1);
|
|
10984
|
+
entry.clickPosSum += clickPos * weight;
|
|
10985
|
+
entry.clickPosWeight += weight;
|
|
10986
|
+
}
|
|
10987
|
+
const imprPos = bingPositionOrNull(row.AvgImpressionPosition);
|
|
10988
|
+
if (imprPos !== null) {
|
|
10989
|
+
const weight = Math.max(impressions, 1);
|
|
10990
|
+
entry.imprPosSum += imprPos * weight;
|
|
10991
|
+
entry.imprPosWeight += weight;
|
|
10992
|
+
}
|
|
10993
|
+
merged.set(id, entry);
|
|
10994
|
+
}
|
|
10995
|
+
return [...merged.values()].map((entry) => ({
|
|
10996
|
+
date: entry.date,
|
|
10997
|
+
key: entry.key,
|
|
10998
|
+
clicks: entry.clicks,
|
|
10999
|
+
impressions: entry.impressions,
|
|
11000
|
+
avgClickPosition: entry.clickPosWeight > 0 ? entry.clickPosSum / entry.clickPosWeight : null,
|
|
11001
|
+
avgImpressionPosition: entry.imprPosWeight > 0 ? entry.imprPosSum / entry.imprPosWeight : null
|
|
11002
|
+
}));
|
|
11003
|
+
}
|
|
11004
|
+
async function executeBingSync(db, runId, projectId, opts) {
|
|
11005
|
+
const reportOutcome = startRunOutcome(db, runId, FeatureNames.bing, "sync");
|
|
11006
|
+
const now = opts.now ?? (() => /* @__PURE__ */ new Date());
|
|
11007
|
+
const sleep3 = opts.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
11008
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt: now().toISOString() }).where(eq13(runs.id, runId)).run();
|
|
11009
|
+
try {
|
|
11010
|
+
const project = db.select().from(projects).where(eq13(projects.id, projectId)).get();
|
|
11011
|
+
if (!project) {
|
|
11012
|
+
throw withOutcomeReason(new Error(`Project not found: ${projectId}`), OutcomeReasonCodes.NOT_FOUND);
|
|
11013
|
+
}
|
|
11014
|
+
const conn = opts.config.bing?.connections?.find((c) => c.domain === project.canonicalDomain);
|
|
11015
|
+
if (!conn) {
|
|
11016
|
+
throw withOutcomeReason(
|
|
11017
|
+
new Error('No Bing connection found for this project. Run "canonry bing connect <project>" first.'),
|
|
11018
|
+
OutcomeReasonCodes.NOT_CONNECTED
|
|
11019
|
+
);
|
|
11020
|
+
}
|
|
11021
|
+
if (!conn.siteUrl) {
|
|
11022
|
+
throw withOutcomeReason(
|
|
11023
|
+
new Error('No Bing site configured. Run "canonry bing set-site <project> <url>" first.'),
|
|
11024
|
+
OutcomeReasonCodes.PROPERTY_NOT_FOUND
|
|
11025
|
+
);
|
|
11026
|
+
}
|
|
11027
|
+
const apiKey = conn.apiKey;
|
|
11028
|
+
const siteUrl = conn.siteUrl;
|
|
11029
|
+
const gate = sharedRateGate(credentialGateKey("bing", apiKey));
|
|
11030
|
+
let throttled = false;
|
|
11031
|
+
async function call(read) {
|
|
11032
|
+
if (throttled) throw new BingSyncThrottled(new Error("Bing throttled this account earlier in the run"));
|
|
11033
|
+
await gate.take(BING_SYNC_SPACING_MS, sleep3);
|
|
11034
|
+
try {
|
|
11035
|
+
return await read();
|
|
11036
|
+
} catch (err) {
|
|
11037
|
+
if (isThrottle(err)) {
|
|
11038
|
+
throttled = true;
|
|
11039
|
+
throw new BingSyncThrottled(err);
|
|
11040
|
+
}
|
|
11041
|
+
throw err;
|
|
11042
|
+
}
|
|
11043
|
+
}
|
|
11044
|
+
const results = [];
|
|
11045
|
+
async function section(name, work) {
|
|
11046
|
+
try {
|
|
11047
|
+
const rows2 = await work();
|
|
11048
|
+
results.push({ section: name, ok: true, rows: rows2 });
|
|
11049
|
+
log10.info("section.completed", { runId, projectId, section: name, rows: rows2 });
|
|
11050
|
+
} catch (err) {
|
|
11051
|
+
const error2 = describeError(err);
|
|
11052
|
+
results.push({ section: name, ok: false, rows: 0, error: error2 });
|
|
11053
|
+
log10.warn("section.failed", { runId, projectId, section: name, error: error2, throttled: err instanceof BingSyncThrottled });
|
|
11054
|
+
}
|
|
11055
|
+
}
|
|
11056
|
+
await section("traffic", async () => {
|
|
11057
|
+
const rows2 = await call(() => getRankAndTrafficStats(apiKey, siteUrl));
|
|
11058
|
+
const syncedAt = now().toISOString();
|
|
11059
|
+
let written = 0;
|
|
11060
|
+
db.transaction((tx) => {
|
|
11061
|
+
for (const row of rows2) {
|
|
11062
|
+
const date = bingDateToDay(row.Date);
|
|
11063
|
+
if (!date) continue;
|
|
11064
|
+
const values = { clicks: count3(row.Clicks), impressions: count3(row.Impressions), syncRunId: runId, syncedAt };
|
|
11065
|
+
tx.insert(bingTrafficDaily).values({ id: crypto15.randomUUID(), projectId, date, ...values }).onConflictDoUpdate({ target: [bingTrafficDaily.projectId, bingTrafficDaily.date], set: values }).run();
|
|
11066
|
+
written += 1;
|
|
11067
|
+
}
|
|
11068
|
+
});
|
|
11069
|
+
return written;
|
|
11070
|
+
});
|
|
11071
|
+
await section("queries", async () => {
|
|
11072
|
+
const merged = mergeWeekly(await call(() => getQueryStats(apiKey, siteUrl)));
|
|
11073
|
+
const syncedAt = now().toISOString();
|
|
11074
|
+
db.transaction((tx) => {
|
|
11075
|
+
for (const row of merged) {
|
|
11076
|
+
const values = { clicks: row.clicks, impressions: row.impressions, avgClickPosition: row.avgClickPosition, avgImpressionPosition: row.avgImpressionPosition, syncRunId: runId, syncedAt };
|
|
11077
|
+
tx.insert(bingQueryStats).values({ id: crypto15.randomUUID(), projectId, date: row.date, query: row.key, ...values }).onConflictDoUpdate({ target: [bingQueryStats.projectId, bingQueryStats.date, bingQueryStats.query], set: values }).run();
|
|
11078
|
+
}
|
|
11079
|
+
});
|
|
11080
|
+
return merged.length;
|
|
11081
|
+
});
|
|
11082
|
+
await section("pages", async () => {
|
|
11083
|
+
const merged = mergeWeekly(await call(() => getPageStats(apiKey, siteUrl)));
|
|
11084
|
+
const syncedAt = now().toISOString();
|
|
11085
|
+
db.transaction((tx) => {
|
|
11086
|
+
for (const row of merged) {
|
|
11087
|
+
const values = { clicks: row.clicks, impressions: row.impressions, avgClickPosition: row.avgClickPosition, avgImpressionPosition: row.avgImpressionPosition, syncRunId: runId, syncedAt };
|
|
11088
|
+
tx.insert(bingPageStats).values({ id: crypto15.randomUUID(), projectId, date: row.date, page: row.key, ...values }).onConflictDoUpdate({ target: [bingPageStats.projectId, bingPageStats.date, bingPageStats.page], set: values }).run();
|
|
11089
|
+
}
|
|
11090
|
+
});
|
|
11091
|
+
return merged.length;
|
|
11092
|
+
});
|
|
11093
|
+
await section("crawl-stats", async () => {
|
|
11094
|
+
const rows2 = await call(() => getCrawlStats(apiKey, siteUrl));
|
|
11095
|
+
const syncedAt = now().toISOString();
|
|
11096
|
+
let written = 0;
|
|
11097
|
+
db.transaction((tx) => {
|
|
11098
|
+
for (const row of rows2) {
|
|
11099
|
+
const date = bingDateToDay(row.Date);
|
|
11100
|
+
if (!date) continue;
|
|
11101
|
+
const values = {
|
|
11102
|
+
crawledPages: count3(row.CrawledPages),
|
|
11103
|
+
inIndex: count3(row.InIndex),
|
|
11104
|
+
crawlErrors: count3(row.CrawlErrors),
|
|
11105
|
+
inLinks: count3(row.InLinks),
|
|
11106
|
+
code2xx: count3(row.Code2xx),
|
|
11107
|
+
code301: count3(row.Code301),
|
|
11108
|
+
code302: count3(row.Code302),
|
|
11109
|
+
code4xx: count3(row.Code4xx),
|
|
11110
|
+
code5xx: count3(row.Code5xx),
|
|
11111
|
+
allOtherCodes: count3(row.AllOtherCodes),
|
|
11112
|
+
blockedByRobotsTxt: count3(row.BlockedByRobotsTxt),
|
|
11113
|
+
connectionTimeout: count3(row.ConnectionTimeout),
|
|
11114
|
+
containsMalware: count3(row.ContainsMalware),
|
|
11115
|
+
dnsFailures: count3(row.DnsFailures),
|
|
11116
|
+
syncRunId: runId,
|
|
11117
|
+
syncedAt
|
|
11118
|
+
};
|
|
11119
|
+
tx.insert(bingCrawlStats).values({ id: crypto15.randomUUID(), projectId, date, ...values }).onConflictDoUpdate({ target: [bingCrawlStats.projectId, bingCrawlStats.date], set: values }).run();
|
|
11120
|
+
written += 1;
|
|
11121
|
+
}
|
|
11122
|
+
});
|
|
11123
|
+
return written;
|
|
11124
|
+
});
|
|
11125
|
+
await section("sitemaps", async () => {
|
|
11126
|
+
const top = await call(() => getFeeds(apiKey, siteUrl));
|
|
11127
|
+
const collected = /* @__PURE__ */ new Map();
|
|
11128
|
+
for (const feed of top) {
|
|
11129
|
+
if (feed.Url) collected.set(feed.Url, { feed, parentUrl: null });
|
|
11130
|
+
}
|
|
11131
|
+
const indexes = top.filter((feed) => /index/i.test(feed.Type ?? "")).slice(0, BING_SITEMAP_INDEX_MAX_PER_SYNC);
|
|
11132
|
+
for (const index of indexes) {
|
|
11133
|
+
const details = await call(() => getFeedDetails(apiKey, siteUrl, index.Url));
|
|
11134
|
+
for (const child of details) {
|
|
11135
|
+
if (!child.Url || child.Url === index.Url || collected.has(child.Url)) continue;
|
|
11136
|
+
collected.set(child.Url, { feed: child, parentUrl: index.Url });
|
|
11137
|
+
}
|
|
11138
|
+
}
|
|
11139
|
+
const syncedAt = now().toISOString();
|
|
11140
|
+
const urls = [...collected.keys()];
|
|
11141
|
+
db.transaction((tx) => {
|
|
11142
|
+
for (const { feed, parentUrl } of collected.values()) {
|
|
11143
|
+
const values = {
|
|
11144
|
+
parentUrl,
|
|
11145
|
+
type: feed.Type ?? "Unknown",
|
|
11146
|
+
status: feed.Status ?? "Unknown",
|
|
11147
|
+
urlCount: count3(feed.UrlCount),
|
|
11148
|
+
fileSize: count3(feed.FileSize),
|
|
11149
|
+
compressed: feed.Compressed === true,
|
|
11150
|
+
submittedAt: parseBingDate(feed.Submitted ?? null),
|
|
11151
|
+
lastCrawledAt: parseBingDate(feed.LastCrawled ?? null),
|
|
11152
|
+
syncRunId: runId,
|
|
11153
|
+
syncedAt
|
|
11154
|
+
};
|
|
11155
|
+
tx.insert(bingSitemaps).values({ id: crypto15.randomUUID(), projectId, url: feed.Url, ...values }).onConflictDoUpdate({ target: [bingSitemaps.projectId, bingSitemaps.url], set: values }).run();
|
|
11156
|
+
}
|
|
11157
|
+
tx.delete(bingSitemaps).where(urls.length > 0 ? and9(eq13(bingSitemaps.projectId, projectId), notInArray2(bingSitemaps.url, urls)) : eq13(bingSitemaps.projectId, projectId)).run();
|
|
11158
|
+
tx.insert(sitemapListSyncs).values({ projectId, engine: "bing", syncedAt, syncRunId: runId }).onConflictDoUpdate({ target: [sitemapListSyncs.projectId, sitemapListSyncs.engine], set: { syncedAt, syncRunId: runId } }).run();
|
|
11159
|
+
});
|
|
11160
|
+
return collected.size;
|
|
11161
|
+
});
|
|
11162
|
+
await section("site", async () => {
|
|
11163
|
+
const reads = [
|
|
11164
|
+
{
|
|
11165
|
+
kind: BingSiteSnapshotKinds["url-submission-quota"],
|
|
11166
|
+
read: async () => {
|
|
11167
|
+
const quota = await getUrlSubmissionQuota(apiKey, siteUrl);
|
|
11168
|
+
return { dailyQuota: count3(quota?.DailyQuota), monthlyQuota: count3(quota?.MonthlyQuota) };
|
|
11169
|
+
}
|
|
11170
|
+
},
|
|
11171
|
+
{
|
|
11172
|
+
kind: BingSiteSnapshotKinds["content-submission-quota"],
|
|
11173
|
+
read: async () => {
|
|
11174
|
+
const quota = await getContentSubmissionQuota(apiKey, siteUrl);
|
|
11175
|
+
return { dailyQuota: count3(quota?.DailyQuota), monthlyQuota: count3(quota?.MonthlyQuota) };
|
|
11176
|
+
}
|
|
11177
|
+
},
|
|
11178
|
+
{
|
|
11179
|
+
kind: BingSiteSnapshotKinds["crawl-settings"],
|
|
11180
|
+
read: async () => {
|
|
11181
|
+
const settings = await getCrawlSettings(apiKey, siteUrl);
|
|
11182
|
+
return {
|
|
11183
|
+
crawlBoostAvailable: settings?.CrawlBoostAvailable === true,
|
|
11184
|
+
crawlBoostEnabled: settings?.CrawlBoostEnabled === true,
|
|
11185
|
+
crawlRate: Array.isArray(settings?.CrawlRate) ? settings.CrawlRate.filter((v) => typeof v === "number") : []
|
|
11186
|
+
};
|
|
11187
|
+
}
|
|
11188
|
+
},
|
|
11189
|
+
{
|
|
11190
|
+
kind: BingSiteSnapshotKinds["crawl-issues"],
|
|
11191
|
+
read: async () => (await getCrawlIssues(apiKey, siteUrl)).map((issue) => ({
|
|
11192
|
+
url: issue.Url,
|
|
11193
|
+
httpCode: typeof issue.HttpCode === "number" ? issue.HttpCode : null,
|
|
11194
|
+
inLinks: typeof issue.InLinks === "number" ? issue.InLinks : null,
|
|
11195
|
+
issues: issue.Issues ?? issue.IssueType ?? null
|
|
11196
|
+
}))
|
|
11197
|
+
},
|
|
11198
|
+
{
|
|
11199
|
+
kind: BingSiteSnapshotKinds["blocked-urls"],
|
|
11200
|
+
read: async () => (await getBlockedUrls(apiKey, siteUrl)).map((row) => ({
|
|
11201
|
+
url: row.Url,
|
|
11202
|
+
date: parseBingDate(row.Date ?? null),
|
|
11203
|
+
daysToExpire: typeof row.DaysToExpire === "number" ? row.DaysToExpire : null,
|
|
11204
|
+
entityType: enumWord(BLOCKED_ENTITY_TYPES, row.EntityType),
|
|
11205
|
+
requestType: enumWord(BLOCKED_REQUEST_TYPES, row.RequestType)
|
|
11206
|
+
}))
|
|
11207
|
+
},
|
|
11208
|
+
{
|
|
11209
|
+
kind: BingSiteSnapshotKinds["page-preview-blocks"],
|
|
11210
|
+
read: async () => (await getActivePagePreviewBlocks(apiKey, siteUrl)).map(convertDates)
|
|
11211
|
+
},
|
|
11212
|
+
{
|
|
11213
|
+
kind: BingSiteSnapshotKinds["deep-link-blocks"],
|
|
11214
|
+
read: async () => (await getDeepLinkBlocks(apiKey, siteUrl)).map(convertDates)
|
|
11215
|
+
},
|
|
11216
|
+
{
|
|
11217
|
+
kind: BingSiteSnapshotKinds["query-parameters"],
|
|
11218
|
+
read: async () => (await getQueryParameters(apiKey, siteUrl)).map((row) => ({
|
|
11219
|
+
parameter: row.Parameter,
|
|
11220
|
+
isEnabled: typeof row.IsEnabled === "boolean" ? row.IsEnabled : null,
|
|
11221
|
+
date: parseBingDate(row.Date ?? null),
|
|
11222
|
+
source: row.Source ?? null
|
|
11223
|
+
}))
|
|
11224
|
+
},
|
|
11225
|
+
{
|
|
11226
|
+
kind: BingSiteSnapshotKinds["country-region-settings"],
|
|
11227
|
+
read: async () => (await getCountryRegionSettings(apiKey, siteUrl)).map((row) => ({
|
|
11228
|
+
url: row.Url,
|
|
11229
|
+
countryCode: row.TwoLetterIsoCountryCode,
|
|
11230
|
+
type: enumWord(COUNTRY_REGION_TYPES, row.Type),
|
|
11231
|
+
date: parseBingDate(row.Date ?? null)
|
|
11232
|
+
}))
|
|
11233
|
+
},
|
|
11234
|
+
{
|
|
11235
|
+
kind: BingSiteSnapshotKinds["fetched-urls"],
|
|
11236
|
+
read: async () => (await getFetchedUrls(apiKey, siteUrl)).map((row) => ({
|
|
11237
|
+
url: row.Url,
|
|
11238
|
+
date: parseBingDate(row.Date ?? null),
|
|
11239
|
+
expired: typeof row.Expired === "boolean" ? row.Expired : null,
|
|
11240
|
+
fetched: typeof row.Fetched === "boolean" ? row.Fetched : null
|
|
11241
|
+
}))
|
|
11242
|
+
},
|
|
11243
|
+
{
|
|
11244
|
+
kind: BingSiteSnapshotKinds["connected-pages"],
|
|
11245
|
+
read: async () => (await getConnectedPages(apiKey, siteUrl)).map(convertDates)
|
|
11246
|
+
},
|
|
11247
|
+
{
|
|
11248
|
+
kind: BingSiteSnapshotKinds["site-roles"],
|
|
11249
|
+
read: async () => (await getSiteRoles(apiKey, siteUrl)).map((row) => ({
|
|
11250
|
+
site: row.Site ?? null,
|
|
11251
|
+
email: row.Email ?? null,
|
|
11252
|
+
role: enumWord(SITE_ROLES, row.Role),
|
|
11253
|
+
date: parseBingDate(row.Date ?? null),
|
|
11254
|
+
expired: typeof row.Expired === "boolean" ? row.Expired : null
|
|
11255
|
+
}))
|
|
11256
|
+
}
|
|
11257
|
+
];
|
|
11258
|
+
let failed2 = 0;
|
|
11259
|
+
let stored = 0;
|
|
11260
|
+
for (const { kind, read } of reads) {
|
|
11261
|
+
let status2 = "ok";
|
|
11262
|
+
let payload = null;
|
|
11263
|
+
let error2 = null;
|
|
11264
|
+
try {
|
|
11265
|
+
payload = await call(read);
|
|
11266
|
+
} catch (err) {
|
|
11267
|
+
if (err instanceof BingSyncThrottled) throw err;
|
|
11268
|
+
const unauthorized = err instanceof BingApiError && err.bingErrorCode === BING_NOT_AUTHORIZED;
|
|
11269
|
+
status2 = unauthorized ? "unauthorized" : "failed";
|
|
11270
|
+
error2 = unauthorized ? "This Bing key is not allowed to read this setting (owner-only)." : describeError(err);
|
|
11271
|
+
if (!unauthorized) failed2 += 1;
|
|
11272
|
+
}
|
|
11273
|
+
const syncedAt = now().toISOString();
|
|
11274
|
+
const previous = status2 === "failed" ? db.select({ payload: bingSiteSnapshots.payload }).from(bingSiteSnapshots).where(and9(eq13(bingSiteSnapshots.projectId, projectId), eq13(bingSiteSnapshots.kind, kind))).get() : void 0;
|
|
11275
|
+
const values = { status: status2, payload: status2 === "failed" ? previous?.payload ?? null : payload, error: error2, syncRunId: runId, syncedAt };
|
|
11276
|
+
db.insert(bingSiteSnapshots).values({ id: crypto15.randomUUID(), projectId, kind, ...values }).onConflictDoUpdate({ target: [bingSiteSnapshots.projectId, bingSiteSnapshots.kind], set: values }).run();
|
|
11277
|
+
stored += 1;
|
|
11278
|
+
}
|
|
11279
|
+
if (failed2 > 0) throw new Error(`${failed2} of ${reads.length} Bing site settings could not be read`);
|
|
11280
|
+
return stored;
|
|
11281
|
+
});
|
|
11282
|
+
await section("keyword-demand", async () => {
|
|
11283
|
+
const market = bingKeywordMarket(project.country, project.language);
|
|
11284
|
+
const tracked = db.select({ query: queries.query }).from(queries).where(eq13(queries.projectId, projectId)).all();
|
|
11285
|
+
if (tracked.length === 0) return 0;
|
|
11286
|
+
const cutoff = new Date(now().getTime() - BING_KEYWORD_DEMAND_REFRESH_DAYS * 864e5).toISOString();
|
|
11287
|
+
const lastSynced = new Map(db.select({ query: bingKeywordDemand.query, syncedAt: sql5`MAX(${bingKeywordDemand.syncedAt})` }).from(bingKeywordDemand).where(and9(
|
|
11288
|
+
eq13(bingKeywordDemand.projectId, projectId),
|
|
11289
|
+
eq13(bingKeywordDemand.country, market.country),
|
|
11290
|
+
eq13(bingKeywordDemand.language, market.language)
|
|
11291
|
+
)).groupBy(bingKeywordDemand.query).all().map((row) => [row.query, row.syncedAt]));
|
|
11292
|
+
const due = tracked.map((row) => ({ query: row.query, syncedAt: lastSynced.get(row.query) ?? null })).filter((row) => row.syncedAt === null || row.syncedAt < cutoff).sort((a, b) => (a.syncedAt ?? "").localeCompare(b.syncedAt ?? "")).slice(0, BING_KEYWORD_DEMAND_MAX_PER_SYNC);
|
|
11293
|
+
let written = 0;
|
|
11294
|
+
let failures = 0;
|
|
11295
|
+
let lastError;
|
|
11296
|
+
for (const { query } of due) {
|
|
11297
|
+
let history;
|
|
11298
|
+
try {
|
|
11299
|
+
history = await call(() => getKeywordHistory(apiKey, query, market));
|
|
11300
|
+
} catch (err) {
|
|
11301
|
+
if (err instanceof BingSyncThrottled) throw err;
|
|
11302
|
+
failures += 1;
|
|
11303
|
+
lastError = err;
|
|
11304
|
+
continue;
|
|
11305
|
+
}
|
|
11306
|
+
const syncedAt = now().toISOString();
|
|
11307
|
+
db.transaction((tx) => {
|
|
11308
|
+
for (const week of history) {
|
|
11309
|
+
const date = bingDateToDay(week.Date);
|
|
11310
|
+
if (!date) continue;
|
|
11311
|
+
const values = { impressions: count3(week.Impressions), broadImpressions: count3(week.BroadImpressions), syncRunId: runId, syncedAt };
|
|
11312
|
+
tx.insert(bingKeywordDemand).values({ id: crypto15.randomUUID(), projectId, query, country: market.country, language: market.language, date, ...values }).onConflictDoUpdate({
|
|
11313
|
+
target: [bingKeywordDemand.projectId, bingKeywordDemand.query, bingKeywordDemand.country, bingKeywordDemand.language, bingKeywordDemand.date],
|
|
11314
|
+
set: values
|
|
11315
|
+
}).run();
|
|
11316
|
+
written += 1;
|
|
11317
|
+
}
|
|
11318
|
+
if (history.length === 0) {
|
|
11319
|
+
const date = now().toISOString().slice(0, 10);
|
|
11320
|
+
tx.insert(bingKeywordDemand).values({ id: crypto15.randomUUID(), projectId, query, country: market.country, language: market.language, date, impressions: 0, broadImpressions: 0, syncRunId: runId, syncedAt }).onConflictDoNothing().run();
|
|
11321
|
+
}
|
|
11322
|
+
});
|
|
11323
|
+
}
|
|
11324
|
+
if (failures > 0) throw new Error(`${failures} of ${due.length} keyword demand reads failed; last: ${describeError(lastError)}`);
|
|
11325
|
+
return written;
|
|
11326
|
+
});
|
|
11327
|
+
const failed = results.filter((r) => !r.ok);
|
|
11328
|
+
const status = failed.length === 0 ? RunStatuses.completed : failed.length === results.length ? RunStatuses.failed : RunStatuses.partial;
|
|
11329
|
+
const error = failed.length === 0 ? null : `${throttled ? "Bing throttled this account; stopped early. " : ""}${failed.map((r) => `${r.section}: ${r.error}`).join("; ")}`.slice(0, 2e3);
|
|
11330
|
+
db.update(runs).set({ status, error, finishedAt: now().toISOString() }).where(eq13(runs.id, runId)).run();
|
|
11331
|
+
const rows = results.reduce((sum, r) => sum + r.rows, 0);
|
|
11332
|
+
log10.info("sync.completed", { runId, projectId, status, rows, failedSections: failed.map((r) => r.section) });
|
|
11333
|
+
if (status === RunStatuses.completed) {
|
|
11334
|
+
reportOutcome({ status: OutcomeStatuses.succeeded, counts: { rows, items: results.length } });
|
|
11335
|
+
} else {
|
|
11336
|
+
reportOutcome({
|
|
11337
|
+
status: status === RunStatuses.partial ? OutcomeStatuses.partial : OutcomeStatuses.failed,
|
|
11338
|
+
...runFailure(new Error(error ?? "Bing sync failed"), throttled ? OutcomeReasonCodes.RATE_LIMITED : void 0),
|
|
11339
|
+
counts: { rows, items: results.length, failures: failed.length }
|
|
11340
|
+
});
|
|
11341
|
+
}
|
|
11342
|
+
} catch (err) {
|
|
11343
|
+
const message = describeError(err);
|
|
11344
|
+
db.update(runs).set({ status: RunStatuses.failed, error: message, finishedAt: now().toISOString() }).where(eq13(runs.id, runId)).run();
|
|
11345
|
+
log10.error("sync.failed", { runId, projectId, error: message });
|
|
11346
|
+
reportOutcome({ status: OutcomeStatuses.failed, ...runFailure(err) });
|
|
11347
|
+
throw err;
|
|
11348
|
+
}
|
|
11349
|
+
}
|
|
11350
|
+
|
|
11351
|
+
// src/coverage-refresh.ts
|
|
11352
|
+
import crypto16 from "crypto";
|
|
11353
|
+
import { and as and10, desc as desc6, eq as eq14, inArray as inArray6 } from "drizzle-orm";
|
|
11354
|
+
var log11 = createLogger("CoverageRefresh");
|
|
10853
11355
|
var COVERAGE_REFRESH_MIN_INTERVAL_MS = 60 * 60 * 1e3;
|
|
10854
11356
|
var COVERAGE_REFRESH_MANUAL_MIN_INTERVAL_MS = 2 * 60 * 1e3;
|
|
10855
11357
|
var ACTIVE_OR_DONE_STATUSES = [
|
|
@@ -10860,44 +11362,44 @@ var ACTIVE_OR_DONE_STATUSES = [
|
|
|
10860
11362
|
];
|
|
10861
11363
|
var IN_FLIGHT_STATUSES = [RunStatuses.queued, RunStatuses.running];
|
|
10862
11364
|
function runWasUserInitiated(db, runId) {
|
|
10863
|
-
const row = db.select({ trigger: runs.trigger }).from(runs).where(
|
|
11365
|
+
const row = db.select({ trigger: runs.trigger }).from(runs).where(eq14(runs.id, runId)).get();
|
|
10864
11366
|
return row?.trigger === RunTriggers.manual;
|
|
10865
11367
|
}
|
|
10866
11368
|
var defaultDeps = { executeInspectSitemap };
|
|
10867
11369
|
async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps, nowMs = Date.now(), opts = {}) {
|
|
10868
|
-
const project = db.select({ canonicalDomain: projects.canonicalDomain }).from(projects).where(
|
|
11370
|
+
const project = db.select({ canonicalDomain: projects.canonicalDomain }).from(projects).where(eq14(projects.id, projectId)).get();
|
|
10869
11371
|
if (!project) return null;
|
|
10870
11372
|
const { clientId, clientSecret } = getGoogleAuthConfig(config);
|
|
10871
11373
|
if (!clientId || !clientSecret) return null;
|
|
10872
11374
|
const conn = getGoogleConnection(config, project.canonicalDomain, "gsc");
|
|
10873
11375
|
if (!conn?.refreshToken || !conn.propertyId) return null;
|
|
10874
11376
|
const inFlight = db.select({ createdAt: runs.createdAt }).from(runs).where(
|
|
10875
|
-
|
|
10876
|
-
|
|
10877
|
-
|
|
11377
|
+
and10(
|
|
11378
|
+
eq14(runs.projectId, projectId),
|
|
11379
|
+
eq14(runs.kind, RunKinds["inspect-sitemap"]),
|
|
10878
11380
|
inArray6(runs.status, IN_FLIGHT_STATUSES)
|
|
10879
11381
|
)
|
|
10880
11382
|
).orderBy(desc6(runs.createdAt)).limit(1).get();
|
|
10881
11383
|
if (inFlight) {
|
|
10882
|
-
|
|
11384
|
+
log11.info("skip.in-flight", { projectId });
|
|
10883
11385
|
return null;
|
|
10884
11386
|
}
|
|
10885
11387
|
const windowMs = opts.userInitiated ? COVERAGE_REFRESH_MANUAL_MIN_INTERVAL_MS : COVERAGE_REFRESH_MIN_INTERVAL_MS;
|
|
10886
11388
|
const recent = db.select({ createdAt: runs.createdAt }).from(runs).where(
|
|
10887
|
-
|
|
10888
|
-
|
|
10889
|
-
|
|
11389
|
+
and10(
|
|
11390
|
+
eq14(runs.projectId, projectId),
|
|
11391
|
+
eq14(runs.kind, RunKinds["inspect-sitemap"]),
|
|
10890
11392
|
inArray6(runs.status, ACTIVE_OR_DONE_STATUSES)
|
|
10891
11393
|
)
|
|
10892
11394
|
).orderBy(desc6(runs.createdAt)).limit(1).get();
|
|
10893
11395
|
if (recent) {
|
|
10894
11396
|
const ageMs = nowMs - Date.parse(recent.createdAt);
|
|
10895
11397
|
if (Number.isFinite(ageMs) && ageMs < windowMs) {
|
|
10896
|
-
|
|
11398
|
+
log11.info("skip.recent", { projectId, ageMs, windowMs, userInitiated: opts.userInitiated === true });
|
|
10897
11399
|
return null;
|
|
10898
11400
|
}
|
|
10899
11401
|
}
|
|
10900
|
-
const runId =
|
|
11402
|
+
const runId = crypto16.randomUUID();
|
|
10901
11403
|
db.insert(runs).values({
|
|
10902
11404
|
id: runId,
|
|
10903
11405
|
projectId,
|
|
@@ -10906,11 +11408,11 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
10906
11408
|
trigger: RunTriggers.scheduled,
|
|
10907
11409
|
createdAt: new Date(nowMs).toISOString()
|
|
10908
11410
|
}).run();
|
|
10909
|
-
|
|
11411
|
+
log11.info("refresh.start", { projectId, runId });
|
|
10910
11412
|
try {
|
|
10911
11413
|
await deps.executeInspectSitemap(db, runId, projectId, { config });
|
|
10912
11414
|
} catch (err) {
|
|
10913
|
-
|
|
11415
|
+
log11.error("refresh.failed", {
|
|
10914
11416
|
projectId,
|
|
10915
11417
|
runId,
|
|
10916
11418
|
error: describeError(err)
|
|
@@ -10920,10 +11422,10 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
10920
11422
|
}
|
|
10921
11423
|
|
|
10922
11424
|
// src/commoncrawl-sync.ts
|
|
10923
|
-
import
|
|
11425
|
+
import crypto17 from "crypto";
|
|
10924
11426
|
import path4 from "path";
|
|
10925
|
-
import { and as
|
|
10926
|
-
var
|
|
11427
|
+
import { and as and11, eq as eq15, sql as sql6 } from "drizzle-orm";
|
|
11428
|
+
var log12 = createLogger("CommonCrawlSync");
|
|
10927
11429
|
var INSERT_CHUNK_SIZE = 1e4;
|
|
10928
11430
|
function defaultDeps2() {
|
|
10929
11431
|
return {
|
|
@@ -10957,7 +11459,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
10957
11459
|
phaseDetail: "downloading vertices + edges",
|
|
10958
11460
|
updatedAt: downloadStartedAt,
|
|
10959
11461
|
error: null
|
|
10960
|
-
}).where(
|
|
11462
|
+
}).where(eq15(ccReleaseSyncs.id, syncId)).run();
|
|
10961
11463
|
const paths = ccReleasePaths(release);
|
|
10962
11464
|
const releaseCacheDir = path4.join(deps.cacheDir, release);
|
|
10963
11465
|
const vertexPath = path4.join(releaseCacheDir, paths.vertexFilename);
|
|
@@ -10980,7 +11482,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
10980
11482
|
vertexSha256: vertex.sha256,
|
|
10981
11483
|
edgesSha256: edges.sha256,
|
|
10982
11484
|
updatedAt: downloadFinishedAt
|
|
10983
|
-
}).where(
|
|
11485
|
+
}).where(eq15(ccReleaseSyncs.id, syncId)).run();
|
|
10984
11486
|
const allProjects = db.select().from(projects).all();
|
|
10985
11487
|
const targets = Array.from(new Set(allProjects.map((p) => p.canonicalDomain)));
|
|
10986
11488
|
let rows = [];
|
|
@@ -10996,15 +11498,15 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
10996
11498
|
}
|
|
10997
11499
|
const queriedAt = deps.now().toISOString();
|
|
10998
11500
|
db.transaction((tx) => {
|
|
10999
|
-
tx.delete(backlinkDomains).where(
|
|
11000
|
-
tx.delete(backlinkSummaries).where(
|
|
11501
|
+
tx.delete(backlinkDomains).where(eq15(backlinkDomains.releaseSyncId, syncId)).run();
|
|
11502
|
+
tx.delete(backlinkSummaries).where(eq15(backlinkSummaries.releaseSyncId, syncId)).run();
|
|
11001
11503
|
const expanded = [];
|
|
11002
11504
|
for (const r of rows) {
|
|
11003
11505
|
const projectIds = projectsByDomain.get(r.targetDomain);
|
|
11004
11506
|
if (!projectIds) continue;
|
|
11005
11507
|
for (const projectId of projectIds) {
|
|
11006
11508
|
expanded.push({
|
|
11007
|
-
id:
|
|
11509
|
+
id: crypto17.randomUUID(),
|
|
11008
11510
|
projectId,
|
|
11009
11511
|
releaseSyncId: syncId,
|
|
11010
11512
|
release,
|
|
@@ -11024,7 +11526,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
11024
11526
|
const projectRows = rowsByProject.get(p.id) ?? [];
|
|
11025
11527
|
const summary = computeBacklinkSummaryMetrics(projectRows);
|
|
11026
11528
|
tx.insert(backlinkSummaries).values({
|
|
11027
|
-
id:
|
|
11529
|
+
id: crypto17.randomUUID(),
|
|
11028
11530
|
projectId: p.id,
|
|
11029
11531
|
releaseSyncId: syncId,
|
|
11030
11532
|
source: BacklinkSources.commoncrawl,
|
|
@@ -11057,8 +11559,8 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
11057
11559
|
domainsDiscovered: rows.length,
|
|
11058
11560
|
updatedAt: finishedAt,
|
|
11059
11561
|
error: null
|
|
11060
|
-
}).where(
|
|
11061
|
-
|
|
11562
|
+
}).where(eq15(ccReleaseSyncs.id, syncId)).run();
|
|
11563
|
+
log12.info("sync.completed", {
|
|
11062
11564
|
syncId,
|
|
11063
11565
|
release,
|
|
11064
11566
|
projectsProcessed: allProjects.length,
|
|
@@ -11076,7 +11578,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
11076
11578
|
try {
|
|
11077
11579
|
deps.enqueueAutoExtract({ projectId: p.id, release });
|
|
11078
11580
|
} catch (err) {
|
|
11079
|
-
|
|
11581
|
+
log12.error("auto-extract.enqueue-failed", {
|
|
11080
11582
|
syncId,
|
|
11081
11583
|
release,
|
|
11082
11584
|
projectId: p.id,
|
|
@@ -11093,8 +11595,8 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
11093
11595
|
error: errorMsg,
|
|
11094
11596
|
phaseDetail: null,
|
|
11095
11597
|
updatedAt: finishedAt
|
|
11096
|
-
}).where(
|
|
11097
|
-
|
|
11598
|
+
}).where(eq15(ccReleaseSyncs.id, syncId)).run();
|
|
11599
|
+
log12.error("sync.failed", { syncId, release, error: errorMsg });
|
|
11098
11600
|
trackFeatureCompleted({
|
|
11099
11601
|
...outcome,
|
|
11100
11602
|
status: OutcomeStatuses.failed,
|
|
@@ -11126,7 +11628,7 @@ async function syncLatestReleaseOnSchedule(projectName, deps) {
|
|
|
11126
11628
|
if (probed === null) report({ status: OutcomeStatuses.failed, reasonCode: OutcomeReasonCodes.NOT_FOUND });
|
|
11127
11629
|
return;
|
|
11128
11630
|
}
|
|
11129
|
-
const alreadySynced = deps.db.select().from(ccReleaseSyncs).where(
|
|
11631
|
+
const alreadySynced = deps.db.select().from(ccReleaseSyncs).where(and11(eq15(ccReleaseSyncs.release, probed.release), eq15(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready))).limit(1).get();
|
|
11130
11632
|
if (alreadySynced) {
|
|
11131
11633
|
deps.log.info({ projectName, release: probed.release }, "Scheduled backlinks sync: already up to date, skipping");
|
|
11132
11634
|
report({ status: OutcomeStatuses.skipped, reasonCode: OutcomeReasonCodes.NOT_DUE });
|
|
@@ -11157,10 +11659,10 @@ function groupByProject(rows, projectsByDomain) {
|
|
|
11157
11659
|
}
|
|
11158
11660
|
|
|
11159
11661
|
// src/backlink-extract.ts
|
|
11160
|
-
import
|
|
11662
|
+
import crypto18 from "crypto";
|
|
11161
11663
|
import fs4 from "fs";
|
|
11162
|
-
import { and as
|
|
11163
|
-
var
|
|
11664
|
+
import { and as and12, desc as desc7, eq as eq16 } from "drizzle-orm";
|
|
11665
|
+
var log13 = createLogger("BacklinkExtract");
|
|
11164
11666
|
function defaultDeps3() {
|
|
11165
11667
|
return {
|
|
11166
11668
|
queryBacklinks,
|
|
@@ -11173,14 +11675,14 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
11173
11675
|
const startedAt = deps.now().toISOString();
|
|
11174
11676
|
const reportOutcome = startRunOutcome(db, runId, "backlinks", "extract");
|
|
11175
11677
|
let refusal;
|
|
11176
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
11678
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq16(runs.id, runId)).run();
|
|
11177
11679
|
try {
|
|
11178
|
-
const project = db.select().from(projects).where(
|
|
11680
|
+
const project = db.select().from(projects).where(eq16(projects.id, projectId)).get();
|
|
11179
11681
|
if (!project) {
|
|
11180
11682
|
refusal = OutcomeReasonCodes.NOT_FOUND;
|
|
11181
11683
|
throw new Error(`Project not found: ${projectId}`);
|
|
11182
11684
|
}
|
|
11183
|
-
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(
|
|
11685
|
+
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(eq16(ccReleaseSyncs.release, opts.release)).get() : db.select().from(ccReleaseSyncs).where(eq16(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).orderBy(desc7(ccReleaseSyncs.createdAt)).limit(1).get();
|
|
11184
11686
|
refusal = OutcomeReasonCodes.NO_DATA;
|
|
11185
11687
|
if (!sync) {
|
|
11186
11688
|
throw new Error("No ready release sync available \u2014 run `canonry backlinks sync` first");
|
|
@@ -11210,15 +11712,15 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
11210
11712
|
const targetDomain = project.canonicalDomain;
|
|
11211
11713
|
db.transaction((tx) => {
|
|
11212
11714
|
tx.delete(backlinkDomains).where(
|
|
11213
|
-
|
|
11214
|
-
|
|
11215
|
-
|
|
11216
|
-
|
|
11715
|
+
and12(
|
|
11716
|
+
eq16(backlinkDomains.projectId, projectId),
|
|
11717
|
+
eq16(backlinkDomains.source, BacklinkSources.commoncrawl),
|
|
11718
|
+
eq16(backlinkDomains.release, release)
|
|
11217
11719
|
)
|
|
11218
11720
|
).run();
|
|
11219
11721
|
if (rows.length > 0) {
|
|
11220
11722
|
const values = rows.map((r) => ({
|
|
11221
|
-
id:
|
|
11723
|
+
id: crypto18.randomUUID(),
|
|
11222
11724
|
projectId,
|
|
11223
11725
|
releaseSyncId: syncId,
|
|
11224
11726
|
source: BacklinkSources.commoncrawl,
|
|
@@ -11232,7 +11734,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
11232
11734
|
}
|
|
11233
11735
|
const summary2 = computeSummary(rows);
|
|
11234
11736
|
tx.insert(backlinkSummaries).values({
|
|
11235
|
-
id:
|
|
11737
|
+
id: crypto18.randomUUID(),
|
|
11236
11738
|
projectId,
|
|
11237
11739
|
releaseSyncId: syncId,
|
|
11238
11740
|
source: BacklinkSources.commoncrawl,
|
|
@@ -11256,8 +11758,8 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
11256
11758
|
}).run();
|
|
11257
11759
|
});
|
|
11258
11760
|
const finishedAt = deps.now().toISOString();
|
|
11259
|
-
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
11260
|
-
|
|
11761
|
+
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq16(runs.id, runId)).run();
|
|
11762
|
+
log13.info("extract.completed", { runId, projectId, release, rows: rows.length });
|
|
11261
11763
|
const summary = computeSummary(rows);
|
|
11262
11764
|
reportOutcome({ status: OutcomeStatuses.succeeded, counts: { domains: summary.totalLinkingDomains, links: summary.totalHosts } });
|
|
11263
11765
|
} catch (err) {
|
|
@@ -11267,8 +11769,8 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
11267
11769
|
status: RunStatuses.failed,
|
|
11268
11770
|
error: errorMsg,
|
|
11269
11771
|
finishedAt
|
|
11270
|
-
}).where(
|
|
11271
|
-
|
|
11772
|
+
}).where(eq16(runs.id, runId)).run();
|
|
11773
|
+
log13.error("extract.failed", { runId, projectId, error: errorMsg });
|
|
11272
11774
|
reportOutcome({ status: OutcomeStatuses.failed, ...outcomeFailure(err, refusal) });
|
|
11273
11775
|
throw err;
|
|
11274
11776
|
}
|
|
@@ -11278,9 +11780,9 @@ function computeSummary(rows) {
|
|
|
11278
11780
|
}
|
|
11279
11781
|
|
|
11280
11782
|
// src/discovery-run.ts
|
|
11281
|
-
import
|
|
11282
|
-
import { and as
|
|
11283
|
-
var
|
|
11783
|
+
import crypto19 from "crypto";
|
|
11784
|
+
import { and as and13, eq as eq17 } from "drizzle-orm";
|
|
11785
|
+
var log14 = createLogger("DiscoveryRun");
|
|
11284
11786
|
var DEFAULT_SEED_COUNT = 30;
|
|
11285
11787
|
var EMBED_RETRY_MAX_RETRIES = 3;
|
|
11286
11788
|
var EMBED_RETRY_MAX_DELAY_MS = 1e4;
|
|
@@ -11312,7 +11814,7 @@ async function embedWithRetry(fn) {
|
|
|
11312
11814
|
isRetryable: isRetryableHttpError,
|
|
11313
11815
|
maxDelayMs: EMBED_RETRY_MAX_DELAY_MS,
|
|
11314
11816
|
onRetry: ({ attempt, delayMs }) => {
|
|
11315
|
-
|
|
11817
|
+
log14.warn("discovery.embed.retry", { attempt, delayMs });
|
|
11316
11818
|
}
|
|
11317
11819
|
}
|
|
11318
11820
|
);
|
|
@@ -11322,11 +11824,11 @@ async function executeDiscoveryRun(opts) {
|
|
|
11322
11824
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
11323
11825
|
const reportOutcome = startRunOutcome(opts.db, opts.runId, "discovery", "run");
|
|
11324
11826
|
let notConfigured = false;
|
|
11325
|
-
opts.db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
11827
|
+
opts.db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq17(runs.id, opts.runId)).run();
|
|
11326
11828
|
try {
|
|
11327
|
-
const projectRow = opts.db.select().from(projects).where(
|
|
11829
|
+
const projectRow = opts.db.select().from(projects).where(eq17(projects.id, opts.projectId)).get();
|
|
11328
11830
|
if (!projectRow) throw new Error(`Project ${opts.projectId} not found`);
|
|
11329
|
-
const projectCompetitors = opts.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
11831
|
+
const projectCompetitors = opts.db.select({ domain: competitors.domain }).from(competitors).where(eq17(competitors.projectId, opts.projectId)).orderBy(competitors.domain).all().map((r) => r.domain.toLowerCase());
|
|
11330
11832
|
const canonicalDomains = effectiveDomains({
|
|
11331
11833
|
canonicalDomain: projectRow.canonicalDomain,
|
|
11332
11834
|
ownedDomains: projectRow.ownedDomains
|
|
@@ -11368,8 +11870,8 @@ async function executeDiscoveryRun(opts) {
|
|
|
11368
11870
|
seedProvider: result.seedProvider,
|
|
11369
11871
|
result
|
|
11370
11872
|
});
|
|
11371
|
-
opts.db.update(runs).set({ status: RunStatuses.completed, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
11372
|
-
|
|
11873
|
+
opts.db.update(runs).set({ status: RunStatuses.completed, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq17(runs.id, opts.runId)).run();
|
|
11874
|
+
log14.info("discovery.completed", {
|
|
11373
11875
|
runId: opts.runId,
|
|
11374
11876
|
sessionId: opts.sessionId,
|
|
11375
11877
|
buckets: result.buckets,
|
|
@@ -11386,13 +11888,13 @@ async function executeDiscoveryRun(opts) {
|
|
|
11386
11888
|
});
|
|
11387
11889
|
} catch (err) {
|
|
11388
11890
|
const errorMsg = describeError(err);
|
|
11389
|
-
|
|
11891
|
+
log14.error("discovery.failed", { runId: opts.runId, sessionId: opts.sessionId, error: errorMsg });
|
|
11390
11892
|
markSessionFailed(opts.db, opts.sessionId, errorMsg);
|
|
11391
11893
|
opts.db.update(runs).set({
|
|
11392
11894
|
status: RunStatuses.failed,
|
|
11393
11895
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
11394
11896
|
error: errorMsg
|
|
11395
|
-
}).where(
|
|
11897
|
+
}).where(eq17(runs.id, opts.runId)).run();
|
|
11396
11898
|
const failure2 = classifyProviderOutcomeError(err);
|
|
11397
11899
|
reportOutcome({
|
|
11398
11900
|
status: OutcomeStatuses.failed,
|
|
@@ -11661,13 +12163,13 @@ function writeDiscoveryInsight(db, input) {
|
|
|
11661
12163
|
totalProbes
|
|
11662
12164
|
});
|
|
11663
12165
|
db.transaction((tx) => {
|
|
11664
|
-
tx.update(insights).set({ dismissed: true }).where(
|
|
11665
|
-
|
|
11666
|
-
|
|
11667
|
-
|
|
12166
|
+
tx.update(insights).set({ dismissed: true }).where(and13(
|
|
12167
|
+
eq17(insights.projectId, input.projectId),
|
|
12168
|
+
eq17(insights.type, "discovery.basket-divergence"),
|
|
12169
|
+
eq17(insights.dismissed, false)
|
|
11668
12170
|
)).run();
|
|
11669
12171
|
tx.insert(insights).values({
|
|
11670
|
-
id:
|
|
12172
|
+
id: crypto19.randomUUID(),
|
|
11671
12173
|
projectId: input.projectId,
|
|
11672
12174
|
runId: input.runId,
|
|
11673
12175
|
type: "discovery.basket-divergence",
|
|
@@ -11703,8 +12205,8 @@ function buildDiscoveryInsightTitle(input) {
|
|
|
11703
12205
|
}
|
|
11704
12206
|
|
|
11705
12207
|
// src/execute-site-audit.ts
|
|
11706
|
-
import
|
|
11707
|
-
import { and as
|
|
12208
|
+
import crypto20 from "crypto";
|
|
12209
|
+
import { and as and15, eq as eq19, ne as ne2, sql as sql7 } from "drizzle-orm";
|
|
11708
12210
|
import { runSiteCrawl } from "@canonry/aeo-audit";
|
|
11709
12211
|
|
|
11710
12212
|
// src/site-audit-root.ts
|
|
@@ -11892,7 +12394,7 @@ async function requestPinnedSiteAuditRoot(target, options) {
|
|
|
11892
12394
|
}
|
|
11893
12395
|
|
|
11894
12396
|
// src/site-crawl-template-links.ts
|
|
11895
|
-
import { and as
|
|
12397
|
+
import { and as and14, asc as asc3, eq as eq18, gt as gt2, inArray as inArray7 } from "drizzle-orm";
|
|
11896
12398
|
var TEMPLATE_LINK_READ_BATCH = 2e3;
|
|
11897
12399
|
var TEMPLATE_LINK_WRITE_BATCH = 500;
|
|
11898
12400
|
function placementOccurrencesOf(row) {
|
|
@@ -11914,10 +12416,10 @@ function* streamAttemptEdges(db, scope) {
|
|
|
11914
12416
|
placementNavigationOccurrences: siteCrawlEdges.placementNavigationOccurrences,
|
|
11915
12417
|
placementContentOccurrences: siteCrawlEdges.placementContentOccurrences,
|
|
11916
12418
|
placementUnknownOccurrences: siteCrawlEdges.placementUnknownOccurrences
|
|
11917
|
-
}).from(siteCrawlEdges).where(
|
|
11918
|
-
|
|
11919
|
-
|
|
11920
|
-
|
|
12419
|
+
}).from(siteCrawlEdges).where(and14(
|
|
12420
|
+
eq18(siteCrawlEdges.projectId, scope.projectId),
|
|
12421
|
+
eq18(siteCrawlEdges.runId, scope.runId),
|
|
12422
|
+
eq18(siteCrawlEdges.attemptId, scope.attemptId),
|
|
11921
12423
|
gt2(siteCrawlEdges.edgeKey, after)
|
|
11922
12424
|
)).orderBy(asc3(siteCrawlEdges.edgeKey)).limit(TEMPLATE_LINK_READ_BATCH).all();
|
|
11923
12425
|
if (rows.length === 0) return;
|
|
@@ -11942,10 +12444,10 @@ function writeGroupKey(isTemplate, templateRatio) {
|
|
|
11942
12444
|
return `${isTemplate ? "1" : "0"}${templateRatio ?? ""}`;
|
|
11943
12445
|
}
|
|
11944
12446
|
function classifySiteCrawlTemplateLinks(db, scope, pagesFetched, placementRulesetVersion) {
|
|
11945
|
-
const scopeFilter =
|
|
11946
|
-
|
|
11947
|
-
|
|
11948
|
-
|
|
12447
|
+
const scopeFilter = and14(
|
|
12448
|
+
eq18(siteCrawlEdges.projectId, scope.projectId),
|
|
12449
|
+
eq18(siteCrawlEdges.runId, scope.runId),
|
|
12450
|
+
eq18(siteCrawlEdges.attemptId, scope.attemptId)
|
|
11949
12451
|
);
|
|
11950
12452
|
const placementAvailable = templateLinkPlacementAvailable(placementRulesetVersion);
|
|
11951
12453
|
const ubiquityAvailable = templateLinkUbiquityAvailable(pagesFetched);
|
|
@@ -11980,7 +12482,7 @@ function classifySiteCrawlTemplateLinks(db, scope, pagesFetched, placementRulese
|
|
|
11980
12482
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
11981
12483
|
for (const group of edgeKeysByValue.values()) {
|
|
11982
12484
|
for (const chunk of chunked(group.edgeKeys)) {
|
|
11983
|
-
db.update(siteCrawlEdges).set({ isTemplate: group.isTemplate, templateRatio: group.templateRatio, updatedAt: now }).where(
|
|
12485
|
+
db.update(siteCrawlEdges).set({ isTemplate: group.isTemplate, templateRatio: group.templateRatio, updatedAt: now }).where(and14(scopeFilter, inArray7(siteCrawlEdges.edgeKey, chunk))).run();
|
|
11984
12486
|
}
|
|
11985
12487
|
}
|
|
11986
12488
|
return {
|
|
@@ -12016,7 +12518,7 @@ function buildSiteCrawlIssueEvidence(siteIssues, pages) {
|
|
|
12016
12518
|
}
|
|
12017
12519
|
|
|
12018
12520
|
// src/execute-site-audit.ts
|
|
12019
|
-
var
|
|
12521
|
+
var log15 = createLogger("SiteAudit");
|
|
12020
12522
|
var SiteAuditCancelledError = class extends Error {
|
|
12021
12523
|
name = "SiteAuditCancelledError";
|
|
12022
12524
|
};
|
|
@@ -12168,7 +12670,7 @@ function trackSiteAuditOutcome(db, input) {
|
|
|
12168
12670
|
);
|
|
12169
12671
|
const pagesAudited = input.crawl?.pagesAudited ?? 0;
|
|
12170
12672
|
if (pagesAudited === 0 || input.trigger === RunTriggers.probe) return;
|
|
12171
|
-
const prior = db.select({ id: siteAuditSnapshots.id }).from(siteAuditSnapshots).where(
|
|
12673
|
+
const prior = db.select({ id: siteAuditSnapshots.id }).from(siteAuditSnapshots).where(and15(eq19(siteAuditSnapshots.projectId, input.projectId), ne2(siteAuditSnapshots.runId, input.runId))).limit(1).get();
|
|
12172
12674
|
if (prior) return;
|
|
12173
12675
|
trackEvent("activation.completed", {
|
|
12174
12676
|
flowVersion: ONBOARDING_FLOW_VERSION,
|
|
@@ -12177,18 +12679,18 @@ function trackSiteAuditOutcome(db, input) {
|
|
|
12177
12679
|
pagesAuditedBucket: bucketOnboardingCount(pagesAudited)
|
|
12178
12680
|
});
|
|
12179
12681
|
} catch (error) {
|
|
12180
|
-
|
|
12682
|
+
log15.warn("telemetry.failed", { runId: input.runId, projectId: input.projectId, error: describeError(error) });
|
|
12181
12683
|
}
|
|
12182
12684
|
}
|
|
12183
12685
|
async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
12184
12686
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
12185
|
-
const claim = db.update(runs).set({ status: "running", startedAt }).where(
|
|
12687
|
+
const claim = db.update(runs).set({ status: "running", startedAt }).where(and15(eq19(runs.id, runId), eq19(runs.projectId, projectId), eq19(runs.status, "queued"))).run();
|
|
12186
12688
|
if (claim.changes === 0) return;
|
|
12187
12689
|
const startTime = Date.now();
|
|
12188
|
-
const trigger = db.select({ trigger: runs.trigger }).from(runs).where(
|
|
12690
|
+
const trigger = db.select({ trigger: runs.trigger }).from(runs).where(eq19(runs.id, runId)).get()?.trigger ?? null;
|
|
12189
12691
|
let canonicalDomain = null;
|
|
12190
|
-
const attemptNumber = (db.select({ value:
|
|
12191
|
-
const attemptId =
|
|
12692
|
+
const attemptNumber = (db.select({ value: sql7`coalesce(max(${siteCrawlAttempts.attemptNumber}), 0)` }).from(siteCrawlAttempts).where(eq19(siteCrawlAttempts.runId, runId)).get()?.value ?? 0) + 1;
|
|
12693
|
+
const attemptId = crypto20.randomUUID();
|
|
12192
12694
|
db.insert(siteCrawlAttempts).values({
|
|
12193
12695
|
id: attemptId,
|
|
12194
12696
|
projectId,
|
|
@@ -12222,22 +12724,22 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
12222
12724
|
const nodeKey = nodeKeyByUrl.get(url);
|
|
12223
12725
|
if (!nodeKey) continue;
|
|
12224
12726
|
const scope = [
|
|
12225
|
-
|
|
12226
|
-
|
|
12227
|
-
|
|
12727
|
+
eq19(siteCrawlEdges.projectId, projectId),
|
|
12728
|
+
eq19(siteCrawlEdges.runId, runId),
|
|
12729
|
+
eq19(siteCrawlEdges.attemptId, attemptId)
|
|
12228
12730
|
];
|
|
12229
|
-
tx.update(siteCrawlEdges).set({ sourceNodeKey: nodeKey, updatedAt: now }).where(
|
|
12731
|
+
tx.update(siteCrawlEdges).set({ sourceNodeKey: nodeKey, updatedAt: now }).where(and15(
|
|
12230
12732
|
...scope,
|
|
12231
|
-
|
|
12733
|
+
eq19(siteCrawlEdges.sourceUrl, url)
|
|
12232
12734
|
)).run();
|
|
12233
|
-
tx.update(siteCrawlEdges).set({ targetNodeKey: nodeKey, updatedAt: now }).where(
|
|
12735
|
+
tx.update(siteCrawlEdges).set({ targetNodeKey: nodeKey, updatedAt: now }).where(and15(
|
|
12234
12736
|
...scope,
|
|
12235
|
-
|
|
12737
|
+
eq19(siteCrawlEdges.targetUrl, url)
|
|
12236
12738
|
)).run();
|
|
12237
12739
|
const pageScope = [
|
|
12238
|
-
|
|
12239
|
-
|
|
12240
|
-
|
|
12740
|
+
eq19(siteCrawlPages.projectId, projectId),
|
|
12741
|
+
eq19(siteCrawlPages.runId, runId),
|
|
12742
|
+
eq19(siteCrawlPages.attemptId, attemptId)
|
|
12241
12743
|
];
|
|
12242
12744
|
const canonicalTargets = tx.select({
|
|
12243
12745
|
id: siteCrawlPages.id,
|
|
@@ -12245,13 +12747,13 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
12245
12747
|
fetchState: siteCrawlPages.fetchState,
|
|
12246
12748
|
indexabilityState: siteCrawlPages.indexabilityState,
|
|
12247
12749
|
indexabilityReasons: siteCrawlPages.indexabilityReasons
|
|
12248
|
-
}).from(siteCrawlPages).where(
|
|
12750
|
+
}).from(siteCrawlPages).where(and15(...pageScope, eq19(siteCrawlPages.canonicalUrl, url))).all();
|
|
12249
12751
|
for (const target of canonicalTargets) {
|
|
12250
12752
|
tx.update(siteCrawlPages).set({
|
|
12251
12753
|
canonicalNodeKey: nodeKey,
|
|
12252
12754
|
healthState: deriveSiteHealthState({ ...target, canonicalNodeKey: nodeKey }),
|
|
12253
12755
|
updatedAt: now
|
|
12254
|
-
}).where(
|
|
12756
|
+
}).where(eq19(siteCrawlPages.id, target.id)).run();
|
|
12255
12757
|
}
|
|
12256
12758
|
}
|
|
12257
12759
|
};
|
|
@@ -12267,7 +12769,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
12267
12769
|
nodeKey: page.key
|
|
12268
12770
|
});
|
|
12269
12771
|
tx.insert(siteCrawlPages).values({
|
|
12270
|
-
id:
|
|
12772
|
+
id: crypto20.randomUUID(),
|
|
12271
12773
|
projectId,
|
|
12272
12774
|
runId,
|
|
12273
12775
|
attemptId,
|
|
@@ -12318,7 +12820,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
12318
12820
|
discoveryProvenance: [{ discoveredFrom: page.provenance.discoveredFrom, sitemapSources: page.provenance.sitemapSources, root: page.provenance.root }],
|
|
12319
12821
|
sitemapMetadata: { sources: page.provenance.sitemapSources },
|
|
12320
12822
|
fetchState: page.state,
|
|
12321
|
-
fetchedAt: page.state === "discovered" || page.state === "robots-blocked" ? null :
|
|
12823
|
+
fetchedAt: page.state === "discovered" || page.state === "robots-blocked" ? null : sql7`coalesce(${siteCrawlPages.fetchedAt}, ${now})`,
|
|
12322
12824
|
httpStatus: page.statusCode,
|
|
12323
12825
|
contentType: page.contentType,
|
|
12324
12826
|
finalUrl: page.finalUrl,
|
|
@@ -12362,7 +12864,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
12362
12864
|
if (isSelfLink(edge.from, edge.to)) return;
|
|
12363
12865
|
observedEdges.set(edge.key, edge);
|
|
12364
12866
|
tx.insert(siteCrawlEdges).values({
|
|
12365
|
-
id:
|
|
12867
|
+
id: crypto20.randomUUID(),
|
|
12366
12868
|
projectId,
|
|
12367
12869
|
runId,
|
|
12368
12870
|
attemptId,
|
|
@@ -12410,20 +12912,20 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
12410
12912
|
linkScoreRaw: metrics.linkScoreRaw,
|
|
12411
12913
|
linkScoreNormalized: metrics.linkScore,
|
|
12412
12914
|
updatedAt: now
|
|
12413
|
-
}).where(
|
|
12414
|
-
|
|
12415
|
-
|
|
12416
|
-
|
|
12417
|
-
|
|
12915
|
+
}).where(and15(
|
|
12916
|
+
eq19(siteCrawlPages.projectId, projectId),
|
|
12917
|
+
eq19(siteCrawlPages.runId, runId),
|
|
12918
|
+
eq19(siteCrawlPages.attemptId, attemptId),
|
|
12919
|
+
eq19(siteCrawlPages.nodeKey, key)
|
|
12418
12920
|
)).run();
|
|
12419
12921
|
};
|
|
12420
12922
|
const persistEvent = async (event) => {
|
|
12421
12923
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
12422
12924
|
db.transaction((tx) => {
|
|
12423
|
-
const receipt = tx.select({ checksum: siteCrawlEventReceipts.checksum }).from(siteCrawlEventReceipts).where(
|
|
12424
|
-
|
|
12425
|
-
|
|
12426
|
-
|
|
12925
|
+
const receipt = tx.select({ checksum: siteCrawlEventReceipts.checksum }).from(siteCrawlEventReceipts).where(and15(
|
|
12926
|
+
eq19(siteCrawlEventReceipts.attemptId, attemptId),
|
|
12927
|
+
eq19(siteCrawlEventReceipts.sequence, event.sequence),
|
|
12928
|
+
eq19(siteCrawlEventReceipts.batchId, event.batchId)
|
|
12427
12929
|
)).get();
|
|
12428
12930
|
if (receipt) {
|
|
12429
12931
|
if (receipt.checksum !== event.checksum) {
|
|
@@ -12432,7 +12934,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
12432
12934
|
return;
|
|
12433
12935
|
}
|
|
12434
12936
|
tx.insert(siteCrawlEventReceipts).values({
|
|
12435
|
-
id:
|
|
12937
|
+
id: crypto20.randomUUID(),
|
|
12436
12938
|
projectId,
|
|
12437
12939
|
runId,
|
|
12438
12940
|
attemptId,
|
|
@@ -12452,24 +12954,24 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
12452
12954
|
lastEventChecksum: event.checksum,
|
|
12453
12955
|
pagesDiscovered: progress.pagesDiscovered,
|
|
12454
12956
|
pagesFetched: progress.pagesFetched,
|
|
12455
|
-
pagesEligible: event.type === "summary" ? observedPages.size === 0 ? 0 : [...observedPages.values()].filter(isInventoryEligible).length :
|
|
12456
|
-
pagesErrored: event.type === "summary" ? observedErrorCount(observedPages.values()) :
|
|
12957
|
+
pagesEligible: event.type === "summary" ? observedPages.size === 0 ? 0 : [...observedPages.values()].filter(isInventoryEligible).length : sql7`${siteCrawlAttempts.pagesEligible}`,
|
|
12958
|
+
pagesErrored: event.type === "summary" ? observedErrorCount(observedPages.values()) : sql7`${siteCrawlAttempts.pagesErrored}`,
|
|
12457
12959
|
// Only the summary knows how many URL versions the crawl left unfetched.
|
|
12458
|
-
urlVersionsSkipped: event.type === "summary" ? event.summary.urlVersions?.skipped ?? 0 :
|
|
12960
|
+
urlVersionsSkipped: event.type === "summary" ? event.summary.urlVersions?.skipped ?? 0 : sql7`${siteCrawlAttempts.urlVersionsSkipped}`,
|
|
12459
12961
|
edgesDiscovered: progress.edgesObserved,
|
|
12460
12962
|
updatedAt: now
|
|
12461
|
-
}).where(
|
|
12963
|
+
}).where(eq19(siteCrawlAttempts.id, attemptId)).run();
|
|
12462
12964
|
} else {
|
|
12463
12965
|
tx.update(siteCrawlAttempts).set({
|
|
12464
12966
|
lastEventSequence: event.sequence,
|
|
12465
12967
|
lastEventChecksum: event.checksum,
|
|
12466
12968
|
updatedAt: now
|
|
12467
|
-
}).where(
|
|
12969
|
+
}).where(eq19(siteCrawlAttempts.id, attemptId)).run();
|
|
12468
12970
|
}
|
|
12469
12971
|
});
|
|
12470
12972
|
};
|
|
12471
12973
|
try {
|
|
12472
|
-
const project = db.select().from(projects).where(
|
|
12974
|
+
const project = db.select().from(projects).where(eq19(projects.id, projectId)).get();
|
|
12473
12975
|
if (!project) throw new Error(`Project not found: ${projectId}`);
|
|
12474
12976
|
canonicalDomain = project.canonicalDomain;
|
|
12475
12977
|
const homepageUrl = toHomepageUrl(project.canonicalDomain);
|
|
@@ -12481,7 +12983,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
12481
12983
|
});
|
|
12482
12984
|
const crawlRootUrl = resolvedRoot.effectiveUrl;
|
|
12483
12985
|
if (opts.sitemapUrl) await assertSiteAuditUrlAllowed(opts.sitemapUrl, "sitemapUrl");
|
|
12484
|
-
|
|
12986
|
+
log15.info("start", {
|
|
12485
12987
|
runId,
|
|
12486
12988
|
projectId,
|
|
12487
12989
|
homepageUrl,
|
|
@@ -12515,7 +13017,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
12515
13017
|
crawlSummary.pagesFetched,
|
|
12516
13018
|
placementRulesetVersion
|
|
12517
13019
|
);
|
|
12518
|
-
|
|
13020
|
+
log15.info("template-links", {
|
|
12519
13021
|
runId,
|
|
12520
13022
|
projectId,
|
|
12521
13023
|
detection: templateLinks.detection,
|
|
@@ -12547,7 +13049,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
12547
13049
|
try {
|
|
12548
13050
|
persistSiteCrawlGraphLayout(db, { projectId, runId, attemptId }, graphLayout, finishedAt);
|
|
12549
13051
|
} catch (error) {
|
|
12550
|
-
|
|
13052
|
+
log15.warn("graph-layout.persist-failed", {
|
|
12551
13053
|
runId,
|
|
12552
13054
|
projectId,
|
|
12553
13055
|
error: describeError(error)
|
|
@@ -12565,7 +13067,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
12565
13067
|
edges: []
|
|
12566
13068
|
}, finishedAt);
|
|
12567
13069
|
} catch (markerError) {
|
|
12568
|
-
|
|
13070
|
+
log15.warn("graph-layout.marker-persist-failed", {
|
|
12569
13071
|
runId,
|
|
12570
13072
|
projectId,
|
|
12571
13073
|
error: describeError(markerError)
|
|
@@ -12573,11 +13075,11 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
12573
13075
|
}
|
|
12574
13076
|
}
|
|
12575
13077
|
const published = db.transaction((tx) => {
|
|
12576
|
-
const claim2 = tx.update(runs).set({ status: terminalStatus, finishedAt }).where(
|
|
13078
|
+
const claim2 = tx.update(runs).set({ status: terminalStatus, finishedAt }).where(and15(eq19(runs.id, runId), eq19(runs.status, "running"))).run();
|
|
12577
13079
|
if (claim2.changes === 0) return false;
|
|
12578
13080
|
if (aggregateScore !== null) {
|
|
12579
13081
|
tx.insert(siteAuditSnapshots).values({
|
|
12580
|
-
id:
|
|
13082
|
+
id: crypto20.randomUUID(),
|
|
12581
13083
|
projectId,
|
|
12582
13084
|
runId,
|
|
12583
13085
|
sitemapUrl: opts.sitemapUrl ?? new URL("/sitemap.xml", crawlRootUrl).href,
|
|
@@ -12595,7 +13097,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
12595
13097
|
for (const page of observedPages.values()) {
|
|
12596
13098
|
if (!isLegacyScorecardPage(page)) continue;
|
|
12597
13099
|
tx.insert(siteAuditPages).values({
|
|
12598
|
-
id:
|
|
13100
|
+
id: crypto20.randomUUID(),
|
|
12599
13101
|
projectId,
|
|
12600
13102
|
runId,
|
|
12601
13103
|
url: page.finalUrl ?? page.requestedUrl,
|
|
@@ -12608,7 +13110,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
12608
13110
|
}
|
|
12609
13111
|
}
|
|
12610
13112
|
tx.insert(siteCrawlSnapshots).values({
|
|
12611
|
-
id:
|
|
13113
|
+
id: crypto20.randomUUID(),
|
|
12612
13114
|
projectId,
|
|
12613
13115
|
runId,
|
|
12614
13116
|
attemptId,
|
|
@@ -12657,7 +13159,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
12657
13159
|
if (opts.checkDeadLinks) {
|
|
12658
13160
|
for (const finding of deadLinkFindings) {
|
|
12659
13161
|
tx.insert(siteCrawlFindings).values({
|
|
12660
|
-
id:
|
|
13162
|
+
id: crypto20.randomUUID(),
|
|
12661
13163
|
projectId,
|
|
12662
13164
|
runId,
|
|
12663
13165
|
attemptId,
|
|
@@ -12683,24 +13185,24 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
12683
13185
|
edgesDiscovered: crawlSummary.edgesObserved,
|
|
12684
13186
|
finishedAt,
|
|
12685
13187
|
updatedAt: finishedAt
|
|
12686
|
-
}).where(
|
|
13188
|
+
}).where(eq19(siteCrawlAttempts.id, attemptId)).run();
|
|
12687
13189
|
return true;
|
|
12688
13190
|
});
|
|
12689
13191
|
if (!published) {
|
|
12690
13192
|
try {
|
|
12691
13193
|
deleteSiteCrawlGraphLayout(db, { projectId, runId, attemptId });
|
|
12692
13194
|
} catch (cleanupError) {
|
|
12693
|
-
|
|
13195
|
+
log15.warn("graph-layout.cleanup-failed", {
|
|
12694
13196
|
runId,
|
|
12695
13197
|
projectId,
|
|
12696
13198
|
error: describeError(cleanupError)
|
|
12697
13199
|
});
|
|
12698
13200
|
}
|
|
12699
|
-
const current = db.select({ status: runs.status }).from(runs).where(
|
|
13201
|
+
const current = db.select({ status: runs.status }).from(runs).where(eq19(runs.id, runId)).get();
|
|
12700
13202
|
if (current?.status === "cancelled") throw new SiteAuditCancelledError(`Site audit cancelled before publication: ${runId}`);
|
|
12701
13203
|
throw new Error(`Site audit run was no longer running before publication: ${runId}`);
|
|
12702
13204
|
}
|
|
12703
|
-
|
|
13205
|
+
log15.info("completed", {
|
|
12704
13206
|
runId,
|
|
12705
13207
|
projectId,
|
|
12706
13208
|
status: terminalStatus,
|
|
@@ -12744,14 +13246,14 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
12744
13246
|
error: message,
|
|
12745
13247
|
finishedAt,
|
|
12746
13248
|
updatedAt: finishedAt
|
|
12747
|
-
}).where(
|
|
13249
|
+
}).where(eq19(siteCrawlAttempts.id, attemptId)).run();
|
|
12748
13250
|
tx.update(runs).set({
|
|
12749
13251
|
status: cancelled ? "cancelled" : "failed",
|
|
12750
13252
|
error: message,
|
|
12751
13253
|
finishedAt
|
|
12752
|
-
}).where(
|
|
13254
|
+
}).where(and15(eq19(runs.id, runId), eq19(runs.status, "running"))).run();
|
|
12753
13255
|
});
|
|
12754
|
-
|
|
13256
|
+
log15.error("failed", { runId, projectId, cancelled, error: message });
|
|
12755
13257
|
trackSiteAuditOutcome(db, {
|
|
12756
13258
|
runId,
|
|
12757
13259
|
projectId,
|
|
@@ -12787,7 +13289,7 @@ function createAnswerFieldRecomputeQueue(opts) {
|
|
|
12787
13289
|
|
|
12788
13290
|
// src/competitor-auto-alias-runner.ts
|
|
12789
13291
|
import { setImmediate as yieldToEventLoop3 } from "timers/promises";
|
|
12790
|
-
var
|
|
13292
|
+
var log16 = createLogger("CompetitorAutoAliases");
|
|
12791
13293
|
function createCompetitorAutoAliasRunner(opts) {
|
|
12792
13294
|
const reasons = /* @__PURE__ */ new Map();
|
|
12793
13295
|
const queue = createProjectPassQueue(async (projectId) => {
|
|
@@ -12798,7 +13300,7 @@ function createCompetitorAutoAliasRunner(opts) {
|
|
|
12798
13300
|
if (!result) return null;
|
|
12799
13301
|
const added = result.detection.competitors.reduce((sum, competitor) => sum + competitor.added.length, 0);
|
|
12800
13302
|
const removed = result.detection.competitors.reduce((sum, competitor) => sum + competitor.removed.length, 0);
|
|
12801
|
-
|
|
13303
|
+
log16.info("detection.completed", {
|
|
12802
13304
|
projectId,
|
|
12803
13305
|
reason,
|
|
12804
13306
|
runs: result.detection.scan.runs,
|
|
@@ -12811,7 +13313,7 @@ function createCompetitorAutoAliasRunner(opts) {
|
|
|
12811
13313
|
try {
|
|
12812
13314
|
opts.onNamesChanged(projectId, result.detection.project);
|
|
12813
13315
|
} catch (err) {
|
|
12814
|
-
|
|
13316
|
+
log16.error("names-changed.failed", { projectId, error: describeError(err) });
|
|
12815
13317
|
}
|
|
12816
13318
|
}
|
|
12817
13319
|
return result;
|
|
@@ -12823,14 +13325,14 @@ function createCompetitorAutoAliasRunner(opts) {
|
|
|
12823
13325
|
return {
|
|
12824
13326
|
schedule(projectId, reason) {
|
|
12825
13327
|
enqueue(projectId, reason).catch((err) => {
|
|
12826
|
-
|
|
13328
|
+
log16.error("detection.failed", { projectId, reason, error: describeError(err) });
|
|
12827
13329
|
});
|
|
12828
13330
|
},
|
|
12829
13331
|
async request(projectId, reason) {
|
|
12830
13332
|
try {
|
|
12831
13333
|
return await enqueue(projectId, reason);
|
|
12832
13334
|
} catch (err) {
|
|
12833
|
-
|
|
13335
|
+
log16.error("detection.failed", { projectId, reason, error: describeError(err) });
|
|
12834
13336
|
throw err;
|
|
12835
13337
|
}
|
|
12836
13338
|
},
|
|
@@ -12947,8 +13449,8 @@ function derivedRouteAdapter(adapter, route) {
|
|
|
12947
13449
|
}
|
|
12948
13450
|
|
|
12949
13451
|
// src/install-state-telemetry.ts
|
|
12950
|
-
import { and as
|
|
12951
|
-
var
|
|
13452
|
+
import { and as and16, count as count4, eq as eq20, gte as gte2, inArray as inArray8, isNotNull as isNotNull2, isNull as isNull2, ne as ne3, sql as sql8 } from "drizzle-orm";
|
|
13453
|
+
var log17 = createLogger("InstallStateTelemetry");
|
|
12952
13454
|
var INSTALL_STATE_FIRST_DELAY_MS = 5 * 6e4;
|
|
12953
13455
|
var INSTALL_STATE_INTERVAL_MS = 24 * 60 * 6e4;
|
|
12954
13456
|
var KNOWN_PROVIDERS2 = new Set(PROVIDER_NAMES);
|
|
@@ -12970,31 +13472,31 @@ function buildInstallState(sources, now, startedAt) {
|
|
|
12970
13472
|
const since = new Date(now - INSTALL_STATE_INTERVAL_MS).toISOString();
|
|
12971
13473
|
const projectIds = db.select({ id: projects.id }).from(projects);
|
|
12972
13474
|
const total = (query) => whole(query.get()?.value);
|
|
12973
|
-
const googleByType = new Map(db.select({ type: googleConnections.connectionType, value:
|
|
12974
|
-
const gaServiceAccounts = total(db.select({ value:
|
|
12975
|
-
const trafficByType = db.select({ type: trafficSources.sourceType, value:
|
|
12976
|
-
const webhookSources = db.select({ config: notifications.config }).from(notifications).where(
|
|
13475
|
+
const googleByType = new Map(db.select({ type: googleConnections.connectionType, value: count4() }).from(googleConnections).groupBy(googleConnections.connectionType).all().map((row) => [row.type, whole(row.value)]));
|
|
13476
|
+
const gaServiceAccounts = total(db.select({ value: count4() }).from(gaConnections));
|
|
13477
|
+
const trafficByType = db.select({ type: trafficSources.sourceType, value: count4() }).from(trafficSources).where(and16(isNull2(trafficSources.archivedAt), ne3(trafficSources.status, TrafficSourceStatuses.archived))).groupBy(trafficSources.sourceType).all();
|
|
13478
|
+
const webhookSources = db.select({ config: notifications.config }).from(notifications).where(eq20(notifications.enabled, true)).all().map((row) => row.config.source);
|
|
12977
13479
|
const agentWebhooks = webhookSources.filter((source) => source === AGENT_WEBHOOK_SOURCE).length;
|
|
12978
13480
|
const webhooks = webhookSources.length - agentWebhooks;
|
|
12979
|
-
const bing = total(db.select({ value:
|
|
12980
|
-
const openAiAds = total(db.select({ value:
|
|
12981
|
-
const googleAdsConnected = total(db.select({ value:
|
|
12982
|
-
const backlinkSources = total(db.select({ value:
|
|
13481
|
+
const bing = total(db.select({ value: count4() }).from(bingConnections));
|
|
13482
|
+
const openAiAds = total(db.select({ value: count4() }).from(adsConnections));
|
|
13483
|
+
const googleAdsConnected = total(db.select({ value: count4() }).from(googleAdsConnections));
|
|
13484
|
+
const backlinkSources = total(db.select({ value: count4() }).from(
|
|
12983
13485
|
db.selectDistinct({ projectId: backlinkSummaries.projectId, source: backlinkSummaries.source }).from(backlinkSummaries).as("backlink_sources")
|
|
12984
13486
|
));
|
|
12985
13487
|
const counts = {
|
|
12986
|
-
projects: total(db.select({ value:
|
|
12987
|
-
queries: total(db.select({ value:
|
|
12988
|
-
competitors: total(db.select({ value:
|
|
12989
|
-
locations: total(db.select({ value:
|
|
12990
|
-
schedules: total(db.select({ value:
|
|
13488
|
+
projects: total(db.select({ value: count4() }).from(projects)),
|
|
13489
|
+
queries: total(db.select({ value: count4() }).from(queries)),
|
|
13490
|
+
competitors: total(db.select({ value: count4() }).from(competitors)),
|
|
13491
|
+
locations: total(db.select({ value: sql8`coalesce(sum(json_array_length(${projects.locations})), 0)` }).from(projects)),
|
|
13492
|
+
schedules: total(db.select({ value: count4() }).from(schedules).where(eq20(schedules.enabled, true))),
|
|
12991
13493
|
webhooks,
|
|
12992
13494
|
agentWebhooks,
|
|
12993
13495
|
trafficSources: trafficByType.reduce((sum, row) => sum + whole(row.value), 0),
|
|
12994
13496
|
googleConnections: [...googleByType.values()].reduce((sum, value) => sum + value, 0) + gaServiceAccounts,
|
|
12995
13497
|
bingConnections: bing,
|
|
12996
13498
|
backlinkSources,
|
|
12997
|
-
adsAccounts: openAiAds + total(db.select({ value:
|
|
13499
|
+
adsAccounts: openAiAds + total(db.select({ value: count4() }).from(googleAdsConnections).where(isNotNull2(googleAdsConnections.selectedCustomerId)))
|
|
12998
13500
|
};
|
|
12999
13501
|
const connected = /* @__PURE__ */ new Set();
|
|
13000
13502
|
if ((googleByType.get("gsc") ?? 0) > 0) connected.add(IntegrationNames.gsc);
|
|
@@ -13002,7 +13504,7 @@ function buildInstallState(sources, now, startedAt) {
|
|
|
13002
13504
|
if ((googleByType.get("gbp") ?? 0) > 0) connected.add(IntegrationNames.gbp);
|
|
13003
13505
|
if (bing > 0) connected.add(IntegrationNames.bing);
|
|
13004
13506
|
if (googleAdsConnected > 0) connected.add(IntegrationNames.google_ads);
|
|
13005
|
-
if (total(db.select({ value:
|
|
13507
|
+
if (total(db.select({ value: count4() }).from(gtmConnections)) > 0) connected.add(IntegrationNames.gtm);
|
|
13006
13508
|
if (openAiAds > 0) connected.add(IntegrationNames.openai_ads);
|
|
13007
13509
|
for (const row of trafficByType) {
|
|
13008
13510
|
const integration = TRAFFIC_INTEGRATIONS[row.type];
|
|
@@ -13020,25 +13522,25 @@ function buildInstallState(sources, now, startedAt) {
|
|
|
13020
13522
|
if (!KNOWN_PROVIDERS2.has(connection)) return;
|
|
13021
13523
|
providerCalls.set(connection, (providerCalls.get(connection) ?? 0) + calls);
|
|
13022
13524
|
};
|
|
13023
|
-
const usagePath = (field) =>
|
|
13525
|
+
const usagePath = (field) => sql8`json_extract(${querySnapshots.usage}, ${`$.${field}`})`;
|
|
13024
13526
|
const answers = db.select({
|
|
13025
13527
|
provider: querySnapshots.provider,
|
|
13026
|
-
answers:
|
|
13027
|
-
measured:
|
|
13028
|
-
unpriced:
|
|
13029
|
-
inputTokens:
|
|
13030
|
-
outputTokens:
|
|
13031
|
-
costMicros:
|
|
13032
|
-
}).from(querySnapshots).where(gte2(querySnapshots.createdAt, since)).groupBy(
|
|
13528
|
+
answers: count4(),
|
|
13529
|
+
measured: sql8`sum(case when ${usagePath("inputTokens")} is not null then 1 else 0 end)`,
|
|
13530
|
+
unpriced: sql8`sum(case when ${usagePath("inputTokens")} is not null and ${usagePath("estimatedCostMicros")} is null then 1 else 0 end)`,
|
|
13531
|
+
inputTokens: sql8`coalesce(sum(${usagePath("inputTokens")}), 0)`,
|
|
13532
|
+
outputTokens: sql8`coalesce(sum(${usagePath("outputTokens")}), 0)`,
|
|
13533
|
+
costMicros: sql8`coalesce(sum(${usagePath("estimatedCostMicros")}), 0)`
|
|
13534
|
+
}).from(querySnapshots).where(gte2(querySnapshots.createdAt, since)).groupBy(sql8`+${querySnapshots.provider}`).all();
|
|
13033
13535
|
for (const row of answers) addCalls(row.provider, whole(row.answers));
|
|
13034
|
-
for (const row of db.select({ provider: researchRuns.provider, answered:
|
|
13536
|
+
for (const row of db.select({ provider: researchRuns.provider, answered: sql8`coalesce(sum(${researchRuns.completedQueries}), 0)` }).from(researchRuns).where(and16(inArray8(researchRuns.projectId, projectIds), gte2(researchRuns.createdAt, since))).groupBy(researchRuns.provider).all()) {
|
|
13035
13537
|
addCalls(row.provider, whole(row.answered));
|
|
13036
13538
|
}
|
|
13037
13539
|
for (const session of db.select({
|
|
13038
13540
|
seedProviders: discoverySessions.seedProviders,
|
|
13039
13541
|
seedCountRaw: discoverySessions.seedCountRaw,
|
|
13040
13542
|
probeCount: discoverySessions.probeCount
|
|
13041
|
-
}).from(discoverySessions).where(
|
|
13543
|
+
}).from(discoverySessions).where(and16(inArray8(discoverySessions.projectId, projectIds), gte2(discoverySessions.createdAt, since))).all()) {
|
|
13042
13544
|
if (session.seedCountRaw !== null) {
|
|
13043
13545
|
for (const provider of session.seedProviders?.length ? session.seedProviders : [ProviderNames.gemini]) addCalls(provider, 1);
|
|
13044
13546
|
}
|
|
@@ -13047,15 +13549,15 @@ function buildInstallState(sources, now, startedAt) {
|
|
|
13047
13549
|
const answered = answers.reduce((sum, row) => sum + whole(row.answers), 0);
|
|
13048
13550
|
const measured = answers.reduce((sum, row) => sum + whole(row.measured), 0);
|
|
13049
13551
|
const usage24h = {
|
|
13050
|
-
sweeps: total(db.select({ value:
|
|
13552
|
+
sweeps: total(db.select({ value: count4() }).from(runs).where(and16(
|
|
13051
13553
|
inArray8(runs.projectId, projectIds),
|
|
13052
|
-
|
|
13554
|
+
eq20(runs.kind, RunKinds["answer-visibility"]),
|
|
13053
13555
|
gte2(runs.createdAt, since),
|
|
13054
13556
|
ne3(runs.trigger, RunTriggers.probe)
|
|
13055
13557
|
))),
|
|
13056
|
-
audits: total(db.select({ value:
|
|
13558
|
+
audits: total(db.select({ value: count4() }).from(runs).where(and16(
|
|
13057
13559
|
inArray8(runs.projectId, projectIds),
|
|
13058
|
-
|
|
13560
|
+
eq20(runs.kind, RunKinds["site-audit"]),
|
|
13059
13561
|
gte2(runs.createdAt, since)
|
|
13060
13562
|
))),
|
|
13061
13563
|
providerCalls: [...providerCalls.values()].reduce((sum, calls) => sum + calls, 0)
|
|
@@ -13068,18 +13570,18 @@ function buildInstallState(sources, now, startedAt) {
|
|
|
13068
13570
|
}
|
|
13069
13571
|
}
|
|
13070
13572
|
const aero = db.select({
|
|
13071
|
-
calls:
|
|
13072
|
-
inputTokens:
|
|
13073
|
-
outputTokens:
|
|
13074
|
-
costMillicents:
|
|
13075
|
-
}).from(llmUsageEvents).where(
|
|
13573
|
+
calls: count4(),
|
|
13574
|
+
inputTokens: sql8`coalesce(sum(${llmUsageEvents.inputTokens}), 0)`,
|
|
13575
|
+
outputTokens: sql8`coalesce(sum(${llmUsageEvents.outputTokens}), 0)`,
|
|
13576
|
+
costMillicents: sql8`coalesce(sum(${llmUsageEvents.costMillicents}), 0)`
|
|
13577
|
+
}).from(llmUsageEvents).where(and16(eq20(llmUsageEvents.feature, AeroLlmUsageFeatures.turn), gte2(llmUsageEvents.createdAt, since))).get();
|
|
13076
13578
|
const aeroCalls = whole(aero?.calls);
|
|
13077
13579
|
usage24h.aeroModelCalls = aeroCalls;
|
|
13078
13580
|
usage24h.aeroInputTokens = whole(aero?.inputTokens);
|
|
13079
13581
|
usage24h.aeroOutputTokens = whole(aero?.outputTokens);
|
|
13080
13582
|
const aeroMillicents = whole(aero?.costMillicents);
|
|
13081
13583
|
if (aeroCalls === 0 || aeroMillicents > 0) usage24h.aeroCostMicros = aeroMillicents * 10;
|
|
13082
|
-
usage24h.webhookDeliveries = total(db.select({ value:
|
|
13584
|
+
usage24h.webhookDeliveries = total(db.select({ value: count4() }).from(auditLog).where(and16(gte2(auditLog.createdAt, since), eq20(auditLog.action, WEBHOOK_DELIVERED_ACTION))));
|
|
13083
13585
|
const providerCalls24h = Object.fromEntries(PROVIDER_NAMES.flatMap((name) => providerCalls.has(name) ? [[name, providerCalls.get(name)]] : []));
|
|
13084
13586
|
return {
|
|
13085
13587
|
providers: registeredProviderNames(config).filter((name) => KNOWN_PROVIDERS2.has(name)),
|
|
@@ -13113,7 +13615,7 @@ function startInstallStateTelemetry(sources) {
|
|
|
13113
13615
|
try {
|
|
13114
13616
|
if (isTelemetryEnabled()) trackInstallState(buildInstallState(sources, Date.now(), startedAt));
|
|
13115
13617
|
} catch (err) {
|
|
13116
|
-
|
|
13618
|
+
log17.warn("telemetry.install-state-failed", { error: describeError(err) });
|
|
13117
13619
|
}
|
|
13118
13620
|
schedule(INSTALL_STATE_INTERVAL_MS);
|
|
13119
13621
|
};
|
|
@@ -13209,7 +13711,7 @@ function configuredProviderEntries(config, apiAdapters, cdpAdapter) {
|
|
|
13209
13711
|
|
|
13210
13712
|
// src/provider-reload-batch-guard.ts
|
|
13211
13713
|
import { isDeepStrictEqual } from "util";
|
|
13212
|
-
import { and as
|
|
13714
|
+
import { and as and17, inArray as inArray9 } from "drizzle-orm";
|
|
13213
13715
|
function pendingBatchIdentity(provider) {
|
|
13214
13716
|
const { quotaPolicy: _quotaPolicy, ...identity } = resolveRegistration(provider.adapter, provider.config).config;
|
|
13215
13717
|
return identity;
|
|
@@ -13223,7 +13725,7 @@ function assertProviderReloadKeepsPendingBatches(db, previous, next, executing =
|
|
|
13223
13725
|
const changed = previous.filter(changedRegistration).map((provider) => provider.adapter.name);
|
|
13224
13726
|
const active = executing.filter((snapshot) => changedRegistration(snapshot.registration));
|
|
13225
13727
|
if (changed.length === 0 && active.length === 0) return;
|
|
13226
|
-
const pending = changed.length === 0 ? [] : db.select({ id: providerBatches.id, provider: providerBatches.provider, runId: providerBatches.runId }).from(providerBatches).where(
|
|
13728
|
+
const pending = changed.length === 0 ? [] : db.select({ id: providerBatches.id, provider: providerBatches.provider, runId: providerBatches.runId }).from(providerBatches).where(and17(
|
|
13227
13729
|
inArray9(providerBatches.provider, changed),
|
|
13228
13730
|
inArray9(providerBatches.status, [ProviderBatchStatuses.submitting, ProviderBatchStatuses.submitted, ProviderBatchStatuses.ended])
|
|
13229
13731
|
)).all();
|
|
@@ -13303,10 +13805,10 @@ function createProviderModelCatalog(registry) {
|
|
|
13303
13805
|
}
|
|
13304
13806
|
|
|
13305
13807
|
// src/scheduler.ts
|
|
13306
|
-
import
|
|
13808
|
+
import crypto21 from "crypto";
|
|
13307
13809
|
import cron from "node-cron";
|
|
13308
|
-
import { and as
|
|
13309
|
-
var
|
|
13810
|
+
import { and as and18, eq as eq21, inArray as inArray10, notExists, sql as sql9 } from "drizzle-orm";
|
|
13811
|
+
var log18 = createLogger("Scheduler");
|
|
13310
13812
|
var SLOT_FIRED = { status: OutcomeStatuses.succeeded };
|
|
13311
13813
|
var slotSkipped = (reasonCode) => ({ status: OutcomeStatuses.skipped, reasonCode });
|
|
13312
13814
|
var slotFailed = (reasonCode) => ({ status: OutcomeStatuses.failed, reasonCode });
|
|
@@ -13315,7 +13817,7 @@ var SCHEDULE_SPREAD_MS = 12e4;
|
|
|
13315
13817
|
var UNSPREAD_KINDS = /* @__PURE__ */ new Set([SchedulableRunKinds["answer-visibility"]]);
|
|
13316
13818
|
function scheduleSpreadMs(projectId, kind, spreadMs = SCHEDULE_SPREAD_MS) {
|
|
13317
13819
|
if (spreadMs <= 0) return 0;
|
|
13318
|
-
return
|
|
13820
|
+
return crypto21.createHash("sha256").update(`${projectId}:${kind}`).digest().readUInt32BE(0) % spreadMs;
|
|
13319
13821
|
}
|
|
13320
13822
|
function shortestCronGapMs(cronExpr, timezone, from = /* @__PURE__ */ new Date()) {
|
|
13321
13823
|
let previous = nextRunFromCron(cronExpr, timezone, from);
|
|
@@ -13330,7 +13832,7 @@ function shortestCronGapMs(cronExpr, timezone, from = /* @__PURE__ */ new Date()
|
|
|
13330
13832
|
}
|
|
13331
13833
|
function ensureDefaultHealthSchedule(db, projectId, now = (/* @__PURE__ */ new Date()).toISOString()) {
|
|
13332
13834
|
const result = db.insert(schedules).values({
|
|
13333
|
-
id:
|
|
13835
|
+
id: crypto21.randomUUID(),
|
|
13334
13836
|
projectId,
|
|
13335
13837
|
kind: SchedulableRunKinds.doctor,
|
|
13336
13838
|
cronExpr: DEFAULT_HEALTH_CRON,
|
|
@@ -13384,9 +13886,9 @@ var Scheduler = class {
|
|
|
13384
13886
|
*/
|
|
13385
13887
|
ensureHealthSchedules() {
|
|
13386
13888
|
const projectsWithoutHealth = this.db.select({ id: projects.id }).from(projects).where(notExists(
|
|
13387
|
-
this.db.select({ one:
|
|
13388
|
-
|
|
13389
|
-
|
|
13889
|
+
this.db.select({ one: sql9`1` }).from(schedules).where(and18(
|
|
13890
|
+
eq21(schedules.projectId, projects.id),
|
|
13891
|
+
eq21(schedules.kind, SchedulableRunKinds.doctor)
|
|
13390
13892
|
))
|
|
13391
13893
|
)).all();
|
|
13392
13894
|
if (projectsWithoutHealth.length === 0) return;
|
|
@@ -13396,7 +13898,7 @@ var Scheduler = class {
|
|
|
13396
13898
|
if (ensureDefaultHealthSchedule(this.db, project.id, now)) seeded += 1;
|
|
13397
13899
|
}
|
|
13398
13900
|
if (seeded > 0) {
|
|
13399
|
-
|
|
13901
|
+
log18.info("health-schedule.seeded", { projectCount: seeded, cron: DEFAULT_HEALTH_CRON });
|
|
13400
13902
|
}
|
|
13401
13903
|
}
|
|
13402
13904
|
/**
|
|
@@ -13420,16 +13922,16 @@ var Scheduler = class {
|
|
|
13420
13922
|
const current = ensureCurrentQueryBasketRevision(this.db, project.id);
|
|
13421
13923
|
if (current !== null && current.revision !== previous) minted += 1;
|
|
13422
13924
|
} catch (err) {
|
|
13423
|
-
|
|
13925
|
+
log18.warn("query-basket.mint-failed", { projectId: project.id, err: String(err) });
|
|
13424
13926
|
}
|
|
13425
13927
|
}
|
|
13426
|
-
if (minted > 0)
|
|
13928
|
+
if (minted > 0) log18.info("query-basket.ensured", { projectCount: minted });
|
|
13427
13929
|
}
|
|
13428
13930
|
/** Load all enabled schedules from DB and register cron jobs. */
|
|
13429
13931
|
start() {
|
|
13430
13932
|
this.ensureHealthSchedules();
|
|
13431
13933
|
this.ensureQueryBaskets();
|
|
13432
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
13934
|
+
const allSchedules = this.db.select().from(schedules).where(eq21(schedules.enabled, true)).all();
|
|
13433
13935
|
for (const schedule of allSchedules) {
|
|
13434
13936
|
const missedRunAt = schedule.nextRunAt;
|
|
13435
13937
|
const recurrence = scheduleRecurrence(schedule);
|
|
@@ -13437,12 +13939,12 @@ var Scheduler = class {
|
|
|
13437
13939
|
if (registered && missedRunAt && new Date(missedRunAt) < /* @__PURE__ */ new Date()) {
|
|
13438
13940
|
const answerRecurrence = recurrence && schedule.kind === SchedulableRunKinds["answer-visibility"];
|
|
13439
13941
|
if (!recurrence || answerRecurrence || this.claimCalendarOccurrence(schedule, missedRunAt, /* @__PURE__ */ new Date())) {
|
|
13440
|
-
|
|
13942
|
+
log18.info("run.catch-up", { projectId: schedule.projectId, kind: schedule.kind, missedRunAt });
|
|
13441
13943
|
this.triggerRun(schedule.id, schedule.projectId, schedule.kind, answerRecurrence ? missedRunAt : void 0, OutcomeTriggers.startup);
|
|
13442
13944
|
}
|
|
13443
13945
|
}
|
|
13444
13946
|
}
|
|
13445
|
-
|
|
13947
|
+
log18.info("started", { scheduleCount: allSchedules.length });
|
|
13446
13948
|
}
|
|
13447
13949
|
/** Stop all cron tasks for graceful shutdown. */
|
|
13448
13950
|
stop() {
|
|
@@ -13463,7 +13965,7 @@ var Scheduler = class {
|
|
|
13463
13965
|
this.stopTask(key, existing, "Stopped");
|
|
13464
13966
|
this.tasks.delete(key);
|
|
13465
13967
|
}
|
|
13466
|
-
const schedule = this.db.select().from(schedules).where(
|
|
13968
|
+
const schedule = this.db.select().from(schedules).where(and18(eq21(schedules.projectId, projectId), eq21(schedules.kind, kind))).get();
|
|
13467
13969
|
if (schedule && schedule.enabled) {
|
|
13468
13970
|
this.registerCronTask(schedule);
|
|
13469
13971
|
}
|
|
@@ -13491,17 +13993,17 @@ var Scheduler = class {
|
|
|
13491
13993
|
clearTimeout(pending);
|
|
13492
13994
|
this.spreadTimers.delete(key);
|
|
13493
13995
|
}
|
|
13494
|
-
|
|
13996
|
+
log18.info(`task.${verb.toLowerCase()}`, { key });
|
|
13495
13997
|
}
|
|
13496
13998
|
/** Runtime timing writes must not restore a version already issued to a settings caller. */
|
|
13497
13999
|
updateScheduleTiming(scheduleId, timing) {
|
|
13498
14000
|
this.db.transaction((tx) => {
|
|
13499
|
-
const current = tx.select({ updatedAt: schedules.updatedAt }).from(schedules).where(
|
|
14001
|
+
const current = tx.select({ updatedAt: schedules.updatedAt }).from(schedules).where(eq21(schedules.id, scheduleId)).get();
|
|
13500
14002
|
if (!current) return;
|
|
13501
14003
|
tx.update(schedules).set({
|
|
13502
14004
|
...timing,
|
|
13503
14005
|
updatedAt: nextScheduleUpdatedAt(current.updatedAt)
|
|
13504
|
-
}).where(
|
|
14006
|
+
}).where(eq21(schedules.id, scheduleId)).run();
|
|
13505
14007
|
});
|
|
13506
14008
|
}
|
|
13507
14009
|
registerCronTask(schedule, options = {}) {
|
|
@@ -13511,13 +14013,13 @@ var Scheduler = class {
|
|
|
13511
14013
|
return true;
|
|
13512
14014
|
}
|
|
13513
14015
|
if (schedule.recurrence) {
|
|
13514
|
-
|
|
14016
|
+
log18.error("calendar.invalid", { projectId: schedule.projectId, kind: schedule.kind });
|
|
13515
14017
|
return false;
|
|
13516
14018
|
}
|
|
13517
14019
|
const { id: scheduleId, projectId, cronExpr, timezone } = schedule;
|
|
13518
14020
|
const kind = schedule.kind;
|
|
13519
14021
|
if (!cron.validate(cronExpr)) {
|
|
13520
|
-
|
|
14022
|
+
log18.error("cron.invalid", { projectId, kind, cronExpr });
|
|
13521
14023
|
return false;
|
|
13522
14024
|
}
|
|
13523
14025
|
const key = taskKey(projectId, kind);
|
|
@@ -13547,7 +14049,7 @@ var Scheduler = class {
|
|
|
13547
14049
|
});
|
|
13548
14050
|
}
|
|
13549
14051
|
const label = schedule.preset ?? cronExpr;
|
|
13550
|
-
|
|
14052
|
+
log18.info("cron.registered", { projectId, kind, schedule: label, timezone });
|
|
13551
14053
|
return true;
|
|
13552
14054
|
}
|
|
13553
14055
|
/** Calendar schedules are checked at most once a minute so long intervals
|
|
@@ -13577,7 +14079,7 @@ var Scheduler = class {
|
|
|
13577
14079
|
};
|
|
13578
14080
|
const tick = () => {
|
|
13579
14081
|
if (stopped) return;
|
|
13580
|
-
const current = this.db.select().from(schedules).where(
|
|
14082
|
+
const current = this.db.select().from(schedules).where(eq21(schedules.id, scheduleId)).get();
|
|
13581
14083
|
if (!current || !current.enabled || !scheduleRecurrence(current)) {
|
|
13582
14084
|
this.remove(projectId, kind);
|
|
13583
14085
|
return;
|
|
@@ -13595,7 +14097,7 @@ var Scheduler = class {
|
|
|
13595
14097
|
this.triggerRun(scheduleId, projectId, kind);
|
|
13596
14098
|
}
|
|
13597
14099
|
}
|
|
13598
|
-
arm(this.db.select({ nextRunAt: schedules.nextRunAt }).from(schedules).where(
|
|
14100
|
+
arm(this.db.select({ nextRunAt: schedules.nextRunAt }).from(schedules).where(eq21(schedules.id, scheduleId)).get()?.nextRunAt);
|
|
13599
14101
|
};
|
|
13600
14102
|
this.tasks.set(key, task);
|
|
13601
14103
|
let registeredNextRunAt = schedule.nextRunAt;
|
|
@@ -13605,12 +14107,12 @@ var Scheduler = class {
|
|
|
13605
14107
|
}
|
|
13606
14108
|
if (options.preserveNextRunAt) arm(registeredNextRunAt);
|
|
13607
14109
|
else timer = setTimeout(tick, 0);
|
|
13608
|
-
|
|
14110
|
+
log18.info("calendar.registered", { projectId, kind, recurrence, timezone });
|
|
13609
14111
|
}
|
|
13610
14112
|
/** Atomically advance a due callback-only calendar occurrence before dispatch. */
|
|
13611
14113
|
claimCalendarOccurrence(schedule, dueAt, now) {
|
|
13612
14114
|
return this.db.transaction((tx) => {
|
|
13613
|
-
const current = tx.select().from(schedules).where(
|
|
14115
|
+
const current = tx.select().from(schedules).where(eq21(schedules.id, schedule.id)).get();
|
|
13614
14116
|
if (!current || !current.enabled || current.updatedAt !== schedule.updatedAt || current.nextRunAt !== dueAt) return false;
|
|
13615
14117
|
const recurrence = scheduleRecurrence(current);
|
|
13616
14118
|
if (!recurrence) return false;
|
|
@@ -13619,11 +14121,11 @@ var Scheduler = class {
|
|
|
13619
14121
|
const result = tx.update(schedules).set({
|
|
13620
14122
|
nextRunAt,
|
|
13621
14123
|
updatedAt: nextScheduleUpdatedAt(current.updatedAt)
|
|
13622
|
-
}).where(
|
|
13623
|
-
|
|
13624
|
-
|
|
13625
|
-
|
|
13626
|
-
|
|
14124
|
+
}).where(and18(
|
|
14125
|
+
eq21(schedules.id, current.id),
|
|
14126
|
+
eq21(schedules.enabled, true),
|
|
14127
|
+
eq21(schedules.nextRunAt, dueAt),
|
|
14128
|
+
eq21(schedules.updatedAt, schedule.updatedAt)
|
|
13627
14129
|
)).run();
|
|
13628
14130
|
return result.changes === 1;
|
|
13629
14131
|
});
|
|
@@ -13645,15 +14147,15 @@ var Scheduler = class {
|
|
|
13645
14147
|
fireSlot(scheduleId, projectId, kind, claimedOccurrence) {
|
|
13646
14148
|
try {
|
|
13647
14149
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
13648
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
14150
|
+
const currentSchedule = this.db.select().from(schedules).where(eq21(schedules.id, scheduleId)).get();
|
|
13649
14151
|
if (!currentSchedule || !currentSchedule.enabled) {
|
|
13650
|
-
|
|
14152
|
+
log18.warn("schedule.stale", { scheduleId, projectId, kind, msg: "schedule no longer exists or is disabled" });
|
|
13651
14153
|
this.remove(projectId, kind);
|
|
13652
14154
|
return slotSkipped(currentSchedule ? OutcomeReasonCodes.NOT_DUE : OutcomeReasonCodes.NOT_FOUND);
|
|
13653
14155
|
}
|
|
13654
14156
|
const recurrence = scheduleRecurrence(currentSchedule);
|
|
13655
14157
|
if (claimedOccurrence && (!recurrence || currentSchedule.nextRunAt !== claimedOccurrence)) {
|
|
13656
|
-
|
|
14158
|
+
log18.info("calendar.stale-claim", { projectId, scheduleId, kind });
|
|
13657
14159
|
return slotSkipped(OutcomeReasonCodes.NOT_DUE);
|
|
13658
14160
|
}
|
|
13659
14161
|
const nextRunAt = nextRunFromSchedule({
|
|
@@ -13661,36 +14163,36 @@ var Scheduler = class {
|
|
|
13661
14163
|
timezone: currentSchedule.timezone,
|
|
13662
14164
|
recurrence
|
|
13663
14165
|
});
|
|
13664
|
-
const project = this.db.select().from(projects).where(
|
|
14166
|
+
const project = this.db.select().from(projects).where(eq21(projects.id, projectId)).get();
|
|
13665
14167
|
if (!project) {
|
|
13666
|
-
|
|
14168
|
+
log18.error("project.not-found", { projectId, kind, msg: "skipping scheduled run" });
|
|
13667
14169
|
this.remove(projectId, kind);
|
|
13668
14170
|
return slotSkipped(OutcomeReasonCodes.NOT_FOUND);
|
|
13669
14171
|
}
|
|
13670
14172
|
if (kind === SchedulableRunKinds["traffic-sync"]) {
|
|
13671
14173
|
const sourceId = currentSchedule.sourceId;
|
|
13672
14174
|
if (!sourceId) {
|
|
13673
|
-
|
|
14175
|
+
log18.warn("traffic-sync.missing-source", { scheduleId, projectId });
|
|
13674
14176
|
return slotFailed(OutcomeReasonCodes.VALIDATION);
|
|
13675
14177
|
}
|
|
13676
14178
|
if (!this.callbacks.onTrafficSyncRequested) {
|
|
13677
|
-
|
|
14179
|
+
log18.warn("traffic-sync.no-callback", { scheduleId, projectId, msg: "host did not register onTrafficSyncRequested" });
|
|
13678
14180
|
return slotFailed(OutcomeReasonCodes.UNSUPPORTED);
|
|
13679
14181
|
}
|
|
13680
14182
|
this.updateScheduleTiming(currentSchedule.id, {
|
|
13681
14183
|
lastRunAt: now,
|
|
13682
14184
|
nextRunAt
|
|
13683
14185
|
});
|
|
13684
|
-
|
|
14186
|
+
log18.info("traffic-sync.triggered", { projectName: project.name, sourceId });
|
|
13685
14187
|
this.callbacks.onTrafficSyncRequested(project.name, sourceId);
|
|
13686
14188
|
return SLOT_FIRED;
|
|
13687
14189
|
}
|
|
13688
14190
|
if (kind === SchedulableRunKinds["gbp-sync"]) {
|
|
13689
14191
|
if (!this.callbacks.onGbpSyncRequested) {
|
|
13690
|
-
|
|
14192
|
+
log18.warn("gbp-sync.no-callback", { scheduleId, projectId, msg: "host did not register onGbpSyncRequested" });
|
|
13691
14193
|
return slotFailed(OutcomeReasonCodes.UNSUPPORTED);
|
|
13692
14194
|
}
|
|
13693
|
-
const runId2 =
|
|
14195
|
+
const runId2 = crypto21.randomUUID();
|
|
13694
14196
|
this.db.insert(runs).values({
|
|
13695
14197
|
id: runId2,
|
|
13696
14198
|
projectId,
|
|
@@ -13703,26 +14205,26 @@ var Scheduler = class {
|
|
|
13703
14205
|
lastRunAt: now,
|
|
13704
14206
|
nextRunAt
|
|
13705
14207
|
});
|
|
13706
|
-
|
|
14208
|
+
log18.info("gbp-sync.triggered", { runId: runId2, projectName: project.name });
|
|
13707
14209
|
this.callbacks.onGbpSyncRequested(runId2, projectId);
|
|
13708
14210
|
return SLOT_FIRED;
|
|
13709
14211
|
}
|
|
13710
14212
|
if (kind === SchedulableRunKinds["ads-sync"]) {
|
|
13711
14213
|
if (!this.callbacks.onAdsSyncRequested) {
|
|
13712
|
-
|
|
14214
|
+
log18.warn("ads-sync.no-callback", { scheduleId, projectId, msg: "host did not register onAdsSyncRequested" });
|
|
13713
14215
|
return slotFailed(OutcomeReasonCodes.UNSUPPORTED);
|
|
13714
14216
|
}
|
|
13715
|
-
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(
|
|
13716
|
-
|
|
13717
|
-
|
|
14217
|
+
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(and18(
|
|
14218
|
+
eq21(runs.projectId, projectId),
|
|
14219
|
+
eq21(runs.kind, RunKinds["ads-sync"]),
|
|
13718
14220
|
inArray10(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
13719
14221
|
)).get();
|
|
13720
14222
|
if (activeAdsRun) {
|
|
13721
|
-
|
|
14223
|
+
log18.info("ads-sync.skipped-active", { projectName: project.name, activeRunId: activeAdsRun.id });
|
|
13722
14224
|
this.updateScheduleTiming(currentSchedule.id, { nextRunAt });
|
|
13723
14225
|
return slotSkipped(OutcomeReasonCodes.OPERATION_IN_PROGRESS);
|
|
13724
14226
|
}
|
|
13725
|
-
const runId2 =
|
|
14227
|
+
const runId2 = crypto21.randomUUID();
|
|
13726
14228
|
this.db.insert(runs).values({
|
|
13727
14229
|
id: runId2,
|
|
13728
14230
|
projectId,
|
|
@@ -13735,22 +14237,22 @@ var Scheduler = class {
|
|
|
13735
14237
|
lastRunAt: now,
|
|
13736
14238
|
nextRunAt
|
|
13737
14239
|
});
|
|
13738
|
-
|
|
14240
|
+
log18.info("ads-sync.triggered", { runId: runId2, projectName: project.name });
|
|
13739
14241
|
this.callbacks.onAdsSyncRequested(runId2, projectId);
|
|
13740
14242
|
return SLOT_FIRED;
|
|
13741
14243
|
}
|
|
13742
14244
|
if (kind === SchedulableRunKinds["seo-sync"]) {
|
|
13743
14245
|
if (!this.callbacks.onSeoSyncRequested) {
|
|
13744
|
-
|
|
14246
|
+
log18.warn("seo-sync.no-callback", { scheduleId, projectId, msg: "host did not register onSeoSyncRequested" });
|
|
13745
14247
|
return slotFailed(OutcomeReasonCodes.UNSUPPORTED);
|
|
13746
14248
|
}
|
|
13747
|
-
const activeSeoRun = this.db.select({ id: runs.id }).from(runs).where(
|
|
13748
|
-
|
|
13749
|
-
|
|
14249
|
+
const activeSeoRun = this.db.select({ id: runs.id }).from(runs).where(and18(
|
|
14250
|
+
eq21(runs.projectId, projectId),
|
|
14251
|
+
eq21(runs.kind, RunKinds["seo-sync"]),
|
|
13750
14252
|
inArray10(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
13751
14253
|
)).get();
|
|
13752
14254
|
if (activeSeoRun) {
|
|
13753
|
-
|
|
14255
|
+
log18.info("seo-sync.skipped-active", { projectName: project.name, activeRunId: activeSeoRun.id });
|
|
13754
14256
|
this.updateScheduleTiming(currentSchedule.id, { nextRunAt });
|
|
13755
14257
|
return slotSkipped(OutcomeReasonCodes.OPERATION_IN_PROGRESS);
|
|
13756
14258
|
}
|
|
@@ -13758,65 +14260,65 @@ var Scheduler = class {
|
|
|
13758
14260
|
lastRunAt: now,
|
|
13759
14261
|
nextRunAt
|
|
13760
14262
|
});
|
|
13761
|
-
|
|
14263
|
+
log18.info("seo-sync.triggered", { projectName: project.name });
|
|
13762
14264
|
this.callbacks.onSeoSyncRequested({ scheduleId, projectId, occurrence: now.slice(0, 16) });
|
|
13763
14265
|
return SLOT_FIRED;
|
|
13764
14266
|
}
|
|
13765
14267
|
if (kind === SchedulableRunKinds["data-refresh"]) {
|
|
13766
14268
|
if (!this.callbacks.onDataRefreshRequested) {
|
|
13767
|
-
|
|
14269
|
+
log18.warn("data-refresh.no-callback", { scheduleId, projectId, msg: "host did not register onDataRefreshRequested" });
|
|
13768
14270
|
return slotFailed(OutcomeReasonCodes.UNSUPPORTED);
|
|
13769
14271
|
}
|
|
13770
14272
|
this.updateScheduleTiming(currentSchedule.id, {
|
|
13771
14273
|
lastRunAt: now,
|
|
13772
14274
|
nextRunAt
|
|
13773
14275
|
});
|
|
13774
|
-
|
|
14276
|
+
log18.info("data-refresh.triggered", { projectName: project.name });
|
|
13775
14277
|
this.callbacks.onDataRefreshRequested(project.name);
|
|
13776
14278
|
return SLOT_FIRED;
|
|
13777
14279
|
}
|
|
13778
14280
|
if (kind === SchedulableRunKinds.doctor) {
|
|
13779
14281
|
if (!this.callbacks.onDoctorRequested) {
|
|
13780
|
-
|
|
14282
|
+
log18.warn("doctor.no-callback", { scheduleId, projectId, msg: "host did not register onDoctorRequested" });
|
|
13781
14283
|
return slotFailed(OutcomeReasonCodes.UNSUPPORTED);
|
|
13782
14284
|
}
|
|
13783
14285
|
this.updateScheduleTiming(currentSchedule.id, {
|
|
13784
14286
|
lastRunAt: now,
|
|
13785
14287
|
nextRunAt
|
|
13786
14288
|
});
|
|
13787
|
-
|
|
14289
|
+
log18.info("doctor.triggered", { projectName: project.name });
|
|
13788
14290
|
this.callbacks.onDoctorRequested(project.name);
|
|
13789
14291
|
return SLOT_FIRED;
|
|
13790
14292
|
}
|
|
13791
14293
|
if (kind === SchedulableRunKinds["backlinks-sync"]) {
|
|
13792
14294
|
if (!this.callbacks.onBacklinksSyncRequested) {
|
|
13793
|
-
|
|
14295
|
+
log18.warn("backlinks-sync.no-callback", { scheduleId, projectId, msg: "host did not register onBacklinksSyncRequested" });
|
|
13794
14296
|
return slotFailed(OutcomeReasonCodes.UNSUPPORTED);
|
|
13795
14297
|
}
|
|
13796
14298
|
this.updateScheduleTiming(currentSchedule.id, {
|
|
13797
14299
|
lastRunAt: now,
|
|
13798
14300
|
nextRunAt
|
|
13799
14301
|
});
|
|
13800
|
-
|
|
14302
|
+
log18.info("backlinks-sync.triggered", { projectName: project.name });
|
|
13801
14303
|
this.callbacks.onBacklinksSyncRequested(project.name);
|
|
13802
14304
|
return SLOT_FIRED;
|
|
13803
14305
|
}
|
|
13804
14306
|
if (kind === SchedulableRunKinds["site-audit"]) {
|
|
13805
14307
|
if (!this.callbacks.onSiteAuditRequested) {
|
|
13806
|
-
|
|
14308
|
+
log18.warn("site-audit.no-callback", { scheduleId, projectId, msg: "host did not register onSiteAuditRequested" });
|
|
13807
14309
|
return slotFailed(OutcomeReasonCodes.UNSUPPORTED);
|
|
13808
14310
|
}
|
|
13809
|
-
const active = this.db.select({ id: runs.id }).from(runs).where(
|
|
13810
|
-
|
|
13811
|
-
|
|
14311
|
+
const active = this.db.select({ id: runs.id }).from(runs).where(and18(
|
|
14312
|
+
eq21(runs.projectId, projectId),
|
|
14313
|
+
eq21(runs.kind, RunKinds["site-audit"]),
|
|
13812
14314
|
inArray10(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
13813
14315
|
)).get();
|
|
13814
14316
|
if (active) {
|
|
13815
|
-
|
|
14317
|
+
log18.info("site-audit.skipped-active", { projectName: project.name, activeRunId: active.id });
|
|
13816
14318
|
this.updateScheduleTiming(currentSchedule.id, { nextRunAt });
|
|
13817
14319
|
return slotSkipped(OutcomeReasonCodes.OPERATION_IN_PROGRESS);
|
|
13818
14320
|
}
|
|
13819
|
-
const runId2 =
|
|
14321
|
+
const runId2 = crypto21.randomUUID();
|
|
13820
14322
|
const effectiveRequest = normalizeSiteAuditRunRequest({}, project.siteAuditMaxPages);
|
|
13821
14323
|
this.db.transaction((tx) => {
|
|
13822
14324
|
tx.insert(runs).values({
|
|
@@ -13839,7 +14341,7 @@ var Scheduler = class {
|
|
|
13839
14341
|
lastRunAt: now,
|
|
13840
14342
|
nextRunAt
|
|
13841
14343
|
});
|
|
13842
|
-
|
|
14344
|
+
log18.info("site-audit.triggered", { runId: runId2, projectName: project.name });
|
|
13843
14345
|
this.callbacks.onSiteAuditRequested(runId2, projectId, { maxPages: effectiveRequest.maxPages });
|
|
13844
14346
|
return SLOT_FIRED;
|
|
13845
14347
|
}
|
|
@@ -13848,7 +14350,7 @@ var Scheduler = class {
|
|
|
13848
14350
|
if (project.defaultLocation) {
|
|
13849
14351
|
const loc = projectLocations.find((l) => l.label === project.defaultLocation);
|
|
13850
14352
|
if (!loc) {
|
|
13851
|
-
|
|
14353
|
+
log18.warn("default-location.stale", { scheduleId, projectId, label: project.defaultLocation });
|
|
13852
14354
|
return slotFailed(OutcomeReasonCodes.VALIDATION);
|
|
13853
14355
|
}
|
|
13854
14356
|
resolvedLocation = loc;
|
|
@@ -13857,7 +14359,7 @@ var Scheduler = class {
|
|
|
13857
14359
|
const scheduleProviders = currentSchedule.providers;
|
|
13858
14360
|
const providers = scheduleProviders.length > 0 ? scheduleProviders : void 0;
|
|
13859
14361
|
if (claimedOccurrence && recurrence && !nextRunAt) {
|
|
13860
|
-
|
|
14362
|
+
log18.error("calendar.invalid", { projectId, kind, scheduleId });
|
|
13861
14363
|
return slotFailed(OutcomeReasonCodes.VALIDATION);
|
|
13862
14364
|
}
|
|
13863
14365
|
const queueResult = queueRunIfProjectIdle(this.db, {
|
|
@@ -13885,11 +14387,11 @@ var Scheduler = class {
|
|
|
13885
14387
|
});
|
|
13886
14388
|
if (queueResult.conflict) {
|
|
13887
14389
|
if (queueResult.scheduleClaimed === false) {
|
|
13888
|
-
|
|
14390
|
+
log18.info("calendar.skipped-claimed", { projectName: project.name, scheduleId: currentSchedule.id });
|
|
13889
14391
|
return slotSkipped(OutcomeReasonCodes.NOT_DUE);
|
|
13890
14392
|
}
|
|
13891
14393
|
const batchPending = hasOutstandingProviderBatch(this.db, queueResult.activeRunId);
|
|
13892
|
-
|
|
14394
|
+
log18.info("run.skipped-active", {
|
|
13893
14395
|
projectName: project.name,
|
|
13894
14396
|
activeRunId: queueResult.activeRunId,
|
|
13895
14397
|
...batchPending ? { reason: "batch-pending" } : {}
|
|
@@ -13898,7 +14400,7 @@ var Scheduler = class {
|
|
|
13898
14400
|
return slotSkipped(OutcomeReasonCodes.OPERATION_IN_PROGRESS);
|
|
13899
14401
|
}
|
|
13900
14402
|
if (queueResult.refused) {
|
|
13901
|
-
|
|
14403
|
+
log18.warn("run.skipped-providers-failing", {
|
|
13902
14404
|
projectName: project.name,
|
|
13903
14405
|
scheduleId: currentSchedule.id,
|
|
13904
14406
|
...queueResult.refused
|
|
@@ -13917,7 +14419,7 @@ var Scheduler = class {
|
|
|
13917
14419
|
const runId = queueResult.runId;
|
|
13918
14420
|
for (const [provider, reason] of Object.entries(queueResult.dispatch.ineligible)) {
|
|
13919
14421
|
if (!reason) continue;
|
|
13920
|
-
|
|
14422
|
+
log18.warn("run.dispatch-sync-fallback", {
|
|
13921
14423
|
runId,
|
|
13922
14424
|
projectName: project.name,
|
|
13923
14425
|
provider,
|
|
@@ -13931,7 +14433,7 @@ var Scheduler = class {
|
|
|
13931
14433
|
lastRunAt: now,
|
|
13932
14434
|
nextRunAt
|
|
13933
14435
|
});
|
|
13934
|
-
|
|
14436
|
+
log18.info("run.triggered", {
|
|
13935
14437
|
runId,
|
|
13936
14438
|
projectName: project.name,
|
|
13937
14439
|
providers: providers ?? "all",
|
|
@@ -13940,14 +14442,14 @@ var Scheduler = class {
|
|
|
13940
14442
|
this.callbacks.onRunCreated(runId, projectId, providers, resolvedLocation);
|
|
13941
14443
|
return SLOT_FIRED;
|
|
13942
14444
|
} catch (err) {
|
|
13943
|
-
|
|
14445
|
+
log18.error("trigger.error", { scheduleId, projectId, kind, error: describeError(err) });
|
|
13944
14446
|
return { status: OutcomeStatuses.failed, ...outcomeFailure(err) };
|
|
13945
14447
|
}
|
|
13946
14448
|
}
|
|
13947
14449
|
};
|
|
13948
14450
|
|
|
13949
14451
|
// src/site-liveness-loop.ts
|
|
13950
|
-
var
|
|
14452
|
+
var log19 = createLogger("SiteLiveness");
|
|
13951
14453
|
var SITE_LIVENESS_INTERVAL_MS = 10 * 6e4;
|
|
13952
14454
|
async function runSiteLivenessPass(deps) {
|
|
13953
14455
|
const now = deps.now ?? (() => (/* @__PURE__ */ new Date()).toISOString());
|
|
@@ -13964,10 +14466,10 @@ async function runSiteLivenessPass(deps) {
|
|
|
13964
14466
|
const event = await deps.notify(project.id, { check, checkedAt: now() });
|
|
13965
14467
|
if (event) {
|
|
13966
14468
|
events += 1;
|
|
13967
|
-
|
|
14469
|
+
log19.info("site-liveness.notified", { projectName: project.name, event, code: check.code });
|
|
13968
14470
|
}
|
|
13969
14471
|
} catch (err) {
|
|
13970
|
-
|
|
14472
|
+
log19.warn("site-liveness.project-failed", { projectName: project.name, err: String(err) });
|
|
13971
14473
|
if (failed === 0) firstFailure = err;
|
|
13972
14474
|
failed += 1;
|
|
13973
14475
|
}
|
|
@@ -13999,14 +14501,14 @@ function startSiteLivenessLoop(deps, intervalMs = SITE_LIVENESS_INTERVAL_MS) {
|
|
|
13999
14501
|
};
|
|
14000
14502
|
const timer = setInterval(() => {
|
|
14001
14503
|
if (running) {
|
|
14002
|
-
|
|
14504
|
+
log19.warn("site-liveness.pass-still-running", { intervalMs });
|
|
14003
14505
|
report({ status: OutcomeStatuses.skipped, reasonCode: OutcomeReasonCodes.OPERATION_IN_PROGRESS });
|
|
14004
14506
|
return;
|
|
14005
14507
|
}
|
|
14006
14508
|
running = true;
|
|
14007
14509
|
const elapsed = startOutcomeTimer();
|
|
14008
14510
|
void runSiteLivenessPass(deps).then((result) => report({ ...livenessOutcome(result), durationBucket: elapsed() })).catch((err) => {
|
|
14009
|
-
|
|
14511
|
+
log19.warn("site-liveness.pass-failed", { err: String(err) });
|
|
14010
14512
|
report({ status: OutcomeStatuses.failed, ...outcomeFailure(err), durationBucket: elapsed() });
|
|
14011
14513
|
}).finally(() => {
|
|
14012
14514
|
running = false;
|
|
@@ -14018,8 +14520,8 @@ function startSiteLivenessLoop(deps, intervalMs = SITE_LIVENESS_INTERVAL_MS) {
|
|
|
14018
14520
|
|
|
14019
14521
|
// src/review-webhook-delivery.ts
|
|
14020
14522
|
import { createHash as createHash7 } from "crypto";
|
|
14021
|
-
import { and as
|
|
14022
|
-
var
|
|
14523
|
+
import { and as and19, asc as asc4, eq as eq22, isNull as isNull3, lte as lte2, or } from "drizzle-orm";
|
|
14524
|
+
var log20 = createLogger("ReviewDelivery");
|
|
14023
14525
|
var hash = (value) => createHash7("sha256").update(JSON.stringify(value)).digest("hex");
|
|
14024
14526
|
var REVIEW_DELIVERY_MAX_ATTEMPTS = 10;
|
|
14025
14527
|
var REVIEW_CHAT_SENDS_PER_PASS = 3;
|
|
@@ -14046,20 +14548,20 @@ function reviewEventId(projectId, review) {
|
|
|
14046
14548
|
function reviewEventRefusal(db, projectId, payload, notification, targetDigest, now) {
|
|
14047
14549
|
if (!notification?.enabled || !notification.config.events.includes("review.negative")) return "destination_disabled";
|
|
14048
14550
|
if (targetDigest !== void 0 && hash(notification.config.url) !== targetDigest) return "destination_changed";
|
|
14049
|
-
const project = db.select().from(projects).where(
|
|
14551
|
+
const project = db.select().from(projects).where(eq22(projects.id, projectId)).get();
|
|
14050
14552
|
if (!project || project.name !== payload.project.name || project.canonicalDomain !== payload.project.canonicalDomain) return "project_changed";
|
|
14051
14553
|
const review = payload.reviews[0];
|
|
14052
|
-
const location = db.select().from(gbpLocations).where(
|
|
14053
|
-
|
|
14054
|
-
|
|
14055
|
-
|
|
14554
|
+
const location = db.select().from(gbpLocations).where(and19(
|
|
14555
|
+
eq22(gbpLocations.projectId, projectId),
|
|
14556
|
+
eq22(gbpLocations.locationName, review.location.name),
|
|
14557
|
+
eq22(gbpLocations.selected, true)
|
|
14056
14558
|
)).get();
|
|
14057
14559
|
if (!location || review.origin === "gbp" && !review.reviewName.startsWith(`${location.accountName}/${location.locationName}/reviews/`)) return "location_removed_or_deselected";
|
|
14058
|
-
const current = db.select().from(gbpReviews).where(
|
|
14059
|
-
|
|
14060
|
-
|
|
14061
|
-
|
|
14062
|
-
|
|
14560
|
+
const current = db.select().from(gbpReviews).where(and19(
|
|
14561
|
+
eq22(gbpReviews.projectId, projectId),
|
|
14562
|
+
eq22(gbpReviews.origin, review.origin),
|
|
14563
|
+
eq22(gbpReviews.reviewName, review.reviewName),
|
|
14564
|
+
eq22(gbpReviews.locationName, review.location.name)
|
|
14063
14565
|
)).get();
|
|
14064
14566
|
if (!current) return "review_removed";
|
|
14065
14567
|
if (!isWithinAlertWindow(review.updateTime, now) || !isWithinAlertWindow(current.updateTime, now)) return "review_expired";
|
|
@@ -14067,10 +14569,10 @@ function reviewEventRefusal(db, projectId, payload, notification, targetDigest,
|
|
|
14067
14569
|
if (!isNegativeReviewRating(current.starRating, threshold) || !isNegativeReviewRating(review.starRating, threshold)) return "review_no_longer_negative";
|
|
14068
14570
|
if (["baseline", "skipped", "suppressed", "stale", "none"].includes(current.alertState)) return "review_suppressed";
|
|
14069
14571
|
if (review.origin === "places") {
|
|
14070
|
-
const v4HasWorked = db.select({ id: gbpReviewRatings.id }).from(gbpReviewRatings).where(
|
|
14071
|
-
|
|
14072
|
-
|
|
14073
|
-
|
|
14572
|
+
const v4HasWorked = db.select({ id: gbpReviewRatings.id }).from(gbpReviewRatings).where(and19(
|
|
14573
|
+
eq22(gbpReviewRatings.projectId, projectId),
|
|
14574
|
+
eq22(gbpReviewRatings.locationName, review.location.name),
|
|
14575
|
+
eq22(gbpReviewRatings.origin, "gbp")
|
|
14074
14576
|
)).get() !== void 0;
|
|
14075
14577
|
if (businessProfileReviewsCover(location.reviewsAccess, v4HasWorked)) return "review_origin_superseded";
|
|
14076
14578
|
}
|
|
@@ -14078,9 +14580,9 @@ function reviewEventRefusal(db, projectId, payload, notification, targetDigest,
|
|
|
14078
14580
|
}
|
|
14079
14581
|
function cancelReviewDelivery(db, id, reason) {
|
|
14080
14582
|
db.transaction((tx) => {
|
|
14081
|
-
const canceled = tx.update(reviewWebhookDeliveries).set({ state: ReviewDeliveryStates.Canceled, retryAt: null }).where(
|
|
14082
|
-
|
|
14083
|
-
|
|
14583
|
+
const canceled = tx.update(reviewWebhookDeliveries).set({ state: ReviewDeliveryStates.Canceled, retryAt: null }).where(and19(
|
|
14584
|
+
eq22(reviewWebhookDeliveries.id, id),
|
|
14585
|
+
eq22(reviewWebhookDeliveries.state, ReviewDeliveryStates.Pending)
|
|
14084
14586
|
)).returning({ projectId: reviewWebhookDeliveries.projectId, eventId: reviewWebhookDeliveries.eventId, notificationId: reviewWebhookDeliveries.notificationId, targetDigest: reviewWebhookDeliveries.targetDigest }).get();
|
|
14085
14587
|
if (canceled) writeAuditLog(tx, {
|
|
14086
14588
|
projectId: canceled.projectId,
|
|
@@ -14096,10 +14598,10 @@ function deferReviewDelivery(db, saved, now) {
|
|
|
14096
14598
|
if (saved.state !== ReviewDeliveryStates.Pending && saved.state !== ReviewDeliveryStates.Attempting) return;
|
|
14097
14599
|
const waited = now - Date.parse(saved.createdAt);
|
|
14098
14600
|
const delay = Math.min(Math.max(RETRY_BASE_MS, Number.isFinite(waited) ? waited / 4 : RETRY_BASE_MS), RETRY_MAX_MS);
|
|
14099
|
-
db.update(reviewWebhookDeliveries).set({ retryAt: new Date(now + delay).toISOString() }).where(
|
|
14100
|
-
|
|
14101
|
-
|
|
14102
|
-
|
|
14601
|
+
db.update(reviewWebhookDeliveries).set({ retryAt: new Date(now + delay).toISOString() }).where(and19(
|
|
14602
|
+
eq22(reviewWebhookDeliveries.id, saved.id),
|
|
14603
|
+
eq22(reviewWebhookDeliveries.state, saved.state),
|
|
14604
|
+
eq22(reviewWebhookDeliveries.attempts, saved.attempts)
|
|
14103
14605
|
)).run();
|
|
14104
14606
|
}
|
|
14105
14607
|
function auditUndelivered(db, saved, outcome) {
|
|
@@ -14116,8 +14618,8 @@ function dueReviewDeliveryProjects(db, now) {
|
|
|
14116
14618
|
const at = new Date(now).toISOString();
|
|
14117
14619
|
const staleAttempt = new Date(now - 6e4).toISOString();
|
|
14118
14620
|
return db.selectDistinct({ projectId: reviewWebhookDeliveries.projectId }).from(reviewWebhookDeliveries).where(or(
|
|
14119
|
-
|
|
14120
|
-
|
|
14621
|
+
and19(eq22(reviewWebhookDeliveries.state, ReviewDeliveryStates.Pending), or(isNull3(reviewWebhookDeliveries.retryAt), lte2(reviewWebhookDeliveries.retryAt, at))),
|
|
14622
|
+
and19(eq22(reviewWebhookDeliveries.state, ReviewDeliveryStates.Attempting), lte2(reviewWebhookDeliveries.attemptedAt, staleAttempt))
|
|
14121
14623
|
)).all().map((row) => row.projectId);
|
|
14122
14624
|
}
|
|
14123
14625
|
function startReviewDeliveryLoop(drain, intervalMs = REVIEW_DELIVERY_INTERVAL_MS) {
|
|
@@ -14126,7 +14628,7 @@ function startReviewDeliveryLoop(drain, intervalMs = REVIEW_DELIVERY_INTERVAL_MS
|
|
|
14126
14628
|
if (running) return;
|
|
14127
14629
|
running = true;
|
|
14128
14630
|
void drain().catch((err) => {
|
|
14129
|
-
|
|
14631
|
+
log20.warn("review-delivery.drain-failed", { error: describeError(err) });
|
|
14130
14632
|
}).finally(() => {
|
|
14131
14633
|
running = false;
|
|
14132
14634
|
});
|
|
@@ -14138,12 +14640,12 @@ async function drainReviewDeliveries(options) {
|
|
|
14138
14640
|
const now = (options.now ?? Date.now)();
|
|
14139
14641
|
const at = new Date(now).toISOString();
|
|
14140
14642
|
const staleAttempt = new Date(now - 6e4).toISOString();
|
|
14141
|
-
const ready = options.db.select().from(reviewWebhookDeliveries).where(
|
|
14142
|
-
|
|
14643
|
+
const ready = options.db.select().from(reviewWebhookDeliveries).where(and19(
|
|
14644
|
+
eq22(reviewWebhookDeliveries.projectId, options.projectId),
|
|
14143
14645
|
or(isNull3(reviewWebhookDeliveries.retryAt), lte2(reviewWebhookDeliveries.retryAt, at)),
|
|
14144
14646
|
or(
|
|
14145
|
-
|
|
14146
|
-
|
|
14647
|
+
eq22(reviewWebhookDeliveries.state, ReviewDeliveryStates.Pending),
|
|
14648
|
+
and19(eq22(reviewWebhookDeliveries.state, ReviewDeliveryStates.Attempting), lte2(reviewWebhookDeliveries.attemptedAt, staleAttempt))
|
|
14147
14649
|
)
|
|
14148
14650
|
)).orderBy(asc4(reviewWebhookDeliveries.retryAt), asc4(reviewWebhookDeliveries.attemptedAt), asc4(reviewWebhookDeliveries.createdAt), asc4(reviewWebhookDeliveries.id)).limit(100).all();
|
|
14149
14651
|
let delivered = 0;
|
|
@@ -14167,15 +14669,15 @@ async function deliverReviewEvent(options) {
|
|
|
14167
14669
|
if (!payload.eventId || !/^evt_[a-f0-9]{64}$/u.test(payload.eventId) || payload.reviews.length !== 1)
|
|
14168
14670
|
return false;
|
|
14169
14671
|
const notification = db.select().from(notifications).where(
|
|
14170
|
-
|
|
14171
|
-
|
|
14172
|
-
|
|
14672
|
+
and19(
|
|
14673
|
+
eq22(notifications.id, notificationId),
|
|
14674
|
+
eq22(notifications.projectId, projectId)
|
|
14173
14675
|
)
|
|
14174
14676
|
).get();
|
|
14175
14677
|
const id = hash([projectId, notificationId, payload.eventId]);
|
|
14176
14678
|
const now = options.now ?? Date.now;
|
|
14177
14679
|
const at = new Date(now()).toISOString();
|
|
14178
|
-
let saved = db.select().from(reviewWebhookDeliveries).where(
|
|
14680
|
+
let saved = db.select().from(reviewWebhookDeliveries).where(eq22(reviewWebhookDeliveries.id, id)).get();
|
|
14179
14681
|
const event = saved?.payload ?? payload;
|
|
14180
14682
|
const refusal = reviewEventRefusal(db, projectId, event, notification, saved?.targetDigest, new Date(now()));
|
|
14181
14683
|
if (refusal) {
|
|
@@ -14194,7 +14696,7 @@ async function deliverReviewEvent(options) {
|
|
|
14194
14696
|
payload,
|
|
14195
14697
|
createdAt: at
|
|
14196
14698
|
}).onConflictDoNothing().run();
|
|
14197
|
-
saved = db.select().from(reviewWebhookDeliveries).where(
|
|
14699
|
+
saved = db.select().from(reviewWebhookDeliveries).where(eq22(reviewWebhookDeliveries.id, id)).get();
|
|
14198
14700
|
if (!saved || saved.targetDigest !== targetDigest) return false;
|
|
14199
14701
|
if (saved.state === ReviewDeliveryStates.Sent) return true;
|
|
14200
14702
|
if (saved.state === ReviewDeliveryStates.Uncertain || saved.state === ReviewDeliveryStates.Canceled) return false;
|
|
@@ -14209,17 +14711,17 @@ async function deliverReviewEvent(options) {
|
|
|
14209
14711
|
return false;
|
|
14210
14712
|
const state2 = destination.signed ? ReviewDeliveryStates.Pending : ReviewDeliveryStates.Uncertain;
|
|
14211
14713
|
const moved = db.update(reviewWebhookDeliveries).set({ state: state2 }).where(
|
|
14212
|
-
|
|
14213
|
-
|
|
14214
|
-
|
|
14215
|
-
|
|
14216
|
-
|
|
14714
|
+
and19(
|
|
14715
|
+
eq22(reviewWebhookDeliveries.id, id),
|
|
14716
|
+
eq22(reviewWebhookDeliveries.state, ReviewDeliveryStates.Attempting),
|
|
14717
|
+
eq22(reviewWebhookDeliveries.attempts, saved.attempts),
|
|
14718
|
+
eq22(reviewWebhookDeliveries.attemptedAt, saved.attemptedAt)
|
|
14217
14719
|
)
|
|
14218
14720
|
).returning({ id: reviewWebhookDeliveries.id }).get();
|
|
14219
14721
|
if (moved && state2 === ReviewDeliveryStates.Uncertain) {
|
|
14220
14722
|
auditUndelivered(db, saved, { state: state2, reason: "interrupted_attempt", attempts: saved.attempts });
|
|
14221
14723
|
}
|
|
14222
|
-
saved = db.select().from(reviewWebhookDeliveries).where(
|
|
14724
|
+
saved = db.select().from(reviewWebhookDeliveries).where(eq22(reviewWebhookDeliveries.id, id)).get();
|
|
14223
14725
|
if (!saved || saved.state !== ReviewDeliveryStates.Pending) return false;
|
|
14224
14726
|
}
|
|
14225
14727
|
if (saved.retryAt && Date.parse(saved.retryAt) > now()) return false;
|
|
@@ -14241,9 +14743,9 @@ async function deliverReviewEvent(options) {
|
|
|
14241
14743
|
return false;
|
|
14242
14744
|
}
|
|
14243
14745
|
const latest = db.select().from(notifications).where(
|
|
14244
|
-
|
|
14245
|
-
|
|
14246
|
-
|
|
14746
|
+
and19(
|
|
14747
|
+
eq22(notifications.id, notificationId),
|
|
14748
|
+
eq22(notifications.projectId, projectId)
|
|
14247
14749
|
)
|
|
14248
14750
|
).get();
|
|
14249
14751
|
const latestRefusal = reviewEventRefusal(db, projectId, saved.payload, latest, targetDigest, new Date(now()));
|
|
@@ -14256,10 +14758,10 @@ async function deliverReviewEvent(options) {
|
|
|
14256
14758
|
attempts: saved.attempts + 1,
|
|
14257
14759
|
attemptedAt: new Date(now()).toISOString()
|
|
14258
14760
|
}).where(
|
|
14259
|
-
|
|
14260
|
-
|
|
14261
|
-
|
|
14262
|
-
|
|
14761
|
+
and19(
|
|
14762
|
+
eq22(reviewWebhookDeliveries.id, id),
|
|
14763
|
+
eq22(reviewWebhookDeliveries.state, ReviewDeliveryStates.Pending),
|
|
14764
|
+
eq22(reviewWebhookDeliveries.attempts, saved.attempts)
|
|
14263
14765
|
)
|
|
14264
14766
|
).returning({ id: reviewWebhookDeliveries.id }).get();
|
|
14265
14767
|
if (!claimed) return false;
|
|
@@ -14286,10 +14788,10 @@ async function deliverReviewEvent(options) {
|
|
|
14286
14788
|
deliveredAt: sent ? new Date(now()).toISOString() : null,
|
|
14287
14789
|
retryAt: state === ReviewDeliveryStates.Pending ? new Date(now() + retryDelayMs(attempts, result.retryAfterSeconds)).toISOString() : null
|
|
14288
14790
|
}).where(
|
|
14289
|
-
|
|
14290
|
-
|
|
14291
|
-
|
|
14292
|
-
|
|
14791
|
+
and19(
|
|
14792
|
+
eq22(reviewWebhookDeliveries.id, id),
|
|
14793
|
+
eq22(reviewWebhookDeliveries.state, ReviewDeliveryStates.Attempting),
|
|
14794
|
+
eq22(reviewWebhookDeliveries.attempts, attempts)
|
|
14293
14795
|
)
|
|
14294
14796
|
).run();
|
|
14295
14797
|
if (state === ReviewDeliveryStates.Uncertain || state === ReviewDeliveryStates.Failed) {
|
|
@@ -14301,8 +14803,8 @@ async function deliverReviewEvent(options) {
|
|
|
14301
14803
|
}
|
|
14302
14804
|
|
|
14303
14805
|
// src/unrecognized-agent-alerts.ts
|
|
14304
|
-
import { and as
|
|
14305
|
-
var
|
|
14806
|
+
import { and as and20, desc as desc8, eq as eq23, gte as gte3, sql as sql10 } from "drizzle-orm";
|
|
14807
|
+
var log21 = createLogger("UnrecognizedAgentAlerts");
|
|
14306
14808
|
var UNRECOGNIZED_AGENT_ALERT_WINDOW_MS = 7 * 864e5;
|
|
14307
14809
|
var UNRECOGNIZED_AGENT_ALERT_MIN_HITS = 3;
|
|
14308
14810
|
var UNRECOGNIZED_AGENT_ALERT_INTERVAL_MS = 15 * 6e4;
|
|
@@ -14317,13 +14819,13 @@ function projectsWithRecentUnrecognizedAgents(db, now) {
|
|
|
14317
14819
|
return db.selectDistinct({ projectId: unrecognizedAgentEventsHourly.projectId }).from(unrecognizedAgentEventsHourly).where(gte3(unrecognizedAgentEventsHourly.tsHour, since)).all().map((row) => row.projectId);
|
|
14318
14820
|
}
|
|
14319
14821
|
function undecidedUnrecognizedAgents(db, projectId, now) {
|
|
14320
|
-
const decided = new Set(db.select({ agentName: unrecognizedAgentNotifyState.agentName }).from(unrecognizedAgentNotifyState).where(
|
|
14822
|
+
const decided = new Set(db.select({ agentName: unrecognizedAgentNotifyState.agentName }).from(unrecognizedAgentNotifyState).where(eq23(unrecognizedAgentNotifyState.projectId, projectId)).all().map((row) => row.agentName));
|
|
14321
14823
|
const totals = db.select({
|
|
14322
14824
|
name: unrecognizedAgentEventsHourly.agentName,
|
|
14323
|
-
hits:
|
|
14324
|
-
firstSeenAt:
|
|
14325
|
-
lastSeenAt:
|
|
14326
|
-
}).from(unrecognizedAgentEventsHourly).where(
|
|
14825
|
+
hits: sql10`SUM(${unrecognizedAgentEventsHourly.hits})`,
|
|
14826
|
+
firstSeenAt: sql10`MIN(${unrecognizedAgentEventsHourly.tsHour})`,
|
|
14827
|
+
lastSeenAt: sql10`MAX(${unrecognizedAgentEventsHourly.tsHour})`
|
|
14828
|
+
}).from(unrecognizedAgentEventsHourly).where(eq23(unrecognizedAgentEventsHourly.projectId, projectId)).groupBy(unrecognizedAgentEventsHourly.agentName).all();
|
|
14327
14829
|
const windowStart = new Date(now - UNRECOGNIZED_AGENT_ALERT_WINDOW_MS).toISOString();
|
|
14328
14830
|
const recordingStart = totals.reduce((earliest, total) => earliest === null || total.firstSeenAt < earliest ? total.firstSeenAt : earliest, null);
|
|
14329
14831
|
const baselineEnd = recordingStart === null ? null : new Date(Date.parse(recordingStart) + UNRECOGNIZED_AGENT_ALERT_WINDOW_MS).toISOString();
|
|
@@ -14341,7 +14843,7 @@ function undecidedUnrecognizedAgents(db, projectId, now) {
|
|
|
14341
14843
|
continue;
|
|
14342
14844
|
}
|
|
14343
14845
|
if (Number(total.hits) < UNRECOGNIZED_AGENT_ALERT_MIN_HITS) continue;
|
|
14344
|
-
const latest = db.select({ sampledUserAgent: unrecognizedAgentEventsHourly.sampledUserAgent, sampledPath: unrecognizedAgentEventsHourly.sampledPath }).from(unrecognizedAgentEventsHourly).where(
|
|
14846
|
+
const latest = db.select({ sampledUserAgent: unrecognizedAgentEventsHourly.sampledUserAgent, sampledPath: unrecognizedAgentEventsHourly.sampledPath }).from(unrecognizedAgentEventsHourly).where(and20(eq23(unrecognizedAgentEventsHourly.projectId, projectId), eq23(unrecognizedAgentEventsHourly.agentName, total.name))).orderBy(desc8(unrecognizedAgentEventsHourly.tsHour)).limit(1).get();
|
|
14345
14847
|
due.push({
|
|
14346
14848
|
name: total.name,
|
|
14347
14849
|
hits: Number(total.hits),
|
|
@@ -14366,7 +14868,7 @@ function startUnrecognizedAgentAlertLoop(dispatch, intervalMs = UNRECOGNIZED_AGE
|
|
|
14366
14868
|
if (running) return;
|
|
14367
14869
|
running = true;
|
|
14368
14870
|
void dispatch().catch((err) => {
|
|
14369
|
-
|
|
14871
|
+
log21.warn("unrecognized-agent-alerts.dispatch-failed", { error: describeError(err) });
|
|
14370
14872
|
}).finally(() => {
|
|
14371
14873
|
running = false;
|
|
14372
14874
|
});
|
|
@@ -14376,8 +14878,8 @@ function startUnrecognizedAgentAlertLoop(dispatch, intervalMs = UNRECOGNIZED_AGE
|
|
|
14376
14878
|
}
|
|
14377
14879
|
|
|
14378
14880
|
// src/provider-batch-poller.ts
|
|
14379
|
-
import { and as
|
|
14380
|
-
var
|
|
14881
|
+
import { and as and21, eq as eq24, inArray as inArray11, isNotNull as isNotNull3, isNull as isNull4, notExists as notExists2, sql as sql11 } from "drizzle-orm";
|
|
14882
|
+
var log22 = createLogger("ProviderBatchPoller");
|
|
14381
14883
|
var PROVIDER_BATCH_POLL_INITIAL_MS = 3e4;
|
|
14382
14884
|
var PROVIDER_BATCH_POLL_MAX_MS = 10 * 6e4;
|
|
14383
14885
|
var PROVIDER_BATCH_CANCEL_GRACE_MS = 60 * 6e4;
|
|
@@ -14401,7 +14903,7 @@ var ProviderBatchPoller = class {
|
|
|
14401
14903
|
try {
|
|
14402
14904
|
if (await this.advance(row)) settledRuns.set(row.runId, row.projectId);
|
|
14403
14905
|
} catch (err) {
|
|
14404
|
-
|
|
14906
|
+
log22.warn("batch.advance-failed", { batchId: row.id, runId: row.runId, provider: row.provider, error: describeError(err) });
|
|
14405
14907
|
this.reschedule(row.id, null);
|
|
14406
14908
|
}
|
|
14407
14909
|
}
|
|
@@ -14410,7 +14912,7 @@ var ProviderBatchPoller = class {
|
|
|
14410
14912
|
try {
|
|
14411
14913
|
this.deps.runner.finalizeBatchRun(runId, projectId);
|
|
14412
14914
|
} catch (err) {
|
|
14413
|
-
|
|
14915
|
+
log22.warn("batch.finalize-failed", { runId, error: describeError(err) });
|
|
14414
14916
|
}
|
|
14415
14917
|
}
|
|
14416
14918
|
}
|
|
@@ -14419,7 +14921,7 @@ var ProviderBatchPoller = class {
|
|
|
14419
14921
|
* outstanding (ingested or cancelled), so its run may now finalize.
|
|
14420
14922
|
*/
|
|
14421
14923
|
async advance(row) {
|
|
14422
|
-
const runStatus = this.deps.db.select({ status: runs.status }).from(runs).where(
|
|
14924
|
+
const runStatus = this.deps.db.select({ status: runs.status }).from(runs).where(eq24(runs.id, row.runId)).get()?.status;
|
|
14423
14925
|
if (runStatus !== RunStatuses.running) {
|
|
14424
14926
|
await this.deps.runner.abandonProviderBatches([row.id], "Cancelled because its run is no longer running.");
|
|
14425
14927
|
this.schedule.delete(row.id);
|
|
@@ -14436,15 +14938,15 @@ var ProviderBatchPoller = class {
|
|
|
14436
14938
|
status = poll.status;
|
|
14437
14939
|
if (poll.status === "ended") {
|
|
14438
14940
|
const endedAt = poll.endedAt ?? new Date(this.now()).toISOString();
|
|
14439
|
-
const ended = this.deps.db.update(providerBatches).set({ status: ProviderBatchStatuses.ended, endedAt, resultsExpireAt: poll.resultsExpireAt ?? null, updatedAt: endedAt }).where(
|
|
14941
|
+
const ended = this.deps.db.update(providerBatches).set({ status: ProviderBatchStatuses.ended, endedAt, resultsExpireAt: poll.resultsExpireAt ?? null, updatedAt: endedAt }).where(and21(eq24(providerBatches.id, row.id), eq24(providerBatches.status, ProviderBatchStatuses.submitted))).run().changes === 1;
|
|
14440
14942
|
this.schedule.delete(row.id);
|
|
14441
14943
|
return ended ? this.ingest({ ...row, status: ProviderBatchStatuses.ended, resultsExpireAt: poll.resultsExpireAt ?? null }) : false;
|
|
14442
14944
|
}
|
|
14443
14945
|
} catch (err) {
|
|
14444
|
-
|
|
14946
|
+
log22.warn("batch.poll-failed", { batchId: row.id, runId: row.runId, provider: row.provider, error: describeError(err) });
|
|
14445
14947
|
}
|
|
14446
14948
|
} else {
|
|
14447
|
-
|
|
14949
|
+
log22.warn("batch.poll-unavailable", { batchId: row.id, runId: row.runId, provider: row.provider });
|
|
14448
14950
|
}
|
|
14449
14951
|
this.reschedule(row.id, status);
|
|
14450
14952
|
return this.enforceDeadline(row, registered);
|
|
@@ -14460,18 +14962,18 @@ var ProviderBatchPoller = class {
|
|
|
14460
14962
|
const at = new Date(now).toISOString();
|
|
14461
14963
|
if (row.cancelRequestedAt === null) {
|
|
14462
14964
|
if (now < Date.parse(row.deadlineAt)) return false;
|
|
14463
|
-
const claimed = this.deps.db.update(providerBatches).set({ cancelRequestedAt: at, updatedAt: at }).where(
|
|
14464
|
-
|
|
14465
|
-
|
|
14965
|
+
const claimed = this.deps.db.update(providerBatches).set({ cancelRequestedAt: at, updatedAt: at }).where(and21(
|
|
14966
|
+
eq24(providerBatches.id, row.id),
|
|
14967
|
+
eq24(providerBatches.status, ProviderBatchStatuses.submitted),
|
|
14466
14968
|
isNull4(providerBatches.cancelRequestedAt)
|
|
14467
14969
|
)).run().changes === 1;
|
|
14468
14970
|
if (!claimed) return false;
|
|
14469
|
-
|
|
14971
|
+
log22.warn("batch.deadline-passed", { batchId: row.id, runId: row.runId, provider: row.provider, deadlineAt: row.deadlineAt });
|
|
14470
14972
|
if (registered?.adapter.batch && row.providerBatchId) {
|
|
14471
14973
|
try {
|
|
14472
14974
|
await registered.adapter.batch.cancel(row.providerBatchId, registered.config);
|
|
14473
14975
|
} catch (err) {
|
|
14474
|
-
|
|
14976
|
+
log22.warn("batch.cancel-failed", { batchId: row.id, provider: row.provider, error: describeError(err) });
|
|
14475
14977
|
}
|
|
14476
14978
|
}
|
|
14477
14979
|
return false;
|
|
@@ -14481,9 +14983,9 @@ var ProviderBatchPoller = class {
|
|
|
14481
14983
|
status: ProviderBatchStatuses.cancelled,
|
|
14482
14984
|
error: `The provider batch did not end within an hour of being cancelled at its deadline; its ${row.requestCount} answer(s) were not recorded.`,
|
|
14483
14985
|
updatedAt: at
|
|
14484
|
-
}).where(
|
|
14986
|
+
}).where(and21(eq24(providerBatches.id, row.id), eq24(providerBatches.status, ProviderBatchStatuses.submitted))).run().changes === 1;
|
|
14485
14987
|
this.schedule.delete(row.id);
|
|
14486
|
-
if (gaveUp)
|
|
14988
|
+
if (gaveUp) log22.warn("batch.abandoned-after-deadline", { batchId: row.id, runId: row.runId, provider: row.provider });
|
|
14487
14989
|
return gaveUp;
|
|
14488
14990
|
}
|
|
14489
14991
|
async ingest(row) {
|
|
@@ -14492,7 +14994,7 @@ var ProviderBatchPoller = class {
|
|
|
14492
14994
|
return result.kind !== "skipped";
|
|
14493
14995
|
} catch (err) {
|
|
14494
14996
|
const error = describeError(err);
|
|
14495
|
-
|
|
14997
|
+
log22.warn("batch.ingest-failed", { batchId: row.id, runId: row.runId, provider: row.provider, error });
|
|
14496
14998
|
const now = this.now();
|
|
14497
14999
|
const unreadable = row.resultsExpireAt !== null && now >= Date.parse(row.resultsExpireAt);
|
|
14498
15000
|
if (unreadable || now >= Date.parse(row.deadlineAt) + PROVIDER_BATCH_CANCEL_GRACE_MS) {
|
|
@@ -14521,10 +15023,10 @@ var ProviderBatchPoller = class {
|
|
|
14521
15023
|
/** Runs the sweep handed off whose batches have all settled. */
|
|
14522
15024
|
finalizableRuns() {
|
|
14523
15025
|
const holding = [ProviderBatchStatuses.submitting, ...OUTSTANDING_PROVIDER_BATCH_STATUSES];
|
|
14524
|
-
return this.deps.db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
14525
|
-
|
|
15026
|
+
return this.deps.db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(and21(
|
|
15027
|
+
eq24(runs.status, RunStatuses.running),
|
|
14526
15028
|
isNotNull3(runs.pendingProviderErrors),
|
|
14527
|
-
notExists2(this.deps.db.select({ one:
|
|
15029
|
+
notExists2(this.deps.db.select({ one: sql11`1` }).from(providerBatches).where(and21(eq24(providerBatches.runId, runs.id), inArray11(providerBatches.status, holding))))
|
|
14528
15030
|
)).all();
|
|
14529
15031
|
}
|
|
14530
15032
|
};
|
|
@@ -14534,7 +15036,7 @@ function startProviderBatchPoller(deps, intervalMs = PROVIDER_BATCH_TICK_MS) {
|
|
|
14534
15036
|
const timer = setInterval(() => {
|
|
14535
15037
|
if (running) return;
|
|
14536
15038
|
running = true;
|
|
14537
|
-
void poller.tick().catch((err) =>
|
|
15039
|
+
void poller.tick().catch((err) => log22.warn("batch.pass-failed", { error: describeError(err) })).finally(() => {
|
|
14538
15040
|
running = false;
|
|
14539
15041
|
});
|
|
14540
15042
|
}, intervalMs);
|
|
@@ -14543,7 +15045,7 @@ function startProviderBatchPoller(deps, intervalMs = PROVIDER_BATCH_TICK_MS) {
|
|
|
14543
15045
|
}
|
|
14544
15046
|
|
|
14545
15047
|
// src/data-refresh.ts
|
|
14546
|
-
var
|
|
15048
|
+
var log23 = createLogger("DataRefresh");
|
|
14547
15049
|
function gaRefreshDays(now = /* @__PURE__ */ new Date()) {
|
|
14548
15050
|
const [reportMonth] = reportMonthsForDoctor(void 0, now);
|
|
14549
15051
|
const reach = inclusiveDayCount(`${reportMonth}-01`, now.toISOString().slice(0, 10)) ?? 0;
|
|
@@ -14562,6 +15064,7 @@ async function refreshAllIntegrations(client, projectName, now = /* @__PURE__ */
|
|
|
14562
15064
|
const integrations = [
|
|
14563
15065
|
{ name: "gsc", kind: RunKinds["gsc-sync"], completion: "queued", run: () => client.gscSync(projectName, {}) },
|
|
14564
15066
|
{ name: "bing", kind: RunKinds["bing-inspect-sitemap"], completion: "queued", run: () => client.bingInspectSitemap(projectName, {}) },
|
|
15067
|
+
{ name: "bing-sync", kind: RunKinds["bing-sync"], completion: "queued", run: () => client.bingSync(projectName) },
|
|
14565
15068
|
{ name: "ga", kind: RunKinds["ga-sync"], completion: "synchronous", run: () => client.gaSync(projectName, { days: gaRefreshDays(now) }) },
|
|
14566
15069
|
{ name: "gbp", kind: RunKinds["gbp-sync"], completion: "queued", run: () => client.triggerGbpSync(projectName, {}) },
|
|
14567
15070
|
// `ads` remains the OpenAI / ChatGPT Ads integration. For scheduled
|
|
@@ -14584,15 +15087,15 @@ async function refreshAllIntegrations(client, projectName, now = /* @__PURE__ */
|
|
|
14584
15087
|
if (result.status === "rejected") {
|
|
14585
15088
|
const reason = result.reason;
|
|
14586
15089
|
const message = describeError(reason);
|
|
14587
|
-
|
|
15090
|
+
log23.warn("integration.refresh-failed", { projectName, integration, error: message });
|
|
14588
15091
|
return;
|
|
14589
15092
|
}
|
|
14590
15093
|
switch (completion) {
|
|
14591
15094
|
case "synchronous":
|
|
14592
|
-
|
|
15095
|
+
log23.info("integration.refreshed", { projectName, integration });
|
|
14593
15096
|
return;
|
|
14594
15097
|
case "queued":
|
|
14595
|
-
|
|
15098
|
+
log23.info("integration.queued", { projectName, integration, ...queuedRun(result.value) });
|
|
14596
15099
|
return;
|
|
14597
15100
|
default: {
|
|
14598
15101
|
const _exhaustive = completion;
|
|
@@ -14603,9 +15106,9 @@ async function refreshAllIntegrations(client, projectName, now = /* @__PURE__ */
|
|
|
14603
15106
|
}
|
|
14604
15107
|
|
|
14605
15108
|
// src/notifier.ts
|
|
14606
|
-
import { eq as
|
|
14607
|
-
import
|
|
14608
|
-
var
|
|
15109
|
+
import { eq as eq25, desc as desc9, and as and22, inArray as inArray12, or as or2, isNull as isNull5 } from "drizzle-orm";
|
|
15110
|
+
import crypto22 from "crypto";
|
|
15111
|
+
var log24 = createLogger("Notifier");
|
|
14609
15112
|
var sampleDelivery = createOutcomeSampler({ burst: 20, refillMs: 6e4, now: () => Date.now() });
|
|
14610
15113
|
function trackWebhookDelivery(input) {
|
|
14611
15114
|
const sample = sampleDelivery(input.failure ? "failed" : "succeeded");
|
|
@@ -14725,26 +15228,26 @@ var Notifier = class {
|
|
|
14725
15228
|
}
|
|
14726
15229
|
/** Called after a run completes (success, partial, or failed). */
|
|
14727
15230
|
async onRunCompleted(runId, projectId, opts) {
|
|
14728
|
-
|
|
14729
|
-
const notifs = this.db.select().from(notifications).where(
|
|
15231
|
+
log24.info("run.completed", { runId, projectId, origin: opts?.origin ?? "sweep" });
|
|
15232
|
+
const notifs = this.db.select().from(notifications).where(eq25(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
14730
15233
|
if (notifs.length === 0) {
|
|
14731
|
-
|
|
15234
|
+
log24.info("notifications.none-enabled", { projectId });
|
|
14732
15235
|
return;
|
|
14733
15236
|
}
|
|
14734
|
-
|
|
14735
|
-
const run = this.db.select().from(runs).where(
|
|
15237
|
+
log24.info("notifications.found", { projectId, count: notifs.length });
|
|
15238
|
+
const run = this.db.select().from(runs).where(eq25(runs.id, runId)).get();
|
|
14736
15239
|
if (!run) {
|
|
14737
|
-
|
|
15240
|
+
log24.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
14738
15241
|
return;
|
|
14739
15242
|
}
|
|
14740
|
-
const project = this.db.select().from(projects).where(
|
|
15243
|
+
const project = this.db.select().from(projects).where(eq25(projects.id, projectId)).get();
|
|
14741
15244
|
if (!project) {
|
|
14742
|
-
|
|
15245
|
+
log24.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
14743
15246
|
return;
|
|
14744
15247
|
}
|
|
14745
15248
|
const transitions = this.computeTransitions(runId, projectId);
|
|
14746
15249
|
const events = [];
|
|
14747
|
-
|
|
15250
|
+
log24.info("run.status", { runId: run.id, status: run.status, projectId });
|
|
14748
15251
|
if ((run.status === "completed" || run.status === "partial") && opts?.origin !== "fill") {
|
|
14749
15252
|
events.push("run.completed");
|
|
14750
15253
|
}
|
|
@@ -14760,7 +15263,7 @@ var Notifier = class {
|
|
|
14760
15263
|
if (!config.url) continue;
|
|
14761
15264
|
const subscribedEvents = config.events;
|
|
14762
15265
|
const matchingEvents = events.filter((e) => subscribedEvents.includes(e));
|
|
14763
|
-
|
|
15266
|
+
log24.info("notification.match", { notificationId: notif.id, subscribedEvents, matchedEvents: matchingEvents });
|
|
14764
15267
|
if (matchingEvents.length === 0) continue;
|
|
14765
15268
|
for (const event of matchingEvents) {
|
|
14766
15269
|
const relevantTransitions = event === "citation.lost" ? lostTransitions : event === "citation.gained" ? gainedTransitions : transitions;
|
|
@@ -14816,9 +15319,9 @@ var Notifier = class {
|
|
|
14816
15319
|
* than inferred from the absence of a webhook.
|
|
14817
15320
|
*/
|
|
14818
15321
|
async onHealthChecked(projectId, report) {
|
|
14819
|
-
const project = this.db.select().from(projects).where(
|
|
15322
|
+
const project = this.db.select().from(projects).where(eq25(projects.id, projectId)).get();
|
|
14820
15323
|
if (!project) {
|
|
14821
|
-
|
|
15324
|
+
log24.error("project.not-found", { projectId, msg: "skipping health notification" });
|
|
14822
15325
|
return null;
|
|
14823
15326
|
}
|
|
14824
15327
|
const rank = healthRank;
|
|
@@ -14849,7 +15352,7 @@ var Notifier = class {
|
|
|
14849
15352
|
const status = noSignal ? "warn" : worst ? worst.status : "ok";
|
|
14850
15353
|
const code = noSignal ? HEALTH_NO_SIGNAL_CODE : worst?.code ?? "health.ok";
|
|
14851
15354
|
const summary = noSignal ? `No health check produced a result (${healthChecks.length} skipped) \u2014 health is unknown, not confirmed.` : worst?.summary ?? `All ${graded.length} health check(s) passing.`;
|
|
14852
|
-
const previous = this.db.select().from(doctorHealthState).where(
|
|
15355
|
+
const previous = this.db.select().from(doctorHealthState).where(eq25(doctorHealthState.projectId, projectId)).get();
|
|
14853
15356
|
const previousStatus = previous?.status ?? null;
|
|
14854
15357
|
const failingSignature = failing.map((c) => `${c.status}:${c.code}`).join(",");
|
|
14855
15358
|
const failingNow = failing.map((c) => ({ id: c.id, code: c.code, rank: rank(c.status) }));
|
|
@@ -14919,7 +15422,7 @@ var Notifier = class {
|
|
|
14919
15422
|
const urgentById = new Map(urgentItems.map((item) => [item.id, item]));
|
|
14920
15423
|
const generalEvent = event;
|
|
14921
15424
|
const urgentRecord = previous?.urgentNotified ?? null;
|
|
14922
|
-
const enabled = generalEvent || urgentItems.length > 0 || urgentRecord ? this.db.select().from(notifications).where(
|
|
15425
|
+
const enabled = generalEvent || urgentItems.length > 0 || urgentRecord ? this.db.select().from(notifications).where(eq25(notifications.projectId, projectId)).all().filter((n) => n.enabled) : [];
|
|
14923
15426
|
const listedEvents = (notif) => notif.config.events;
|
|
14924
15427
|
const hasUrl = (notif) => Boolean(notif.config.url);
|
|
14925
15428
|
const urgentSubscribers = enabled.filter((n) => hasUrl(n) && (listedEvents(n) ?? []).some((e) => URGENT_HEALTH_EVENTS.has(e)));
|
|
@@ -14967,7 +15470,7 @@ var Notifier = class {
|
|
|
14967
15470
|
if (previous === void 0) {
|
|
14968
15471
|
this.db.insert(doctorHealthState).values({ ...observation, notifiedAt: null }).run();
|
|
14969
15472
|
} else {
|
|
14970
|
-
this.db.update(doctorHealthState).set(observation).where(
|
|
15473
|
+
this.db.update(doctorHealthState).set(observation).where(eq25(doctorHealthState.projectId, projectId)).run();
|
|
14971
15474
|
}
|
|
14972
15475
|
for (const group of urgentGroups) {
|
|
14973
15476
|
const { urgentEvent, urgentTold, urgentNew, urgentCleared, urgentNotifs, urgentSettled } = group;
|
|
@@ -15011,10 +15514,10 @@ var Notifier = class {
|
|
|
15011
15514
|
}
|
|
15012
15515
|
if (accepted > 0) {
|
|
15013
15516
|
recordUrgentReceipts(urgentReceipts, group.subscribers, formatUrgentBreaches(urgentSettled) || null);
|
|
15014
|
-
this.db.update(doctorHealthState).set({ urgentNotified: doctorUrgentSummary(urgentReceipts), urgentReceipts }).where(
|
|
15517
|
+
this.db.update(doctorHealthState).set({ urgentNotified: doctorUrgentSummary(urgentReceipts), urgentReceipts }).where(eq25(doctorHealthState.projectId, projectId)).run();
|
|
15015
15518
|
}
|
|
15016
15519
|
if (urgentNotifs.length > 0) {
|
|
15017
|
-
|
|
15520
|
+
log24.info("health.urgent-notified", {
|
|
15018
15521
|
projectId,
|
|
15019
15522
|
event: urgentEvent,
|
|
15020
15523
|
codes: urgentEvent === "health.urgent" ? breachCodes(urgentNew) : breachCodes(urgentCleared),
|
|
@@ -15025,9 +15528,9 @@ var Notifier = class {
|
|
|
15025
15528
|
}
|
|
15026
15529
|
if (!event) {
|
|
15027
15530
|
if (clearedThisPass.length > 0 || easedThisPass.length > 0) {
|
|
15028
|
-
|
|
15531
|
+
log24.info("health.improved", { projectId, status, code, cleared: clearedThisPass, eased: easedThisPass });
|
|
15029
15532
|
} else {
|
|
15030
|
-
|
|
15533
|
+
log24.info("health.unchanged", { projectId, status, code });
|
|
15031
15534
|
}
|
|
15032
15535
|
return null;
|
|
15033
15536
|
}
|
|
@@ -15055,9 +15558,9 @@ var Notifier = class {
|
|
|
15055
15558
|
if (await this.sendWebhook(config.url, payload, notif.id, projectId, notif.webhookSecret ?? null)) delivered += 1;
|
|
15056
15559
|
}
|
|
15057
15560
|
if (delivered > 0) {
|
|
15058
|
-
this.db.update(doctorHealthState).set({ notifiedAt: now, notifiedSignature: formatBreaches(failingNow), pendingSignature: null }).where(
|
|
15561
|
+
this.db.update(doctorHealthState).set({ notifiedAt: now, notifiedSignature: formatBreaches(failingNow), pendingSignature: null }).where(eq25(doctorHealthState.projectId, projectId)).run();
|
|
15059
15562
|
}
|
|
15060
|
-
|
|
15563
|
+
log24.info("health.notified", {
|
|
15061
15564
|
projectId,
|
|
15062
15565
|
event,
|
|
15063
15566
|
status,
|
|
@@ -15085,12 +15588,12 @@ var Notifier = class {
|
|
|
15085
15588
|
async onSiteLivenessChecked(projectId, result) {
|
|
15086
15589
|
const { check, checkedAt } = result;
|
|
15087
15590
|
if (check.status !== "ok" && check.status !== "fail") return null;
|
|
15088
|
-
const project = this.db.select().from(projects).where(
|
|
15591
|
+
const project = this.db.select().from(projects).where(eq25(projects.id, projectId)).get();
|
|
15089
15592
|
if (!project) {
|
|
15090
|
-
|
|
15593
|
+
log24.error("project.not-found", { projectId, msg: "skipping site liveness notification" });
|
|
15091
15594
|
return null;
|
|
15092
15595
|
}
|
|
15093
|
-
const previous = this.db.select().from(siteLivenessState).where(
|
|
15596
|
+
const previous = this.db.select().from(siteLivenessState).where(eq25(siteLivenessState.projectId, projectId)).get();
|
|
15094
15597
|
const sincePrevious = previous ? Date.parse(checkedAt) - Date.parse(previous.checkedAt) : Number.NaN;
|
|
15095
15598
|
const countsAsRepeat = Number.isFinite(sincePrevious) && sincePrevious >= SITE_LIVENESS_MIN_PASS_GAP_MS && sincePrevious <= SITE_LIVENESS_MAX_PASS_GAP_MS;
|
|
15096
15599
|
const failures = check.status === "fail" ? countsAsRepeat ? (previous?.consecutiveFailures ?? 0) + 1 : 1 : 0;
|
|
@@ -15100,7 +15603,7 @@ var Notifier = class {
|
|
|
15100
15603
|
if (check.status === "ok" && outagePaged) event = "health.recovered";
|
|
15101
15604
|
const confirmed = check.status === "fail" && failures >= SITE_LIVENESS_FAILURES_TO_PAGE;
|
|
15102
15605
|
const urgentRecord = previous?.urgentNotifiedAt ?? null;
|
|
15103
|
-
const enabled = event !== null || confirmed || urgentRecord !== null ? this.db.select().from(notifications).where(
|
|
15606
|
+
const enabled = event !== null || confirmed || urgentRecord !== null ? this.db.select().from(notifications).where(eq25(notifications.projectId, projectId)).all().filter((n) => n.enabled) : [];
|
|
15104
15607
|
const listedEvents = (notif) => notif.config.events;
|
|
15105
15608
|
const hasUrl = (notif) => Boolean(notif.config.url);
|
|
15106
15609
|
const scoped = enabled.filter((n) => hasUrl(n) && (listedEvents(n) ?? []).some((e) => URGENT_HEALTH_EVENTS.has(e)));
|
|
@@ -15133,7 +15636,7 @@ var Notifier = class {
|
|
|
15133
15636
|
if (previous === void 0) {
|
|
15134
15637
|
this.db.insert(siteLivenessState).values(observation).run();
|
|
15135
15638
|
} else {
|
|
15136
|
-
this.db.update(siteLivenessState).set(observation).where(
|
|
15639
|
+
this.db.update(siteLivenessState).set(observation).where(eq25(siteLivenessState.projectId, projectId)).run();
|
|
15137
15640
|
}
|
|
15138
15641
|
if (!event && urgentGroups.every((group) => group.urgentEvent === null)) return null;
|
|
15139
15642
|
const payloadFor = (name) => {
|
|
@@ -15169,9 +15672,9 @@ var Notifier = class {
|
|
|
15169
15672
|
}
|
|
15170
15673
|
}
|
|
15171
15674
|
if (event === "health.degraded" && delivered > 0) {
|
|
15172
|
-
this.db.update(siteLivenessState).set({ notifiedAt: checkedAt }).where(
|
|
15675
|
+
this.db.update(siteLivenessState).set({ notifiedAt: checkedAt }).where(eq25(siteLivenessState.projectId, projectId)).run();
|
|
15173
15676
|
}
|
|
15174
|
-
|
|
15677
|
+
log24.info("site-liveness.notified", { projectId, event, code: check.code, subscribers: notifs.length, delivered });
|
|
15175
15678
|
}
|
|
15176
15679
|
for (const group of urgentGroups) {
|
|
15177
15680
|
const { urgentEvent, urgentNotifs, generalIds } = group;
|
|
@@ -15183,10 +15686,10 @@ var Notifier = class {
|
|
|
15183
15686
|
}
|
|
15184
15687
|
if (accepted > 0) {
|
|
15185
15688
|
recordUrgentReceipts(urgentReceipts, group.subscribers, group.settled);
|
|
15186
|
-
this.db.update(siteLivenessState).set({ urgentNotifiedAt: siteUrgentSummary(urgentReceipts), urgentReceipts }).where(
|
|
15689
|
+
this.db.update(siteLivenessState).set({ urgentNotifiedAt: siteUrgentSummary(urgentReceipts), urgentReceipts }).where(eq25(siteLivenessState.projectId, projectId)).run();
|
|
15187
15690
|
}
|
|
15188
15691
|
if (urgentNotifs.length > 0 || generalIds.size > 0) {
|
|
15189
|
-
|
|
15692
|
+
log24.info("site-liveness.urgent-notified", { projectId, event: urgentEvent, code: check.code, subscribers: urgentNotifs.length + generalIds.size, delivered: accepted });
|
|
15190
15693
|
}
|
|
15191
15694
|
}
|
|
15192
15695
|
return event;
|
|
@@ -15209,7 +15712,7 @@ var Notifier = class {
|
|
|
15209
15712
|
*/
|
|
15210
15713
|
isNewInsightNews(projectId, insight) {
|
|
15211
15714
|
const key = insightNotifyKey(projectId, insight);
|
|
15212
|
-
const previous = this.db.select().from(insightNotifyState).where(
|
|
15715
|
+
const previous = this.db.select().from(insightNotifyState).where(eq25(insightNotifyState.key, key)).get();
|
|
15213
15716
|
if (!previous) return true;
|
|
15214
15717
|
if (severityRank(insight.severity) > severityRank(previous.severity)) return true;
|
|
15215
15718
|
const magnitude = insightMagnitude(insight);
|
|
@@ -15231,9 +15734,9 @@ var Notifier = class {
|
|
|
15231
15734
|
magnitude: insightMagnitude(insight),
|
|
15232
15735
|
notifiedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
15233
15736
|
};
|
|
15234
|
-
const existing = this.db.select().from(insightNotifyState).where(
|
|
15737
|
+
const existing = this.db.select().from(insightNotifyState).where(eq25(insightNotifyState.key, key)).get();
|
|
15235
15738
|
if (existing) {
|
|
15236
|
-
this.db.update(insightNotifyState).set(row).where(
|
|
15739
|
+
this.db.update(insightNotifyState).set(row).where(eq25(insightNotifyState.key, key)).run();
|
|
15237
15740
|
} else {
|
|
15238
15741
|
this.db.insert(insightNotifyState).values(row).run();
|
|
15239
15742
|
}
|
|
@@ -15247,11 +15750,11 @@ var Notifier = class {
|
|
|
15247
15750
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
15248
15751
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
15249
15752
|
if (insightEvents.length === 0) return;
|
|
15250
|
-
const notifs = this.db.select().from(notifications).where(
|
|
15753
|
+
const notifs = this.db.select().from(notifications).where(eq25(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
15251
15754
|
if (notifs.length === 0) return;
|
|
15252
|
-
const run = this.db.select().from(runs).where(
|
|
15755
|
+
const run = this.db.select().from(runs).where(eq25(runs.id, runId)).get();
|
|
15253
15756
|
if (!run) return;
|
|
15254
|
-
const project = this.db.select().from(projects).where(
|
|
15757
|
+
const project = this.db.select().from(projects).where(eq25(projects.id, projectId)).get();
|
|
15255
15758
|
if (!project) return;
|
|
15256
15759
|
for (const notif of notifs) {
|
|
15257
15760
|
const config = notif.config;
|
|
@@ -15332,9 +15835,9 @@ var Notifier = class {
|
|
|
15332
15835
|
return delivered;
|
|
15333
15836
|
}
|
|
15334
15837
|
async dispatchQueuedReviewAlerts(projectId) {
|
|
15335
|
-
const pendingReviews = this.db.select().from(gbpReviews).where(
|
|
15336
|
-
const pendingRatings = this.db.select().from(gbpReviewRatings).where(
|
|
15337
|
-
const project = this.db.select().from(projects).where(
|
|
15838
|
+
const pendingReviews = this.db.select().from(gbpReviews).where(and22(eq25(gbpReviews.projectId, projectId), eq25(gbpReviews.alertState, "pending"))).all();
|
|
15839
|
+
const pendingRatings = this.db.select().from(gbpReviewRatings).where(and22(eq25(gbpReviewRatings.projectId, projectId), eq25(gbpReviewRatings.alertState, "pending"))).all();
|
|
15840
|
+
const project = this.db.select().from(projects).where(eq25(projects.id, projectId)).get();
|
|
15338
15841
|
if (!project) return { reviews: 0, ratingDrops: 0, delivered: 0 };
|
|
15339
15842
|
const chatSendsLeft = /* @__PURE__ */ new Map();
|
|
15340
15843
|
const now = /* @__PURE__ */ new Date();
|
|
@@ -15346,10 +15849,10 @@ var Notifier = class {
|
|
|
15346
15849
|
displayName: gbpLocations.displayName,
|
|
15347
15850
|
mapsUri: gbpLocations.mapsUri,
|
|
15348
15851
|
reviewsAccess: gbpLocations.reviewsAccess
|
|
15349
|
-
}).from(gbpLocations).where(
|
|
15852
|
+
}).from(gbpLocations).where(eq25(gbpLocations.projectId, projectId)).all().map((loc) => [loc.locationName, loc])
|
|
15350
15853
|
);
|
|
15351
15854
|
const v4HasWorked = new Set(
|
|
15352
|
-
this.db.select({ locationName: gbpReviewRatings.locationName }).from(gbpReviewRatings).where(
|
|
15855
|
+
this.db.select({ locationName: gbpReviewRatings.locationName }).from(gbpReviewRatings).where(and22(eq25(gbpReviewRatings.projectId, projectId), eq25(gbpReviewRatings.origin, "gbp"))).all().map((row) => row.locationName)
|
|
15353
15856
|
);
|
|
15354
15857
|
const coveredByBusinessProfile = (locationName) => businessProfileReviewsCover(locations.get(locationName)?.reviewsAccess ?? null, v4HasWorked.has(locationName));
|
|
15355
15858
|
const locationRef = (locationName) => {
|
|
@@ -15378,7 +15881,7 @@ var Notifier = class {
|
|
|
15378
15881
|
else if (row.rating === null || row.previousRating === null) setRatingState([row.id], "none");
|
|
15379
15882
|
else ratingsToSend.push(row);
|
|
15380
15883
|
}
|
|
15381
|
-
const enabled = this.db.select().from(notifications).where(
|
|
15884
|
+
const enabled = this.db.select().from(notifications).where(eq25(notifications.projectId, projectId)).all().filter((n) => n.enabled && n.config.url);
|
|
15382
15885
|
const subscribedTo = (event) => enabled.filter((n) => (n.config.events ?? []).includes(event));
|
|
15383
15886
|
const projectRef = { name: project.name, canonicalDomain: project.canonicalDomain };
|
|
15384
15887
|
const dashboardUrl = `${this.serverUrl}/projects/${project.name}`;
|
|
@@ -15432,13 +15935,13 @@ var Notifier = class {
|
|
|
15432
15935
|
if (!sent) all = false;
|
|
15433
15936
|
}
|
|
15434
15937
|
if (all) {
|
|
15435
|
-
this.db.update(gbpReviews).set({ alertState: "sent", alertStateAt: nowIso }).where(
|
|
15436
|
-
|
|
15437
|
-
|
|
15438
|
-
|
|
15439
|
-
review.alertStateAt === null ? isNull5(gbpReviews.alertStateAt) :
|
|
15440
|
-
review.comment === null ? isNull5(gbpReviews.comment) :
|
|
15441
|
-
review.starRating === null ? isNull5(gbpReviews.starRating) :
|
|
15938
|
+
this.db.update(gbpReviews).set({ alertState: "sent", alertStateAt: nowIso }).where(and22(
|
|
15939
|
+
eq25(gbpReviews.id, review.id),
|
|
15940
|
+
eq25(gbpReviews.alertState, "pending"),
|
|
15941
|
+
eq25(gbpReviews.updateTime, review.updateTime),
|
|
15942
|
+
review.alertStateAt === null ? isNull5(gbpReviews.alertStateAt) : eq25(gbpReviews.alertStateAt, review.alertStateAt),
|
|
15943
|
+
review.comment === null ? isNull5(gbpReviews.comment) : eq25(gbpReviews.comment, review.comment),
|
|
15944
|
+
review.starRating === null ? isNull5(gbpReviews.starRating) : eq25(gbpReviews.starRating, review.starRating)
|
|
15442
15945
|
)).run();
|
|
15443
15946
|
}
|
|
15444
15947
|
}
|
|
@@ -15470,7 +15973,7 @@ var Notifier = class {
|
|
|
15470
15973
|
delivered += accepted;
|
|
15471
15974
|
}
|
|
15472
15975
|
}
|
|
15473
|
-
|
|
15976
|
+
log24.info("reviews.dispatched", {
|
|
15474
15977
|
projectId,
|
|
15475
15978
|
reviews: reviewsToSend.length,
|
|
15476
15979
|
ratingDrops: ratingsToSend.length,
|
|
@@ -15479,21 +15982,21 @@ var Notifier = class {
|
|
|
15479
15982
|
return { reviews: reviewsToSend.length, ratingDrops: ratingsToSend.length, delivered };
|
|
15480
15983
|
}
|
|
15481
15984
|
computeTransitions(runId, projectId) {
|
|
15482
|
-
const thisRun = this.db.select().from(runs).where(
|
|
15985
|
+
const thisRun = this.db.select().from(runs).where(eq25(runs.id, runId)).get();
|
|
15483
15986
|
if (!thisRun) return [];
|
|
15484
15987
|
const completeness = measurementRunCompleteness(this.db, runId);
|
|
15485
15988
|
if (completeness.planned && !completeness.complete) {
|
|
15486
|
-
|
|
15989
|
+
log24.info("transitions.skipped-incomplete", {
|
|
15487
15990
|
runId,
|
|
15488
15991
|
executed: completeness.executed,
|
|
15489
15992
|
expected: completeness.expected
|
|
15490
15993
|
});
|
|
15491
15994
|
return [];
|
|
15492
15995
|
}
|
|
15493
|
-
const groupSiblings = this.db.select().from(runs).where(
|
|
15494
|
-
|
|
15495
|
-
|
|
15496
|
-
|
|
15996
|
+
const groupSiblings = this.db.select().from(runs).where(and22(
|
|
15997
|
+
eq25(runs.projectId, projectId),
|
|
15998
|
+
eq25(runs.kind, thisRun.kind),
|
|
15999
|
+
eq25(runs.createdAt, thisRun.createdAt),
|
|
15497
16000
|
notProbeRun()
|
|
15498
16001
|
)).all();
|
|
15499
16002
|
const stillPending = groupSiblings.some((r) => r.status === "queued" || r.status === "running");
|
|
@@ -15510,17 +16013,17 @@ var Notifier = class {
|
|
|
15510
16013
|
return candidate.id > best.id ? candidate : best;
|
|
15511
16014
|
});
|
|
15512
16015
|
if (winner.id !== runId) return [];
|
|
15513
|
-
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
16016
|
+
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(eq25(projects.id, projectId)).get();
|
|
15514
16017
|
const locationCount = Math.max(
|
|
15515
16018
|
1,
|
|
15516
16019
|
(projectLocations?.locations ?? []).length
|
|
15517
16020
|
);
|
|
15518
16021
|
const RECENT_FETCH_LIMIT = Math.max(8, locationCount * 4);
|
|
15519
16022
|
const recentRuns = this.db.select().from(runs).where(
|
|
15520
|
-
|
|
15521
|
-
|
|
15522
|
-
|
|
15523
|
-
or2(
|
|
16023
|
+
and22(
|
|
16024
|
+
eq25(runs.projectId, projectId),
|
|
16025
|
+
eq25(runs.kind, thisRun.kind),
|
|
16026
|
+
or2(eq25(runs.status, "completed"), eq25(runs.status, "partial")),
|
|
15524
16027
|
notProbeRun()
|
|
15525
16028
|
)
|
|
15526
16029
|
).orderBy(desc9(runs.createdAt), desc9(runs.id)).limit(RECENT_FETCH_LIMIT).all();
|
|
@@ -15538,7 +16041,7 @@ var Notifier = class {
|
|
|
15538
16041
|
provider: querySnapshots.provider,
|
|
15539
16042
|
location: querySnapshots.location,
|
|
15540
16043
|
citationState: querySnapshots.citationState
|
|
15541
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
16044
|
+
}).from(querySnapshots).leftJoin(queries, eq25(querySnapshots.queryId, queries.id)).where(inArray12(querySnapshots.runId, currentRunIds)).all();
|
|
15542
16045
|
const previousSnapshots = this.db.select({
|
|
15543
16046
|
queryId: querySnapshots.queryId,
|
|
15544
16047
|
provider: querySnapshots.provider,
|
|
@@ -15589,12 +16092,12 @@ var Notifier = class {
|
|
|
15589
16092
|
recordUnrecognizedAgentDecisions(this.db, projectId, baseline, UnrecognizedAgentNotifyStates.baseline, now);
|
|
15590
16093
|
if (due.length === 0) continue;
|
|
15591
16094
|
const names = due.map((agent) => agent.name);
|
|
15592
|
-
const subscribed = this.db.select().from(notifications).where(
|
|
16095
|
+
const subscribed = this.db.select().from(notifications).where(eq25(notifications.projectId, projectId)).all().filter((n) => n.enabled && n.config.url && n.config.events.includes("traffic.agent.discovered"));
|
|
15593
16096
|
if (subscribed.length === 0) {
|
|
15594
16097
|
recordUnrecognizedAgentDecisions(this.db, projectId, names, UnrecognizedAgentNotifyStates.skipped, now);
|
|
15595
16098
|
continue;
|
|
15596
16099
|
}
|
|
15597
|
-
const project = this.db.select().from(projects).where(
|
|
16100
|
+
const project = this.db.select().from(projects).where(eq25(projects.id, projectId)).get();
|
|
15598
16101
|
if (!project) continue;
|
|
15599
16102
|
const payload = {
|
|
15600
16103
|
source: "canonry",
|
|
@@ -15625,7 +16128,7 @@ var Notifier = class {
|
|
|
15625
16128
|
const targetLabel = redactNotificationUrl(url).urlDisplay;
|
|
15626
16129
|
const targetCheck = await resolveWebhookTarget(url, { allowLoopback: this.allowLoopbackWebhooks });
|
|
15627
16130
|
if (!targetCheck.ok) {
|
|
15628
|
-
|
|
16131
|
+
log24.error("webhook.ssrf-blocked", { url: targetLabel, reason: targetCheck.message });
|
|
15629
16132
|
this.logDelivery(projectId, notificationId, payload.event, "failed", `SSRF: ${targetCheck.message}`);
|
|
15630
16133
|
const reasonCode = webhookTargetRefusalReason(targetCheck);
|
|
15631
16134
|
trackWebhookDelivery({
|
|
@@ -15638,7 +16141,7 @@ var Notifier = class {
|
|
|
15638
16141
|
});
|
|
15639
16142
|
return false;
|
|
15640
16143
|
}
|
|
15641
|
-
|
|
16144
|
+
log24.info("webhook.send", { event: payload.event, url: targetLabel });
|
|
15642
16145
|
const maxRetries = 3;
|
|
15643
16146
|
const delays = [1e3, 4e3, 16e3];
|
|
15644
16147
|
let lastStatusClass = "network";
|
|
@@ -15647,13 +16150,13 @@ var Notifier = class {
|
|
|
15647
16150
|
try {
|
|
15648
16151
|
const response = await deliverWebhook(targetCheck.target, body, signingSecret);
|
|
15649
16152
|
if (response.status >= 200 && response.status < 300) {
|
|
15650
|
-
|
|
16153
|
+
log24.info("webhook.delivered", { event: payload.event, url: targetLabel, httpStatus: response.status });
|
|
15651
16154
|
this.logDelivery(projectId, notificationId, payload.event, "sent", null);
|
|
15652
16155
|
trackWebhookDelivery({ url, event: payload.event, attempts: attempt + 1, durationBucket: elapsed(), statusClass: "2xx" });
|
|
15653
16156
|
return true;
|
|
15654
16157
|
}
|
|
15655
16158
|
const errorDetail = response.error ?? `HTTP ${response.status}`;
|
|
15656
|
-
|
|
16159
|
+
log24.warn("webhook.attempt-failed", { event: payload.event, url: targetLabel, attempt: attempt + 1, maxRetries, httpStatus: response.status, error: errorDetail });
|
|
15657
16160
|
lastStatusClass = response.status === 0 && response.timedOut ? "timeout" : statusClassOf(response.status);
|
|
15658
16161
|
lastFailure = { reasonCode: webhookResponseReason(response) ?? "UNKNOWN" };
|
|
15659
16162
|
if (attempt === maxRetries - 1) {
|
|
@@ -15665,7 +16168,7 @@ var Notifier = class {
|
|
|
15665
16168
|
lastFailure = outcomeFailure(err);
|
|
15666
16169
|
if (attempt === maxRetries - 1) {
|
|
15667
16170
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
15668
|
-
|
|
16171
|
+
log24.error("webhook.exhausted", { event: payload.event, url: targetLabel, maxRetries, error: errorDetail });
|
|
15669
16172
|
}
|
|
15670
16173
|
}
|
|
15671
16174
|
if (attempt < maxRetries - 1) {
|
|
@@ -15684,7 +16187,7 @@ var Notifier = class {
|
|
|
15684
16187
|
}
|
|
15685
16188
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
15686
16189
|
this.db.insert(auditLog).values({
|
|
15687
|
-
id:
|
|
16190
|
+
id: crypto22.randomUUID(),
|
|
15688
16191
|
projectId,
|
|
15689
16192
|
actor: "scheduler",
|
|
15690
16193
|
action: `notification.${status}`,
|
|
@@ -15697,8 +16200,8 @@ var Notifier = class {
|
|
|
15697
16200
|
};
|
|
15698
16201
|
|
|
15699
16202
|
// src/run-coordinator.ts
|
|
15700
|
-
import { eq as
|
|
15701
|
-
var
|
|
16203
|
+
import { eq as eq26 } from "drizzle-orm";
|
|
16204
|
+
var log25 = createLogger("RunCoordinator");
|
|
15702
16205
|
function reportInsightsGenerated(runTrigger, outcome) {
|
|
15703
16206
|
trackFeatureCompleted({
|
|
15704
16207
|
feature: "insights",
|
|
@@ -15731,22 +16234,22 @@ var RunCoordinator = class {
|
|
|
15731
16234
|
*/
|
|
15732
16235
|
onAnswersRecorded;
|
|
15733
16236
|
async onRunCompleted(runId, projectId, opts) {
|
|
15734
|
-
const runRow = this.db.select().from(runs).where(
|
|
16237
|
+
const runRow = this.db.select().from(runs).where(eq26(runs.id, runId)).get();
|
|
15735
16238
|
const kind = runRow?.kind ?? RunKinds["answer-visibility"];
|
|
15736
16239
|
if (runRow?.trigger === RunTriggers.probe) {
|
|
15737
|
-
|
|
16240
|
+
log25.info("probe.skip-side-effects", { runId, projectId, kind });
|
|
15738
16241
|
return;
|
|
15739
16242
|
}
|
|
15740
16243
|
if (kind === RunKinds["answer-visibility"] && (runRow?.status === RunStatuses.completed || runRow?.status === RunStatuses.partial) && this.onAnswersRecorded) {
|
|
15741
16244
|
try {
|
|
15742
16245
|
await this.onAnswersRecorded(runId, projectId);
|
|
15743
16246
|
} catch (err) {
|
|
15744
|
-
|
|
16247
|
+
log25.error("answers-recorded.failed", { runId, projectId, error: describeError(err) });
|
|
15745
16248
|
}
|
|
15746
16249
|
}
|
|
15747
16250
|
const completeness = measurementRunCompleteness(this.db, runId);
|
|
15748
16251
|
if (completeness.planned && !completeness.complete) {
|
|
15749
|
-
|
|
16252
|
+
log25.info("measurement.incomplete-skip-derived", {
|
|
15750
16253
|
runId,
|
|
15751
16254
|
projectId,
|
|
15752
16255
|
executed: completeness.executed,
|
|
@@ -15755,7 +16258,7 @@ var RunCoordinator = class {
|
|
|
15755
16258
|
try {
|
|
15756
16259
|
await this.notifier.onRunCompleted(runId, projectId);
|
|
15757
16260
|
} catch (err) {
|
|
15758
|
-
|
|
16261
|
+
log25.error("notifier.failed", { runId, error: describeError(err) });
|
|
15759
16262
|
}
|
|
15760
16263
|
return;
|
|
15761
16264
|
}
|
|
@@ -15763,7 +16266,7 @@ var RunCoordinator = class {
|
|
|
15763
16266
|
try {
|
|
15764
16267
|
await this.onSentimentCompleted(runId, projectId);
|
|
15765
16268
|
} catch {
|
|
15766
|
-
|
|
16269
|
+
log25.error("sentiment.wakeup-failed", { runId, projectId });
|
|
15767
16270
|
}
|
|
15768
16271
|
}
|
|
15769
16272
|
let insightCount = 0;
|
|
@@ -15783,12 +16286,12 @@ var RunCoordinator = class {
|
|
|
15783
16286
|
try {
|
|
15784
16287
|
await this.onInsightsGenerated(runId, projectId, result);
|
|
15785
16288
|
} catch (err) {
|
|
15786
|
-
|
|
16289
|
+
log25.error("insight-webhook.failed", { runId, error: describeError(err) });
|
|
15787
16290
|
}
|
|
15788
16291
|
}
|
|
15789
16292
|
}
|
|
15790
16293
|
} catch (err) {
|
|
15791
|
-
|
|
16294
|
+
log25.error("intelligence.failed", { runId, error: describeError(err) });
|
|
15792
16295
|
insightsOutcome = { status: OutcomeStatuses.failed, ...outcomeFailure(err), durationBucket: elapsed() };
|
|
15793
16296
|
}
|
|
15794
16297
|
reportInsightsGenerated(runRow?.trigger, insightsOutcome);
|
|
@@ -15806,24 +16309,24 @@ var RunCoordinator = class {
|
|
|
15806
16309
|
try {
|
|
15807
16310
|
await this.onInsightsGenerated(runId, projectId, analysisResultFromInsights(gbpInsights));
|
|
15808
16311
|
} catch (err) {
|
|
15809
|
-
|
|
16312
|
+
log25.error("gbp-insight-webhook.failed", { runId, error: describeError(err) });
|
|
15810
16313
|
}
|
|
15811
16314
|
}
|
|
15812
16315
|
} catch (err) {
|
|
15813
|
-
|
|
16316
|
+
log25.error("gbp-intelligence.failed", { runId, error: describeError(err) });
|
|
15814
16317
|
insightsOutcome = { status: OutcomeStatuses.failed, ...outcomeFailure(err), durationBucket: elapsed() };
|
|
15815
16318
|
}
|
|
15816
16319
|
reportInsightsGenerated(runRow?.trigger, insightsOutcome);
|
|
15817
16320
|
try {
|
|
15818
16321
|
await this.notifier.dispatchReviewAlerts(projectId);
|
|
15819
16322
|
} catch (err) {
|
|
15820
|
-
|
|
16323
|
+
log25.error("review-webhook.failed", { runId, error: describeError(err) });
|
|
15821
16324
|
}
|
|
15822
16325
|
}
|
|
15823
16326
|
try {
|
|
15824
16327
|
await (opts ? this.notifier.onRunCompleted(runId, projectId, opts) : this.notifier.onRunCompleted(runId, projectId));
|
|
15825
16328
|
} catch (err) {
|
|
15826
|
-
|
|
16329
|
+
log25.error("notifier.failed", { runId, error: describeError(err) });
|
|
15827
16330
|
}
|
|
15828
16331
|
if (this.onAeroEvent) {
|
|
15829
16332
|
try {
|
|
@@ -15836,7 +16339,7 @@ var RunCoordinator = class {
|
|
|
15836
16339
|
};
|
|
15837
16340
|
await this.onAeroEvent(ctx);
|
|
15838
16341
|
} catch (err) {
|
|
15839
|
-
|
|
16342
|
+
log25.error("aero.failed", { runId, error: describeError(err) });
|
|
15840
16343
|
}
|
|
15841
16344
|
}
|
|
15842
16345
|
}
|
|
@@ -15851,7 +16354,7 @@ var RunCoordinator = class {
|
|
|
15851
16354
|
* so the Aero queue is never starved of a follow-up.
|
|
15852
16355
|
*/
|
|
15853
16356
|
buildDiscoveryAeroContext(runId, projectId, status, error) {
|
|
15854
|
-
const session = this.db.select().from(discoverySessions).where(
|
|
16357
|
+
const session = this.db.select().from(discoverySessions).where(eq26(discoverySessions.runId, runId)).get();
|
|
15855
16358
|
const competitorMap = session ? session.competitorMap : [];
|
|
15856
16359
|
return {
|
|
15857
16360
|
kind: RunKinds["aeo-discover-probe"],
|
|
@@ -15893,13 +16396,13 @@ function analysisResultFromInsights(insights2) {
|
|
|
15893
16396
|
}
|
|
15894
16397
|
|
|
15895
16398
|
// src/agent/session-registry.ts
|
|
15896
|
-
import
|
|
15897
|
-
import { eq as
|
|
16399
|
+
import crypto24 from "crypto";
|
|
16400
|
+
import { eq as eq29 } from "drizzle-orm";
|
|
15898
16401
|
|
|
15899
16402
|
// src/agent/remote-mcp.ts
|
|
15900
16403
|
import { Type } from "@earendil-works/pi-ai";
|
|
15901
16404
|
import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
|
|
15902
|
-
var
|
|
16405
|
+
var log26 = createLogger("RemoteMcp");
|
|
15903
16406
|
var AERO_EXCLUDED_MCP_TOOLS = /* @__PURE__ */ new Set([]);
|
|
15904
16407
|
async function connectStreamableHttp(server) {
|
|
15905
16408
|
const transport = new StreamableHTTPClientTransport(new URL(server.url), {
|
|
@@ -15967,7 +16470,7 @@ async function loadExternalMcpTools(servers, opts = {}) {
|
|
|
15967
16470
|
try {
|
|
15968
16471
|
client = await connect(server);
|
|
15969
16472
|
} catch (err) {
|
|
15970
|
-
|
|
16473
|
+
log26.error("external-mcp.connect-failed", {
|
|
15971
16474
|
label,
|
|
15972
16475
|
error: describeError(err)
|
|
15973
16476
|
});
|
|
@@ -15977,7 +16480,7 @@ async function loadExternalMcpTools(servers, opts = {}) {
|
|
|
15977
16480
|
try {
|
|
15978
16481
|
listed = await client.listTools();
|
|
15979
16482
|
} catch (err) {
|
|
15980
|
-
|
|
16483
|
+
log26.error("external-mcp.list-failed", {
|
|
15981
16484
|
label,
|
|
15982
16485
|
error: describeError(err)
|
|
15983
16486
|
});
|
|
@@ -15988,7 +16491,7 @@ async function loadExternalMcpTools(servers, opts = {}) {
|
|
|
15988
16491
|
if (!isReadOnly(tool)) continue;
|
|
15989
16492
|
tools.push(adaptRemoteTool(client, tool));
|
|
15990
16493
|
}
|
|
15991
|
-
|
|
16494
|
+
log26.info("external-mcp.loaded", {
|
|
15992
16495
|
label,
|
|
15993
16496
|
discovered: listed.tools.length,
|
|
15994
16497
|
adopted: tools.length
|
|
@@ -15998,8 +16501,8 @@ async function loadExternalMcpTools(servers, opts = {}) {
|
|
|
15998
16501
|
}
|
|
15999
16502
|
|
|
16000
16503
|
// src/agent/memory-store.ts
|
|
16001
|
-
import
|
|
16002
|
-
import { and as
|
|
16504
|
+
import crypto23 from "crypto";
|
|
16505
|
+
import { and as and23, desc as desc10, eq as eq27, like, sql as sql12, or as or3 } from "drizzle-orm";
|
|
16003
16506
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
16004
16507
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
16005
16508
|
function rowToDto(row) {
|
|
@@ -16013,7 +16516,7 @@ function rowToDto(row) {
|
|
|
16013
16516
|
};
|
|
16014
16517
|
}
|
|
16015
16518
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
16016
|
-
const query = db.select().from(agentMemory).where(
|
|
16519
|
+
const query = db.select().from(agentMemory).where(eq27(agentMemory.projectId, projectId)).orderBy(desc10(agentMemory.updatedAt));
|
|
16017
16520
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
16018
16521
|
return rows.map(rowToDto);
|
|
16019
16522
|
}
|
|
@@ -16027,7 +16530,7 @@ function upsertMemoryEntry(db, args) {
|
|
|
16027
16530
|
throw new Error(`memory key prefix "${COMPACTION_KEY_PREFIX}" is reserved for compaction notes`);
|
|
16028
16531
|
}
|
|
16029
16532
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
16030
|
-
const id =
|
|
16533
|
+
const id = crypto23.randomUUID();
|
|
16031
16534
|
db.insert(agentMemory).values({
|
|
16032
16535
|
id,
|
|
16033
16536
|
projectId: args.projectId,
|
|
@@ -16044,21 +16547,21 @@ function upsertMemoryEntry(db, args) {
|
|
|
16044
16547
|
updatedAt: now
|
|
16045
16548
|
}
|
|
16046
16549
|
}).run();
|
|
16047
|
-
const row = db.select().from(agentMemory).where(
|
|
16550
|
+
const row = db.select().from(agentMemory).where(and23(eq27(agentMemory.projectId, args.projectId), eq27(agentMemory.key, args.key))).get();
|
|
16048
16551
|
if (!row) throw new Error("memory upsert produced no row");
|
|
16049
16552
|
return rowToDto(row);
|
|
16050
16553
|
}
|
|
16051
16554
|
function deleteMemoryEntry(db, projectId, key) {
|
|
16052
|
-
const result = db.delete(agentMemory).where(
|
|
16555
|
+
const result = db.delete(agentMemory).where(and23(eq27(agentMemory.projectId, projectId), eq27(agentMemory.key, key))).run();
|
|
16053
16556
|
const changes = result.changes ?? 0;
|
|
16054
16557
|
return changes > 0;
|
|
16055
16558
|
}
|
|
16056
16559
|
function loadRecentForHydrate(db, projectId, limit, sessionId) {
|
|
16057
16560
|
if (sessionId === void 0) return listMemoryEntries(db, projectId, { limit });
|
|
16058
16561
|
const sessionPrefix = `compaction:${sessionId ?? ""}:`;
|
|
16059
|
-
return db.select().from(agentMemory).where(
|
|
16060
|
-
|
|
16061
|
-
or3(
|
|
16562
|
+
return db.select().from(agentMemory).where(and23(
|
|
16563
|
+
eq27(agentMemory.projectId, projectId),
|
|
16564
|
+
or3(sql12`substr(${agentMemory.key}, 1, ${COMPACTION_KEY_PREFIX.length}) <> ${COMPACTION_KEY_PREFIX}`, sessionId ? sql12`substr(${agentMemory.key}, 1, ${sessionPrefix.length}) = ${sessionPrefix}` : void 0)
|
|
16062
16565
|
)).orderBy(desc10(agentMemory.updatedAt)).limit(limit).all().map(rowToDto);
|
|
16063
16566
|
}
|
|
16064
16567
|
function writeCompactionNote(db, args) {
|
|
@@ -16069,7 +16572,7 @@ function writeCompactionNote(db, args) {
|
|
|
16069
16572
|
}
|
|
16070
16573
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
16071
16574
|
const key = `${COMPACTION_KEY_PREFIX}${args.sessionId}:${now}`;
|
|
16072
|
-
const id =
|
|
16575
|
+
const id = crypto23.randomUUID();
|
|
16073
16576
|
let inserted;
|
|
16074
16577
|
db.transaction((tx) => {
|
|
16075
16578
|
tx.insert(agentMemory).values({
|
|
@@ -16083,16 +16586,16 @@ function writeCompactionNote(db, args) {
|
|
|
16083
16586
|
}).run();
|
|
16084
16587
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
16085
16588
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
16086
|
-
|
|
16087
|
-
|
|
16589
|
+
and23(
|
|
16590
|
+
eq27(agentMemory.projectId, args.projectId),
|
|
16088
16591
|
like(agentMemory.key, `${sessionPrefix}%`)
|
|
16089
16592
|
)
|
|
16090
16593
|
).orderBy(desc10(agentMemory.updatedAt)).all();
|
|
16091
16594
|
const stale = existing.slice(COMPACTION_NOTES_PER_SESSION).map((r) => r.id);
|
|
16092
16595
|
if (stale.length > 0) {
|
|
16093
|
-
tx.delete(agentMemory).where(
|
|
16596
|
+
tx.delete(agentMemory).where(sql12`${agentMemory.id} IN (${sql12.join(stale.map((s) => sql12`${s}`), sql12`, `)})`).run();
|
|
16094
16597
|
}
|
|
16095
|
-
const row = tx.select().from(agentMemory).where(
|
|
16598
|
+
const row = tx.select().from(agentMemory).where(and23(eq27(agentMemory.projectId, args.projectId), eq27(agentMemory.key, key))).get();
|
|
16096
16599
|
if (row) inserted = rowToDto(row);
|
|
16097
16600
|
});
|
|
16098
16601
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -16166,7 +16669,7 @@ function trackAeroTurn(agent, turn) {
|
|
|
16166
16669
|
}
|
|
16167
16670
|
|
|
16168
16671
|
// src/agent/project-shape.ts
|
|
16169
|
-
import { and as
|
|
16672
|
+
import { and as and24, count as count5, eq as eq28 } from "drizzle-orm";
|
|
16170
16673
|
var ADVANCED_TOOLS = [
|
|
16171
16674
|
"canonry_measurement_portfolio_summary",
|
|
16172
16675
|
"canonry_measurement_overview",
|
|
@@ -16193,10 +16696,10 @@ var ADVANCED_ROUTES = [
|
|
|
16193
16696
|
];
|
|
16194
16697
|
function aeroProjectShape(db, projectId) {
|
|
16195
16698
|
try {
|
|
16196
|
-
const pointer = db.select().from(measurementPlans).where(
|
|
16197
|
-
const version = pointer ? db.select().from(measurementPlanVersions).where(
|
|
16198
|
-
|
|
16199
|
-
|
|
16699
|
+
const pointer = db.select().from(measurementPlans).where(eq28(measurementPlans.projectId, projectId)).get();
|
|
16700
|
+
const version = pointer ? db.select().from(measurementPlanVersions).where(and24(
|
|
16701
|
+
eq28(measurementPlanVersions.projectId, projectId),
|
|
16702
|
+
eq28(measurementPlanVersions.id, pointer.activeVersionId)
|
|
16200
16703
|
)).get() : void 0;
|
|
16201
16704
|
if (version) {
|
|
16202
16705
|
const plan = JSON.parse(version.canonicalJson);
|
|
@@ -16223,7 +16726,7 @@ canonry_visibility_report describes the whole project, not one Property. canonry
|
|
|
16223
16726
|
pinned: ADVANCED_TOOLS
|
|
16224
16727
|
};
|
|
16225
16728
|
}
|
|
16226
|
-
const tracked = db.select({ total:
|
|
16729
|
+
const tracked = db.select({ total: count5() }).from(queries).where(eq28(queries.projectId, projectId)).get()?.total ?? 0;
|
|
16227
16730
|
return {
|
|
16228
16731
|
prompt: `
|
|
16229
16732
|
|
|
@@ -16363,7 +16866,7 @@ async function compactMessages(args) {
|
|
|
16363
16866
|
}
|
|
16364
16867
|
|
|
16365
16868
|
// src/agent/session-registry.ts
|
|
16366
|
-
var
|
|
16869
|
+
var log27 = createLogger("SessionRegistry");
|
|
16367
16870
|
var MAX_HYDRATE_NOTES = 20;
|
|
16368
16871
|
var MAX_HYDRATE_BYTES = 32 * 1024;
|
|
16369
16872
|
function escapeMemoryFragment(value) {
|
|
@@ -16422,7 +16925,7 @@ var SessionRegistry = class {
|
|
|
16422
16925
|
if (!this.externalToolsPromise) {
|
|
16423
16926
|
this.externalToolsPromise = loadExternalMcpTools(this.opts.config.externalMcpServers).catch(
|
|
16424
16927
|
(err) => {
|
|
16425
|
-
|
|
16928
|
+
log27.error("external-mcp.load-failed", {
|
|
16426
16929
|
error: describeError(err)
|
|
16427
16930
|
});
|
|
16428
16931
|
return [];
|
|
@@ -16479,7 +16982,7 @@ var SessionRegistry = class {
|
|
|
16479
16982
|
modelProvider: effectiveProvider,
|
|
16480
16983
|
modelId: effectiveModelId,
|
|
16481
16984
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
16482
|
-
}).where(
|
|
16985
|
+
}).where(eq29(agentSessions.projectId, projectId)).run();
|
|
16483
16986
|
}
|
|
16484
16987
|
const agent2 = createAeroSession({
|
|
16485
16988
|
projectName,
|
|
@@ -16508,7 +17011,7 @@ var SessionRegistry = class {
|
|
|
16508
17011
|
return agent2;
|
|
16509
17012
|
}
|
|
16510
17013
|
const { provider, modelId } = resolveSessionProviderAndModel(this.opts.config, preferences);
|
|
16511
|
-
const sessionId =
|
|
17014
|
+
const sessionId = crypto24.randomUUID();
|
|
16512
17015
|
const agent = createAeroSession({
|
|
16513
17016
|
projectName,
|
|
16514
17017
|
client: this.opts.client,
|
|
@@ -16698,13 +17201,13 @@ ${lines.join("\n")}
|
|
|
16698
17201
|
agent.state.messages = result.messages;
|
|
16699
17202
|
setAeroSystemPrompt(agent, this.buildHydratedSystemPrompt(projectId, row.systemPrompt));
|
|
16700
17203
|
this.save(projectName);
|
|
16701
|
-
|
|
17204
|
+
log27.info("compaction.completed", {
|
|
16702
17205
|
projectName,
|
|
16703
17206
|
removedCount: result.removedCount,
|
|
16704
17207
|
summaryBytes: Buffer.byteLength(result.summary, "utf8")
|
|
16705
17208
|
});
|
|
16706
17209
|
} catch (err) {
|
|
16707
|
-
|
|
17210
|
+
log27.error("compaction.failed", {
|
|
16708
17211
|
projectName,
|
|
16709
17212
|
error: describeError(err)
|
|
16710
17213
|
});
|
|
@@ -16798,7 +17301,7 @@ ${lines.join("\n")}
|
|
|
16798
17301
|
modelProvider: nextProvider,
|
|
16799
17302
|
modelId: nextModelId,
|
|
16800
17303
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
16801
|
-
}).where(
|
|
17304
|
+
}).where(eq29(agentSessions.projectId, projectId)).run();
|
|
16802
17305
|
}
|
|
16803
17306
|
/** Snapshot pending work without consuming it before an atomic conversation switch. */
|
|
16804
17307
|
prepareConversationChange(projectName) {
|
|
@@ -16882,7 +17385,7 @@ ${lines.join("\n")}
|
|
|
16882
17385
|
}
|
|
16883
17386
|
trackAeroTurn(agent, { trigger: OutcomeTriggers.agent });
|
|
16884
17387
|
} catch (err) {
|
|
16885
|
-
|
|
17388
|
+
log27.error("drain.failed", {
|
|
16886
17389
|
projectName,
|
|
16887
17390
|
error: describeError(err)
|
|
16888
17391
|
});
|
|
@@ -16979,18 +17482,18 @@ ${lines.join("\n")}
|
|
|
16979
17482
|
return id;
|
|
16980
17483
|
}
|
|
16981
17484
|
tryResolveProjectId(projectName) {
|
|
16982
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
17485
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq29(projects.name, projectName)).get();
|
|
16983
17486
|
return row?.id;
|
|
16984
17487
|
}
|
|
16985
17488
|
loadRow(projectId) {
|
|
16986
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
17489
|
+
const row = this.opts.db.select().from(agentSessions).where(eq29(agentSessions.projectId, projectId)).get();
|
|
16987
17490
|
return row ?? null;
|
|
16988
17491
|
}
|
|
16989
17492
|
insertRow(params) {
|
|
16990
17493
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
16991
17494
|
const modelProvider = params.provider ?? params.modelProvider ?? AgentProviderIds.claude;
|
|
16992
17495
|
this.opts.db.insert(agentSessions).values({
|
|
16993
|
-
id: params.id ??
|
|
17496
|
+
id: params.id ?? crypto24.randomUUID(),
|
|
16994
17497
|
projectId: params.projectId,
|
|
16995
17498
|
systemPrompt: params.systemPrompt,
|
|
16996
17499
|
modelProvider,
|
|
@@ -17003,7 +17506,7 @@ ${lines.join("\n")}
|
|
|
17003
17506
|
}
|
|
17004
17507
|
updateRow(projectId, patch) {
|
|
17005
17508
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
17006
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
17509
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq29(agentSessions.projectId, projectId)).run();
|
|
17007
17510
|
}
|
|
17008
17511
|
/**
|
|
17009
17512
|
* Store a refreshed prompt snapshot without touching `updatedAt`, which the
|
|
@@ -17012,13 +17515,13 @@ ${lines.join("\n")}
|
|
|
17012
17515
|
* recently used right after an upgrade.
|
|
17013
17516
|
*/
|
|
17014
17517
|
persistPromptSnapshot(projectId, systemPrompt) {
|
|
17015
|
-
this.opts.db.update(agentSessions).set({ systemPrompt }).where(
|
|
17518
|
+
this.opts.db.update(agentSessions).set({ systemPrompt }).where(eq29(agentSessions.projectId, projectId)).run();
|
|
17016
17519
|
}
|
|
17017
17520
|
};
|
|
17018
17521
|
|
|
17019
17522
|
// src/agent/viewer-sessions.ts
|
|
17020
|
-
import
|
|
17021
|
-
import { and as
|
|
17523
|
+
import crypto25 from "crypto";
|
|
17524
|
+
import { and as and25, eq as eq30, isNotNull as isNotNull4, like as like2 } from "drizzle-orm";
|
|
17022
17525
|
var AERO_VIEWER_EXCLUDED_MCP_TOOLS = /* @__PURE__ */ new Set([
|
|
17023
17526
|
CanonryMcpToolNames.canonry_ads_account,
|
|
17024
17527
|
CanonryMcpToolNames.canonry_ads_geo_search,
|
|
@@ -17071,8 +17574,8 @@ function trimViewerTranscript(messages, max2 = MAX_VIEWER_MESSAGES) {
|
|
|
17071
17574
|
}
|
|
17072
17575
|
var VIEWER_IDLE_MS = 12 * 60 * 60 * 1e3;
|
|
17073
17576
|
function mintViewerAeroKey(db, userId, now = /* @__PURE__ */ new Date()) {
|
|
17074
|
-
const raw = `cnry_${
|
|
17075
|
-
const id =
|
|
17577
|
+
const raw = `cnry_${crypto25.randomBytes(24).toString("hex")}`;
|
|
17578
|
+
const id = crypto25.randomUUID();
|
|
17076
17579
|
db.insert(apiKeys).values({
|
|
17077
17580
|
id,
|
|
17078
17581
|
name: `${VIEWER_AERO_KEY_PREFIX}${userId}`,
|
|
@@ -17131,8 +17634,8 @@ var ViewerAeroSessions = class {
|
|
|
17131
17634
|
if (this.isBusy(project.name, userId)) throw agentBusy(project.name);
|
|
17132
17635
|
const today = new Date(this.now()).toISOString().slice(0, 10);
|
|
17133
17636
|
const used = this.turnsToday.get(key);
|
|
17134
|
-
const
|
|
17135
|
-
if (
|
|
17637
|
+
const count6 = used?.date === today ? used.count : 0;
|
|
17638
|
+
if (count6 >= VIEWER_AERO_DAILY_TURN_LIMIT) throw viewerAeroDailyLimitExceeded(VIEWER_AERO_DAILY_TURN_LIMIT);
|
|
17136
17639
|
const minted = mintViewerAeroKey(this.opts.db, userId, new Date(this.now()));
|
|
17137
17640
|
this.acquisitions.add(key);
|
|
17138
17641
|
let handedOff = false;
|
|
@@ -17147,7 +17650,7 @@ var ViewerAeroSessions = class {
|
|
|
17147
17650
|
setAeroSystemPrompt(agent, loadAeroSystemPrompt(void 0, { extras: false }) + VIEWER_AERO_PROMPT + shape.prompt + aeroViewPrompt(preferences.context));
|
|
17148
17651
|
configureAeroRuntime(agent, [...buildViewerAeroTools(client, project.name, this.opts.managedSweeps), buildAeroViewTool(view, evidence)], void 0, true, shape.pinned);
|
|
17149
17652
|
this.sessions.set(key, { agent, lastUsedAt: this.now(), updatedAt: this.sessions.get(key)?.updatedAt ?? null });
|
|
17150
|
-
this.turnsToday.set(key, { date: today, count:
|
|
17653
|
+
this.turnsToday.set(key, { date: today, count: count6 + 1 });
|
|
17151
17654
|
handedOff = true;
|
|
17152
17655
|
let released = false;
|
|
17153
17656
|
return {
|
|
@@ -17195,12 +17698,12 @@ var ViewerAeroSessions = class {
|
|
|
17195
17698
|
*/
|
|
17196
17699
|
removeKey(id) {
|
|
17197
17700
|
try {
|
|
17198
|
-
this.opts.db.delete(apiKeys).where(
|
|
17701
|
+
this.opts.db.delete(apiKeys).where(eq30(apiKeys.id, id)).run();
|
|
17199
17702
|
} catch {
|
|
17200
17703
|
try {
|
|
17201
17704
|
this.opts.db.transaction((tx) => {
|
|
17202
|
-
tx.delete(dashboardSessions).where(
|
|
17203
|
-
tx.update(apiKeys).set({ revokedAt: new Date(this.now()).toISOString() }).where(
|
|
17705
|
+
tx.delete(dashboardSessions).where(eq30(dashboardSessions.apiKeyId, id)).run();
|
|
17706
|
+
tx.update(apiKeys).set({ revokedAt: new Date(this.now()).toISOString() }).where(eq30(apiKeys.id, id)).run();
|
|
17204
17707
|
});
|
|
17205
17708
|
} catch {
|
|
17206
17709
|
}
|
|
@@ -17213,7 +17716,7 @@ var ViewerAeroSessions = class {
|
|
|
17213
17716
|
*/
|
|
17214
17717
|
revokeOrphanedKeys() {
|
|
17215
17718
|
try {
|
|
17216
|
-
this.opts.db.delete(apiKeys).where(
|
|
17719
|
+
this.opts.db.delete(apiKeys).where(and25(like2(apiKeys.name, `${VIEWER_AERO_KEY_PREFIX}%`), isNotNull4(apiKeys.delegatedUserId))).run();
|
|
17217
17720
|
} catch {
|
|
17218
17721
|
}
|
|
17219
17722
|
}
|
|
@@ -17228,9 +17731,9 @@ var ViewerAeroSessions = class {
|
|
|
17228
17731
|
};
|
|
17229
17732
|
|
|
17230
17733
|
// src/mcp-http.ts
|
|
17231
|
-
import
|
|
17734
|
+
import crypto26 from "crypto";
|
|
17232
17735
|
import { isIP } from "net";
|
|
17233
|
-
import { and as
|
|
17736
|
+
import { and as and26, eq as eq31, isNull as isNull6 } from "drizzle-orm";
|
|
17234
17737
|
import { NodeStreamableHTTPServerTransport, toNodeHandler, toWebRequest } from "@modelcontextprotocol/node";
|
|
17235
17738
|
import { McpServer, createMcpHandler, isInitializeRequest, isLegacyRequest } from "@modelcontextprotocol/server";
|
|
17236
17739
|
var SESSION_IDLE_MS = 10 * 60 * 1e3;
|
|
@@ -17239,8 +17742,8 @@ var MAX_CLEANUP_RETRIES_PER_SWEEP = 8;
|
|
|
17239
17742
|
var MAX_PENDING_KEY_RETIREMENTS = 256;
|
|
17240
17743
|
var MAX_CLEANUP_BACKOFF_MS = 10 * 60 * 1e3;
|
|
17241
17744
|
function mintSessionKey(db, scopes, userId, oauthGrantId, createdAt) {
|
|
17242
|
-
const raw = `cnry_${
|
|
17243
|
-
const id =
|
|
17745
|
+
const raw = `cnry_${crypto26.randomBytes(24).toString("hex")}`;
|
|
17746
|
+
const id = crypto26.randomUUID();
|
|
17244
17747
|
db.insert(apiKeys).values({
|
|
17245
17748
|
id,
|
|
17246
17749
|
name: `mcp-session:${userId}`,
|
|
@@ -17336,12 +17839,12 @@ function registerMcpHttpRoutes(scope, opts) {
|
|
|
17336
17839
|
try {
|
|
17337
17840
|
withoutLockWait(() => {
|
|
17338
17841
|
opts.db.transaction((tx) => {
|
|
17339
|
-
tx.delete(dashboardSessions).where(
|
|
17340
|
-
tx.update(apiKeys).set({ revokedAt: new Date(now()).toISOString() }).where(
|
|
17842
|
+
tx.delete(dashboardSessions).where(eq31(dashboardSessions.apiKeyId, id)).run();
|
|
17843
|
+
tx.update(apiKeys).set({ revokedAt: new Date(now()).toISOString() }).where(and26(eq31(apiKeys.id, id), isNull6(apiKeys.revokedAt))).run();
|
|
17341
17844
|
});
|
|
17342
17845
|
if (remove) {
|
|
17343
17846
|
try {
|
|
17344
|
-
opts.db.delete(apiKeys).where(
|
|
17847
|
+
opts.db.delete(apiKeys).where(eq31(apiKeys.id, id)).run();
|
|
17345
17848
|
} catch (error) {
|
|
17346
17849
|
scope.log.warn({ err: error, keyId: id }, "Retained revoked MCP request key after deletion failed");
|
|
17347
17850
|
}
|
|
@@ -17392,14 +17895,14 @@ function registerMcpHttpRoutes(scope, opts) {
|
|
|
17392
17895
|
withoutLockWait(() => {
|
|
17393
17896
|
const cutoff = now() - SESSION_IDLE_MS;
|
|
17394
17897
|
for (const session of legacyServers) {
|
|
17395
|
-
const key = session.sessionKeyId ? opts.db.select({ revokedAt: apiKeys.revokedAt }).from(apiKeys).where(
|
|
17898
|
+
const key = session.sessionKeyId ? opts.db.select({ revokedAt: apiKeys.revokedAt }).from(apiKeys).where(eq31(apiKeys.id, session.sessionKeyId)).get() : void 0;
|
|
17396
17899
|
if (session.lastSeenAt < cutoff || session.expiresAt !== void 0 && session.expiresAt <= now() || session.sessionKeyId && (!key || key.revokedAt)) {
|
|
17397
17900
|
void session.close().catch((error) => scope.log.error({ err: error }, "Failed to close expired MCP session"));
|
|
17398
17901
|
}
|
|
17399
17902
|
}
|
|
17400
17903
|
for (const exchange of modernExchanges) {
|
|
17401
17904
|
if (!exchange.sessionKey) continue;
|
|
17402
|
-
const key = opts.db.select({ revokedAt: apiKeys.revokedAt }).from(apiKeys).where(
|
|
17905
|
+
const key = opts.db.select({ revokedAt: apiKeys.revokedAt }).from(apiKeys).where(eq31(apiKeys.id, exchange.sessionKey.id)).get();
|
|
17403
17906
|
if (!key || key.revokedAt || exchange.sessionKey.expiresAt <= now()) {
|
|
17404
17907
|
void exchange.close().catch((error) => scope.log.error({ err: error }, "Failed to close expired MCP request"));
|
|
17405
17908
|
}
|
|
@@ -17454,7 +17957,7 @@ function registerMcpHttpRoutes(scope, opts) {
|
|
|
17454
17957
|
clientName: `canonry-mcp/${PACKAGE_VERSION}`,
|
|
17455
17958
|
surface: "mcp-http",
|
|
17456
17959
|
// Correlation only: this is never accepted as caller identity or authority.
|
|
17457
|
-
...modern ? {} : { actorSession:
|
|
17960
|
+
...modern ? {} : { actorSession: crypto26.randomUUID() }
|
|
17458
17961
|
});
|
|
17459
17962
|
let server;
|
|
17460
17963
|
try {
|
|
@@ -17479,7 +17982,7 @@ function registerMcpHttpRoutes(scope, opts) {
|
|
|
17479
17982
|
if (!bearer) return null;
|
|
17480
17983
|
const { server, sessionKey } = createAuthorizedServer(request, segment, bearer);
|
|
17481
17984
|
const transport = new NodeStreamableHTTPServerTransport({
|
|
17482
|
-
sessionIdGenerator: () =>
|
|
17985
|
+
sessionIdGenerator: () => crypto26.randomUUID(),
|
|
17483
17986
|
onsessioninitialized: (id) => {
|
|
17484
17987
|
if (!shuttingDown && !closing) sessions.set(id, session);
|
|
17485
17988
|
}
|
|
@@ -17649,7 +18152,7 @@ function registerMcpHttpRoutes(scope, opts) {
|
|
|
17649
18152
|
}
|
|
17650
18153
|
|
|
17651
18154
|
// src/agent/agent-routes.ts
|
|
17652
|
-
import { eq as
|
|
18155
|
+
import { eq as eq32 } from "drizzle-orm";
|
|
17653
18156
|
function viewerAeroCaller(request, opts) {
|
|
17654
18157
|
if (!opts.viewerSessions) return null;
|
|
17655
18158
|
const principal = request.principal;
|
|
@@ -17689,7 +18192,7 @@ function turnAttribution(request) {
|
|
|
17689
18192
|
};
|
|
17690
18193
|
}
|
|
17691
18194
|
function resolveProject(db, name) {
|
|
17692
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
18195
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq32(projects.name, name)).get();
|
|
17693
18196
|
if (!row) throw notFound("project", name);
|
|
17694
18197
|
return row;
|
|
17695
18198
|
}
|
|
@@ -17808,7 +18311,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
17808
18311
|
}
|
|
17809
18312
|
requireInstanceAdministrator(request);
|
|
17810
18313
|
const project = resolveProject(opts.db, request.params.name);
|
|
17811
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
18314
|
+
const row = opts.db.select().from(agentSessions).where(eq32(agentSessions.projectId, project.id)).get();
|
|
17812
18315
|
const nextTurn = opts.sessionRegistry.nextTurnModel(project.name);
|
|
17813
18316
|
if (!row) {
|
|
17814
18317
|
return { conversationId: null, messages: [], modelProvider: null, modelId: null, nextTurn, updatedAt: null };
|
|
@@ -17850,7 +18353,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
17850
18353
|
const project = resolveProject(opts.db, request.params.name);
|
|
17851
18354
|
if (opts.sessionRegistry.isBusy(project.name)) throw agentBusy(project.name);
|
|
17852
18355
|
opts.sessionRegistry.reset(project.name);
|
|
17853
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
18356
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq32(agentSessions.projectId, project.id)).run();
|
|
17854
18357
|
return { status: "reset" };
|
|
17855
18358
|
}
|
|
17856
18359
|
);
|
|
@@ -17874,7 +18377,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
17874
18377
|
return streamViewerTurn(opts, project, viewerId, body, reply, turnAttribution(request));
|
|
17875
18378
|
}
|
|
17876
18379
|
if (body.conversationId !== void 0) {
|
|
17877
|
-
const current = opts.db.select({ id: agentSessions.id }).from(agentSessions).where(
|
|
18380
|
+
const current = opts.db.select({ id: agentSessions.id }).from(agentSessions).where(eq32(agentSessions.projectId, project.id)).get();
|
|
17878
18381
|
if ((current?.id ?? null) !== body.conversationId) throw validationError("The active conversation changed. Reload the conversation before sending.");
|
|
17879
18382
|
}
|
|
17880
18383
|
const promptText = body.prompt;
|
|
@@ -18285,7 +18788,7 @@ function stripHtml(html) {
|
|
|
18285
18788
|
}
|
|
18286
18789
|
|
|
18287
18790
|
// src/snapshot-service.ts
|
|
18288
|
-
var
|
|
18791
|
+
var log28 = createLogger("Snapshot");
|
|
18289
18792
|
var ANALYSIS_PROVIDER_PRIORITY = ["openai", "claude", "gemini", "perplexity", "muse", "local"];
|
|
18290
18793
|
var SNAPSHOT_QUERY_COUNT = 6;
|
|
18291
18794
|
var SnapshotService = class {
|
|
@@ -18387,7 +18890,7 @@ var SnapshotService = class {
|
|
|
18387
18890
|
return mapAuditReport(report);
|
|
18388
18891
|
} catch (err) {
|
|
18389
18892
|
const message = describeError(err);
|
|
18390
|
-
|
|
18893
|
+
log28.warn("audit.failed", { homepageUrl, error: message });
|
|
18391
18894
|
return {
|
|
18392
18895
|
url: homepageUrl,
|
|
18393
18896
|
finalUrl: homepageUrl,
|
|
@@ -18416,7 +18919,7 @@ var SnapshotService = class {
|
|
|
18416
18919
|
queries: parsedQueries
|
|
18417
18920
|
};
|
|
18418
18921
|
} catch (err) {
|
|
18419
|
-
|
|
18922
|
+
log28.warn("profile.generation-failed", {
|
|
18420
18923
|
domain: ctx.domain,
|
|
18421
18924
|
provider: ctx.analysisProvider.adapter.name,
|
|
18422
18925
|
error: describeError(err)
|
|
@@ -18564,7 +19067,7 @@ var SnapshotService = class {
|
|
|
18564
19067
|
recommendedActions: uniqueStrings2(parsed.recommendedActions ?? []).slice(0, 4)
|
|
18565
19068
|
};
|
|
18566
19069
|
} catch (err) {
|
|
18567
|
-
|
|
19070
|
+
log28.warn("response.analysis-failed", {
|
|
18568
19071
|
provider: ctx.analysisProvider.adapter.name,
|
|
18569
19072
|
error: describeError(err)
|
|
18570
19073
|
});
|
|
@@ -18690,7 +19193,7 @@ function buildSnapshotSummary(companyName, queries2, providers, queryResults, au
|
|
|
18690
19193
|
competitorCounts.set(competitor, (competitorCounts.get(competitor) ?? 0) + 1);
|
|
18691
19194
|
}
|
|
18692
19195
|
}
|
|
18693
|
-
const topCompetitors = [...competitorCounts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 10).map(([name,
|
|
19196
|
+
const topCompetitors = [...competitorCounts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 10).map(([name, count6]) => ({ name, count: count6 }));
|
|
18694
19197
|
const defaultMeaning = totalComparisons > 0 ? `${companyName} was mentioned in ${mentionCount}/${totalComparisons} successful provider-query response${totalComparisons === 1 ? "" : "s"} across ${queries2.length} category queries.` : `No successful provider responses were returned across ${queries2.length} category queries.`;
|
|
18695
19198
|
const failureNote = failedComparisons > 0 ? [
|
|
18696
19199
|
`${failedComparisons} provider response${failedComparisons === 1 ? "" : "s"} failed and ${failedComparisons === 1 ? "was" : "were"} excluded from visibility totals.`
|
|
@@ -18834,7 +19337,7 @@ function clipText(value, length) {
|
|
|
18834
19337
|
}
|
|
18835
19338
|
|
|
18836
19339
|
// src/research-runner.ts
|
|
18837
|
-
import { and as
|
|
19340
|
+
import { and as and27, eq as eq33, inArray as inArray13, sql as sql13 } from "drizzle-orm";
|
|
18838
19341
|
var unfinishedResearchQueryStatuses = [ResearchQueryStatuses.queued, ResearchQueryStatuses.running];
|
|
18839
19342
|
function finalResearchRunStatus(completed, failed, waiting) {
|
|
18840
19343
|
if (waiting > 0) return ResearchRunStatuses["waiting-upstream"];
|
|
@@ -18842,13 +19345,13 @@ function finalResearchRunStatus(completed, failed, waiting) {
|
|
|
18842
19345
|
return completed > 0 ? ResearchRunStatuses.partial : ResearchRunStatuses.failed;
|
|
18843
19346
|
}
|
|
18844
19347
|
async function executeResearchRun(db, registry, runId, projectId, options = {}) {
|
|
18845
|
-
const run = db.select().from(researchRuns).where(
|
|
19348
|
+
const run = db.select().from(researchRuns).where(and27(eq33(researchRuns.id, runId), eq33(researchRuns.projectId, projectId))).get();
|
|
18846
19349
|
if (!run || run.status !== ResearchRunStatuses.queued) return;
|
|
18847
|
-
const project = db.select().from(projects).where(
|
|
19350
|
+
const project = db.select().from(projects).where(eq33(projects.id, projectId)).get();
|
|
18848
19351
|
if (!project) return;
|
|
18849
19352
|
const provider = registry.get(run.provider);
|
|
18850
19353
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
18851
|
-
const claim = db.update(researchRuns).set({ status: ResearchRunStatuses.running, startedAt: now }).where(
|
|
19354
|
+
const claim = db.update(researchRuns).set({ status: ResearchRunStatuses.running, startedAt: now }).where(and27(eq33(researchRuns.id, runId), eq33(researchRuns.status, ResearchRunStatuses.queued))).run();
|
|
18852
19355
|
if (claim.changes !== 1) return;
|
|
18853
19356
|
const elapsed = startOutcomeTimer();
|
|
18854
19357
|
const requested = currentOutcomeOrigin();
|
|
@@ -18882,8 +19385,8 @@ async function executeResearchRun(db, registry, runId, projectId, options = {})
|
|
|
18882
19385
|
}
|
|
18883
19386
|
reserved = run.totalQueries;
|
|
18884
19387
|
reservation = { scope, period };
|
|
18885
|
-
const rows = db.select().from(researchRunQueries).where(
|
|
18886
|
-
const competitorRows = db.select(competitorIdentityColumns).from(competitors).where(
|
|
19388
|
+
const rows = db.select().from(researchRunQueries).where(eq33(researchRunQueries.researchRunId, runId)).orderBy(researchRunQueries.position).all();
|
|
19389
|
+
const competitorRows = db.select(competitorIdentityColumns).from(competitors).where(eq33(competitors.projectId, projectId)).orderBy(competitors.domain).all();
|
|
18887
19390
|
const competitorDomains = competitorRows.map((row) => row.domain);
|
|
18888
19391
|
const competitorAliases = new Map(competitorRows.map((row) => [row.domain, competitorIdentityAliases(row)]).filter(([, aliases]) => aliases.length > 0));
|
|
18889
19392
|
const domains = effectiveDomains(project);
|
|
@@ -18893,7 +19396,7 @@ async function executeResearchRun(db, registry, runId, projectId, options = {})
|
|
|
18893
19396
|
await mapWithConcurrency(rows, Math.max(1, provider.config.quotaPolicy.maxConcurrency), async (row) => {
|
|
18894
19397
|
try {
|
|
18895
19398
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
18896
|
-
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.running, startedAt }).where(
|
|
19399
|
+
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.running, startedAt }).where(and27(eq33(researchRunQueries.id, row.id), eq33(researchRunQueries.status, ResearchQueryStatuses.queued))).run();
|
|
18897
19400
|
const raw = await gate.run(async () => {
|
|
18898
19401
|
dispatched++;
|
|
18899
19402
|
return provider.adapter.executeTrackedQuery({ query: row.queryText, canonicalDomains: domains, competitorDomains, ...run.location ? { location: run.location } : {} }, config);
|
|
@@ -18924,7 +19427,7 @@ async function executeResearchRun(db, registry, runId, projectId, options = {})
|
|
|
18924
19427
|
rawResponse: raw.rawResponse,
|
|
18925
19428
|
answerEvidence: raw.evidence ?? null,
|
|
18926
19429
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
18927
|
-
}).where(
|
|
19430
|
+
}).where(and27(eq33(researchRunQueries.id, row.id), eq33(researchRunQueries.status, ResearchQueryStatuses.running))).run();
|
|
18928
19431
|
if (completed.changes === 1) incrementResearchProgress(db, runId, "completedQueries");
|
|
18929
19432
|
} catch (error) {
|
|
18930
19433
|
if (!(error instanceof IncompleteAnswerError && error.resumable)) queryFailure ??= error;
|
|
@@ -18975,11 +19478,11 @@ function researchOutcomeStatus(status) {
|
|
|
18975
19478
|
}
|
|
18976
19479
|
}
|
|
18977
19480
|
function incrementResearchProgress(db, runId, column) {
|
|
18978
|
-
db.update(researchRuns).set({ [column]:
|
|
19481
|
+
db.update(researchRuns).set({ [column]: sql13`${researchRuns[column]} + 1` }).where(eq33(researchRuns.id, runId)).run();
|
|
18979
19482
|
}
|
|
18980
19483
|
function markResearchQueryFailed(db, runId, queryId, error) {
|
|
18981
19484
|
const message = describeError(error);
|
|
18982
|
-
const unfinished =
|
|
19485
|
+
const unfinished = and27(eq33(researchRunQueries.id, queryId), inArray13(researchRunQueries.status, unfinishedResearchQueryStatuses));
|
|
18983
19486
|
if (error instanceof IncompleteAnswerError && error.resumable) {
|
|
18984
19487
|
db.update(researchRunQueries).set({
|
|
18985
19488
|
status: ResearchQueryStatuses["waiting-upstream"],
|
|
@@ -18995,10 +19498,10 @@ function markResearchQueryFailed(db, runId, queryId, error) {
|
|
|
18995
19498
|
if (failed.changes === 1) incrementResearchProgress(db, runId, "failedQueries");
|
|
18996
19499
|
}
|
|
18997
19500
|
function markUnfinishedResearchQueriesFailed(db, runId, error) {
|
|
18998
|
-
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
19501
|
+
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and27(eq33(researchRunQueries.researchRunId, runId), inArray13(researchRunQueries.status, unfinishedResearchQueryStatuses))).run();
|
|
18999
19502
|
}
|
|
19000
19503
|
function finalizeResearchRun(db, runId, fatalError) {
|
|
19001
|
-
const rows = db.select({ status: researchRunQueries.status }).from(researchRunQueries).where(
|
|
19504
|
+
const rows = db.select({ status: researchRunQueries.status }).from(researchRunQueries).where(eq33(researchRunQueries.researchRunId, runId)).all();
|
|
19002
19505
|
const completed = rows.filter((row) => row.status === ResearchQueryStatuses.completed).length;
|
|
19003
19506
|
const failed = rows.filter((row) => row.status === ResearchQueryStatuses.failed).length;
|
|
19004
19507
|
const waiting = rows.filter((row) => row.status === ResearchQueryStatuses["waiting-upstream"]).length;
|
|
@@ -19010,12 +19513,12 @@ function finalizeResearchRun(db, runId, fatalError) {
|
|
|
19010
19513
|
error: fatalError ?? (failed === rows.length ? "Every research query failed." : null),
|
|
19011
19514
|
// A run waiting on upstream jobs has not finished.
|
|
19012
19515
|
finishedAt: waiting > 0 ? null : (/* @__PURE__ */ new Date()).toISOString()
|
|
19013
|
-
}).where(
|
|
19516
|
+
}).where(eq33(researchRuns.id, runId)).run();
|
|
19014
19517
|
return { status, total: rows.length, completed, failed };
|
|
19015
19518
|
}
|
|
19016
19519
|
|
|
19017
19520
|
// src/google-marketing-sync.ts
|
|
19018
|
-
import { and as
|
|
19521
|
+
import { and as and28, eq as eq34, isNull as isNull7, sql as sql14 } from "drizzle-orm";
|
|
19019
19522
|
var RUNTIME_FAILURE_REASONS = {
|
|
19020
19523
|
INVALID_INPUT: OutcomeReasonCodes.VALIDATION,
|
|
19021
19524
|
NOT_CONFIGURED: OutcomeReasonCodes.NOT_CONNECTED,
|
|
@@ -19052,10 +19555,10 @@ function assertSnapshotContext(metadata, expected) {
|
|
|
19052
19555
|
}
|
|
19053
19556
|
}
|
|
19054
19557
|
function requireSyncRun(db, runId, projectId, kind) {
|
|
19055
|
-
const run = db.select().from(runs).where(
|
|
19056
|
-
|
|
19057
|
-
|
|
19058
|
-
|
|
19558
|
+
const run = db.select().from(runs).where(and28(
|
|
19559
|
+
eq34(runs.id, runId),
|
|
19560
|
+
eq34(runs.projectId, projectId),
|
|
19561
|
+
eq34(runs.kind, kind)
|
|
19059
19562
|
)).get();
|
|
19060
19563
|
if (!run) throw withOutcomeReason(new Error(`Run ${runId} is not a ${kind} run for this project.`), OutcomeReasonCodes.NOT_FOUND);
|
|
19061
19564
|
if (run.status !== RunStatuses.queued) {
|
|
@@ -19071,19 +19574,19 @@ function markFailed(db, runId, projectId, kind, expectedStatus, error) {
|
|
|
19071
19574
|
status: RunStatuses.failed,
|
|
19072
19575
|
error: serializeRunError({ message: safeErrorMessage(error) }),
|
|
19073
19576
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
19074
|
-
}).where(
|
|
19075
|
-
|
|
19076
|
-
|
|
19077
|
-
|
|
19078
|
-
|
|
19577
|
+
}).where(and28(
|
|
19578
|
+
eq34(runs.id, runId),
|
|
19579
|
+
eq34(runs.projectId, projectId),
|
|
19580
|
+
eq34(runs.kind, kind),
|
|
19581
|
+
eq34(runs.status, expectedStatus)
|
|
19079
19582
|
)).run();
|
|
19080
19583
|
}
|
|
19081
19584
|
function claimSyncRun(db, runId, projectId, kind, startedAt) {
|
|
19082
|
-
const claim = db.update(runs).set({ status: RunStatuses.running, startedAt, error: null }).where(
|
|
19083
|
-
|
|
19084
|
-
|
|
19085
|
-
|
|
19086
|
-
|
|
19585
|
+
const claim = db.update(runs).set({ status: RunStatuses.running, startedAt, error: null }).where(and28(
|
|
19586
|
+
eq34(runs.id, runId),
|
|
19587
|
+
eq34(runs.projectId, projectId),
|
|
19588
|
+
eq34(runs.kind, kind),
|
|
19589
|
+
eq34(runs.status, RunStatuses.queued)
|
|
19087
19590
|
)).run();
|
|
19088
19591
|
return claim.changes === 1;
|
|
19089
19592
|
}
|
|
@@ -19093,9 +19596,9 @@ async function executeGoogleAdsMarketingSync(db, runtime, runId, projectId) {
|
|
|
19093
19596
|
let evidenceCollected = false;
|
|
19094
19597
|
try {
|
|
19095
19598
|
requireSyncRun(db, runId, projectId, RunKinds["google-ads-sync"]);
|
|
19096
|
-
const project = db.select().from(projects).where(
|
|
19599
|
+
const project = db.select().from(projects).where(eq34(projects.id, projectId)).get();
|
|
19097
19600
|
if (!project) throw withOutcomeReason(new Error(`Project ${projectId} was not found.`), OutcomeReasonCodes.NOT_FOUND);
|
|
19098
|
-
const connection = db.select().from(googleAdsConnections).where(
|
|
19601
|
+
const connection = db.select().from(googleAdsConnections).where(eq34(googleAdsConnections.projectId, projectId)).get();
|
|
19099
19602
|
if (!connection?.selectedCustomerId) {
|
|
19100
19603
|
throw withOutcomeReason(new Error("Google Ads needs a selected customer before it can sync."), OutcomeReasonCodes.ACCOUNT_NOT_FOUND);
|
|
19101
19604
|
}
|
|
@@ -19147,11 +19650,11 @@ async function executeGoogleAdsMarketingSync(db, runtime, runId, projectId) {
|
|
|
19147
19650
|
status: RunStatuses.completed,
|
|
19148
19651
|
finishedAt,
|
|
19149
19652
|
error: null
|
|
19150
|
-
}).where(
|
|
19151
|
-
|
|
19152
|
-
|
|
19153
|
-
|
|
19154
|
-
|
|
19653
|
+
}).where(and28(
|
|
19654
|
+
eq34(runs.id, runId),
|
|
19655
|
+
eq34(runs.projectId, projectId),
|
|
19656
|
+
eq34(runs.kind, RunKinds["google-ads-sync"]),
|
|
19657
|
+
eq34(runs.status, RunStatuses.running)
|
|
19155
19658
|
)).run();
|
|
19156
19659
|
if (terminalClaim.changes !== 1) {
|
|
19157
19660
|
throw withOutcomeReason(
|
|
@@ -19171,12 +19674,12 @@ async function executeGoogleAdsMarketingSync(db, runtime, runId, projectId) {
|
|
|
19171
19674
|
lastMetricsSnapshotAt: result.metrics?.metadata.capturedAt ?? connection.lastMetricsSnapshotAt,
|
|
19172
19675
|
lastMetricsSnapshotId: result.metrics?.metadata.id ?? connection.lastMetricsSnapshotId,
|
|
19173
19676
|
updatedAt: finishedAt
|
|
19174
|
-
}).where(
|
|
19175
|
-
|
|
19176
|
-
|
|
19177
|
-
|
|
19178
|
-
connection.selectedLoginCustomerId === null ? isNull7(googleAdsConnections.selectedLoginCustomerId) :
|
|
19179
|
-
|
|
19677
|
+
}).where(and28(
|
|
19678
|
+
eq34(googleAdsConnections.id, connection.id),
|
|
19679
|
+
eq34(googleAdsConnections.projectId, projectId),
|
|
19680
|
+
eq34(googleAdsConnections.selectedCustomerId, connection.selectedCustomerId),
|
|
19681
|
+
connection.selectedLoginCustomerId === null ? isNull7(googleAdsConnections.selectedLoginCustomerId) : eq34(googleAdsConnections.selectedLoginCustomerId, connection.selectedLoginCustomerId),
|
|
19682
|
+
eq34(googleAdsConnections.selectionGeneration, connection.selectionGeneration)
|
|
19180
19683
|
)).run();
|
|
19181
19684
|
if (connectionClaim.changes !== 1) {
|
|
19182
19685
|
throw withOutcomeReason(
|
|
@@ -19216,9 +19719,9 @@ async function executeGtmMarketingSync(db, runtime, runId, projectId) {
|
|
|
19216
19719
|
let evidenceCollected = false;
|
|
19217
19720
|
try {
|
|
19218
19721
|
requireSyncRun(db, runId, projectId, RunKinds["gtm-sync"]);
|
|
19219
|
-
const project = db.select().from(projects).where(
|
|
19722
|
+
const project = db.select().from(projects).where(eq34(projects.id, projectId)).get();
|
|
19220
19723
|
if (!project) throw withOutcomeReason(new Error(`Project ${projectId} was not found.`), OutcomeReasonCodes.NOT_FOUND);
|
|
19221
|
-
const connection = db.select().from(gtmConnections).where(
|
|
19724
|
+
const connection = db.select().from(gtmConnections).where(eq34(gtmConnections.projectId, projectId)).get();
|
|
19222
19725
|
if (!connection?.selectedAccountId || !connection.selectedContainerId) {
|
|
19223
19726
|
throw withOutcomeReason(new Error("GTM needs a selected account and container before it can sync."), OutcomeReasonCodes.ACCOUNT_NOT_FOUND);
|
|
19224
19727
|
}
|
|
@@ -19256,11 +19759,11 @@ async function executeGtmMarketingSync(db, runtime, runId, projectId) {
|
|
|
19256
19759
|
status: RunStatuses.completed,
|
|
19257
19760
|
finishedAt,
|
|
19258
19761
|
error: null
|
|
19259
|
-
}).where(
|
|
19260
|
-
|
|
19261
|
-
|
|
19262
|
-
|
|
19263
|
-
|
|
19762
|
+
}).where(and28(
|
|
19763
|
+
eq34(runs.id, runId),
|
|
19764
|
+
eq34(runs.projectId, projectId),
|
|
19765
|
+
eq34(runs.kind, RunKinds["gtm-sync"]),
|
|
19766
|
+
eq34(runs.status, RunStatuses.running)
|
|
19264
19767
|
)).run();
|
|
19265
19768
|
if (terminalClaim.changes !== 1) {
|
|
19266
19769
|
throw withOutcomeReason(
|
|
@@ -19275,19 +19778,19 @@ async function executeGtmMarketingSync(db, runtime, runId, projectId) {
|
|
|
19275
19778
|
selectedWorkspaceName: selectedWorkspace?.name ?? null,
|
|
19276
19779
|
...workspaceReplaced ? {
|
|
19277
19780
|
selectedWorkspaceId: workspaceId,
|
|
19278
|
-
selectionGeneration:
|
|
19781
|
+
selectionGeneration: sql14`${gtmConnections.selectionGeneration} + 1`
|
|
19279
19782
|
} : {},
|
|
19280
19783
|
lastValidatedAt: snapshot.metadata.capturedAt,
|
|
19281
19784
|
lastSnapshotAt: snapshot.metadata.capturedAt,
|
|
19282
19785
|
lastSnapshotId: snapshot.metadata.id,
|
|
19283
19786
|
updatedAt: finishedAt
|
|
19284
|
-
}).where(
|
|
19285
|
-
|
|
19286
|
-
|
|
19287
|
-
|
|
19288
|
-
|
|
19289
|
-
connection.selectedWorkspaceId === null ? isNull7(gtmConnections.selectedWorkspaceId) :
|
|
19290
|
-
|
|
19787
|
+
}).where(and28(
|
|
19788
|
+
eq34(gtmConnections.id, connection.id),
|
|
19789
|
+
eq34(gtmConnections.projectId, projectId),
|
|
19790
|
+
eq34(gtmConnections.selectedAccountId, connection.selectedAccountId),
|
|
19791
|
+
eq34(gtmConnections.selectedContainerId, connection.selectedContainerId),
|
|
19792
|
+
connection.selectedWorkspaceId === null ? isNull7(gtmConnections.selectedWorkspaceId) : eq34(gtmConnections.selectedWorkspaceId, connection.selectedWorkspaceId),
|
|
19793
|
+
eq34(gtmConnections.selectionGeneration, connection.selectionGeneration)
|
|
19291
19794
|
)).run();
|
|
19292
19795
|
if (connectionClaim.changes !== 1) {
|
|
19293
19796
|
throw withOutcomeReason(
|
|
@@ -19325,7 +19828,7 @@ async function executeGtmMarketingSync(db, runtime, runId, projectId) {
|
|
|
19325
19828
|
// src/server.ts
|
|
19326
19829
|
var _require = createRequire(import.meta.url);
|
|
19327
19830
|
var { version: PKG_VERSION } = _require("../package.json");
|
|
19328
|
-
var
|
|
19831
|
+
var log29 = createLogger("Server");
|
|
19329
19832
|
var runtimeStartupByServer = /* @__PURE__ */ new WeakMap();
|
|
19330
19833
|
function waitForServerRuntimeStartup(app) {
|
|
19331
19834
|
const startup = runtimeStartupByServer.get(app);
|
|
@@ -19490,8 +19993,8 @@ var DASHBOARD_SCRYPT_KEYLEN = 64;
|
|
|
19490
19993
|
var DASHBOARD_SCRYPT_COST = 1 << 15;
|
|
19491
19994
|
var DASHBOARD_SCRYPT_MAXMEM = 64 * 1024 * 1024;
|
|
19492
19995
|
function hashDashboardPassword(password) {
|
|
19493
|
-
const salt =
|
|
19494
|
-
const derived =
|
|
19996
|
+
const salt = crypto27.randomBytes(16);
|
|
19997
|
+
const derived = crypto27.scryptSync(password, salt, DASHBOARD_SCRYPT_KEYLEN, {
|
|
19495
19998
|
N: DASHBOARD_SCRYPT_COST,
|
|
19496
19999
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
19497
20000
|
});
|
|
@@ -19512,14 +20015,14 @@ function verifyDashboardPassword(password, storedHash) {
|
|
|
19512
20015
|
} catch {
|
|
19513
20016
|
return { ok: false, needsRehash: false };
|
|
19514
20017
|
}
|
|
19515
|
-
const derived =
|
|
20018
|
+
const derived = crypto27.scryptSync(password, salt, expected.length, {
|
|
19516
20019
|
N: DASHBOARD_SCRYPT_COST,
|
|
19517
20020
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
19518
20021
|
});
|
|
19519
20022
|
if (derived.length !== expected.length)
|
|
19520
20023
|
return { ok: false, needsRehash: false };
|
|
19521
20024
|
return {
|
|
19522
|
-
ok:
|
|
20025
|
+
ok: crypto27.timingSafeEqual(derived, expected),
|
|
19523
20026
|
needsRehash: false
|
|
19524
20027
|
};
|
|
19525
20028
|
}
|
|
@@ -19528,7 +20031,7 @@ function verifyDashboardPassword(password, storedHash) {
|
|
|
19528
20031
|
const expected = Buffer.from(storedHash, "hex");
|
|
19529
20032
|
if (candidate.length !== expected.length)
|
|
19530
20033
|
return { ok: false, needsRehash: false };
|
|
19531
|
-
const ok =
|
|
20034
|
+
const ok = crypto27.timingSafeEqual(candidate, expected);
|
|
19532
20035
|
return { ok, needsRehash: ok };
|
|
19533
20036
|
}
|
|
19534
20037
|
return { ok: false, needsRehash: false };
|
|
@@ -19575,7 +20078,7 @@ function applyLegacyCredentials(rows, config, persistConfigPatch = saveConfigPat
|
|
|
19575
20078
|
}
|
|
19576
20079
|
if (migratedGoogle > 0) {
|
|
19577
20080
|
persistConfigPatch({ google: config.google });
|
|
19578
|
-
|
|
20081
|
+
log29.info("credentials.migrated", { type: "google", count: migratedGoogle });
|
|
19579
20082
|
}
|
|
19580
20083
|
let migratedGa4 = 0;
|
|
19581
20084
|
for (const row of rows.ga4) {
|
|
@@ -19593,7 +20096,7 @@ function applyLegacyCredentials(rows, config, persistConfigPatch = saveConfigPat
|
|
|
19593
20096
|
}
|
|
19594
20097
|
if (migratedGa4 > 0) {
|
|
19595
20098
|
persistConfigPatch({ ga4: config.ga4 });
|
|
19596
|
-
|
|
20099
|
+
log29.info("credentials.migrated", { type: "ga4", count: migratedGa4 });
|
|
19597
20100
|
}
|
|
19598
20101
|
}
|
|
19599
20102
|
function isLoopbackPeerAddress(address) {
|
|
@@ -19695,7 +20198,7 @@ function cloneGtmConfig(config) {
|
|
|
19695
20198
|
function createGoogleMarketingConfigCredentialStore(input) {
|
|
19696
20199
|
const persistConfigPatch = input.saveConfigPatch ?? saveConfigPatch;
|
|
19697
20200
|
const env = input.env ?? process.env;
|
|
19698
|
-
const randomUUID2 = input.randomUUID ??
|
|
20201
|
+
const randomUUID2 = input.randomUUID ?? crypto27.randomUUID;
|
|
19699
20202
|
const snapshot = () => ({
|
|
19700
20203
|
googleAds: cloneGoogleAdsConfig(input.config.googleAds),
|
|
19701
20204
|
gtm: cloneGtmConfig(input.config.gtm)
|
|
@@ -19812,7 +20315,7 @@ async function createServer(opts) {
|
|
|
19812
20315
|
const trustProxy = resolveTrustProxy(process.env.CANONRY_TRUST_PROXY);
|
|
19813
20316
|
const app = Fastify({
|
|
19814
20317
|
loggerInstance: logger,
|
|
19815
|
-
genReqId: () =>
|
|
20318
|
+
genReqId: () => crypto27.randomUUID(),
|
|
19816
20319
|
trustProxy
|
|
19817
20320
|
});
|
|
19818
20321
|
if (!ownsDefaultConfig) {
|
|
@@ -19876,17 +20379,17 @@ async function createServer(opts) {
|
|
|
19876
20379
|
applyLegacyCredentials(legacyRows, opts.config, persistConfigPatch);
|
|
19877
20380
|
dropLegacyCredentialColumns(opts.db);
|
|
19878
20381
|
} catch (err) {
|
|
19879
|
-
|
|
20382
|
+
log29.warn("credentials.migration.failed", {
|
|
19880
20383
|
error: describeError(err)
|
|
19881
20384
|
});
|
|
19882
20385
|
}
|
|
19883
20386
|
}
|
|
19884
|
-
|
|
20387
|
+
log29.info("providers.configured", {
|
|
19885
20388
|
providers: registeredProviderNames({ providers, cdp: opts.config.cdp })
|
|
19886
20389
|
});
|
|
19887
20390
|
registry.replace(configuredProviderEntries(opts.config, API_ADAPTERS, cdpChatgptAdapter));
|
|
19888
20391
|
for (const name of providersWithUnsupportedBatch(providers, (provider) => adapterMap[provider])) {
|
|
19889
|
-
|
|
20392
|
+
log29.warn("provider.batch.unsupported", {
|
|
19890
20393
|
provider: name,
|
|
19891
20394
|
message: `providers.${name}.batch.enabled is true, but the ${name} adapter has no batch API; its sweeps run sync.`
|
|
19892
20395
|
});
|
|
@@ -19943,7 +20446,7 @@ async function createServer(opts) {
|
|
|
19943
20446
|
async (ctx) => {
|
|
19944
20447
|
if (!sessionRegistry) return;
|
|
19945
20448
|
if (!agentProactive) return;
|
|
19946
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
20449
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq35(projects.id, ctx.projectId)).get();
|
|
19947
20450
|
if (!project) return;
|
|
19948
20451
|
let content;
|
|
19949
20452
|
if (ctx.kind === RunKinds["aeo-discover-probe"]) {
|
|
@@ -20417,7 +20920,7 @@ async function createServer(opts) {
|
|
|
20417
20920
|
void (async () => {
|
|
20418
20921
|
try {
|
|
20419
20922
|
const report = await schedulerClient.runDoctor({ project: projectName, checkIds: scheduledHealthCheckIds() });
|
|
20420
|
-
const project = opts.db.select().from(projects).where(
|
|
20923
|
+
const project = opts.db.select().from(projects).where(eq35(projects.name, projectName)).get();
|
|
20421
20924
|
if (!project) {
|
|
20422
20925
|
app.log.warn({ projectName }, "doctor schedule fired for an unknown project");
|
|
20423
20926
|
return;
|
|
@@ -20611,7 +21114,7 @@ async function createServer(opts) {
|
|
|
20611
21114
|
return removed;
|
|
20612
21115
|
}
|
|
20613
21116
|
};
|
|
20614
|
-
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ??
|
|
21117
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto27.randomBytes(32).toString("hex");
|
|
20615
21118
|
const googleConnectionStore = {
|
|
20616
21119
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
20617
21120
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -20709,11 +21212,11 @@ async function createServer(opts) {
|
|
|
20709
21212
|
})();
|
|
20710
21213
|
if (opts.config.apiKey) {
|
|
20711
21214
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
20712
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
21215
|
+
const existing = opts.db.select().from(apiKeys).where(eq35(apiKeys.keyHash, keyHash)).get();
|
|
20713
21216
|
if (!existing) {
|
|
20714
21217
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
20715
21218
|
opts.db.insert(apiKeys).values({
|
|
20716
|
-
id: `key_${
|
|
21219
|
+
id: `key_${crypto27.randomBytes(8).toString("hex")}`,
|
|
20717
21220
|
name: "default",
|
|
20718
21221
|
keyHash,
|
|
20719
21222
|
keyPrefix: prefix,
|
|
@@ -20728,14 +21231,14 @@ async function createServer(opts) {
|
|
|
20728
21231
|
);
|
|
20729
21232
|
const currentPasswordFingerprint = () => hashSessionToken(opts.config.dashboardPasswordHash ?? "");
|
|
20730
21233
|
opts.db.delete(dashboardSessions).where(or4(
|
|
20731
|
-
|
|
20732
|
-
|
|
21234
|
+
eq35(dashboardSessions.apiKeyHash, ""),
|
|
21235
|
+
and29(
|
|
20733
21236
|
isNotNull5(dashboardSessions.passwordFingerprint),
|
|
20734
21237
|
ne4(dashboardSessions.passwordFingerprint, currentPasswordFingerprint())
|
|
20735
21238
|
)
|
|
20736
21239
|
)).run();
|
|
20737
21240
|
if (!opts.config.dashboardPasswordHash) {
|
|
20738
|
-
const lastPasswordEventAt = (action) => opts.db.select({ at: max(auditLog.createdAt) }).from(auditLog).where(
|
|
21241
|
+
const lastPasswordEventAt = (action) => opts.db.select({ at: max(auditLog.createdAt) }).from(auditLog).where(and29(eq35(auditLog.entityType, "dashboard-password"), eq35(auditLog.action, action))).get()?.at ?? null;
|
|
20739
21242
|
const createdAt = lastPasswordEventAt("dashboard-password.created");
|
|
20740
21243
|
const clearedAt = lastPasswordEventAt("dashboard-password.cleared");
|
|
20741
21244
|
if (createdAt !== null && (clearedAt === null || clearedAt < createdAt)) {
|
|
@@ -20750,7 +21253,7 @@ async function createServer(opts) {
|
|
|
20750
21253
|
const createSession = (key, passwordBased = false, db = opts.db) => {
|
|
20751
21254
|
const now = /* @__PURE__ */ new Date();
|
|
20752
21255
|
db.delete(dashboardSessions).where(lte3(dashboardSessions.expiresAt, now.toISOString())).run();
|
|
20753
|
-
const sessionId =
|
|
21256
|
+
const sessionId = crypto27.randomBytes(32).toString("hex");
|
|
20754
21257
|
db.insert(dashboardSessions).values({
|
|
20755
21258
|
tokenHash: hashSessionToken(sessionId),
|
|
20756
21259
|
apiKeyId: key.id,
|
|
@@ -20769,23 +21272,23 @@ async function createServer(opts) {
|
|
|
20769
21272
|
currentApiKeyHash: apiKeys.keyHash,
|
|
20770
21273
|
passwordFingerprint: dashboardSessions.passwordFingerprint,
|
|
20771
21274
|
expiresAt: dashboardSessions.expiresAt
|
|
20772
|
-
}).from(dashboardSessions).innerJoin(apiKeys,
|
|
21275
|
+
}).from(dashboardSessions).innerJoin(apiKeys, eq35(apiKeys.id, dashboardSessions.apiKeyId)).where(and29(eq35(dashboardSessions.tokenHash, tokenHash), isNull8(apiKeys.revokedAt))).get();
|
|
20773
21276
|
if (!session) return null;
|
|
20774
21277
|
const expiresAt = Date.parse(session.expiresAt);
|
|
20775
21278
|
if (!Number.isFinite(expiresAt) || expiresAt <= Date.now() || session.apiKeyHash !== session.currentApiKeyHash || session.passwordFingerprint !== null && session.passwordFingerprint !== currentPasswordFingerprint()) {
|
|
20776
|
-
opts.db.delete(dashboardSessions).where(
|
|
21279
|
+
opts.db.delete(dashboardSessions).where(eq35(dashboardSessions.tokenHash, tokenHash)).run();
|
|
20777
21280
|
return null;
|
|
20778
21281
|
}
|
|
20779
21282
|
return session.apiKeyId;
|
|
20780
21283
|
};
|
|
20781
21284
|
const clearSession = (sessionId) => {
|
|
20782
21285
|
if (sessionId) {
|
|
20783
|
-
opts.db.delete(dashboardSessions).where(
|
|
21286
|
+
opts.db.delete(dashboardSessions).where(eq35(dashboardSessions.tokenHash, hashSessionToken(sessionId))).run();
|
|
20784
21287
|
}
|
|
20785
21288
|
};
|
|
20786
21289
|
const getDefaultApiKey = () => {
|
|
20787
21290
|
if (!opts.config.apiKey) return void 0;
|
|
20788
|
-
return opts.db.select().from(apiKeys).where(
|
|
21291
|
+
return opts.db.select().from(apiKeys).where(eq35(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
20789
21292
|
};
|
|
20790
21293
|
const setSessionCookie = (reply, sessionId) => {
|
|
20791
21294
|
reply.header(
|
|
@@ -20814,7 +21317,7 @@ async function createServer(opts) {
|
|
|
20814
21317
|
if (parts.length !== 2 || parts[0] !== "Bearer" || !parts[1]) return false;
|
|
20815
21318
|
const presented = Buffer.from(hashApiKey(parts[1]), "hex");
|
|
20816
21319
|
const expected = Buffer.from(key.keyHash, "hex");
|
|
20817
|
-
return presented.length === expected.length &&
|
|
21320
|
+
return presented.length === expected.length && crypto27.timingSafeEqual(presented, expected);
|
|
20818
21321
|
};
|
|
20819
21322
|
const namedAccountsInUse = () => anyUsersExist(opts.db);
|
|
20820
21323
|
const NAMED_ACCOUNTS_MESSAGE = "This install uses named accounts. Sign in with your name and password.";
|
|
@@ -20946,12 +21449,12 @@ async function createServer(opts) {
|
|
|
20946
21449
|
return reply.send({ authenticated: true });
|
|
20947
21450
|
}
|
|
20948
21451
|
if (apiKey) {
|
|
20949
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
21452
|
+
const key = opts.db.select().from(apiKeys).where(eq35(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
20950
21453
|
if (!key || key.revokedAt) {
|
|
20951
21454
|
const err2 = authInvalid();
|
|
20952
21455
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
20953
21456
|
}
|
|
20954
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
21457
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq35(apiKeys.id, key.id)).run();
|
|
20955
21458
|
setSessionCookie(reply, createSession(key));
|
|
20956
21459
|
return reply.send({ authenticated: true });
|
|
20957
21460
|
}
|
|
@@ -21017,7 +21520,7 @@ async function createServer(opts) {
|
|
|
21017
21520
|
const sentimentPoller = createSentimentPoller(
|
|
21018
21521
|
() => sentimentWorker.tick(),
|
|
21019
21522
|
() => {
|
|
21020
|
-
|
|
21523
|
+
log29.error("sentiment.worker-failed", { reason: "Stored sentiment worker could not finish its tick." });
|
|
21021
21524
|
}
|
|
21022
21525
|
);
|
|
21023
21526
|
const pollSentiment = sentimentPoller.poll;
|
|
@@ -21029,7 +21532,7 @@ async function createServer(opts) {
|
|
|
21029
21532
|
const recomputeDebts = /* @__PURE__ */ new Map();
|
|
21030
21533
|
let recomputeSequence = 0;
|
|
21031
21534
|
const markAnswerFieldRecompute = (projectId, full) => {
|
|
21032
|
-
opts.db.update(projects).set({ answerFieldsRecompute: full ? "full" :
|
|
21535
|
+
opts.db.update(projects).set({ answerFieldsRecompute: full ? "full" : sql15`CASE WHEN ${projects.answerFieldsRecompute} = 'full' THEN 'full' ELSE 'competitors' END` }).where(eq35(projects.id, projectId)).run();
|
|
21033
21536
|
};
|
|
21034
21537
|
const requestAnswerFieldRecompute = (projectId, projectName, scope, reason) => {
|
|
21035
21538
|
const sequence = ++recomputeSequence;
|
|
@@ -21047,10 +21550,10 @@ async function createServer(opts) {
|
|
|
21047
21550
|
if (pass.full && debt.latestFull <= sequence) debt.latestFull = 0;
|
|
21048
21551
|
try {
|
|
21049
21552
|
if (debt.latest === sequence && debt.latestFull === 0) {
|
|
21050
|
-
opts.db.update(projects).set({ answerFieldsRecompute: null }).where(
|
|
21553
|
+
opts.db.update(projects).set({ answerFieldsRecompute: null }).where(eq35(projects.id, projectId)).run();
|
|
21051
21554
|
if (recomputeDebts.get(projectId) === debt) recomputeDebts.delete(projectId);
|
|
21052
21555
|
} else if (debt.latestFull === 0) {
|
|
21053
|
-
opts.db.update(projects).set({ answerFieldsRecompute: "competitors" }).where(
|
|
21556
|
+
opts.db.update(projects).set({ answerFieldsRecompute: "competitors" }).where(and29(eq35(projects.id, projectId), eq35(projects.answerFieldsRecompute, "full"))).run();
|
|
21054
21557
|
}
|
|
21055
21558
|
} catch (err) {
|
|
21056
21559
|
app.log.error({ err, projectId }, "could not clear the owed answer field recompute");
|
|
@@ -21087,7 +21590,7 @@ async function createServer(opts) {
|
|
|
21087
21590
|
await yieldToEventLoop4();
|
|
21088
21591
|
const reason = (competitorAutoAliasReasons.get(projectId) ?? []).join(",");
|
|
21089
21592
|
competitorAutoAliasReasons.delete(projectId);
|
|
21090
|
-
const project = opts.db.select().from(projects).where(
|
|
21593
|
+
const project = opts.db.select().from(projects).where(eq35(projects.id, projectId)).get();
|
|
21091
21594
|
if (!project || project.competitorAutoAliases === "off") return;
|
|
21092
21595
|
if (project.competitorAutoAliases === "apply") {
|
|
21093
21596
|
await competitorAutoAliases.request(projectId, reason).catch(() => void 0);
|
|
@@ -21287,8 +21790,8 @@ async function createServer(opts) {
|
|
|
21287
21790
|
getAgentPin: () => describeAgentPin(opts.config),
|
|
21288
21791
|
getGoogleMarketingDoctorInput: (ctx) => {
|
|
21289
21792
|
if (!ctx.project) return null;
|
|
21290
|
-
const googleAds = opts.db.select().from(googleAdsConnections).where(
|
|
21291
|
-
const gtm = opts.db.select().from(gtmConnections).where(
|
|
21793
|
+
const googleAds = opts.db.select().from(googleAdsConnections).where(eq35(googleAdsConnections.projectId, ctx.project.id)).get();
|
|
21794
|
+
const gtm = opts.db.select().from(gtmConnections).where(eq35(gtmConnections.projectId, ctx.project.id)).get();
|
|
21292
21795
|
const googleAdsCredential = getGoogleAdsConnection(opts.config, ctx.project.id);
|
|
21293
21796
|
const gtmCredential = getGtmConnection(opts.config, ctx.project.id);
|
|
21294
21797
|
const googleAdsConnected = Boolean(googleAdsCredential?.accessToken);
|
|
@@ -21417,7 +21920,7 @@ async function createServer(opts) {
|
|
|
21417
21920
|
deps: {
|
|
21418
21921
|
enqueueAutoExtract: ({ projectId, release: r }) => {
|
|
21419
21922
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
21420
|
-
const runId =
|
|
21923
|
+
const runId = crypto27.randomUUID();
|
|
21421
21924
|
opts.db.insert(runs).values({
|
|
21422
21925
|
id: runId,
|
|
21423
21926
|
projectId,
|
|
@@ -21538,7 +22041,7 @@ async function createServer(opts) {
|
|
|
21538
22041
|
...inspectOpts,
|
|
21539
22042
|
config: opts.config
|
|
21540
22043
|
}).then(() => {
|
|
21541
|
-
const finished = opts.db.select({ status: runs.status }).from(runs).where(
|
|
22044
|
+
const finished = opts.db.select({ status: runs.status }).from(runs).where(eq35(runs.id, runId)).get();
|
|
21542
22045
|
if (finished?.status === RunStatuses.completed || finished?.status === RunStatuses.partial) {
|
|
21543
22046
|
return maybeRefreshGscCoverage(
|
|
21544
22047
|
opts.db,
|
|
@@ -21554,6 +22057,13 @@ async function createServer(opts) {
|
|
|
21554
22057
|
app.log.error({ runId, err }, "Bing inspect sitemap failed");
|
|
21555
22058
|
});
|
|
21556
22059
|
},
|
|
22060
|
+
onBingSyncRequested: (runId, projectId) => {
|
|
22061
|
+
executeBingSync(opts.db, runId, projectId, { config: opts.config }).catch(
|
|
22062
|
+
(err) => {
|
|
22063
|
+
app.log.error({ runId, err }, "Bing sync failed");
|
|
22064
|
+
}
|
|
22065
|
+
);
|
|
22066
|
+
},
|
|
21557
22067
|
wordpressConnectionStore,
|
|
21558
22068
|
ga4CredentialStore,
|
|
21559
22069
|
cloudRunCredentialStore,
|
|
@@ -21784,7 +22294,7 @@ async function createServer(opts) {
|
|
|
21784
22294
|
}
|
|
21785
22295
|
];
|
|
21786
22296
|
},
|
|
21787
|
-
onGenerateQueries: async (providerName,
|
|
22297
|
+
onGenerateQueries: async (providerName, count6, project) => {
|
|
21788
22298
|
const provider = registry.get(providerName);
|
|
21789
22299
|
if (!provider)
|
|
21790
22300
|
throw new Error(`Provider "${providerName}" is not configured`);
|
|
@@ -21798,10 +22308,10 @@ async function createServer(opts) {
|
|
|
21798
22308
|
language: project.language,
|
|
21799
22309
|
existingQueries: project.existingQueries,
|
|
21800
22310
|
siteText,
|
|
21801
|
-
count:
|
|
22311
|
+
count: count6
|
|
21802
22312
|
});
|
|
21803
22313
|
const raw = await provider.adapter.generateText(prompt, provider.config);
|
|
21804
|
-
return parseQueryResponse(raw,
|
|
22314
|
+
return parseQueryResponse(raw, count6);
|
|
21805
22315
|
},
|
|
21806
22316
|
onSnapshotRequested: async (input) => {
|
|
21807
22317
|
return snapshotService.createReport(input);
|
|
@@ -22039,7 +22549,7 @@ async function createServer(opts) {
|
|
|
22039
22549
|
isBacklinksInstalled: () => isDuckdbInstalled()
|
|
22040
22550
|
});
|
|
22041
22551
|
}
|
|
22042
|
-
for (const run of opts.db.select({ id: researchRuns.id, projectId: researchRuns.projectId }).from(researchRuns).where(
|
|
22552
|
+
for (const run of opts.db.select({ id: researchRuns.id, projectId: researchRuns.projectId }).from(researchRuns).where(eq35(researchRuns.status, ResearchRunStatuses.queued)).all()) {
|
|
22043
22553
|
dispatchResearchRun(run.id, run.projectId, OutcomeTriggers.startup);
|
|
22044
22554
|
}
|
|
22045
22555
|
runtimeStartupSettled = true;
|
|
@@ -22104,7 +22614,7 @@ function buildQueryGenerationPrompt(ctx) {
|
|
|
22104
22614
|
);
|
|
22105
22615
|
return lines.join("\n");
|
|
22106
22616
|
}
|
|
22107
|
-
function parseQueryResponse(raw,
|
|
22617
|
+
function parseQueryResponse(raw, count6) {
|
|
22108
22618
|
const seen = /* @__PURE__ */ new Set();
|
|
22109
22619
|
const results = [];
|
|
22110
22620
|
for (const line of raw.split("\n")) {
|
|
@@ -22120,7 +22630,7 @@ function parseQueryResponse(raw, count5) {
|
|
|
22120
22630
|
if (seen.has(key)) continue;
|
|
22121
22631
|
seen.add(key);
|
|
22122
22632
|
results.push(cleaned);
|
|
22123
|
-
if (results.length >=
|
|
22633
|
+
if (results.length >= count6) break;
|
|
22124
22634
|
}
|
|
22125
22635
|
return results;
|
|
22126
22636
|
}
|