@canonry/canonry 7.89.0 → 7.90.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +12 -0
- package/assets/assets/{AgentSetupPage-BU1supL8.js → AgentSetupPage-BSfsbQXZ.js} +1 -1
- package/assets/assets/{AuditHistoryPanel-D7or_J3j.js → AuditHistoryPanel-YrEggsyA.js} +1 -1
- package/assets/assets/{BacklinksPage-SOls3ew4.js → BacklinksPage-vXUxxplH.js} +1 -1
- package/assets/assets/{HistoryPage-LNKtQ6a8.js → HistoryPage-VJ_JQ7R3.js} +1 -1
- package/assets/assets/MeasurementPropertyPage-BHPa2zdE.js +4 -0
- package/assets/assets/ProjectPage-C1AI_aAQ.js +11 -0
- package/assets/assets/{RunsPage-B8zUD2gr.js → RunsPage-DKIktkY1.js} +1 -1
- package/assets/assets/SettingsPage-CTXfqNZs.js +1 -0
- package/assets/assets/SiteHealthSection-CRwvLfHd.js +2 -0
- package/assets/assets/{ToneBadge-DawS8vMp.js → ToneBadge-B3rD6bDF.js} +5 -5
- package/assets/assets/{TrafficPage-DJUqxtrQ.js → TrafficPage-C4R3X5lf.js} +4 -4
- package/assets/assets/{TrafficSourceDetailPage-OVY_L73m.js → TrafficSourceDetailPage-C79eQilP.js} +1 -1
- package/assets/assets/{account-context-DoI9S9qY.js → account-context-GstYbqpX.js} +1 -1
- package/assets/assets/{agent-trust-copy-CZpfPr4M.js → agent-trust-copy-Ca4ys8J2.js} +1 -1
- package/assets/assets/{card-ClURYmZN.js → card-CgdGerYK.js} +1 -1
- package/assets/assets/{format-helpers-CscAFAMJ.js → format-helpers-ByAY4dv0.js} +1 -1
- package/assets/assets/index-DdG9gR0r.js +60 -0
- package/assets/assets/query-client-DjRrRIsq.js +1 -0
- package/assets/assets/react-query-B8JD_K7g.js +1 -0
- package/assets/assets/{react-sigma_core.esm.min-6J0MSvMJ.js → react-sigma_core.esm.min-BoCIEnkF.js} +1 -1
- package/assets/assets/{server-traffic-C-_D6pdo.js → server-traffic-Bt9bexaP.js} +1 -1
- package/assets/assets/{service-C7qDx_qB.js → service-SgtD5EOS.js} +1 -1
- package/assets/index.html +7 -7
- package/dist/{ads-XEW2LBKV.js → ads-6BAITIBY.js} +3 -3
- package/dist/{agent-VYADMMPT.js → agent-BE3D65U4.js} +7 -7
- package/dist/{agent-ask-4G57GM3S.js → agent-ask-TT6BAV2T.js} +3 -3
- package/dist/{agent-conversations-MRKRCXQI.js → agent-conversations-GJDAWU36.js} +3 -3
- package/dist/{agent-journeys-WEUEYRMK.js → agent-journeys-UEL27GNY.js} +3 -3
- package/dist/{agent-memory-YLAOXQQ6.js → agent-memory-PB74OP46.js} +3 -3
- package/dist/{agent-providers-AWKQRJDZ.js → agent-providers-5WFMNRRK.js} +3 -3
- package/dist/{agent-transcript-HXYFVCUD.js → agent-transcript-IAGB3JXG.js} +3 -3
- package/dist/{analytics-5G5G5OJU.js → analytics-5BVE7KUN.js} +3 -3
- package/dist/{apply-FCKZG7A2.js → apply-YJJWQ4JX.js} +3 -3
- package/dist/{backfill-3CTZHCJS.js → backfill-6OLGZIEV.js} +10 -10
- package/dist/{backlinks-Q53JT63C.js → backlinks-4JK26OIO.js} +3 -3
- package/dist/{bing-FOY2UVIM.js → bing-NZYYPB2M.js} +12 -3
- package/dist/{bootstrap-FXIT7LDW.js → bootstrap-6KHK63W3.js} +10 -10
- package/dist/{cdp-WWQW3LIZ.js → cdp-PTXWVX6T.js} +7 -7
- package/dist/{chunk-ZYEWB5BK.js → chunk-2UULABPI.js} +7 -3
- package/dist/{chunk-JLBJEQPV.js → chunk-7ASKZHS4.js} +1 -1
- package/dist/{chunk-S7D2ELTC.js → chunk-7VJJXLRP.js} +3 -3
- package/dist/{chunk-WPTCWNA6.js → chunk-AFYKUX2X.js} +2 -2
- package/dist/{chunk-7522CQ2E.js → chunk-CHTADAEC.js} +4 -4
- package/dist/{chunk-GDEYX6E3.js → chunk-DF4B6OGX.js} +4 -4
- package/dist/{chunk-RE24YYIY.js → chunk-DZO6CATO.js} +4 -4
- package/dist/{chunk-YKLYWVLL.js → chunk-ETIIPSWL.js} +71 -4
- package/dist/{chunk-UIULMWJX.js → chunk-F442MSX2.js} +2 -2
- package/dist/{chunk-TCANF22P.js → chunk-F6WNKKGG.js} +33 -7
- package/dist/{chunk-DPKA5CWK.js → chunk-GKX5F7NX.js} +1 -1
- package/dist/{chunk-X6XFWQCE.js → chunk-HUQDD26S.js} +7 -7
- package/dist/{chunk-4INNOSWU.js → chunk-I7DAEE3L.js} +5 -5
- package/dist/{chunk-5GQ2AGWS.js → chunk-L2MDMZFT.js} +3 -3
- package/dist/{chunk-3H5VURTU.js → chunk-LQWMNDZR.js} +1 -1
- package/dist/{chunk-2BZJ4W27.js → chunk-LTZNUDSI.js} +3 -3
- package/dist/{chunk-EWR25OVQ.js → chunk-N7LRPMWR.js} +2 -2
- package/dist/{chunk-GSNKBXGN.js → chunk-NKGQCCUS.js} +6 -6
- package/dist/{chunk-VL4AZGUA.js → chunk-NPX5SL54.js} +1 -1
- package/dist/{chunk-A6DKZ4KQ.js → chunk-OOVWMWMT.js} +1 -1
- package/dist/{chunk-DX46HM7S.js → chunk-PID3C4B7.js} +1 -1
- package/dist/{chunk-MXGME7NG.js → chunk-Q6EZEVXE.js} +1 -1
- package/dist/{chunk-J7V7TYX7.js → chunk-QLVTMPFW.js} +2 -2
- package/dist/{chunk-MNLQY5WG.js → chunk-QWNDNZJ3.js} +2 -2
- package/dist/{chunk-HSOZSLIZ.js → chunk-SDXMG4VX.js} +1064 -873
- package/dist/{chunk-WHN4RF2Y.js → chunk-T7OH3YPX.js} +395 -362
- package/dist/{chunk-3IC7PRTW.js → chunk-TA4L7U2M.js} +1 -1
- package/dist/{chunk-ZX4TZNP4.js → chunk-UG7TZJIG.js} +1 -1
- package/dist/{chunk-ZEWOPHTA.js → chunk-USHY3SH6.js} +62 -1
- package/dist/{chunk-VBM6T5ZP.js → chunk-WNDBACO7.js} +25 -3
- package/dist/{chunk-WE6FSG7J.js → chunk-WYMBWCG5.js} +3 -1
- package/dist/{chunk-Y35D3ZCH.js → chunk-XCEA5OMI.js} +2 -2
- package/dist/{chunk-KAGMYJYG.js → chunk-YB2GAK32.js} +5 -5
- package/dist/{chunk-IK3PYAQU.js → chunk-YFZHRYCM.js} +94 -21
- package/dist/{chunk-YW2IQEWS.js → chunk-ZV6PMFF4.js} +1 -1
- package/dist/{citations-ZZ6ODCGZ.js → citations-WX4VO6ZI.js} +3 -3
- package/dist/{cli-catalog-HEX47BKL.js → cli-catalog-BTRNR2UP.js} +3 -3
- package/dist/{cli-commands-6OLN355M.js → cli-commands-EXWAMK7Z.js} +7 -7
- package/dist/cli.js +13 -13
- package/dist/{competitor-6MRHORU2.js → competitor-HZRZWCEN.js} +3 -3
- package/dist/{content-S2UOGTLN.js → content-NOZDHI3M.js} +3 -3
- package/dist/{dashboard-J4H3FVKH.js → dashboard-TZLWHXP4.js} +6 -6
- package/dist/{demo-7PJJQMLO.js → demo-2PYBQD47.js} +1 -1
- package/dist/{demo-server-55I5LNNS.js → demo-server-JEQNGMMV.js} +12 -12
- package/dist/{discover-O4NFDHGP.js → discover-A2XFXHOC.js} +3 -3
- package/dist/{discover-eval-PE2MW42B.js → discover-eval-A6SBUYXF.js} +3 -3
- package/dist/{doctor-GT4MMJQJ.js → doctor-GZACELPF.js} +3 -3
- package/dist/{evidence-Z344KVJX.js → evidence-RVFMUWIL.js} +3 -3
- package/dist/{export-cmd-Z6WWLI6H.js → export-cmd-X2TFZGNJ.js} +3 -3
- package/dist/{ga-SXIGAF5F.js → ga-TOT3V67A.js} +3 -3
- package/dist/{gbp-CSLWSEKF.js → gbp-KP4YZU6S.js} +3 -3
- package/dist/{google-EAQJ46AR.js → google-GCFSTV5N.js} +4 -4
- package/dist/{google-marketing-NVKB4NLG.js → google-marketing-ANS65KQN.js} +1 -1
- package/dist/{health-cmd-6IYRMZS3.js → health-cmd-UKMNLU6D.js} +3 -3
- package/dist/{history-KLZI7IKJ.js → history-TWGYSAZZ.js} +3 -3
- package/dist/index.d.ts +395 -37
- package/dist/index.js +25 -25
- package/dist/{init-LFLCM6KJ.js → init-2WZWYSDW.js} +12 -12
- package/dist/{insights-C7NN64E2.js → insights-7IOC7WIR.js} +3 -3
- package/dist/intelligence-service-42Q3OVI5.js +16 -0
- package/dist/{journeys-C34YDOM6.js → journeys-NWKT45PT.js} +3 -3
- package/dist/{keys-4RDLFHXO.js → keys-ZZYZUHUK.js} +3 -3
- package/dist/{keyword-BZEJEYAG.js → keyword-F6EIX5KM.js} +3 -3
- package/dist/{logs-5FPZRSZM.js → logs-B45FXS6E.js} +3 -3
- package/dist/{mcp-574UA5Y3.js → mcp-WPNCNWA5.js} +4 -4
- package/dist/mcp.js +9 -9
- package/dist/{measurement-plan-2H4KHD2Y.js → measurement-plan-QT5HJ3KC.js} +5 -5
- package/dist/{notify-TFXL4CSH.js → notify-Z3LIF3SO.js} +3 -3
- package/dist/{organic-evidence-R55KSYFN.js → organic-evidence-R5TM7XUB.js} +3 -3
- package/dist/{overview-NJTCLHAB.js → overview-6TZHJLQN.js} +4 -4
- package/dist/{project-PV36DZDG.js → project-VMPVOQ2A.js} +3 -3
- package/dist/{query-5SVITJ4K.js → query-EXSH5QP7.js} +3 -3
- package/dist/{research-JSET5FNY.js → research-6R6BEL4X.js} +3 -3
- package/dist/{results-export-5EFMXTPG.js → results-export-ZJRL3ECU.js} +3 -3
- package/dist/{run-NOSOG3YH.js → run-C4VXAJE3.js} +4 -4
- package/dist/{schedule-I3ACQC2C.js → schedule-KC6ETWMO.js} +3 -3
- package/dist/{search-EE53K7J5.js → search-SUR6BEQD.js} +3 -3
- package/dist/{serve-SDU77HVC.js → serve-VVG3VYMU.js} +26 -26
- package/dist/{session-FYZCUPVX.js → session-D42KUKCZ.js} +13 -13
- package/dist/{settings-CEI7EKPO.js → settings-LTCG3D5B.js} +3 -3
- package/dist/sitemaps-YMTZLF5S.js +46 -0
- package/dist/{skills-TRTFFZU3.js → skills-THMG3L64.js} +2 -2
- package/dist/{snapshot-2RXVTDTZ.js → snapshot-ZEVB3EE4.js} +3 -3
- package/dist/{src-DYJHW2HG.js → src-HPVSTXVF.js} +6 -2
- package/dist/{status-2VHBSIRY.js → status-LPCPTWKO.js} +4 -4
- package/dist/{technical-aeo-3OMGT4ZT.js → technical-aeo-J6LNT45B.js} +3 -3
- package/dist/{telemetry-5F4EHGR2.js → telemetry-BPY34GDV.js} +4 -4
- package/dist/{tools-C3IIMXBJ.js → tools-K3G43ESI.js} +11 -11
- package/dist/{traffic-S3VUYBG6.js → traffic-BOIFAPAR.js} +5 -5
- package/dist/{trends-RQ4B2QWA.js → trends-A2MDRSQQ.js} +3 -3
- package/dist/{users-T6OOG5YB.js → users-VJOQODJK.js} +3 -3
- package/dist/{visibility-stats-2O24MQCV.js → visibility-stats-YNMJYASA.js} +3 -3
- package/dist/{wordpress-SDENOBFI.js → wordpress-BWG6D7VH.js} +3 -3
- package/package.json +1 -1
- package/assets/assets/MeasurementPropertyPage-CTZzlHWE.js +0 -4
- package/assets/assets/ProjectPage-D79zuY-M.js +0 -11
- package/assets/assets/SettingsPage-BOQgT_7N.js +0 -1
- package/assets/assets/SiteHealthSection-G3_MShnD.js +0 -2
- package/assets/assets/index-y0Wx0ZWq.js +0 -60
- package/assets/assets/query-client-Dm_GqsTu.js +0 -1
- package/assets/assets/react-query-BwhgbuS1.js +0 -1
- package/dist/intelligence-service-ENI35JFR.js +0 -16
|
@@ -15,14 +15,14 @@ import {
|
|
|
15
15
|
normalizeCloudflareWorkerEvent,
|
|
16
16
|
sessionizeJourneyRequests,
|
|
17
17
|
verifyRequestSignature
|
|
18
|
-
} from "./chunk-
|
|
18
|
+
} from "./chunk-NPX5SL54.js";
|
|
19
19
|
import {
|
|
20
20
|
currentOutcomeAttribution,
|
|
21
21
|
getRequestContext,
|
|
22
22
|
registerRequestContext,
|
|
23
23
|
requestAuditIdentity,
|
|
24
24
|
sanitizeRequestContext
|
|
25
|
-
} from "./chunk-
|
|
25
|
+
} from "./chunk-ZV6PMFF4.js";
|
|
26
26
|
import {
|
|
27
27
|
RawEventSampleEventTypes,
|
|
28
28
|
SentimentIdempotencyConflict,
|
|
@@ -106,6 +106,7 @@ import {
|
|
|
106
106
|
gscDataWatermarks,
|
|
107
107
|
gscQueryDailyTotals,
|
|
108
108
|
gscSearchData,
|
|
109
|
+
gscSitemaps,
|
|
109
110
|
gscUrlInspections,
|
|
110
111
|
gtmConnections,
|
|
111
112
|
gtmRawSnapshots,
|
|
@@ -171,6 +172,7 @@ import {
|
|
|
171
172
|
siteCrawlPages,
|
|
172
173
|
siteCrawlRunRequests,
|
|
173
174
|
siteCrawlSnapshots,
|
|
175
|
+
sitemapListSyncs,
|
|
174
176
|
trafficEventReceipts,
|
|
175
177
|
trafficSources,
|
|
176
178
|
unrecognizedAgentEventsHourly,
|
|
@@ -178,7 +180,7 @@ import {
|
|
|
178
180
|
userSessions,
|
|
179
181
|
users,
|
|
180
182
|
writeNegativeReviewMaxStars
|
|
181
|
-
} from "./chunk-
|
|
183
|
+
} from "./chunk-USHY3SH6.js";
|
|
182
184
|
import {
|
|
183
185
|
ADS_ACTIVATE_SCOPE,
|
|
184
186
|
ADS_ACTIVATION_ABSOLUTE_MAX_ENTITIES,
|
|
@@ -534,6 +536,7 @@ import {
|
|
|
534
536
|
bingSitemapsResponseDtoSchema,
|
|
535
537
|
bingSitesResponseDtoSchema,
|
|
536
538
|
bingStatusDtoSchema,
|
|
539
|
+
bingSubmitSitemapResponseDtoSchema,
|
|
537
540
|
bingTrafficResponseDtoSchema,
|
|
538
541
|
bingUrlChildrenResponseDtoSchema,
|
|
539
542
|
bingUrlInspectionDtoSchema,
|
|
@@ -1190,6 +1193,7 @@ import {
|
|
|
1190
1193
|
siteHealthStateSchema,
|
|
1191
1194
|
siteHealthSubgraphResponseSchema,
|
|
1192
1195
|
siteHealthTemplateDetectionSchema,
|
|
1196
|
+
sitemapCoverageResponseDtoSchema,
|
|
1193
1197
|
snapshotDiffResponseSchema,
|
|
1194
1198
|
snapshotListResponseSchema,
|
|
1195
1199
|
snapshotReportSchema,
|
|
@@ -1270,7 +1274,7 @@ import {
|
|
|
1270
1274
|
wordpressSchemaDeployResultDtoSchema,
|
|
1271
1275
|
wordpressSchemaStatusResultDtoSchema,
|
|
1272
1276
|
wordpressStatusDtoSchema
|
|
1273
|
-
} from "./chunk-
|
|
1277
|
+
} from "./chunk-ETIIPSWL.js";
|
|
1274
1278
|
|
|
1275
1279
|
// ../api-routes/src/user-session.ts
|
|
1276
1280
|
import crypto2 from "crypto";
|
|
@@ -35635,6 +35639,8 @@ var SCHEMA_TABLE = {
|
|
|
35635
35639
|
BingSiteResponseDto: bingSiteResponseDtoSchema,
|
|
35636
35640
|
BingSitemapsResponseDto: bingSitemapsResponseDtoSchema,
|
|
35637
35641
|
BingTrafficResponseDto: bingTrafficResponseDtoSchema,
|
|
35642
|
+
BingSubmitSitemapResponseDto: bingSubmitSitemapResponseDtoSchema,
|
|
35643
|
+
SitemapCoverageResponseDto: sitemapCoverageResponseDtoSchema,
|
|
35638
35644
|
BingUrlChildrenResponseDto: bingUrlChildrenResponseDtoSchema,
|
|
35639
35645
|
BingUrlTrafficResponseDto: bingUrlTrafficResponseDtoSchema,
|
|
35640
35646
|
BingCoverageSnapshotDto: bingCoverageSnapshotDtoSchema,
|
|
@@ -41807,6 +41813,37 @@ var routeCatalog = [
|
|
|
41807
41813
|
404: errorResponse("Project not found.")
|
|
41808
41814
|
}
|
|
41809
41815
|
},
|
|
41816
|
+
{
|
|
41817
|
+
method: "post",
|
|
41818
|
+
path: "/api/v1/projects/{name}/bing/sitemaps",
|
|
41819
|
+
summary: "Ask Bing to read a sitemap (adds a new one, rereads a listed one)",
|
|
41820
|
+
description: "Calls Bing `SubmitFeed` for the selected site. Bing lists a new sitemap once it has read it, so it shows in `/bing/sitemaps` and `/sitemaps` after the next `bing-sync`.",
|
|
41821
|
+
tags: ["bing"],
|
|
41822
|
+
parameters: [nameParameter],
|
|
41823
|
+
requestBody: {
|
|
41824
|
+
required: true,
|
|
41825
|
+
content: { "application/json": { schema: { type: "object", required: ["sitemapUrl"], properties: { sitemapUrl: stringSchema } } } }
|
|
41826
|
+
},
|
|
41827
|
+
responses: {
|
|
41828
|
+
200: jsonResponse("Bing accepted the sitemap.", "BingSubmitSitemapResponseDto"),
|
|
41829
|
+
400: errorResponse("Invalid sitemap URL, or Bing is not connected or has no site selected for this project."),
|
|
41830
|
+
404: errorResponse("Project not found."),
|
|
41831
|
+
429: errorResponse("Bing is throttling this key."),
|
|
41832
|
+
502: errorResponse("Bing refused the sitemap.")
|
|
41833
|
+
}
|
|
41834
|
+
},
|
|
41835
|
+
{
|
|
41836
|
+
method: "get",
|
|
41837
|
+
path: "/api/v1/projects/{name}/sitemaps",
|
|
41838
|
+
summary: "Compare the site's sitemaps with what Google and Bing list",
|
|
41839
|
+
description: "Stored data only. `site.sitemaps` are the sitemap files the latest Site Health scan read pages from. `google` and `bing` hold each engine's stored list (null when the engine is not connected or its list was never read) and `missing`: sitemaps the site declares, or someone submitted to the other engine, that this engine lacks. A file inside a missing index is left out; Google lists only sitemaps inside the connected property.",
|
|
41840
|
+
tags: ["sitemaps"],
|
|
41841
|
+
parameters: [nameParameter],
|
|
41842
|
+
responses: {
|
|
41843
|
+
200: jsonResponse("Sitemap coverage returned.", "SitemapCoverageResponseDto"),
|
|
41844
|
+
404: errorResponse("Project not found.")
|
|
41845
|
+
}
|
|
41846
|
+
},
|
|
41810
41847
|
{
|
|
41811
41848
|
method: "get",
|
|
41812
41849
|
path: "/api/v1/projects/{name}/bing/traffic",
|
|
@@ -57892,6 +57929,15 @@ async function getChildrenUrlInfo(apiKey, siteUrl, url, page2 = 0, filterPropert
|
|
|
57892
57929
|
async function getFeeds(apiKey, siteUrl) {
|
|
57893
57930
|
return siteList(apiKey, "GetFeeds", siteUrl);
|
|
57894
57931
|
}
|
|
57932
|
+
async function submitFeed(apiKey, siteUrl, feedUrl) {
|
|
57933
|
+
validateApiKey(apiKey);
|
|
57934
|
+
validateSiteUrl2(siteUrl);
|
|
57935
|
+
validateUrl2(feedUrl);
|
|
57936
|
+
await bingFetch(apiKey, "SubmitFeed", {
|
|
57937
|
+
method: "POST",
|
|
57938
|
+
body: { siteUrl, feedUrl }
|
|
57939
|
+
});
|
|
57940
|
+
}
|
|
57895
57941
|
async function getFeedDetails(apiKey, siteUrl, feedUrl) {
|
|
57896
57942
|
validateUrl2(feedUrl);
|
|
57897
57943
|
return siteList(apiKey, "GetFeedDetails", siteUrl, { feedUrl });
|
|
@@ -58090,18 +58136,18 @@ function ctrOf(clicks, impressions) {
|
|
|
58090
58136
|
function round(value) {
|
|
58091
58137
|
return value === null ? null : Math.round(value * 100) / 100;
|
|
58092
58138
|
}
|
|
58093
|
-
function bingReadError(err, what) {
|
|
58139
|
+
function bingReadError(err, what, verb = "read") {
|
|
58094
58140
|
if (!(err instanceof BingApiError)) return err;
|
|
58095
58141
|
if (err.isThrottle || err.status === 429) {
|
|
58096
58142
|
return quotaExceeded("Bing Webmaster API", { reason: "bing-throttled", detail: err.message });
|
|
58097
58143
|
}
|
|
58098
58144
|
if (err.bingErrorCode === 14) {
|
|
58099
|
-
return providerError(`Bing refused to
|
|
58145
|
+
return providerError(`Bing refused to ${verb} ${what}: this API key is not allowed to (owner-only).`, { bingErrorCode: 14 });
|
|
58100
58146
|
}
|
|
58101
58147
|
if (err.status === 400 && err.bingErrorCode === null) {
|
|
58102
58148
|
return validationError(err.message);
|
|
58103
58149
|
}
|
|
58104
|
-
return providerError(`Bing could not
|
|
58150
|
+
return providerError(`Bing could not ${verb} ${what}: ${err.message}`, { bingErrorCode: err.bingErrorCode, httpStatus: err.status });
|
|
58105
58151
|
}
|
|
58106
58152
|
async function liveRead(what, read) {
|
|
58107
58153
|
try {
|
|
@@ -58156,6 +58202,17 @@ async function bingDataRoutes(app, opts) {
|
|
|
58156
58202
|
opts.onSyncRequested?.(runId, project.id);
|
|
58157
58203
|
return app.db.select().from(runs).where(eq79(runs.id, runId)).get();
|
|
58158
58204
|
});
|
|
58205
|
+
app.post("/projects/:name/bing/sitemaps", async (request) => {
|
|
58206
|
+
const project = resolveProject(app.db, request.params.name);
|
|
58207
|
+
const sitemapUrl = requireHttpUrl(request.body?.sitemapUrl, "sitemapUrl");
|
|
58208
|
+
const { apiKey, siteUrl } = opts.requireSite(project.canonicalDomain);
|
|
58209
|
+
try {
|
|
58210
|
+
await submitFeed(apiKey, siteUrl, sitemapUrl);
|
|
58211
|
+
} catch (err) {
|
|
58212
|
+
throw bingReadError(err, "the sitemap", "accept");
|
|
58213
|
+
}
|
|
58214
|
+
return { sitemapUrl, accepted: true };
|
|
58215
|
+
});
|
|
58159
58216
|
app.get("/projects/:name/bing/traffic", async (request) => {
|
|
58160
58217
|
const project = resolveProject(app.db, request.params.name);
|
|
58161
58218
|
const bounds = app.db.select({ latest: sql32`MAX(${bingTrafficDaily.date})`, earliest: sql32`MIN(${bingTrafficDaily.date})` }).from(bingTrafficDaily).where(eq79(bingTrafficDaily.projectId, project.id)).get();
|
|
@@ -59035,18 +59092,147 @@ async function bingRoutes(app, opts) {
|
|
|
59035
59092
|
});
|
|
59036
59093
|
}
|
|
59037
59094
|
|
|
59095
|
+
// ../api-routes/src/sitemaps.ts
|
|
59096
|
+
import { and as and69, asc as asc19, desc as desc36, eq as eq81, inArray as inArray39, sql as sql33 } from "drizzle-orm";
|
|
59097
|
+
|
|
59098
|
+
// ../api-routes/src/sitemap-coverage.ts
|
|
59099
|
+
function sitemapKey(url) {
|
|
59100
|
+
try {
|
|
59101
|
+
const parsed = new URL(url);
|
|
59102
|
+
return `${parsed.protocol}//${parsed.host.toLowerCase()}${parsed.pathname}${parsed.search}`;
|
|
59103
|
+
} catch {
|
|
59104
|
+
return url;
|
|
59105
|
+
}
|
|
59106
|
+
}
|
|
59107
|
+
function googleSitemapState(row) {
|
|
59108
|
+
if (row.isPending) return { state: "pending", statusLabel: "Processing" };
|
|
59109
|
+
if ((row.errors ?? 0) > 0) return { state: "errors", statusLabel: "Errors" };
|
|
59110
|
+
if ((row.warnings ?? 0) > 0) return { state: "warnings", statusLabel: "Warnings" };
|
|
59111
|
+
return { state: "ok", statusLabel: "Success" };
|
|
59112
|
+
}
|
|
59113
|
+
function bingSitemapState(status) {
|
|
59114
|
+
const word = status.trim().toLowerCase();
|
|
59115
|
+
if (word === "success") return "ok";
|
|
59116
|
+
if (word === "pending" || word === "processing") return "pending";
|
|
59117
|
+
if (word.includes("error") || word.includes("fail")) return "errors";
|
|
59118
|
+
if (word.includes("warn")) return "warnings";
|
|
59119
|
+
return "other";
|
|
59120
|
+
}
|
|
59121
|
+
function missingFor(list, other, site, parents, accepts) {
|
|
59122
|
+
const have = new Set(list.sitemaps.map((row) => sitemapKey(row.url)));
|
|
59123
|
+
const candidates = /* @__PURE__ */ new Map();
|
|
59124
|
+
const add = (url, source) => {
|
|
59125
|
+
const key = sitemapKey(url);
|
|
59126
|
+
if (have.has(key) || !accepts(url)) return;
|
|
59127
|
+
const entry = candidates.get(key) ?? { url, foundIn: /* @__PURE__ */ new Set() };
|
|
59128
|
+
entry.foundIn.add(source);
|
|
59129
|
+
candidates.set(key, entry);
|
|
59130
|
+
};
|
|
59131
|
+
for (const url of site) add(url, "site");
|
|
59132
|
+
for (const row of other?.list.sitemaps ?? []) if (row.parentUrl === null && row.submittedAt !== null) add(row.url, other.name);
|
|
59133
|
+
return [...candidates.entries()].filter(([key]) => {
|
|
59134
|
+
const parent2 = parents.get(key);
|
|
59135
|
+
return parent2 === void 0 || !candidates.has(parent2);
|
|
59136
|
+
}).map(([, entry]) => ({ url: entry.url, foundIn: ["site", "google", "bing"].filter((source) => entry.foundIn.has(source)) })).sort((a, b) => a.url.localeCompare(b.url));
|
|
59137
|
+
}
|
|
59138
|
+
function buildSitemapCoverage(input) {
|
|
59139
|
+
const parents = /* @__PURE__ */ new Map();
|
|
59140
|
+
for (const list of [input.google, input.bing]) {
|
|
59141
|
+
for (const row of list?.sitemaps ?? []) if (row.parentUrl !== null) parents.set(sitemapKey(row.url), sitemapKey(row.parentUrl));
|
|
59142
|
+
}
|
|
59143
|
+
const engine = (name, list, otherName, other) => list === null ? null : {
|
|
59144
|
+
syncedAt: list.syncedAt,
|
|
59145
|
+
sitemaps: list.sitemaps,
|
|
59146
|
+
missing: missingFor(list, other ? { name: otherName, list: other } : null, input.site.sitemaps, parents, input.accepts?.[name] ?? (() => true))
|
|
59147
|
+
};
|
|
59148
|
+
return {
|
|
59149
|
+
site: input.site,
|
|
59150
|
+
google: engine("google", input.google, "bing", input.bing),
|
|
59151
|
+
bing: engine("bing", input.bing, "google", input.google)
|
|
59152
|
+
};
|
|
59153
|
+
}
|
|
59154
|
+
|
|
59155
|
+
// ../api-routes/src/sitemaps.ts
|
|
59156
|
+
async function sitemapRoutes(app, opts) {
|
|
59157
|
+
app.get("/projects/:name/sitemaps", async (request) => {
|
|
59158
|
+
const project = resolveProject(app.db, request.params.name);
|
|
59159
|
+
const scan = app.db.select({ runId: siteCrawlSnapshots.runId, attemptId: siteCrawlSnapshots.attemptId, createdAt: siteCrawlSnapshots.createdAt }).from(siteCrawlSnapshots).innerJoin(runs, eq81(siteCrawlSnapshots.runId, runs.id)).where(and69(
|
|
59160
|
+
eq81(siteCrawlSnapshots.projectId, project.id),
|
|
59161
|
+
eq81(runs.projectId, project.id),
|
|
59162
|
+
eq81(runs.kind, RunKinds["site-audit"]),
|
|
59163
|
+
inArray39(runs.status, [RunStatuses.completed, RunStatuses.partial]),
|
|
59164
|
+
notProbeRun()
|
|
59165
|
+
)).orderBy(desc36(siteCrawlSnapshots.createdAt), desc36(siteCrawlSnapshots.runId)).limit(1).get();
|
|
59166
|
+
const siteSitemaps = scan ? app.db.all(sql33`
|
|
59167
|
+
select distinct source.value as url
|
|
59168
|
+
from ${siteCrawlPages}, json_each(${siteCrawlPages.sitemapMetadata}, '$.sources') as source
|
|
59169
|
+
where ${siteCrawlPages.projectId} = ${project.id}
|
|
59170
|
+
and ${siteCrawlPages.runId} = ${scan.runId}
|
|
59171
|
+
and (${scan.attemptId} is null or ${siteCrawlPages.attemptId} = ${scan.attemptId})
|
|
59172
|
+
and source.type = 'text'
|
|
59173
|
+
order by url`).map((row) => row.url) : [];
|
|
59174
|
+
const listedAt = new Map(app.db.select({ engine: sitemapListSyncs.engine, syncedAt: sitemapListSyncs.syncedAt }).from(sitemapListSyncs).where(eq81(sitemapListSyncs.projectId, project.id)).all().map((row) => [row.engine, row.syncedAt]));
|
|
59175
|
+
const googleRows = app.db.select().from(gscSitemaps).where(eq81(gscSitemaps.projectId, project.id)).orderBy(asc19(gscSitemaps.url)).all();
|
|
59176
|
+
const google = listedAt.has("google") ? {
|
|
59177
|
+
syncedAt: listedAt.get("google"),
|
|
59178
|
+
sitemaps: googleRows.map((row) => ({
|
|
59179
|
+
url: row.url,
|
|
59180
|
+
parentUrl: row.parentUrl,
|
|
59181
|
+
isIndex: row.isIndex,
|
|
59182
|
+
type: row.type,
|
|
59183
|
+
...googleSitemapState(row),
|
|
59184
|
+
warnings: row.warnings,
|
|
59185
|
+
errors: row.errors,
|
|
59186
|
+
urlCount: row.urlCount,
|
|
59187
|
+
submittedAt: row.submittedAt,
|
|
59188
|
+
lastReadAt: row.lastDownloadedAt
|
|
59189
|
+
}))
|
|
59190
|
+
} : null;
|
|
59191
|
+
const bingRows = app.db.select().from(bingSitemaps).where(eq81(bingSitemaps.projectId, project.id)).orderBy(asc19(bingSitemaps.url)).all();
|
|
59192
|
+
const bing = listedAt.has("bing") ? {
|
|
59193
|
+
syncedAt: listedAt.get("bing"),
|
|
59194
|
+
sitemaps: bingRows.map((row) => ({
|
|
59195
|
+
url: row.url,
|
|
59196
|
+
parentUrl: row.parentUrl,
|
|
59197
|
+
isIndex: /index/i.test(row.type),
|
|
59198
|
+
type: row.type,
|
|
59199
|
+
state: bingSitemapState(row.status),
|
|
59200
|
+
statusLabel: row.status,
|
|
59201
|
+
warnings: null,
|
|
59202
|
+
errors: null,
|
|
59203
|
+
urlCount: row.urlCount,
|
|
59204
|
+
submittedAt: row.submittedAt,
|
|
59205
|
+
lastReadAt: row.lastCrawledAt
|
|
59206
|
+
}))
|
|
59207
|
+
} : null;
|
|
59208
|
+
const propertyId = opts.googleConnectionStore?.getConnection(project.canonicalDomain, "gsc")?.propertyId ?? null;
|
|
59209
|
+
const bingSite = opts.bingConnectionStore?.getConnection(project.canonicalDomain)?.siteUrl ?? null;
|
|
59210
|
+
return buildSitemapCoverage({
|
|
59211
|
+
site: { scannedAt: scan?.createdAt ?? null, runId: scan?.runId ?? null, sitemaps: siteSitemaps },
|
|
59212
|
+
google,
|
|
59213
|
+
bing,
|
|
59214
|
+
accepts: {
|
|
59215
|
+
// Google rejects a sitemap outside the property, so it is never "missing" there.
|
|
59216
|
+
google: (url) => propertyId === null || isSitemapOwnedByProperty(url, propertyId, project.canonicalDomain),
|
|
59217
|
+
// Bing takes sitemaps on its site's host and subdomains.
|
|
59218
|
+
bing: (url) => bingSite === null || hostMatchesDomain(url, bingSite)
|
|
59219
|
+
}
|
|
59220
|
+
});
|
|
59221
|
+
});
|
|
59222
|
+
}
|
|
59223
|
+
|
|
59038
59224
|
// ../api-routes/src/cdp.ts
|
|
59039
59225
|
import fs from "fs";
|
|
59040
59226
|
import path from "path";
|
|
59041
59227
|
import os from "os";
|
|
59042
|
-
import { eq as
|
|
59228
|
+
import { eq as eq82, and as and70 } from "drizzle-orm";
|
|
59043
59229
|
function getScreenshotDir() {
|
|
59044
59230
|
return path.join(os.homedir(), ".canonry", "screenshots");
|
|
59045
59231
|
}
|
|
59046
59232
|
async function cdpRoutes(app, opts) {
|
|
59047
59233
|
app.get("/screenshots/:snapshotId", async (request, reply) => {
|
|
59048
59234
|
const { snapshotId } = request.params;
|
|
59049
|
-
const snapshot = app.db.select({ screenshotPath: querySnapshots.screenshotPath, projectId: runs.projectId }).from(querySnapshots).innerJoin(runs,
|
|
59235
|
+
const snapshot = app.db.select({ screenshotPath: querySnapshots.screenshotPath, projectId: runs.projectId }).from(querySnapshots).innerJoin(runs, eq82(querySnapshots.runId, runs.id)).where(eq82(querySnapshots.id, snapshotId)).get();
|
|
59050
59236
|
if (!snapshot?.screenshotPath) {
|
|
59051
59237
|
const err = notFound("Screenshot", snapshotId);
|
|
59052
59238
|
return reply.code(err.statusCode).send(err.toJSON());
|
|
@@ -59127,7 +59313,7 @@ async function cdpRoutes(app, opts) {
|
|
|
59127
59313
|
async (request, reply) => {
|
|
59128
59314
|
const project = resolveProject(app.db, request.params.name);
|
|
59129
59315
|
const { runId } = request.params;
|
|
59130
|
-
const run = app.db.select().from(runs).where(
|
|
59316
|
+
const run = app.db.select().from(runs).where(and70(eq82(runs.id, runId), eq82(runs.projectId, project.id))).get();
|
|
59131
59317
|
if (!run) {
|
|
59132
59318
|
const err = notFound("Run", runId);
|
|
59133
59319
|
return reply.code(err.statusCode).send(err.toJSON());
|
|
@@ -59140,8 +59326,8 @@ async function cdpRoutes(app, opts) {
|
|
|
59140
59326
|
citedDomains: querySnapshots.citedDomains,
|
|
59141
59327
|
screenshotPath: querySnapshots.screenshotPath,
|
|
59142
59328
|
rawResponse: querySnapshots.rawResponse
|
|
59143
|
-
}).from(querySnapshots).where(
|
|
59144
|
-
const queryRows2 = app.db.select({ id: queries.id, query: queries.query }).from(queries).where(
|
|
59329
|
+
}).from(querySnapshots).where(eq82(querySnapshots.runId, runId)).all());
|
|
59330
|
+
const queryRows2 = app.db.select({ id: queries.id, query: queries.query }).from(queries).where(eq82(queries.projectId, project.id)).all();
|
|
59145
59331
|
const queryMap = new Map(queryRows2.map((q) => [q.id, q.query]));
|
|
59146
59332
|
const byQuery = /* @__PURE__ */ new Map();
|
|
59147
59333
|
for (const snap of snapshots) {
|
|
@@ -59224,7 +59410,7 @@ async function cdpRoutes(app, opts) {
|
|
|
59224
59410
|
|
|
59225
59411
|
// ../api-routes/src/ga.ts
|
|
59226
59412
|
import crypto51 from "crypto";
|
|
59227
|
-
import { eq as
|
|
59413
|
+
import { eq as eq84, desc as desc38, and as and72, sql as sql35 } from "drizzle-orm";
|
|
59228
59414
|
|
|
59229
59415
|
// ../api-routes/src/ga-session-history.ts
|
|
59230
59416
|
function buildSessionHistory(landingPageDays, dailyTotals) {
|
|
@@ -59389,7 +59575,7 @@ function findBiggestMover(current, prior) {
|
|
|
59389
59575
|
|
|
59390
59576
|
// ../api-routes/src/ga-search-landing-pages.ts
|
|
59391
59577
|
import crypto50 from "crypto";
|
|
59392
|
-
import { and as
|
|
59578
|
+
import { and as and71, asc as asc20, count as count5, desc as desc37, eq as eq83, gte as gte22, lte as lte22, sql as sql34 } from "drizzle-orm";
|
|
59393
59579
|
function persistSearchLandingSnapshot(db, input) {
|
|
59394
59580
|
db.transaction((tx) => {
|
|
59395
59581
|
deleteSearchLandingRows(tx, input.projectId);
|
|
@@ -59453,7 +59639,7 @@ function persistSearchLandingSnapshot(db, input) {
|
|
|
59453
59639
|
}
|
|
59454
59640
|
function persistSearchLandingFailure(db, projectId, propertyId, status, message, attemptedAt) {
|
|
59455
59641
|
db.transaction((tx) => {
|
|
59456
|
-
const storedProperties = tx.selectDistinct({ propertyId: gaSearchLandingWindows.propertyId }).from(gaSearchLandingWindows).where(
|
|
59642
|
+
const storedProperties = tx.selectDistinct({ propertyId: gaSearchLandingWindows.propertyId }).from(gaSearchLandingWindows).where(eq83(gaSearchLandingWindows.projectId, projectId)).all();
|
|
59457
59643
|
const otherProperty = storedProperties.some((stored) => stored.propertyId !== propertyId);
|
|
59458
59644
|
if (otherProperty) deleteSearchLandingRows(tx, projectId);
|
|
59459
59645
|
upsertSearchLandingState(tx, projectId, {
|
|
@@ -59465,8 +59651,8 @@ function persistSearchLandingFailure(db, projectId, propertyId, status, message,
|
|
|
59465
59651
|
});
|
|
59466
59652
|
}
|
|
59467
59653
|
function deleteSearchLandingRows(db, projectId) {
|
|
59468
|
-
db.delete(gaSearchLandingPages).where(
|
|
59469
|
-
db.delete(gaSearchLandingWindows).where(
|
|
59654
|
+
db.delete(gaSearchLandingPages).where(eq83(gaSearchLandingPages.projectId, projectId)).run();
|
|
59655
|
+
db.delete(gaSearchLandingWindows).where(eq83(gaSearchLandingWindows.projectId, projectId)).run();
|
|
59470
59656
|
}
|
|
59471
59657
|
function upsertSearchLandingState(db, projectId, fields, updatedAt) {
|
|
59472
59658
|
db.insert(gaMeasurementSyncStates).values({ projectId, ...fields, updatedAt }).onConflictDoUpdate({
|
|
@@ -59498,13 +59684,13 @@ function buildGaSearchLandingPages(db, projectName, query2, connections) {
|
|
|
59498
59684
|
throw validationError('"offset" must be a non-negative integer');
|
|
59499
59685
|
}
|
|
59500
59686
|
const project = resolveProject(db, projectName);
|
|
59501
|
-
const storedWindow = db.select().from(gaSearchLandingWindows).where(
|
|
59687
|
+
const storedWindow = db.select().from(gaSearchLandingWindows).where(and71(eq83(gaSearchLandingWindows.projectId, project.id), eq83(gaSearchLandingWindows.windowKey, window2))).get();
|
|
59502
59688
|
const otherProperty = storedWindow !== void 0 && storedWindow.propertyId !== resolveCurrentGa4PropertyId(connections, project.name, project.canonicalDomain);
|
|
59503
59689
|
const stored = otherProperty ? void 0 : storedWindow;
|
|
59504
|
-
const state = otherProperty ? void 0 : db.select().from(gaMeasurementSyncStates).where(
|
|
59505
|
-
const pageScope =
|
|
59690
|
+
const state = otherProperty ? void 0 : db.select().from(gaMeasurementSyncStates).where(eq83(gaMeasurementSyncStates.projectId, project.id)).get();
|
|
59691
|
+
const pageScope = and71(eq83(gaSearchLandingPages.projectId, project.id), eq83(gaSearchLandingPages.windowKey, window2));
|
|
59506
59692
|
const totalRows = otherProperty ? 0 : db.select({ value: count5() }).from(gaSearchLandingPages).where(pageScope).get()?.value ?? 0;
|
|
59507
|
-
const rows = otherProperty ? [] : db.select().from(gaSearchLandingPages).where(pageScope).orderBy(
|
|
59693
|
+
const rows = otherProperty ? [] : db.select().from(gaSearchLandingPages).where(pageScope).orderBy(desc37(gaSearchLandingPages.clicks), desc37(gaSearchLandingPages.impressions), asc20(gaSearchLandingPages.landingPage)).limit(limit).offset(offset).all();
|
|
59508
59694
|
const total = stored ? toWireMetrics({
|
|
59509
59695
|
clicks: stored.totalClicks,
|
|
59510
59696
|
impressions: stored.totalImpressions,
|
|
@@ -59541,8 +59727,8 @@ function buildGaSearchLandingPages(db, projectName, query2, connections) {
|
|
|
59541
59727
|
function searchConsoleLinkFor(db, projectId, total, windowStart, windowEnd) {
|
|
59542
59728
|
const stored = db.select({
|
|
59543
59729
|
days: count5(),
|
|
59544
|
-
impressions:
|
|
59545
|
-
}).from(gscDailyTotals).where(
|
|
59730
|
+
impressions: sql34`sum(${gscDailyTotals.impressions})`
|
|
59731
|
+
}).from(gscDailyTotals).where(and71(eq83(gscDailyTotals.projectId, projectId), gte22(gscDailyTotals.date, windowStart), lte22(gscDailyTotals.date, windowEnd))).get();
|
|
59546
59732
|
const searchConsoleImpressions = stored && stored.days > 0 ? Number(stored.impressions ?? 0) : null;
|
|
59547
59733
|
if (gaSearchLandingHasSearchData(total)) return { state: "has-data", searchConsoleImpressions };
|
|
59548
59734
|
if (searchConsoleImpressions === null) return { state: "unknown", searchConsoleImpressions };
|
|
@@ -59567,8 +59753,8 @@ function withSessionShares(rows) {
|
|
|
59567
59753
|
}
|
|
59568
59754
|
function dateRangeConditions(column, range) {
|
|
59569
59755
|
const conditions = [];
|
|
59570
|
-
if (range.startDate) conditions.push(
|
|
59571
|
-
if (range.endDate) conditions.push(
|
|
59756
|
+
if (range.startDate) conditions.push(sql35`${column} >= ${range.startDate}`);
|
|
59757
|
+
if (range.endDate) conditions.push(sql35`${column} <= ${range.endDate}`);
|
|
59572
59758
|
return conditions;
|
|
59573
59759
|
}
|
|
59574
59760
|
var SOCIAL_CHANNEL_GROUPS2 = /* @__PURE__ */ new Set(["Organic Social", "Paid Social"]);
|
|
@@ -59626,10 +59812,10 @@ function keepWinningDimensionRows(rows, groupKey) {
|
|
|
59626
59812
|
function persistAcquisitionMeasurement(db, input) {
|
|
59627
59813
|
const { startDate, endDate } = input.report;
|
|
59628
59814
|
db.transaction((tx) => {
|
|
59629
|
-
tx.delete(gaAcquisitionDaily).where(
|
|
59630
|
-
|
|
59631
|
-
|
|
59632
|
-
|
|
59815
|
+
tx.delete(gaAcquisitionDaily).where(and72(
|
|
59816
|
+
eq84(gaAcquisitionDaily.projectId, input.projectId),
|
|
59817
|
+
sql35`${gaAcquisitionDaily.date} >= ${startDate}`,
|
|
59818
|
+
sql35`${gaAcquisitionDaily.date} <= ${endDate}`
|
|
59633
59819
|
)).run();
|
|
59634
59820
|
for (const row of input.report.rows) {
|
|
59635
59821
|
tx.insert(gaAcquisitionDaily).values({
|
|
@@ -59668,10 +59854,10 @@ function persistAcquisitionMeasurement(db, input) {
|
|
|
59668
59854
|
function persistLeadMeasurement(db, input) {
|
|
59669
59855
|
const { startDate, endDate } = input.report;
|
|
59670
59856
|
db.transaction((tx) => {
|
|
59671
|
-
tx.delete(gaLeadEventsDaily).where(
|
|
59672
|
-
|
|
59673
|
-
|
|
59674
|
-
|
|
59857
|
+
tx.delete(gaLeadEventsDaily).where(and72(
|
|
59858
|
+
eq84(gaLeadEventsDaily.projectId, input.projectId),
|
|
59859
|
+
sql35`${gaLeadEventsDaily.date} >= ${startDate}`,
|
|
59860
|
+
sql35`${gaLeadEventsDaily.date} <= ${endDate}`
|
|
59675
59861
|
)).run();
|
|
59676
59862
|
for (const row of input.report.rows) {
|
|
59677
59863
|
tx.insert(gaLeadEventsDaily).values({
|
|
@@ -59715,7 +59901,7 @@ function persistLeadMeasurement(db, input) {
|
|
|
59715
59901
|
}
|
|
59716
59902
|
function resetLeadMeasurement(db, projectId, updatedAt) {
|
|
59717
59903
|
db.transaction((tx) => {
|
|
59718
|
-
tx.delete(gaLeadEventsDaily).where(
|
|
59904
|
+
tx.delete(gaLeadEventsDaily).where(eq84(gaLeadEventsDaily.projectId, projectId)).run();
|
|
59719
59905
|
tx.insert(gaMeasurementSyncStates).values({ projectId, leadStatus: "never-synced", leadError: null, leadSyncedAt: null, leadAttributionScope: null, leadEventNames: null, updatedAt }).onConflictDoUpdate({
|
|
59720
59906
|
target: gaMeasurementSyncStates.projectId,
|
|
59721
59907
|
set: { leadStatus: "never-synced", leadError: null, leadSyncedAt: null, leadAttributionScope: null, leadEventNames: null, updatedAt }
|
|
@@ -60057,7 +60243,7 @@ async function ga4Routes(app, opts) {
|
|
|
60057
60243
|
if (!connected2) {
|
|
60058
60244
|
return { connected: false, propertyId: null, clientEmail: null, authMethod: null, lastSyncedAt: null };
|
|
60059
60245
|
}
|
|
60060
|
-
const latestSync = app.db.select({ syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(
|
|
60246
|
+
const latestSync = app.db.select({ syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(eq84(gaTrafficSummaries.projectId, project.id)).orderBy(desc38(gaTrafficSummaries.syncedAt)).limit(1).get();
|
|
60061
60247
|
return {
|
|
60062
60248
|
connected: true,
|
|
60063
60249
|
propertyId: saConn?.propertyId ?? oauthConn?.propertyId ?? null,
|
|
@@ -60080,7 +60266,7 @@ async function ga4Routes(app, opts) {
|
|
|
60080
60266
|
const syncAi = !only || only === "ai";
|
|
60081
60267
|
const syncSocial = !only || only === "social";
|
|
60082
60268
|
const syncSearchLanding = !only || only === "search-landing";
|
|
60083
|
-
const measurementState = app.db.select().from(gaMeasurementSyncStates).where(
|
|
60269
|
+
const measurementState = app.db.select().from(gaMeasurementSyncStates).where(eq84(gaMeasurementSyncStates.projectId, project.id)).get();
|
|
60084
60270
|
const acquisitionDays = measurementState?.acquisitionSyncedAt != null ? days : GA4_MAX_SYNC_DAYS;
|
|
60085
60271
|
let leadDays = GA4_MAX_SYNC_DAYS;
|
|
60086
60272
|
let leadEventNames = [];
|
|
@@ -60152,10 +60338,10 @@ async function ga4Routes(app, opts) {
|
|
|
60152
60338
|
app.db.transaction((tx) => {
|
|
60153
60339
|
if (syncTraffic) {
|
|
60154
60340
|
tx.delete(gaTrafficSnapshots).where(
|
|
60155
|
-
|
|
60156
|
-
|
|
60157
|
-
|
|
60158
|
-
|
|
60341
|
+
and72(
|
|
60342
|
+
eq84(gaTrafficSnapshots.projectId, project.id),
|
|
60343
|
+
sql35`${gaTrafficSnapshots.date} >= ${summary3.periodStart}`,
|
|
60344
|
+
sql35`${gaTrafficSnapshots.date} <= ${summary3.periodEnd}`
|
|
60159
60345
|
)
|
|
60160
60346
|
).run();
|
|
60161
60347
|
for (const row of rows) {
|
|
@@ -60175,10 +60361,10 @@ async function ga4Routes(app, opts) {
|
|
|
60175
60361
|
}
|
|
60176
60362
|
}
|
|
60177
60363
|
tx.delete(gaDailyTotals).where(
|
|
60178
|
-
|
|
60179
|
-
|
|
60180
|
-
|
|
60181
|
-
|
|
60364
|
+
and72(
|
|
60365
|
+
eq84(gaDailyTotals.projectId, project.id),
|
|
60366
|
+
sql35`${gaDailyTotals.date} >= ${dailyTotals.startDate}`,
|
|
60367
|
+
sql35`${gaDailyTotals.date} <= ${dailyTotals.endDate}`
|
|
60182
60368
|
)
|
|
60183
60369
|
).run();
|
|
60184
60370
|
for (const row of dailyTotals.rows) {
|
|
@@ -60206,7 +60392,7 @@ async function ga4Routes(app, opts) {
|
|
|
60206
60392
|
createdAt: now
|
|
60207
60393
|
}).run();
|
|
60208
60394
|
}
|
|
60209
|
-
tx.delete(gaWindowMetrics).where(
|
|
60395
|
+
tx.delete(gaWindowMetrics).where(eq84(gaWindowMetrics.projectId, project.id)).run();
|
|
60210
60396
|
for (const windowMetric of windowMetrics) {
|
|
60211
60397
|
for (const [period2, totals] of [
|
|
60212
60398
|
[GaOverviewPeriods.current, windowMetric.current],
|
|
@@ -60237,10 +60423,10 @@ async function ga4Routes(app, opts) {
|
|
|
60237
60423
|
}
|
|
60238
60424
|
if (syncAi) {
|
|
60239
60425
|
tx.delete(gaAiReferrals).where(
|
|
60240
|
-
|
|
60241
|
-
|
|
60242
|
-
|
|
60243
|
-
|
|
60426
|
+
and72(
|
|
60427
|
+
eq84(gaAiReferrals.projectId, project.id),
|
|
60428
|
+
sql35`${gaAiReferrals.date} >= ${summary3.periodStart}`,
|
|
60429
|
+
sql35`${gaAiReferrals.date} <= ${summary3.periodEnd}`
|
|
60244
60430
|
)
|
|
60245
60431
|
).run();
|
|
60246
60432
|
for (const row of aiReferrals) {
|
|
@@ -60269,10 +60455,10 @@ async function ga4Routes(app, opts) {
|
|
|
60269
60455
|
}
|
|
60270
60456
|
if (syncSocial) {
|
|
60271
60457
|
tx.delete(gaSocialReferrals).where(
|
|
60272
|
-
|
|
60273
|
-
|
|
60274
|
-
|
|
60275
|
-
|
|
60458
|
+
and72(
|
|
60459
|
+
eq84(gaSocialReferrals.projectId, project.id),
|
|
60460
|
+
sql35`${gaSocialReferrals.date} >= ${summary3.periodStart}`,
|
|
60461
|
+
sql35`${gaSocialReferrals.date} <= ${summary3.periodEnd}`
|
|
60276
60462
|
)
|
|
60277
60463
|
).run();
|
|
60278
60464
|
for (const row of socialReferrals) {
|
|
@@ -60291,7 +60477,7 @@ async function ga4Routes(app, opts) {
|
|
|
60291
60477
|
}
|
|
60292
60478
|
}
|
|
60293
60479
|
if (syncSummary) {
|
|
60294
|
-
tx.delete(gaTrafficSummaries).where(
|
|
60480
|
+
tx.delete(gaTrafficSummaries).where(eq84(gaTrafficSummaries.projectId, project.id)).run();
|
|
60295
60481
|
tx.insert(gaTrafficSummaries).values({
|
|
60296
60482
|
id: crypto51.randomUUID(),
|
|
60297
60483
|
projectId: project.id,
|
|
@@ -60303,7 +60489,7 @@ async function ga4Routes(app, opts) {
|
|
|
60303
60489
|
syncedAt: now,
|
|
60304
60490
|
syncRunId: runId
|
|
60305
60491
|
}).run();
|
|
60306
|
-
tx.delete(gaTrafficWindowSummaries).where(
|
|
60492
|
+
tx.delete(gaTrafficWindowSummaries).where(eq84(gaTrafficWindowSummaries.projectId, project.id)).run();
|
|
60307
60493
|
for (const ws of windowSummaries) {
|
|
60308
60494
|
tx.insert(gaTrafficWindowSummaries).values({
|
|
60309
60495
|
id: crypto51.randomUUID(),
|
|
@@ -60433,7 +60619,7 @@ async function ga4Routes(app, opts) {
|
|
|
60433
60619
|
return recordSearchLandingFailure("error", error);
|
|
60434
60620
|
}
|
|
60435
60621
|
})();
|
|
60436
|
-
app.db.update(runs).set({ status: RunStatuses.completed, finishedAt: measurementNow }).where(
|
|
60622
|
+
app.db.update(runs).set({ status: RunStatuses.completed, finishedAt: measurementNow }).where(eq84(runs.id, runId)).run();
|
|
60437
60623
|
const syncedComponents = only ? [
|
|
60438
60624
|
...syncTraffic ? ["traffic"] : [],
|
|
60439
60625
|
...syncSummary ? ["summary"] : [],
|
|
@@ -60479,7 +60665,7 @@ async function ga4Routes(app, opts) {
|
|
|
60479
60665
|
} catch (e) {
|
|
60480
60666
|
const msg = describeError(e);
|
|
60481
60667
|
gaLog("error", "sync.fetch-failed", { projectId: project.id, runId, error: msg });
|
|
60482
|
-
app.db.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
60668
|
+
app.db.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq84(runs.id, runId)).run();
|
|
60483
60669
|
outcome2.report({ status: OutcomeStatuses.failed, ...ga4SyncFailure(e, credentialsResolved, opts, project) });
|
|
60484
60670
|
throw e;
|
|
60485
60671
|
}
|
|
@@ -60497,9 +60683,9 @@ async function ga4Routes(app, opts) {
|
|
|
60497
60683
|
totalDirectSessions: gaTrafficWindowSummaries.totalDirectSessions,
|
|
60498
60684
|
totalUsers: gaTrafficWindowSummaries.totalUsers
|
|
60499
60685
|
}).from(gaTrafficWindowSummaries).where(
|
|
60500
|
-
|
|
60501
|
-
|
|
60502
|
-
|
|
60686
|
+
and72(
|
|
60687
|
+
eq84(gaTrafficWindowSummaries.projectId, project.id),
|
|
60688
|
+
eq84(gaTrafficWindowSummaries.windowKey, range.window)
|
|
60503
60689
|
)
|
|
60504
60690
|
).get() : null;
|
|
60505
60691
|
const projectSummaryRow = app.db.select({
|
|
@@ -60508,11 +60694,11 @@ async function ga4Routes(app, opts) {
|
|
|
60508
60694
|
totalSessions: gaTrafficSummaries.totalSessions,
|
|
60509
60695
|
totalOrganicSessions: gaTrafficSummaries.totalOrganicSessions,
|
|
60510
60696
|
totalUsers: gaTrafficSummaries.totalUsers
|
|
60511
|
-
}).from(gaTrafficSummaries).where(
|
|
60697
|
+
}).from(gaTrafficSummaries).where(eq84(gaTrafficSummaries.projectId, project.id)).get();
|
|
60512
60698
|
const retainedBounds = [
|
|
60513
|
-
app.db.select({ min:
|
|
60514
|
-
app.db.select({ min:
|
|
60515
|
-
app.db.select({ min:
|
|
60699
|
+
app.db.select({ min: sql35`MIN(${gaTrafficSnapshots.date})`, max: sql35`MAX(${gaTrafficSnapshots.date})` }).from(gaTrafficSnapshots).where(eq84(gaTrafficSnapshots.projectId, project.id)).get(),
|
|
60700
|
+
app.db.select({ min: sql35`MIN(${gaAiReferrals.date})`, max: sql35`MAX(${gaAiReferrals.date})` }).from(gaAiReferrals).where(eq84(gaAiReferrals.projectId, project.id)).get(),
|
|
60701
|
+
app.db.select({ min: sql35`MIN(${gaSocialReferrals.date})`, max: sql35`MAX(${gaSocialReferrals.date})` }).from(gaSocialReferrals).where(eq84(gaSocialReferrals.projectId, project.id)).get()
|
|
60516
60702
|
];
|
|
60517
60703
|
const retainedDates = retainedBounds.flatMap((bound) => [bound?.min, bound?.max]).filter((date) => Boolean(date));
|
|
60518
60704
|
const projectSummaryCoversAll = Boolean(
|
|
@@ -60521,15 +60707,15 @@ async function ga4Routes(app, opts) {
|
|
|
60521
60707
|
const denominatorRow = windowSummaryRow ?? (range.window === "all" && projectSummaryCoversAll ? projectSummaryRow : null);
|
|
60522
60708
|
const denominatorPeriod = denominatorRow?.periodStart && denominatorRow.periodEnd ? { startDate: denominatorRow.periodStart, endDate: denominatorRow.periodEnd } : null;
|
|
60523
60709
|
const measuredRange = denominatorPeriod ? { ...range, ...denominatorPeriod } : range;
|
|
60524
|
-
const snapshotConditions = [
|
|
60525
|
-
const aiConditions = [
|
|
60526
|
-
const socialConditions = [
|
|
60710
|
+
const snapshotConditions = [eq84(gaTrafficSnapshots.projectId, project.id), ...dateRangeConditions(gaTrafficSnapshots.date, measuredRange)];
|
|
60711
|
+
const aiConditions = [eq84(gaAiReferrals.projectId, project.id), ...dateRangeConditions(gaAiReferrals.date, measuredRange)];
|
|
60712
|
+
const socialConditions = [eq84(gaSocialReferrals.projectId, project.id), ...dateRangeConditions(gaSocialReferrals.date, measuredRange)];
|
|
60527
60713
|
const snapshotTotalsRow = !denominatorRow ? (() => {
|
|
60528
60714
|
const summed = app.db.select({
|
|
60529
|
-
totalSessions:
|
|
60530
|
-
totalOrganicSessions:
|
|
60531
|
-
totalUsers:
|
|
60532
|
-
}).from(gaTrafficSnapshots).where(
|
|
60715
|
+
totalSessions: sql35`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)`,
|
|
60716
|
+
totalOrganicSessions: sql35`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)`,
|
|
60717
|
+
totalUsers: sql35`COALESCE(SUM(${gaTrafficSnapshots.users}), 0)`
|
|
60718
|
+
}).from(gaTrafficSnapshots).where(and72(...snapshotConditions)).get();
|
|
60533
60719
|
return {
|
|
60534
60720
|
totalSessions: summed?.totalSessions ?? 0,
|
|
60535
60721
|
totalOrganicSessions: summed?.totalOrganicSessions ?? 0,
|
|
@@ -60541,35 +60727,35 @@ async function ga4Routes(app, opts) {
|
|
|
60541
60727
|
})() : null;
|
|
60542
60728
|
const summaryRow = denominatorRow ?? snapshotTotalsRow;
|
|
60543
60729
|
const directTotalRow = windowSummaryRow ? { totalDirectSessions: windowSummaryRow.totalDirectSessions } : app.db.select({
|
|
60544
|
-
totalDirectSessions:
|
|
60545
|
-
}).from(gaTrafficSnapshots).where(
|
|
60730
|
+
totalDirectSessions: sql35`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)`
|
|
60731
|
+
}).from(gaTrafficSnapshots).where(and72(...snapshotConditions)).get();
|
|
60546
60732
|
const rows = app.db.select({
|
|
60547
|
-
landingPage:
|
|
60548
|
-
sessions:
|
|
60549
|
-
organicSessions:
|
|
60550
|
-
directSessions:
|
|
60551
|
-
users:
|
|
60552
|
-
}).from(gaTrafficSnapshots).where(
|
|
60733
|
+
landingPage: sql35`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`,
|
|
60734
|
+
sessions: sql35`SUM(${gaTrafficSnapshots.sessions})`,
|
|
60735
|
+
organicSessions: sql35`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
60736
|
+
directSessions: sql35`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)`,
|
|
60737
|
+
users: sql35`SUM(${gaTrafficSnapshots.users})`
|
|
60738
|
+
}).from(gaTrafficSnapshots).where(and72(...snapshotConditions)).groupBy(sql35`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`).orderBy(sql35`SUM(${gaTrafficSnapshots.sessions}) DESC`).limit(limit).all();
|
|
60553
60739
|
const aiReferralRows = app.db.select({
|
|
60554
60740
|
source: gaAiReferrals.source,
|
|
60555
60741
|
medium: gaAiReferrals.medium,
|
|
60556
60742
|
trafficClass: gaAiReferrals.trafficClass,
|
|
60557
60743
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
60558
|
-
sessions:
|
|
60559
|
-
}).from(gaAiReferrals).where(
|
|
60744
|
+
sessions: sql35`SUM(${gaAiReferrals.sessions})`
|
|
60745
|
+
}).from(gaAiReferrals).where(and72(...aiConditions)).groupBy(gaAiReferrals.source, gaAiReferrals.medium, gaAiReferrals.trafficClass, gaAiReferrals.sourceDimension).all();
|
|
60560
60746
|
const aiReferralLandingPageRows = app.db.select({
|
|
60561
60747
|
source: gaAiReferrals.source,
|
|
60562
60748
|
medium: gaAiReferrals.medium,
|
|
60563
60749
|
trafficClass: gaAiReferrals.trafficClass,
|
|
60564
60750
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
60565
|
-
landingPage:
|
|
60566
|
-
sessions:
|
|
60567
|
-
}).from(gaAiReferrals).where(
|
|
60751
|
+
landingPage: sql35`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`,
|
|
60752
|
+
sessions: sql35`SUM(${gaAiReferrals.sessions})`
|
|
60753
|
+
}).from(gaAiReferrals).where(and72(...aiConditions)).groupBy(
|
|
60568
60754
|
gaAiReferrals.source,
|
|
60569
60755
|
gaAiReferrals.medium,
|
|
60570
60756
|
gaAiReferrals.trafficClass,
|
|
60571
60757
|
gaAiReferrals.sourceDimension,
|
|
60572
|
-
|
|
60758
|
+
sql35`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`
|
|
60573
60759
|
).all();
|
|
60574
60760
|
const aiRowsForTotals = app.db.select({
|
|
60575
60761
|
date: gaAiReferrals.date,
|
|
@@ -60579,7 +60765,7 @@ async function ga4Routes(app, opts) {
|
|
|
60579
60765
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
60580
60766
|
channelGroup: gaAiReferrals.channelGroup,
|
|
60581
60767
|
sessions: gaAiReferrals.sessions
|
|
60582
|
-
}).from(gaAiReferrals).where(
|
|
60768
|
+
}).from(gaAiReferrals).where(and72(...aiConditions)).all();
|
|
60583
60769
|
const aiReferrals = keepWinningDimensionRows(aiReferralRows, (r) => r.source);
|
|
60584
60770
|
const aiReferralLandingPages = keepWinningDimensionRows(
|
|
60585
60771
|
aiReferralLandingPageRows,
|
|
@@ -60595,17 +60781,17 @@ async function ga4Routes(app, opts) {
|
|
|
60595
60781
|
// is GA's COUNT DISTINCT at that grain, so summing over the window's
|
|
60596
60782
|
// dates counts a returning visitor once per day they returned. Same
|
|
60597
60783
|
// conclusion the AI-referral surfaces reached in 4.135.0.
|
|
60598
|
-
sessions:
|
|
60599
|
-
}).from(gaSocialReferrals).where(
|
|
60784
|
+
sessions: sql35`SUM(${gaSocialReferrals.sessions})`
|
|
60785
|
+
}).from(gaSocialReferrals).where(and72(...socialConditions)).groupBy(gaSocialReferrals.source, gaSocialReferrals.medium, gaSocialReferrals.channelGroup).orderBy(sql35`SUM(${gaSocialReferrals.sessions}) DESC`).all();
|
|
60600
60786
|
const socialDailyRows = app.db.select({
|
|
60601
60787
|
date: gaSocialReferrals.date,
|
|
60602
60788
|
source: gaSocialReferrals.source,
|
|
60603
|
-
sessions:
|
|
60604
|
-
}).from(gaSocialReferrals).where(
|
|
60789
|
+
sessions: sql35`SUM(${gaSocialReferrals.sessions})`
|
|
60790
|
+
}).from(gaSocialReferrals).where(and72(...socialConditions)).groupBy(gaSocialReferrals.date, gaSocialReferrals.source).all();
|
|
60605
60791
|
const socialTotals = app.db.select({
|
|
60606
|
-
sessions:
|
|
60607
|
-
}).from(gaSocialReferrals).where(
|
|
60608
|
-
const latestSync = app.db.select({ syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(
|
|
60792
|
+
sessions: sql35`SUM(${gaSocialReferrals.sessions})`
|
|
60793
|
+
}).from(gaSocialReferrals).where(and72(...socialConditions)).get();
|
|
60794
|
+
const latestSync = app.db.select({ syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(eq84(gaTrafficSummaries.projectId, project.id)).orderBy(desc38(gaTrafficSummaries.syncedAt)).limit(1).get();
|
|
60609
60795
|
const total = summaryRow?.totalSessions ?? 0;
|
|
60610
60796
|
const totalDirectSessions = directTotalRow?.totalDirectSessions ?? 0;
|
|
60611
60797
|
const totalOrganicSessions = summaryRow?.totalOrganicSessions ?? 0;
|
|
@@ -60728,26 +60914,26 @@ async function ga4Routes(app, opts) {
|
|
|
60728
60914
|
const project = resolveProject(app.db, request.params.name);
|
|
60729
60915
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
60730
60916
|
const range = resolveDateRange(request.query);
|
|
60731
|
-
const conditions = [
|
|
60917
|
+
const conditions = [eq84(gaAiReferrals.projectId, project.id), ...dateRangeConditions(gaAiReferrals.date, range)];
|
|
60732
60918
|
const rows = app.db.select({
|
|
60733
60919
|
date: gaAiReferrals.date,
|
|
60734
60920
|
source: gaAiReferrals.source,
|
|
60735
60921
|
medium: gaAiReferrals.medium,
|
|
60736
60922
|
trafficClass: gaAiReferrals.trafficClass,
|
|
60737
|
-
landingPage:
|
|
60923
|
+
landingPage: sql35`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`,
|
|
60738
60924
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
60739
60925
|
// No `users`: ga-ai-referral-aggregation states the rule for this
|
|
60740
60926
|
// table, that there is no grain at which the stored users column may be
|
|
60741
60927
|
// added up. ga4AiReferralDtoSchema.users was deprecated for this in
|
|
60742
60928
|
// 4.135.0; the history endpoint kept emitting one.
|
|
60743
|
-
sessions:
|
|
60744
|
-
}).from(gaAiReferrals).where(
|
|
60929
|
+
sessions: sql35`SUM(${gaAiReferrals.sessions})`
|
|
60930
|
+
}).from(gaAiReferrals).where(and72(...conditions)).groupBy(
|
|
60745
60931
|
gaAiReferrals.date,
|
|
60746
60932
|
gaAiReferrals.source,
|
|
60747
60933
|
gaAiReferrals.medium,
|
|
60748
60934
|
gaAiReferrals.trafficClass,
|
|
60749
60935
|
gaAiReferrals.sourceDimension,
|
|
60750
|
-
|
|
60936
|
+
sql35`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`
|
|
60751
60937
|
).orderBy(gaAiReferrals.date).all();
|
|
60752
60938
|
return rows;
|
|
60753
60939
|
});
|
|
@@ -60755,7 +60941,7 @@ async function ga4Routes(app, opts) {
|
|
|
60755
60941
|
const project = resolveProject(app.db, request.params.name);
|
|
60756
60942
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
60757
60943
|
const range = resolveDateRange(request.query);
|
|
60758
|
-
const conditions = [
|
|
60944
|
+
const conditions = [eq84(gaAiReferrals.projectId, project.id), ...dateRangeConditions(gaAiReferrals.date, range)];
|
|
60759
60945
|
const rows = app.db.select({
|
|
60760
60946
|
date: gaAiReferrals.date,
|
|
60761
60947
|
source: gaAiReferrals.source,
|
|
@@ -60764,14 +60950,14 @@ async function ga4Routes(app, opts) {
|
|
|
60764
60950
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
60765
60951
|
channelGroup: gaAiReferrals.channelGroup,
|
|
60766
60952
|
sessions: gaAiReferrals.sessions
|
|
60767
|
-
}).from(gaAiReferrals).where(
|
|
60953
|
+
}).from(gaAiReferrals).where(and72(...conditions)).all();
|
|
60768
60954
|
return buildAiReferralDailySeries(rows);
|
|
60769
60955
|
});
|
|
60770
60956
|
app.get("/projects/:name/ga/social-referral-history", async (request, _reply) => {
|
|
60771
60957
|
const project = resolveProject(app.db, request.params.name);
|
|
60772
60958
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
60773
60959
|
const range = resolveDateRange(request.query);
|
|
60774
|
-
const conditions = [
|
|
60960
|
+
const conditions = [eq84(gaSocialReferrals.projectId, project.id), ...dateRangeConditions(gaSocialReferrals.date, range)];
|
|
60775
60961
|
const rows = app.db.select({
|
|
60776
60962
|
date: gaSocialReferrals.date,
|
|
60777
60963
|
source: gaSocialReferrals.source,
|
|
@@ -60779,7 +60965,7 @@ async function ga4Routes(app, opts) {
|
|
|
60779
60965
|
channelGroup: gaSocialReferrals.channelGroup,
|
|
60780
60966
|
sessions: gaSocialReferrals.sessions,
|
|
60781
60967
|
users: gaSocialReferrals.users
|
|
60782
|
-
}).from(gaSocialReferrals).where(
|
|
60968
|
+
}).from(gaSocialReferrals).where(and72(...conditions)).orderBy(gaSocialReferrals.date).all();
|
|
60783
60969
|
return rows;
|
|
60784
60970
|
});
|
|
60785
60971
|
app.get("/projects/:name/ga/social-referral-trend", async (request, _reply) => {
|
|
@@ -60792,10 +60978,10 @@ async function ga4Routes(app, opts) {
|
|
|
60792
60978
|
d.setDate(d.getDate() - n);
|
|
60793
60979
|
return fmt(d);
|
|
60794
60980
|
};
|
|
60795
|
-
const sumSocial = (from, to) => app.db.select({ sessions:
|
|
60796
|
-
|
|
60797
|
-
|
|
60798
|
-
|
|
60981
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql35`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and72(
|
|
60982
|
+
eq84(gaSocialReferrals.projectId, project.id),
|
|
60983
|
+
sql35`${gaSocialReferrals.date} >= ${from}`,
|
|
60984
|
+
sql35`${gaSocialReferrals.date} < ${to}`
|
|
60799
60985
|
)).get();
|
|
60800
60986
|
const current7d = sumSocial(daysAgo(7), fmt(today));
|
|
60801
60987
|
const prev7d = sumSocial(daysAgo(14), daysAgo(7));
|
|
@@ -60803,19 +60989,19 @@ async function ga4Routes(app, opts) {
|
|
|
60803
60989
|
const prev30d = sumSocial(daysAgo(60), daysAgo(30));
|
|
60804
60990
|
const sourceCurrent = app.db.select({
|
|
60805
60991
|
source: gaSocialReferrals.source,
|
|
60806
|
-
sessions:
|
|
60807
|
-
}).from(gaSocialReferrals).where(
|
|
60808
|
-
|
|
60809
|
-
|
|
60810
|
-
|
|
60992
|
+
sessions: sql35`SUM(${gaSocialReferrals.sessions})`
|
|
60993
|
+
}).from(gaSocialReferrals).where(and72(
|
|
60994
|
+
eq84(gaSocialReferrals.projectId, project.id),
|
|
60995
|
+
sql35`${gaSocialReferrals.date} >= ${daysAgo(7)}`,
|
|
60996
|
+
sql35`${gaSocialReferrals.date} < ${fmt(today)}`
|
|
60811
60997
|
)).groupBy(gaSocialReferrals.source).all();
|
|
60812
60998
|
const sourcePrev = app.db.select({
|
|
60813
60999
|
source: gaSocialReferrals.source,
|
|
60814
|
-
sessions:
|
|
60815
|
-
}).from(gaSocialReferrals).where(
|
|
60816
|
-
|
|
60817
|
-
|
|
60818
|
-
|
|
61000
|
+
sessions: sql35`SUM(${gaSocialReferrals.sessions})`
|
|
61001
|
+
}).from(gaSocialReferrals).where(and72(
|
|
61002
|
+
eq84(gaSocialReferrals.projectId, project.id),
|
|
61003
|
+
sql35`${gaSocialReferrals.date} >= ${daysAgo(14)}`,
|
|
61004
|
+
sql35`${gaSocialReferrals.date} < ${daysAgo(7)}`
|
|
60819
61005
|
)).groupBy(gaSocialReferrals.source).all();
|
|
60820
61006
|
const response = {
|
|
60821
61007
|
socialSessions7d: current7d?.sessions ?? 0,
|
|
@@ -60838,16 +61024,16 @@ async function ga4Routes(app, opts) {
|
|
|
60838
61024
|
d.setDate(d.getDate() - n);
|
|
60839
61025
|
return fmt(d);
|
|
60840
61026
|
};
|
|
60841
|
-
const sumTotal = (from, to) => app.db.select({ sessions:
|
|
60842
|
-
const sumOrganic = (from, to) => app.db.select({ sessions:
|
|
60843
|
-
const sumDirect = (from, to) => app.db.select({ sessions:
|
|
60844
|
-
const sumAi = (from, to) => app.db.select({ sessions:
|
|
60845
|
-
|
|
60846
|
-
|
|
60847
|
-
|
|
60848
|
-
|
|
61027
|
+
const sumTotal = (from, to) => app.db.select({ sessions: sql35`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)` }).from(gaTrafficSnapshots).where(and72(eq84(gaTrafficSnapshots.projectId, project.id), sql35`${gaTrafficSnapshots.date} >= ${from}`, sql35`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
61028
|
+
const sumOrganic = (from, to) => app.db.select({ sessions: sql35`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)` }).from(gaTrafficSnapshots).where(and72(eq84(gaTrafficSnapshots.projectId, project.id), sql35`${gaTrafficSnapshots.date} >= ${from}`, sql35`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
61029
|
+
const sumDirect = (from, to) => app.db.select({ sessions: sql35`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)` }).from(gaTrafficSnapshots).where(and72(eq84(gaTrafficSnapshots.projectId, project.id), sql35`${gaTrafficSnapshots.date} >= ${from}`, sql35`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
61030
|
+
const sumAi = (from, to) => app.db.select({ sessions: sql35`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and72(
|
|
61031
|
+
eq84(gaAiReferrals.projectId, project.id),
|
|
61032
|
+
sql35`${gaAiReferrals.date} >= ${from}`,
|
|
61033
|
+
sql35`${gaAiReferrals.date} < ${to}`,
|
|
61034
|
+
eq84(gaAiReferrals.sourceDimension, "session")
|
|
60849
61035
|
)).get();
|
|
60850
|
-
const sumSocial = (from, to) => app.db.select({ sessions:
|
|
61036
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql35`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and72(eq84(gaSocialReferrals.projectId, project.id), sql35`${gaSocialReferrals.date} >= ${from}`, sql35`${gaSocialReferrals.date} < ${to}`)).get();
|
|
60851
61037
|
const todayStr = fmt(today);
|
|
60852
61038
|
const buildTrend = (sum) => {
|
|
60853
61039
|
const c7 = sum(daysAgo(7), todayStr)?.sessions ?? 0;
|
|
@@ -60856,20 +61042,20 @@ async function ga4Routes(app, opts) {
|
|
|
60856
61042
|
const p30 = sum(daysAgo(60), daysAgo(30))?.sessions ?? 0;
|
|
60857
61043
|
return { sessions7d: c7, sessionsPrev7d: p7, trend7dPct: deltaPercent(c7, p7), sessions30d: c30, sessionsPrev30d: p30, trend30dPct: deltaPercent(c30, p30) };
|
|
60858
61044
|
};
|
|
60859
|
-
const aiSourceCurrent = app.db.select({ source: gaAiReferrals.source, sessions:
|
|
60860
|
-
|
|
60861
|
-
|
|
60862
|
-
|
|
60863
|
-
|
|
61045
|
+
const aiSourceCurrent = app.db.select({ source: gaAiReferrals.source, sessions: sql35`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and72(
|
|
61046
|
+
eq84(gaAiReferrals.projectId, project.id),
|
|
61047
|
+
sql35`${gaAiReferrals.date} >= ${daysAgo(7)}`,
|
|
61048
|
+
sql35`${gaAiReferrals.date} < ${todayStr}`,
|
|
61049
|
+
eq84(gaAiReferrals.sourceDimension, "session")
|
|
60864
61050
|
)).groupBy(gaAiReferrals.source).all();
|
|
60865
|
-
const aiSourcePrev = app.db.select({ source: gaAiReferrals.source, sessions:
|
|
60866
|
-
|
|
60867
|
-
|
|
60868
|
-
|
|
60869
|
-
|
|
61051
|
+
const aiSourcePrev = app.db.select({ source: gaAiReferrals.source, sessions: sql35`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and72(
|
|
61052
|
+
eq84(gaAiReferrals.projectId, project.id),
|
|
61053
|
+
sql35`${gaAiReferrals.date} >= ${daysAgo(14)}`,
|
|
61054
|
+
sql35`${gaAiReferrals.date} < ${daysAgo(7)}`,
|
|
61055
|
+
eq84(gaAiReferrals.sourceDimension, "session")
|
|
60870
61056
|
)).groupBy(gaAiReferrals.source).all();
|
|
60871
|
-
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions:
|
|
60872
|
-
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions:
|
|
61057
|
+
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions: sql35`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and72(eq84(gaSocialReferrals.projectId, project.id), sql35`${gaSocialReferrals.date} >= ${daysAgo(7)}`, sql35`${gaSocialReferrals.date} < ${todayStr}`)).groupBy(gaSocialReferrals.source).all();
|
|
61058
|
+
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions: sql35`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and72(eq84(gaSocialReferrals.projectId, project.id), sql35`${gaSocialReferrals.date} >= ${daysAgo(14)}`, sql35`${gaSocialReferrals.date} < ${daysAgo(7)}`)).groupBy(gaSocialReferrals.source).all();
|
|
60873
61059
|
const response = {
|
|
60874
61060
|
total: buildTrend(sumTotal),
|
|
60875
61061
|
organic: buildTrend(sumOrganic),
|
|
@@ -60885,16 +61071,16 @@ async function ga4Routes(app, opts) {
|
|
|
60885
61071
|
const project = resolveProject(app.db, request.params.name);
|
|
60886
61072
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
60887
61073
|
const window2 = parseGaOverviewWindow(request.query.window);
|
|
60888
|
-
const periodRows = app.db.select().from(gaWindowMetrics).where(
|
|
61074
|
+
const periodRows = app.db.select().from(gaWindowMetrics).where(and72(eq84(gaWindowMetrics.projectId, project.id), eq84(gaWindowMetrics.windowKey, window2))).all();
|
|
60889
61075
|
const current = periodRows.find((row) => row.period === GaOverviewPeriods.current) ?? null;
|
|
60890
61076
|
const previous = periodRows.find((row) => row.period === GaOverviewPeriods.previous) ?? null;
|
|
60891
61077
|
const fallback = gaOverviewComparisonRanges(window2);
|
|
60892
61078
|
const from = previous?.startDate ?? fallback.previous.startDate;
|
|
60893
61079
|
const to = current?.endDate ?? fallback.current.endDate;
|
|
60894
|
-
const daily = app.db.select().from(gaDailyTotals).where(
|
|
60895
|
-
|
|
60896
|
-
|
|
60897
|
-
|
|
61080
|
+
const daily = app.db.select().from(gaDailyTotals).where(and72(
|
|
61081
|
+
eq84(gaDailyTotals.projectId, project.id),
|
|
61082
|
+
sql35`${gaDailyTotals.date} >= ${from}`,
|
|
61083
|
+
sql35`${gaDailyTotals.date} <= ${to}`
|
|
60898
61084
|
)).all();
|
|
60899
61085
|
return buildGaOverview({ window: window2, current, previous, daily, fallback });
|
|
60900
61086
|
});
|
|
@@ -60902,15 +61088,15 @@ async function ga4Routes(app, opts) {
|
|
|
60902
61088
|
const project = resolveProject(app.db, request.params.name);
|
|
60903
61089
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
60904
61090
|
const range = resolveDateRange(request.query);
|
|
60905
|
-
const conditions = [
|
|
61091
|
+
const conditions = [eq84(gaTrafficSnapshots.projectId, project.id), ...dateRangeConditions(gaTrafficSnapshots.date, range)];
|
|
60906
61092
|
const rows = app.db.select({
|
|
60907
61093
|
date: gaTrafficSnapshots.date,
|
|
60908
|
-
sessions:
|
|
60909
|
-
organicSessions:
|
|
60910
|
-
users:
|
|
60911
|
-
}).from(gaTrafficSnapshots).where(
|
|
60912
|
-
const totalConditions = [
|
|
60913
|
-
const totals = app.db.select({ date: gaDailyTotals.date, users: gaDailyTotals.users }).from(gaDailyTotals).where(
|
|
61094
|
+
sessions: sql35`SUM(${gaTrafficSnapshots.sessions})`,
|
|
61095
|
+
organicSessions: sql35`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
61096
|
+
users: sql35`SUM(${gaTrafficSnapshots.users})`
|
|
61097
|
+
}).from(gaTrafficSnapshots).where(and72(...conditions)).groupBy(gaTrafficSnapshots.date).orderBy(gaTrafficSnapshots.date).all();
|
|
61098
|
+
const totalConditions = [eq84(gaDailyTotals.projectId, project.id), ...dateRangeConditions(gaDailyTotals.date, range)];
|
|
61099
|
+
const totals = app.db.select({ date: gaDailyTotals.date, users: gaDailyTotals.users }).from(gaDailyTotals).where(and72(...totalConditions)).all();
|
|
60914
61100
|
return buildSessionHistory(
|
|
60915
61101
|
rows.map((r) => ({
|
|
60916
61102
|
date: r.date,
|
|
@@ -60925,11 +61111,11 @@ async function ga4Routes(app, opts) {
|
|
|
60925
61111
|
const project = resolveProject(app.db, request.params.name);
|
|
60926
61112
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
60927
61113
|
const trafficPages = app.db.select({
|
|
60928
|
-
landingPage:
|
|
60929
|
-
sessions:
|
|
60930
|
-
organicSessions:
|
|
60931
|
-
users:
|
|
60932
|
-
}).from(gaTrafficSnapshots).where(
|
|
61114
|
+
landingPage: sql35`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`,
|
|
61115
|
+
sessions: sql35`SUM(${gaTrafficSnapshots.sessions})`,
|
|
61116
|
+
organicSessions: sql35`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
61117
|
+
users: sql35`SUM(${gaTrafficSnapshots.users})`
|
|
61118
|
+
}).from(gaTrafficSnapshots).where(eq84(gaTrafficSnapshots.projectId, project.id)).groupBy(sql35`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`).orderBy(sql35`SUM(${gaTrafficSnapshots.sessions}) DESC`).all();
|
|
60933
61119
|
return {
|
|
60934
61120
|
pages: trafficPages.map((r) => ({
|
|
60935
61121
|
landingPage: r.landingPage,
|
|
@@ -60942,7 +61128,7 @@ async function ga4Routes(app, opts) {
|
|
|
60942
61128
|
}
|
|
60943
61129
|
|
|
60944
61130
|
// ../api-routes/src/ga-acquisition.ts
|
|
60945
|
-
import { and as
|
|
61131
|
+
import { and as and73, asc as asc21, count as count6, desc as desc39, eq as eq85, gte as gte23, lte as lte23, sql as sql36 } from "drizzle-orm";
|
|
60946
61132
|
async function gaAcquisitionRoutes(app) {
|
|
60947
61133
|
app.get("/projects/:name/ga/acquisition", async (request) => {
|
|
60948
61134
|
const project = resolveProject(app.db, request.params.name);
|
|
@@ -60956,7 +61142,7 @@ async function gaAcquisitionRoutes(app) {
|
|
|
60956
61142
|
const groupBy = query2.groupBy === void 0 ? [...gaAcquisitionDimensionSchema.options] : [...new Set(query2.groupBy.split(",").map((key) => gaAcquisitionDimensionSchema.parse(key.trim())))];
|
|
60957
61143
|
const limit = query2.limit ?? 100;
|
|
60958
61144
|
const offset = query2.offset ?? 0;
|
|
60959
|
-
const landingPage =
|
|
61145
|
+
const landingPage = sql36`coalesce(${gaAcquisitionDaily.landingPageNormalized}, ${gaAcquisitionDaily.landingPage})`;
|
|
60960
61146
|
const dimensions = {
|
|
60961
61147
|
date: gaAcquisitionDaily.date,
|
|
60962
61148
|
channelGroup: gaAcquisitionDaily.channelGroup,
|
|
@@ -60965,36 +61151,36 @@ async function gaAcquisitionRoutes(app) {
|
|
|
60965
61151
|
hostName: gaAcquisitionDaily.hostName,
|
|
60966
61152
|
landingPage
|
|
60967
61153
|
};
|
|
60968
|
-
const windowFilters = [
|
|
61154
|
+
const windowFilters = [eq85(gaAcquisitionDaily.projectId, project.id)];
|
|
60969
61155
|
if (range.startDate) windowFilters.push(gte23(gaAcquisitionDaily.date, range.startDate));
|
|
60970
61156
|
if (range.endDate) windowFilters.push(lte23(gaAcquisitionDaily.date, range.endDate));
|
|
60971
61157
|
const filters = [...windowFilters];
|
|
60972
|
-
if (query2.channel !== void 0) filters.push(
|
|
60973
|
-
if (query2.hostName !== void 0) filters.push(
|
|
60974
|
-
if (query2.source !== void 0) filters.push(
|
|
60975
|
-
if (query2.medium !== void 0) filters.push(
|
|
61158
|
+
if (query2.channel !== void 0) filters.push(sql36`lower(${gaAcquisitionDaily.channelGroup}) = lower(${query2.channel})`);
|
|
61159
|
+
if (query2.hostName !== void 0) filters.push(sql36`lower(${gaAcquisitionDaily.hostName}) = lower(${query2.hostName})`);
|
|
61160
|
+
if (query2.source !== void 0) filters.push(eq85(gaAcquisitionDaily.source, query2.source));
|
|
61161
|
+
if (query2.medium !== void 0) filters.push(eq85(gaAcquisitionDaily.medium, query2.medium));
|
|
60976
61162
|
const selectedLandingPage = query2.landingPage === void 0 ? void 0 : normalizeUrlPath(query2.landingPage) ?? query2.landingPage;
|
|
60977
|
-
if (selectedLandingPage !== void 0) filters.push(
|
|
61163
|
+
if (selectedLandingPage !== void 0) filters.push(sql36`${landingPage} = ${selectedLandingPage}`);
|
|
60978
61164
|
const pathPrefix = normalizePathPrefix(query2.pathPrefix);
|
|
60979
61165
|
if (pathPrefix && pathPrefix !== "/") {
|
|
60980
|
-
const pathname =
|
|
61166
|
+
const pathname = sql36`case when instr(${landingPage}, '?') = 0 then ${landingPage} else substr(${landingPage}, 1, instr(${landingPage}, '?') - 1) end`;
|
|
60981
61167
|
filters.push(literalPathPrefixCondition(pathname, pathPrefix));
|
|
60982
61168
|
}
|
|
60983
|
-
const where =
|
|
60984
|
-
const sessions =
|
|
61169
|
+
const where = and73(...filters);
|
|
61170
|
+
const sessions = sql36`coalesce(sum(${gaAcquisitionDaily.sessions}), 0)`.mapWith(Number);
|
|
60985
61171
|
const selected = Object.fromEntries(groupBy.map((key) => [key, dimensions[key]]));
|
|
60986
61172
|
const grouped = app.db.select({ ...selected, sessions }).from(gaAcquisitionDaily).where(where).groupBy(...groupBy.map((key) => dimensions[key]));
|
|
60987
61173
|
const totalRows = app.db.select({ total: count6() }).from(grouped.as("grouped_acquisition")).get().total;
|
|
60988
61174
|
const totalSessions = app.db.select({ total: sessions }).from(gaAcquisitionDaily).where(where).get().total;
|
|
60989
61175
|
const order = [
|
|
60990
|
-
...groupBy.includes(GaAcquisitionDimensions.date) ? [
|
|
60991
|
-
|
|
60992
|
-
...groupBy.filter((key) => key !== GaAcquisitionDimensions.date).map((key) =>
|
|
61176
|
+
...groupBy.includes(GaAcquisitionDimensions.date) ? [desc39(gaAcquisitionDaily.date)] : [],
|
|
61177
|
+
desc39(sessions),
|
|
61178
|
+
...groupBy.filter((key) => key !== GaAcquisitionDimensions.date).map((key) => asc21(dimensions[key]))
|
|
60993
61179
|
];
|
|
60994
61180
|
const rows = grouped.orderBy(...order).limit(limit).offset(offset).all();
|
|
60995
|
-
const coverage = app.db.select({ earliestDate:
|
|
60996
|
-
const channelGroups = app.db.selectDistinct({ channel: gaAcquisitionDaily.channelGroup }).from(gaAcquisitionDaily).where(
|
|
60997
|
-
const state = app.db.select().from(gaMeasurementSyncStates).where(
|
|
61181
|
+
const coverage = app.db.select({ earliestDate: sql36`min(${gaAcquisitionDaily.date})`, latestDate: sql36`max(${gaAcquisitionDaily.date})` }).from(gaAcquisitionDaily).where(eq85(gaAcquisitionDaily.projectId, project.id)).get();
|
|
61182
|
+
const channelGroups = app.db.selectDistinct({ channel: gaAcquisitionDaily.channelGroup }).from(gaAcquisitionDaily).where(and73(...windowFilters)).orderBy(asc21(gaAcquisitionDaily.channelGroup)).all().map((row) => row.channel);
|
|
61183
|
+
const state = app.db.select().from(gaMeasurementSyncStates).where(eq85(gaMeasurementSyncStates.projectId, project.id)).get();
|
|
60998
61184
|
const response = {
|
|
60999
61185
|
source: "ga4-acquisition",
|
|
61000
61186
|
status: state?.acquisitionStatus ?? "never-synced",
|
|
@@ -62726,7 +62912,7 @@ async function wordpressRoutes(app, opts) {
|
|
|
62726
62912
|
|
|
62727
62913
|
// ../api-routes/src/backlinks.ts
|
|
62728
62914
|
import crypto53 from "crypto";
|
|
62729
|
-
import { and as
|
|
62915
|
+
import { and as and76, asc as asc22, desc as desc40, eq as eq87, sql as sql37 } from "drizzle-orm";
|
|
62730
62916
|
|
|
62731
62917
|
// ../integration-commoncrawl/src/constants.ts
|
|
62732
62918
|
import os2 from "os";
|
|
@@ -63063,7 +63249,7 @@ async function queryBacklinks(opts) {
|
|
|
63063
63249
|
const reversed = opts.targets.map(reverseDomain);
|
|
63064
63250
|
const targetList = reversed.map(quote).join(", ");
|
|
63065
63251
|
const limitClause = opts.limitPerTarget ? `QUALIFY row_number() OVER (PARTITION BY t.target_rev_domain ORDER BY v.num_hosts DESC) <= ${Math.floor(opts.limitPerTarget)}` : "";
|
|
63066
|
-
const
|
|
63252
|
+
const sql45 = `
|
|
63067
63253
|
WITH vertices AS (
|
|
63068
63254
|
SELECT * FROM read_csv(
|
|
63069
63255
|
${quote(opts.vertexPath)},
|
|
@@ -63099,7 +63285,7 @@ async function queryBacklinks(opts) {
|
|
|
63099
63285
|
const conn = await instance.connect();
|
|
63100
63286
|
let rows;
|
|
63101
63287
|
try {
|
|
63102
|
-
const reader = await conn.runAndReadAll(
|
|
63288
|
+
const reader = await conn.runAndReadAll(sql45);
|
|
63103
63289
|
rows = reader.getRowObjects();
|
|
63104
63290
|
} finally {
|
|
63105
63291
|
conn.disconnectSync?.();
|
|
@@ -63176,7 +63362,7 @@ function pruneCachedRelease(release, opts = {}) {
|
|
|
63176
63362
|
}
|
|
63177
63363
|
|
|
63178
63364
|
// ../api-routes/src/backlinks-filter.ts
|
|
63179
|
-
import { and as
|
|
63365
|
+
import { and as and74, ne as ne15, notLike } from "drizzle-orm";
|
|
63180
63366
|
var BACKLINK_FILTER_PATTERNS = [
|
|
63181
63367
|
"*.google.com",
|
|
63182
63368
|
"*.googleusercontent.com",
|
|
@@ -63199,23 +63385,23 @@ function backlinkCrawlerExclusionClause() {
|
|
|
63199
63385
|
conditions.push(ne15(backlinkDomains.linkingDomain, pattern));
|
|
63200
63386
|
}
|
|
63201
63387
|
}
|
|
63202
|
-
const combined =
|
|
63388
|
+
const combined = and74(...conditions);
|
|
63203
63389
|
if (!combined) throw new Error("BACKLINK_FILTER_PATTERNS is unexpectedly empty");
|
|
63204
63390
|
return combined;
|
|
63205
63391
|
}
|
|
63206
63392
|
|
|
63207
63393
|
// ../api-routes/src/backlinks-source-state.ts
|
|
63208
|
-
import { and as
|
|
63394
|
+
import { and as and75, eq as eq86, isNotNull as isNotNull8, or as or17 } from "drizzle-orm";
|
|
63209
63395
|
function readCommonCrawlSourceState(db, projectId) {
|
|
63210
63396
|
const completedReleases = db.select({ status: ccReleaseSyncs.status }).from(ccReleaseSyncs).where(or17(
|
|
63211
|
-
|
|
63397
|
+
eq86(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready),
|
|
63212
63398
|
isNotNull8(ccReleaseSyncs.queryFinishedAt)
|
|
63213
63399
|
)).all();
|
|
63214
63400
|
const hasReadyRelease = completedReleases.length > 0;
|
|
63215
63401
|
const hasCurrentReadyRelease = completedReleases.some((release) => release.status === CcReleaseSyncStatuses.ready);
|
|
63216
|
-
const hasProjectData = db.select({ id: backlinkSummaries.id }).from(backlinkSummaries).where(
|
|
63217
|
-
|
|
63218
|
-
|
|
63402
|
+
const hasProjectData = db.select({ id: backlinkSummaries.id }).from(backlinkSummaries).where(and75(
|
|
63403
|
+
eq86(backlinkSummaries.projectId, projectId),
|
|
63404
|
+
eq86(backlinkSummaries.source, BacklinkSources.commoncrawl)
|
|
63219
63405
|
)).limit(1).get() !== void 0;
|
|
63220
63406
|
return { hasReadyRelease, hasCurrentReadyRelease, hasProjectData, connected: hasReadyRelease && hasProjectData };
|
|
63221
63407
|
}
|
|
@@ -63286,11 +63472,11 @@ function mapRunRow(row) {
|
|
|
63286
63472
|
}
|
|
63287
63473
|
function latestSummaryForProject(db, projectId, source, release) {
|
|
63288
63474
|
const conditions = [
|
|
63289
|
-
|
|
63290
|
-
|
|
63475
|
+
eq87(backlinkSummaries.projectId, projectId),
|
|
63476
|
+
eq87(backlinkSummaries.source, source)
|
|
63291
63477
|
];
|
|
63292
|
-
if (release) conditions.push(
|
|
63293
|
-
const rows = db.select().from(backlinkSummaries).where(
|
|
63478
|
+
if (release) conditions.push(eq87(backlinkSummaries.release, release));
|
|
63479
|
+
const rows = db.select().from(backlinkSummaries).where(and76(...conditions)).orderBy(desc40(backlinkSummaries.queriedAt)).all();
|
|
63294
63480
|
return rows.reduce((latest, row) => {
|
|
63295
63481
|
if (!latest) return row;
|
|
63296
63482
|
const newer = isValidReleaseId(row.release) && isValidReleaseId(latest.release) && compareReleaseIds(row.release, latest.release) > 0;
|
|
@@ -63303,21 +63489,21 @@ function parseExcludeCrawlers(value) {
|
|
|
63303
63489
|
return lower === "1" || lower === "true" || lower === "yes";
|
|
63304
63490
|
}
|
|
63305
63491
|
function computeFilteredSummary(db, base) {
|
|
63306
|
-
const baseDomainCondition =
|
|
63307
|
-
|
|
63308
|
-
|
|
63309
|
-
|
|
63492
|
+
const baseDomainCondition = and76(
|
|
63493
|
+
eq87(backlinkDomains.projectId, base.projectId),
|
|
63494
|
+
eq87(backlinkDomains.source, base.source),
|
|
63495
|
+
eq87(backlinkDomains.release, base.release)
|
|
63310
63496
|
);
|
|
63311
|
-
const filteredCondition =
|
|
63497
|
+
const filteredCondition = and76(baseDomainCondition, backlinkCrawlerExclusionClause());
|
|
63312
63498
|
const unfilteredAgg = db.select({
|
|
63313
|
-
count:
|
|
63314
|
-
total:
|
|
63499
|
+
count: sql37`count(*)`,
|
|
63500
|
+
total: sql37`coalesce(sum(${backlinkDomains.numHosts}), 0)`
|
|
63315
63501
|
}).from(backlinkDomains).where(baseDomainCondition).get();
|
|
63316
63502
|
const filteredAgg = db.select({
|
|
63317
|
-
count:
|
|
63318
|
-
total:
|
|
63503
|
+
count: sql37`count(*)`,
|
|
63504
|
+
total: sql37`coalesce(sum(${backlinkDomains.numHosts}), 0)`
|
|
63319
63505
|
}).from(backlinkDomains).where(filteredCondition).get();
|
|
63320
|
-
const top10Rows = db.select({ numHosts: backlinkDomains.numHosts }).from(backlinkDomains).where(filteredCondition).orderBy(
|
|
63506
|
+
const top10Rows = db.select({ numHosts: backlinkDomains.numHosts }).from(backlinkDomains).where(filteredCondition).orderBy(desc40(backlinkDomains.numHosts)).limit(10).all();
|
|
63321
63507
|
const totalLinkingDomains = Number(filteredAgg?.count ?? 0);
|
|
63322
63508
|
const totalHosts = Number(filteredAgg?.total ?? 0);
|
|
63323
63509
|
const unfilteredLinkingDomains = Number(unfilteredAgg?.count ?? 0);
|
|
@@ -63338,8 +63524,8 @@ function computeFilteredSummary(db, base) {
|
|
|
63338
63524
|
};
|
|
63339
63525
|
}
|
|
63340
63526
|
function computeFilteredReleaseTotals(db, projectId, source) {
|
|
63341
|
-
const kept =
|
|
63342
|
-
const rows = db.all(
|
|
63527
|
+
const kept = sql37`case when ${backlinkCrawlerExclusionClause()} then 1 else 0 end`;
|
|
63528
|
+
const rows = db.all(sql37`
|
|
63343
63529
|
SELECT release,
|
|
63344
63530
|
COUNT(*) AS domain_rows,
|
|
63345
63531
|
SUM(kept) AS linking_domains,
|
|
@@ -63349,7 +63535,7 @@ function computeFilteredReleaseTotals(db, projectId, source) {
|
|
|
63349
63535
|
SELECT ${backlinkDomains.release} AS release, ${backlinkDomains.numHosts} AS num_hosts, ${kept} AS kept,
|
|
63350
63536
|
ROW_NUMBER() OVER (PARTITION BY ${backlinkDomains.release}, ${kept} ORDER BY ${backlinkDomains.numHosts} DESC) AS kept_rank
|
|
63351
63537
|
FROM ${backlinkDomains}
|
|
63352
|
-
WHERE ${
|
|
63538
|
+
WHERE ${and76(eq87(backlinkDomains.projectId, projectId), eq87(backlinkDomains.source, source))}
|
|
63353
63539
|
)
|
|
63354
63540
|
GROUP BY release`);
|
|
63355
63541
|
return new Map(rows.map((row) => {
|
|
@@ -63363,13 +63549,13 @@ function computeFilteredReleaseTotals(db, projectId, source) {
|
|
|
63363
63549
|
}));
|
|
63364
63550
|
}
|
|
63365
63551
|
function buildSourceAvailability(db, projectId, source, connected2, excludeCrawlers) {
|
|
63366
|
-
const summary3 = db.select().from(backlinkSummaries).where(
|
|
63552
|
+
const summary3 = db.select().from(backlinkSummaries).where(and76(eq87(backlinkSummaries.projectId, projectId), eq87(backlinkSummaries.source, source))).orderBy(desc40(backlinkSummaries.queriedAt)).limit(1).get();
|
|
63367
63553
|
let totalLinkingDomains = summary3?.totalLinkingDomains ?? 0;
|
|
63368
63554
|
if (summary3 && excludeCrawlers) {
|
|
63369
|
-
const filtered = db.select({ count:
|
|
63370
|
-
|
|
63371
|
-
|
|
63372
|
-
|
|
63555
|
+
const filtered = db.select({ count: sql37`count(*)` }).from(backlinkDomains).where(and76(
|
|
63556
|
+
eq87(backlinkDomains.projectId, projectId),
|
|
63557
|
+
eq87(backlinkDomains.source, source),
|
|
63558
|
+
eq87(backlinkDomains.release, summary3.release),
|
|
63373
63559
|
backlinkCrawlerExclusionClause()
|
|
63374
63560
|
)).get();
|
|
63375
63561
|
totalLinkingDomains = Number(filtered?.count ?? 0);
|
|
@@ -63439,7 +63625,7 @@ async function backlinksRoutes(app, opts) {
|
|
|
63439
63625
|
"@duckdb/node-api is not installed. Run `canonry backlinks install` to enable the backlinks feature."
|
|
63440
63626
|
);
|
|
63441
63627
|
}
|
|
63442
|
-
const existing = app.db.select().from(ccReleaseSyncs).where(
|
|
63628
|
+
const existing = app.db.select().from(ccReleaseSyncs).where(eq87(ccReleaseSyncs.release, release)).get();
|
|
63443
63629
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
63444
63630
|
if (existing) {
|
|
63445
63631
|
if (NON_TERMINAL_SYNC_STATUSES.has(existing.status)) {
|
|
@@ -63450,9 +63636,9 @@ async function backlinksRoutes(app, opts) {
|
|
|
63450
63636
|
phaseDetail: null,
|
|
63451
63637
|
error: null,
|
|
63452
63638
|
updatedAt: now
|
|
63453
|
-
}).where(
|
|
63639
|
+
}).where(eq87(ccReleaseSyncs.id, existing.id)).run();
|
|
63454
63640
|
opts.onReleaseSyncRequested(existing.id, release);
|
|
63455
|
-
const refreshed = app.db.select().from(ccReleaseSyncs).where(
|
|
63641
|
+
const refreshed = app.db.select().from(ccReleaseSyncs).where(eq87(ccReleaseSyncs.id, existing.id)).get();
|
|
63456
63642
|
return reply.status(200).send(mapSyncRow(refreshed));
|
|
63457
63643
|
}
|
|
63458
63644
|
const id = crypto53.randomUUID();
|
|
@@ -63464,15 +63650,15 @@ async function backlinksRoutes(app, opts) {
|
|
|
63464
63650
|
updatedAt: now
|
|
63465
63651
|
}).run();
|
|
63466
63652
|
opts.onReleaseSyncRequested(id, release);
|
|
63467
|
-
const inserted = app.db.select().from(ccReleaseSyncs).where(
|
|
63653
|
+
const inserted = app.db.select().from(ccReleaseSyncs).where(eq87(ccReleaseSyncs.id, id)).get();
|
|
63468
63654
|
return reply.status(201).send(mapSyncRow(inserted));
|
|
63469
63655
|
});
|
|
63470
63656
|
app.get("/backlinks/syncs/latest", async (_request, reply) => {
|
|
63471
|
-
const row = app.db.select().from(ccReleaseSyncs).orderBy(
|
|
63657
|
+
const row = app.db.select().from(ccReleaseSyncs).orderBy(desc40(ccReleaseSyncs.updatedAt)).limit(1).get();
|
|
63472
63658
|
return reply.send(row ? mapSyncRow(row) : null);
|
|
63473
63659
|
});
|
|
63474
63660
|
app.get("/backlinks/syncs", async (_request, reply) => {
|
|
63475
|
-
const rows = app.db.select().from(ccReleaseSyncs).orderBy(
|
|
63661
|
+
const rows = app.db.select().from(ccReleaseSyncs).orderBy(desc40(ccReleaseSyncs.updatedAt)).all();
|
|
63476
63662
|
return reply.send(rows.map(mapSyncRow));
|
|
63477
63663
|
});
|
|
63478
63664
|
app.get("/backlinks/releases", async (_request, reply) => {
|
|
@@ -63522,7 +63708,7 @@ async function backlinksRoutes(app, opts) {
|
|
|
63522
63708
|
createdAt: now
|
|
63523
63709
|
}).run();
|
|
63524
63710
|
opts.onBacklinkExtractRequested(runId, project.id, release);
|
|
63525
|
-
const run = app.db.select().from(runs).where(
|
|
63711
|
+
const run = app.db.select().from(runs).where(eq87(runs.id, runId)).get();
|
|
63526
63712
|
return reply.status(201).send(mapRunRow(run));
|
|
63527
63713
|
});
|
|
63528
63714
|
app.get(
|
|
@@ -63548,18 +63734,18 @@ async function backlinksRoutes(app, opts) {
|
|
|
63548
63734
|
const limit = Math.min(Math.max(parseInt(request.query.limit ?? "50", 10) || 50, 1), 500);
|
|
63549
63735
|
const offset = Math.max(parseInt(request.query.offset ?? "0", 10) || 0, 0);
|
|
63550
63736
|
const excludeCrawlers = parseExcludeCrawlers(request.query.excludeCrawlers);
|
|
63551
|
-
const baseDomainCondition =
|
|
63552
|
-
|
|
63553
|
-
|
|
63554
|
-
|
|
63737
|
+
const baseDomainCondition = and76(
|
|
63738
|
+
eq87(backlinkDomains.projectId, project.id),
|
|
63739
|
+
eq87(backlinkDomains.source, source),
|
|
63740
|
+
eq87(backlinkDomains.release, targetRelease)
|
|
63555
63741
|
);
|
|
63556
|
-
const domainCondition = excludeCrawlers ?
|
|
63557
|
-
const totalRow = app.db.select({ count:
|
|
63742
|
+
const domainCondition = excludeCrawlers ? and76(baseDomainCondition, backlinkCrawlerExclusionClause()) : baseDomainCondition;
|
|
63743
|
+
const totalRow = app.db.select({ count: sql37`count(*)` }).from(backlinkDomains).where(domainCondition).get();
|
|
63558
63744
|
const rows = app.db.select({
|
|
63559
63745
|
linkingDomain: backlinkDomains.linkingDomain,
|
|
63560
63746
|
numHosts: backlinkDomains.numHosts,
|
|
63561
63747
|
source: backlinkDomains.source
|
|
63562
|
-
}).from(backlinkDomains).where(domainCondition).orderBy(
|
|
63748
|
+
}).from(backlinkDomains).where(domainCondition).orderBy(desc40(backlinkDomains.numHosts)).limit(limit).offset(offset).all();
|
|
63563
63749
|
let summary3 = null;
|
|
63564
63750
|
if (summaryRow) {
|
|
63565
63751
|
summary3 = excludeCrawlers ? computeFilteredSummary(app.db, summaryRow) : mapSummaryRow(summaryRow);
|
|
@@ -63577,7 +63763,7 @@ async function backlinksRoutes(app, opts) {
|
|
|
63577
63763
|
async (request, reply) => {
|
|
63578
63764
|
const project = resolveProject(app.db, request.params.name);
|
|
63579
63765
|
const source = parseSourceParam(request.query.source);
|
|
63580
|
-
const rows = app.db.select().from(backlinkSummaries).where(
|
|
63766
|
+
const rows = app.db.select().from(backlinkSummaries).where(and76(eq87(backlinkSummaries.projectId, project.id), eq87(backlinkSummaries.source, source))).orderBy(asc22(backlinkSummaries.queriedAt)).all();
|
|
63581
63767
|
const response = rows.map((r) => ({
|
|
63582
63768
|
release: r.release,
|
|
63583
63769
|
totalLinkingDomains: r.totalLinkingDomains,
|
|
@@ -63604,7 +63790,7 @@ async function backlinksRoutes(app, opts) {
|
|
|
63604
63790
|
import crypto55 from "crypto";
|
|
63605
63791
|
import { isIP as isIP3 } from "net";
|
|
63606
63792
|
import { isDeepStrictEqual } from "util";
|
|
63607
|
-
import { and as
|
|
63793
|
+
import { and as and78, desc as desc41, eq as eq89, gt as gt7, gte as gte24, inArray as inArray40, lt as lt16, lte as lte25, sql as sql38 } from "drizzle-orm";
|
|
63608
63794
|
|
|
63609
63795
|
// ../api-routes/src/traffic-limits.ts
|
|
63610
63796
|
var VERCEL_MAX_SYNC_WINDOW_MS = 24 * 60 * 6e4;
|
|
@@ -64947,7 +65133,7 @@ async function ackCloudflareQueueMessages(client, options) {
|
|
|
64947
65133
|
}
|
|
64948
65134
|
|
|
64949
65135
|
// ../api-routes/src/traffic-sync-lease.ts
|
|
64950
|
-
import { and as
|
|
65136
|
+
import { and as and77, eq as eq88, isNull as isNull13, lte as lte24, or as or18 } from "drizzle-orm";
|
|
64951
65137
|
function leaseExpiresAt(now, ttlMs) {
|
|
64952
65138
|
const nowMs = Date.parse(now);
|
|
64953
65139
|
if (!Number.isFinite(nowMs)) throw new RangeError("Traffic sync lease now must be an ISO instant");
|
|
@@ -64960,12 +65146,12 @@ function tryClaimTrafficSyncLease(input) {
|
|
|
64960
65146
|
const changed = tx.update(trafficSources).set({
|
|
64961
65147
|
syncLeaseOwner: input.owner,
|
|
64962
65148
|
syncLeaseExpiresAt: expiresAt
|
|
64963
|
-
}).where(
|
|
64964
|
-
|
|
65149
|
+
}).where(and77(
|
|
65150
|
+
eq88(trafficSources.id, input.sourceId),
|
|
64965
65151
|
or18(
|
|
64966
65152
|
isNull13(trafficSources.syncLeaseOwner),
|
|
64967
65153
|
lte24(trafficSources.syncLeaseExpiresAt, input.now),
|
|
64968
|
-
|
|
65154
|
+
eq88(trafficSources.syncLeaseOwner, input.owner)
|
|
64969
65155
|
)
|
|
64970
65156
|
)).run();
|
|
64971
65157
|
return changed.changes === 1;
|
|
@@ -64976,9 +65162,9 @@ function releaseTrafficSyncLease(input) {
|
|
|
64976
65162
|
const changed = tx.update(trafficSources).set({
|
|
64977
65163
|
syncLeaseOwner: null,
|
|
64978
65164
|
syncLeaseExpiresAt: null
|
|
64979
|
-
}).where(
|
|
64980
|
-
|
|
64981
|
-
|
|
65165
|
+
}).where(and77(
|
|
65166
|
+
eq88(trafficSources.id, input.sourceId),
|
|
65167
|
+
eq88(trafficSources.syncLeaseOwner, input.owner)
|
|
64982
65168
|
)).run();
|
|
64983
65169
|
return changed.changes === 1;
|
|
64984
65170
|
}, { behavior: "immediate" });
|
|
@@ -65045,14 +65231,14 @@ async function loadQueueReceiptPolicy(db, sourceId, readRetentionSeconds, warn2)
|
|
|
65045
65231
|
if (liveTtlMs === null) return { ttlMsAt: () => CLOUDFLARE_QUEUE_RECEIPT_TTL_MS };
|
|
65046
65232
|
const ttlMs = liveTtlMs;
|
|
65047
65233
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
65048
|
-
const alive =
|
|
65049
|
-
const lifetimeMs =
|
|
65050
|
-
const shortest = db.select({ ms:
|
|
65234
|
+
const alive = and78(eq89(trafficEventReceipts.sourceId, sourceId), gt7(trafficEventReceipts.expiresAt, now));
|
|
65235
|
+
const lifetimeMs = sql38`(julianday(${trafficEventReceipts.expiresAt}) - julianday(${trafficEventReceipts.receivedAt})) * ${DAY_MS4}`;
|
|
65236
|
+
const shortest = db.select({ ms: sql38`min(${lifetimeMs})` }).from(trafficEventReceipts).where(alive).get()?.ms ?? null;
|
|
65051
65237
|
if (shortest !== null && shortest < ttlMs - 1e3) {
|
|
65052
|
-
const extended =
|
|
65053
|
-
db.update(trafficEventReceipts).set({ expiresAt: extended }).where(
|
|
65238
|
+
const extended = sql38`strftime('%Y-%m-%dT%H:%M:%fZ', julianday(${trafficEventReceipts.receivedAt}) + ${ttlMs / DAY_MS4})`;
|
|
65239
|
+
db.update(trafficEventReceipts).set({ expiresAt: extended }).where(and78(alive, lt16(trafficEventReceipts.expiresAt, extended))).run();
|
|
65054
65240
|
}
|
|
65055
|
-
const floorExpiry = db.select({ at:
|
|
65241
|
+
const floorExpiry = db.select({ at: sql38`max(${trafficEventReceipts.expiresAt})` }).from(trafficEventReceipts).where(and78(alive, sql38`${lifetimeMs} < ${CLOUDFLARE_QUEUE_RECEIPT_TTL_MS - 1e3}`)).get()?.at ?? null;
|
|
65056
65242
|
const floorMs = floorExpiry ? Date.parse(floorExpiry) : 0;
|
|
65057
65243
|
return { ttlMsAt: (receivedAt) => Math.max(ttlMs, floorMs - Date.parse(receivedAt)) };
|
|
65058
65244
|
}
|
|
@@ -65107,7 +65293,7 @@ function cloudflareIngestRateLimitKey(request, store, db) {
|
|
|
65107
65293
|
const authenticated = authenticateCloudflareIngest(request, store);
|
|
65108
65294
|
const params = request.params;
|
|
65109
65295
|
if (authenticated && typeof params.name === "string" && authenticated.credential.projectName === params.name) {
|
|
65110
|
-
const source = db.select().from(trafficSources).where(
|
|
65296
|
+
const source = db.select().from(trafficSources).where(eq89(trafficSources.id, authenticated.credential.sourceId)).get();
|
|
65111
65297
|
if (source && source.sourceType === TrafficSourceTypes.cloudflare && source.status === TrafficSourceStatuses.connected && parseDirectPushCloudflareSourceConfig(source.configJson) && timingSafeEqualHex(authenticated.bearerHash, source.ingestTokenHash)) {
|
|
65112
65298
|
return `cloudflare-source:${authenticated.credential.sourceId}`;
|
|
65113
65299
|
}
|
|
@@ -65261,7 +65447,7 @@ function crawlerVerificationProvenance(db, projectId, events) {
|
|
|
65261
65447
|
manifestId: crawlerVerificationManifestsHourly.manifestId,
|
|
65262
65448
|
manifest: crawlerVerificationManifestsHourly.manifestJson,
|
|
65263
65449
|
hits: crawlerVerificationManifestsHourly.hits
|
|
65264
|
-
}).from(crawlerVerificationManifestsHourly).where(
|
|
65450
|
+
}).from(crawlerVerificationManifestsHourly).where(sql38`(
|
|
65265
65451
|
${crawlerVerificationManifestsHourly.projectId},
|
|
65266
65452
|
${crawlerVerificationManifestsHourly.sourceId},
|
|
65267
65453
|
${crawlerVerificationManifestsHourly.tsHour},
|
|
@@ -65295,7 +65481,7 @@ function aiUserFetchVerificationProvenance(db, projectId, events) {
|
|
|
65295
65481
|
manifestId: aiUserFetchVerificationManifestsHourly.manifestId,
|
|
65296
65482
|
manifest: aiUserFetchVerificationManifestsHourly.manifestJson,
|
|
65297
65483
|
hits: aiUserFetchVerificationManifestsHourly.hits
|
|
65298
|
-
}).from(aiUserFetchVerificationManifestsHourly).where(
|
|
65484
|
+
}).from(aiUserFetchVerificationManifestsHourly).where(sql38`(
|
|
65299
65485
|
${aiUserFetchVerificationManifestsHourly.projectId},
|
|
65300
65486
|
${aiUserFetchVerificationManifestsHourly.sourceId},
|
|
65301
65487
|
${aiUserFetchVerificationManifestsHourly.tsHour},
|
|
@@ -65350,7 +65536,7 @@ async function defaultResolveAccessToken(record2) {
|
|
|
65350
65536
|
);
|
|
65351
65537
|
}
|
|
65352
65538
|
function hasConnectedTrafficSourceSibling(db, projectId, sourceId) {
|
|
65353
|
-
return db.select().from(trafficSources).where(
|
|
65539
|
+
return db.select().from(trafficSources).where(eq89(trafficSources.projectId, projectId)).all().some((row) => row.id !== sourceId && row.status === TrafficSourceStatuses.connected);
|
|
65354
65540
|
}
|
|
65355
65541
|
function isAuthoritativeTrafficSource(db, source) {
|
|
65356
65542
|
return (source.status === TrafficSourceStatuses.connected || source.status === TrafficSourceStatuses.error) && !hasConnectedTrafficSourceSibling(db, source.projectId, source.id);
|
|
@@ -65377,13 +65563,13 @@ function wordpressBaseUrlIdentity(value) {
|
|
|
65377
65563
|
}
|
|
65378
65564
|
}
|
|
65379
65565
|
function bindTrafficSyncSchedule(tx, projectId, sourceId, now, createIfMissing = true) {
|
|
65380
|
-
const schedule = tx.select().from(schedules).where(
|
|
65381
|
-
|
|
65382
|
-
|
|
65566
|
+
const schedule = tx.select().from(schedules).where(and78(
|
|
65567
|
+
eq89(schedules.projectId, projectId),
|
|
65568
|
+
eq89(schedules.kind, SchedulableRunKinds["traffic-sync"])
|
|
65383
65569
|
)).get();
|
|
65384
65570
|
if (schedule) {
|
|
65385
65571
|
if (schedule.sourceId === sourceId) return { changed: false, created: false };
|
|
65386
|
-
tx.update(schedules).set({ sourceId, updatedAt: nextScheduleUpdatedAt(schedule.updatedAt, Date.parse(now)) }).where(
|
|
65572
|
+
tx.update(schedules).set({ sourceId, updatedAt: nextScheduleUpdatedAt(schedule.updatedAt, Date.parse(now)) }).where(eq89(schedules.id, schedule.id)).run();
|
|
65387
65573
|
return { changed: true, created: false };
|
|
65388
65574
|
}
|
|
65389
65575
|
if (!createIfMissing) return { changed: false, created: false };
|
|
@@ -65425,7 +65611,7 @@ async function runBackfillTask(options) {
|
|
|
65425
65611
|
};
|
|
65426
65612
|
const renewLease2 = setInterval(() => {
|
|
65427
65613
|
try {
|
|
65428
|
-
const latest = app.db.select().from(trafficSources).where(
|
|
65614
|
+
const latest = app.db.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
65429
65615
|
if (!latest || !isAuthoritativeTrafficSource(app.db, latest) || !isSameTrafficSourceGeneration(latest, sourceRow2)) {
|
|
65430
65616
|
clearInterval(renewLease2);
|
|
65431
65617
|
stopped.abort();
|
|
@@ -65473,10 +65659,10 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65473
65659
|
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
65474
65660
|
try {
|
|
65475
65661
|
app.db.transaction((tx) => {
|
|
65476
|
-
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(
|
|
65477
|
-
const latestSource = tx.select().from(trafficSources).where(
|
|
65662
|
+
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(eq89(runs.id, runId)).run();
|
|
65663
|
+
const latestSource = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
65478
65664
|
if (!preserveSourceState && latestSource && isAuthoritativeTrafficSource(tx, latestSource) && isSameBackfillSource(latestSource, sourceRow2, leaseOwner)) {
|
|
65479
|
-
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(
|
|
65665
|
+
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(eq89(trafficSources.id, sourceRow2.id)).run();
|
|
65480
65666
|
}
|
|
65481
65667
|
});
|
|
65482
65668
|
} catch {
|
|
@@ -65508,7 +65694,7 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65508
65694
|
let emptyOutcome;
|
|
65509
65695
|
try {
|
|
65510
65696
|
const completed = app.db.transaction((tx) => {
|
|
65511
|
-
const latestSource = tx.select().from(trafficSources).where(
|
|
65697
|
+
const latestSource = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
65512
65698
|
const stillAuthoritative = latestSource && isAuthoritativeTrafficSource(tx, latestSource) && isSameBackfillSource(latestSource, sourceRow2, leaseOwner);
|
|
65513
65699
|
if (stillAuthoritative) {
|
|
65514
65700
|
enforceRawEventSampleRetention(tx, sourceRow2.id, finishedAt2);
|
|
@@ -65517,7 +65703,7 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65517
65703
|
status: RunStatuses.failed,
|
|
65518
65704
|
error: "Traffic source was deactivated or reconfigured during backfill",
|
|
65519
65705
|
finishedAt: finishedAt2
|
|
65520
|
-
}).where(
|
|
65706
|
+
}).where(eq89(runs.id, runId)).run();
|
|
65521
65707
|
return Boolean(stillAuthoritative);
|
|
65522
65708
|
});
|
|
65523
65709
|
emptyOutcome = completed ? { status: OutcomeStatuses.skipped, reasonCode: OutcomeReasonCodes.NO_DATA } : { status: OutcomeStatuses.failed, reasonCode: OutcomeReasonCodes.NOT_CONNECTED };
|
|
@@ -65549,13 +65735,13 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65549
65735
|
const nextLastSyncedAt = Math.max(currentLastSyncedMs, windowEnd.getTime()) === windowEnd.getTime() ? windowEndIso : sourceRow2.lastSyncedAt;
|
|
65550
65736
|
try {
|
|
65551
65737
|
const commitOutcome = app.db.transaction((tx) => {
|
|
65552
|
-
const latestSource = tx.select().from(trafficSources).where(
|
|
65738
|
+
const latestSource = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
65553
65739
|
if (latestSource?.sourceType === TrafficSourceTypes.wordpress) {
|
|
65554
65740
|
tx.update(runs).set({
|
|
65555
65741
|
status: RunStatuses.failed,
|
|
65556
65742
|
error: "Generic WordPress replace backfill is unavailable because retained coverage is unproven.",
|
|
65557
65743
|
finishedAt
|
|
65558
|
-
}).where(
|
|
65744
|
+
}).where(eq89(runs.id, runId)).run();
|
|
65559
65745
|
return "wordpress-backfill-unsupported";
|
|
65560
65746
|
}
|
|
65561
65747
|
if (!latestSource || !isAuthoritativeTrafficSource(tx, latestSource) || !isSameBackfillSource(latestSource, sourceRow2, leaseOwner)) {
|
|
@@ -65563,7 +65749,7 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65563
65749
|
status: RunStatuses.failed,
|
|
65564
65750
|
error: "Traffic source was deactivated or reconfigured during backfill",
|
|
65565
65751
|
finishedAt
|
|
65566
|
-
}).where(
|
|
65752
|
+
}).where(eq89(runs.id, runId)).run();
|
|
65567
65753
|
return "source-inactive";
|
|
65568
65754
|
}
|
|
65569
65755
|
const rawSampleCutoff = enforceRawEventSampleRetention(
|
|
@@ -65572,37 +65758,37 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65572
65758
|
finishedAt
|
|
65573
65759
|
);
|
|
65574
65760
|
tx.delete(crawlerEventsHourly).where(
|
|
65575
|
-
|
|
65576
|
-
|
|
65761
|
+
and78(
|
|
65762
|
+
eq89(crawlerEventsHourly.sourceId, sourceRow2.id),
|
|
65577
65763
|
gte24(crawlerEventsHourly.tsHour, windowStartIso),
|
|
65578
65764
|
lte25(crawlerEventsHourly.tsHour, windowEndIso)
|
|
65579
65765
|
)
|
|
65580
65766
|
).run();
|
|
65581
65767
|
tx.delete(aiUserFetchEventsHourly).where(
|
|
65582
|
-
|
|
65583
|
-
|
|
65768
|
+
and78(
|
|
65769
|
+
eq89(aiUserFetchEventsHourly.sourceId, sourceRow2.id),
|
|
65584
65770
|
gte24(aiUserFetchEventsHourly.tsHour, windowStartIso),
|
|
65585
65771
|
lte25(aiUserFetchEventsHourly.tsHour, windowEndIso)
|
|
65586
65772
|
)
|
|
65587
65773
|
).run();
|
|
65588
65774
|
clearLikelyFakeAgentHits(tx, { sourceId: sourceRow2.id, windowStart: windowStartIso, windowEnd: windowEndIso });
|
|
65589
65775
|
tx.delete(aiReferralEventsHourly).where(
|
|
65590
|
-
|
|
65591
|
-
|
|
65776
|
+
and78(
|
|
65777
|
+
eq89(aiReferralEventsHourly.sourceId, sourceRow2.id),
|
|
65592
65778
|
gte24(aiReferralEventsHourly.tsHour, windowStartIso),
|
|
65593
65779
|
lte25(aiReferralEventsHourly.tsHour, windowEndIso)
|
|
65594
65780
|
)
|
|
65595
65781
|
).run();
|
|
65596
65782
|
tx.delete(rawEventSamples).where(
|
|
65597
|
-
|
|
65598
|
-
|
|
65783
|
+
and78(
|
|
65784
|
+
eq89(rawEventSamples.sourceId, sourceRow2.id),
|
|
65599
65785
|
gte24(rawEventSamples.ts, windowStartIso),
|
|
65600
65786
|
lte25(rawEventSamples.ts, windowEndIso)
|
|
65601
65787
|
)
|
|
65602
65788
|
).run();
|
|
65603
65789
|
tx.delete(unrecognizedAgentEventsHourly).where(
|
|
65604
|
-
|
|
65605
|
-
|
|
65790
|
+
and78(
|
|
65791
|
+
eq89(unrecognizedAgentEventsHourly.sourceId, sourceRow2.id),
|
|
65606
65792
|
gte24(unrecognizedAgentEventsHourly.tsHour, windowStartIso),
|
|
65607
65793
|
lte25(unrecognizedAgentEventsHourly.tsHour, windowEndIso)
|
|
65608
65794
|
)
|
|
@@ -65633,7 +65819,7 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65633
65819
|
crawlerEventsHourly.status
|
|
65634
65820
|
],
|
|
65635
65821
|
set: {
|
|
65636
|
-
hits:
|
|
65822
|
+
hits: sql38`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
65637
65823
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
65638
65824
|
updatedAt: finishedAt
|
|
65639
65825
|
}
|
|
@@ -65664,7 +65850,7 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65664
65850
|
crawlerVerificationManifestsHourly.manifestId
|
|
65665
65851
|
],
|
|
65666
65852
|
set: {
|
|
65667
|
-
hits:
|
|
65853
|
+
hits: sql38`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
65668
65854
|
manifestJson: bucket.verificationManifest,
|
|
65669
65855
|
updatedAt: finishedAt
|
|
65670
65856
|
}
|
|
@@ -65697,7 +65883,7 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65697
65883
|
aiUserFetchEventsHourly.status
|
|
65698
65884
|
],
|
|
65699
65885
|
set: {
|
|
65700
|
-
hits:
|
|
65886
|
+
hits: sql38`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
65701
65887
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
65702
65888
|
updatedAt: finishedAt
|
|
65703
65889
|
}
|
|
@@ -65728,7 +65914,7 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65728
65914
|
aiUserFetchVerificationManifestsHourly.manifestId
|
|
65729
65915
|
],
|
|
65730
65916
|
set: {
|
|
65731
|
-
hits:
|
|
65917
|
+
hits: sql38`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
65732
65918
|
manifestJson: bucket.verificationManifest,
|
|
65733
65919
|
updatedAt: finishedAt
|
|
65734
65920
|
}
|
|
@@ -65810,8 +65996,8 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65810
65996
|
lastEventIds: newRingBuffer,
|
|
65811
65997
|
...skipRecovered ? { skippedThroughAt: null } : {},
|
|
65812
65998
|
updatedAt: finishedAt
|
|
65813
|
-
}).where(
|
|
65814
|
-
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
65999
|
+
}).where(eq89(trafficSources.id, sourceRow2.id)).run();
|
|
66000
|
+
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq89(runs.id, runId)).run();
|
|
65815
66001
|
return "committed";
|
|
65816
66002
|
});
|
|
65817
66003
|
reportBackfill(commitOutcome === "committed" ? {
|
|
@@ -65984,7 +66170,7 @@ async function trafficRoutes(app, opts) {
|
|
|
65984
66170
|
};
|
|
65985
66171
|
const fallbackName = displayName ?? `Cloud Run \xB7 ${gcpProjectId}${serviceName ? ` / ${serviceName}` : ""}`;
|
|
65986
66172
|
const { sourceRow: sourceRow2, scheduleChanged } = app.db.transaction((tx) => {
|
|
65987
|
-
const activeSource = tx.select().from(trafficSources).where(
|
|
66173
|
+
const activeSource = tx.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes["cloud-run"] && row.status !== TrafficSourceStatuses.archived);
|
|
65988
66174
|
const sourceId = activeSource?.id ?? crypto55.randomUUID();
|
|
65989
66175
|
const sourceStatus = trafficConnectStatus(tx, project.id, activeSource);
|
|
65990
66176
|
if (activeSource) {
|
|
@@ -65994,7 +66180,7 @@ async function trafficRoutes(app, opts) {
|
|
|
65994
66180
|
lastError: null,
|
|
65995
66181
|
configJson: config,
|
|
65996
66182
|
updatedAt: now
|
|
65997
|
-
}).where(
|
|
66183
|
+
}).where(eq89(trafficSources.id, sourceId)).run();
|
|
65998
66184
|
} else {
|
|
65999
66185
|
tx.insert(trafficSources).values({
|
|
66000
66186
|
id: sourceId,
|
|
@@ -66020,7 +66206,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66020
66206
|
entityId: sourceId
|
|
66021
66207
|
});
|
|
66022
66208
|
return {
|
|
66023
|
-
sourceRow: tx.select().from(trafficSources).where(
|
|
66209
|
+
sourceRow: tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceId)).get(),
|
|
66024
66210
|
scheduleChanged: scheduleBinding.changed
|
|
66025
66211
|
};
|
|
66026
66212
|
}, { behavior: "immediate" });
|
|
@@ -66075,7 +66261,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66075
66261
|
const config = { baseUrl, username };
|
|
66076
66262
|
const fallbackName = displayName ?? `WordPress \xB7 ${new URL(baseUrl).host}`;
|
|
66077
66263
|
const { sourceRow: sourceRow2, scheduleChanged } = app.db.transaction((tx) => {
|
|
66078
|
-
const activeSource = tx.select().from(trafficSources).where(
|
|
66264
|
+
const activeSource = tx.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.wordpress && row.status !== TrafficSourceStatuses.archived);
|
|
66079
66265
|
const endpointChanged = activeSource !== void 0 && wordpressBaseUrlIdentity(activeSource.configJson.baseUrl) !== wordpressBaseUrlIdentity(baseUrl);
|
|
66080
66266
|
const sourceId = endpointChanged ? crypto55.randomUUID() : activeSource?.id ?? crypto55.randomUUID();
|
|
66081
66267
|
const sourceStatus = trafficConnectStatus(tx, project.id, activeSource);
|
|
@@ -66086,14 +66272,14 @@ async function trafficRoutes(app, opts) {
|
|
|
66086
66272
|
lastError: null,
|
|
66087
66273
|
configJson: config,
|
|
66088
66274
|
updatedAt: now
|
|
66089
|
-
}).where(
|
|
66275
|
+
}).where(eq89(trafficSources.id, sourceId)).run();
|
|
66090
66276
|
} else {
|
|
66091
66277
|
if (activeSource) {
|
|
66092
66278
|
tx.update(trafficSources).set({
|
|
66093
66279
|
status: TrafficSourceStatuses.archived,
|
|
66094
66280
|
archivedAt: now,
|
|
66095
66281
|
updatedAt: now
|
|
66096
|
-
}).where(
|
|
66282
|
+
}).where(eq89(trafficSources.id, activeSource.id)).run();
|
|
66097
66283
|
}
|
|
66098
66284
|
tx.insert(trafficSources).values({
|
|
66099
66285
|
id: sourceId,
|
|
@@ -66119,7 +66305,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66119
66305
|
entityId: sourceId
|
|
66120
66306
|
});
|
|
66121
66307
|
return {
|
|
66122
|
-
sourceRow: tx.select().from(trafficSources).where(
|
|
66308
|
+
sourceRow: tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceId)).get(),
|
|
66123
66309
|
scheduleChanged: scheduleBinding.changed
|
|
66124
66310
|
};
|
|
66125
66311
|
}, { behavior: "immediate" });
|
|
@@ -66176,7 +66362,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66176
66362
|
const config = { projectId, teamId, environment };
|
|
66177
66363
|
const fallbackName = displayName ?? `Vercel \xB7 ${projectId}`;
|
|
66178
66364
|
const { sourceRow: sourceRow2, scheduleChanged } = app.db.transaction((tx) => {
|
|
66179
|
-
const activeSource = tx.select().from(trafficSources).where(
|
|
66365
|
+
const activeSource = tx.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all().find((row2) => row2.sourceType === TrafficSourceTypes.vercel && row2.status !== TrafficSourceStatuses.archived);
|
|
66180
66366
|
const sourceStatus = trafficConnectStatus(tx, project.id, activeSource);
|
|
66181
66367
|
let row;
|
|
66182
66368
|
if (activeSource) {
|
|
@@ -66186,8 +66372,8 @@ async function trafficRoutes(app, opts) {
|
|
|
66186
66372
|
lastError: null,
|
|
66187
66373
|
configJson: config,
|
|
66188
66374
|
updatedAt: now
|
|
66189
|
-
}).where(
|
|
66190
|
-
row = tx.select().from(trafficSources).where(
|
|
66375
|
+
}).where(eq89(trafficSources.id, activeSource.id)).run();
|
|
66376
|
+
row = tx.select().from(trafficSources).where(eq89(trafficSources.id, activeSource.id)).get();
|
|
66191
66377
|
} else {
|
|
66192
66378
|
const newId = crypto55.randomUUID();
|
|
66193
66379
|
tx.insert(trafficSources).values({
|
|
@@ -66212,7 +66398,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66212
66398
|
createdAt: now,
|
|
66213
66399
|
updatedAt: now
|
|
66214
66400
|
}).run();
|
|
66215
|
-
row = tx.select().from(trafficSources).where(
|
|
66401
|
+
row = tx.select().from(trafficSources).where(eq89(trafficSources.id, newId)).get();
|
|
66216
66402
|
}
|
|
66217
66403
|
const scheduleBinding = sourceStatus === TrafficSourceStatuses.connected ? bindTrafficSyncSchedule(tx, project.id, row.id, now) : { changed: false, created: false };
|
|
66218
66404
|
writeAuditLog(tx, {
|
|
@@ -66256,7 +66442,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66256
66442
|
const { displayName: displayName2, zoneId: zoneId2, accountId: accountId2, queueId, queueName, retentionSeconds, apiToken } = parsed.data;
|
|
66257
66443
|
const workerRouteHost2 = resolveCloudflareWorkerRouteHost(project.canonicalDomain);
|
|
66258
66444
|
const now2 = (/* @__PURE__ */ new Date()).toISOString();
|
|
66259
|
-
const cloudflareSources = app.db.select().from(trafficSources).where(
|
|
66445
|
+
const cloudflareSources = app.db.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all().filter((row) => row.sourceType === TrafficSourceTypes.cloudflare && row.status !== TrafficSourceStatuses.archived);
|
|
66260
66446
|
const sameModeSource = cloudflareSources.find((row) => parseQueuePullCloudflareSourceConfig(row.configJson));
|
|
66261
66447
|
if (sameModeSource) attempt.update({ action: "reauth" });
|
|
66262
66448
|
const sourceId2 = sameModeSource?.id ?? crypto55.randomUUID();
|
|
@@ -66310,7 +66496,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66310
66496
|
let scheduleChanged = false;
|
|
66311
66497
|
try {
|
|
66312
66498
|
const result = app.db.transaction((tx) => {
|
|
66313
|
-
const currentSameModeSource = tx.select().from(trafficSources).where(
|
|
66499
|
+
const currentSameModeSource = tx.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.cloudflare && row.status !== TrafficSourceStatuses.archived && parseQueuePullCloudflareSourceConfig(row.configJson) !== null);
|
|
66314
66500
|
if ((currentSameModeSource?.id ?? null) !== (sameModeSource?.id ?? null)) {
|
|
66315
66501
|
throw operationInProgress("Cloudflare Queue source changed during connect; retry");
|
|
66316
66502
|
}
|
|
@@ -66328,7 +66514,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66328
66514
|
queueBacklogObservedAt: null
|
|
66329
66515
|
} : {},
|
|
66330
66516
|
updatedAt: now2
|
|
66331
|
-
}).where(
|
|
66517
|
+
}).where(eq89(trafficSources.id, sourceId2)).run();
|
|
66332
66518
|
} else {
|
|
66333
66519
|
tx.insert(trafficSources).values({
|
|
66334
66520
|
id: sourceId2,
|
|
@@ -66348,7 +66534,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66348
66534
|
}
|
|
66349
66535
|
const changed = status === TrafficSourceStatuses.connected ? bindTrafficSyncSchedule(tx, project.id, sourceId2, now2).changed : false;
|
|
66350
66536
|
return {
|
|
66351
|
-
row: tx.select().from(trafficSources).where(
|
|
66537
|
+
row: tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceId2)).get(),
|
|
66352
66538
|
scheduleChanged: changed
|
|
66353
66539
|
};
|
|
66354
66540
|
}, { behavior: "immediate" });
|
|
@@ -66398,7 +66584,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66398
66584
|
const { deliveryMode, displayName, zoneId, accountId } = parsed.data;
|
|
66399
66585
|
const workerRouteHost = resolveCloudflareWorkerRouteHost(project.canonicalDomain);
|
|
66400
66586
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
66401
|
-
const activeSource = app.db.select().from(trafficSources).where(
|
|
66587
|
+
const activeSource = app.db.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.cloudflare && row.status !== TrafficSourceStatuses.archived && parseDirectPushCloudflareSourceConfig(row.configJson) !== null);
|
|
66402
66588
|
if (activeSource) attempt.update({ action: "reauth" });
|
|
66403
66589
|
const sourceId = activeSource?.id ?? crypto55.randomUUID();
|
|
66404
66590
|
const parsedActiveConfig = activeSource ? parseDirectPushCloudflareSourceConfig(activeSource.configJson) : null;
|
|
@@ -66466,7 +66652,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66466
66652
|
let scheduleRemoved = false;
|
|
66467
66653
|
try {
|
|
66468
66654
|
const result = app.db.transaction((tx) => {
|
|
66469
|
-
const currentActiveSource = tx.select().from(trafficSources).where(
|
|
66655
|
+
const currentActiveSource = tx.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all().find((row2) => row2.sourceType === TrafficSourceTypes.cloudflare && row2.status !== TrafficSourceStatuses.archived && parseDirectPushCloudflareSourceConfig(row2.configJson) !== null);
|
|
66470
66656
|
if ((currentActiveSource?.id ?? null) !== (activeSource?.id ?? null)) {
|
|
66471
66657
|
throw operationInProgress("Cloudflare direct-push source changed during connect; retry");
|
|
66472
66658
|
}
|
|
@@ -66480,8 +66666,8 @@ async function trafficRoutes(app, opts) {
|
|
|
66480
66666
|
configJson: config,
|
|
66481
66667
|
ingestTokenHash,
|
|
66482
66668
|
updatedAt: now
|
|
66483
|
-
}).where(
|
|
66484
|
-
row = tx.select().from(trafficSources).where(
|
|
66669
|
+
}).where(eq89(trafficSources.id, currentActiveSource.id)).run();
|
|
66670
|
+
row = tx.select().from(trafficSources).where(eq89(trafficSources.id, currentActiveSource.id)).get();
|
|
66485
66671
|
} else {
|
|
66486
66672
|
tx.insert(trafficSources).values({
|
|
66487
66673
|
id: sourceId,
|
|
@@ -66501,16 +66687,16 @@ async function trafficRoutes(app, opts) {
|
|
|
66501
66687
|
createdAt: now,
|
|
66502
66688
|
updatedAt: now
|
|
66503
66689
|
}).run();
|
|
66504
|
-
row = tx.select().from(trafficSources).where(
|
|
66690
|
+
row = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceId)).get();
|
|
66505
66691
|
}
|
|
66506
66692
|
let removed = false;
|
|
66507
66693
|
if (row.status === TrafficSourceStatuses.connected) {
|
|
66508
|
-
const schedule = tx.select().from(schedules).where(
|
|
66509
|
-
|
|
66510
|
-
|
|
66694
|
+
const schedule = tx.select().from(schedules).where(and78(
|
|
66695
|
+
eq89(schedules.projectId, project.id),
|
|
66696
|
+
eq89(schedules.kind, SchedulableRunKinds["traffic-sync"])
|
|
66511
66697
|
)).get();
|
|
66512
66698
|
if (schedule) {
|
|
66513
|
-
tx.delete(schedules).where(
|
|
66699
|
+
tx.delete(schedules).where(eq89(schedules.id, schedule.id)).run();
|
|
66514
66700
|
removed = true;
|
|
66515
66701
|
}
|
|
66516
66702
|
}
|
|
@@ -66566,47 +66752,47 @@ async function trafficRoutes(app, opts) {
|
|
|
66566
66752
|
};
|
|
66567
66753
|
}));
|
|
66568
66754
|
app.post("/projects/:name/traffic/sources/:id/activate", connectionRoute(app, (request) => {
|
|
66569
|
-
const source = app.db.select({ sourceType: trafficSources.sourceType }).from(trafficSources).where(
|
|
66755
|
+
const source = app.db.select({ sourceType: trafficSources.sourceType }).from(trafficSources).where(eq89(trafficSources.id, request.params.id)).get();
|
|
66570
66756
|
const integration = source ? TRAFFIC_SOURCE_INTEGRATIONS[source.sourceType] : void 0;
|
|
66571
66757
|
return integration ? { integration, action: "select" } : void 0;
|
|
66572
66758
|
}, async (request) => {
|
|
66573
66759
|
const project = resolveProject(app.db, request.params.name);
|
|
66574
66760
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
66575
66761
|
const result = app.db.transaction((tx) => {
|
|
66576
|
-
const target = tx.select().from(trafficSources).where(
|
|
66762
|
+
const target = tx.select().from(trafficSources).where(eq89(trafficSources.id, request.params.id)).get();
|
|
66577
66763
|
if (!target || target.projectId !== project.id || target.status === TrafficSourceStatuses.archived) {
|
|
66578
66764
|
throw notFound("Traffic source", request.params.id);
|
|
66579
66765
|
}
|
|
66580
66766
|
const { usesPullSchedule } = validateTrafficSourceCredential(target, project.name);
|
|
66581
|
-
const siblings = tx.select().from(trafficSources).where(
|
|
66767
|
+
const siblings = tx.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all();
|
|
66582
66768
|
for (const sibling of siblings) {
|
|
66583
66769
|
if (sibling.id === target.id || sibling.status === TrafficSourceStatuses.archived) continue;
|
|
66584
66770
|
tx.update(trafficSources).set({
|
|
66585
66771
|
status: TrafficSourceStatuses.paused,
|
|
66586
66772
|
updatedAt: now
|
|
66587
|
-
}).where(
|
|
66773
|
+
}).where(eq89(trafficSources.id, sibling.id)).run();
|
|
66588
66774
|
}
|
|
66589
66775
|
tx.update(trafficSources).set({
|
|
66590
66776
|
status: TrafficSourceStatuses.connected,
|
|
66591
66777
|
lastError: null,
|
|
66592
66778
|
archivedAt: null,
|
|
66593
66779
|
updatedAt: now
|
|
66594
|
-
}).where(
|
|
66780
|
+
}).where(eq89(trafficSources.id, target.id)).run();
|
|
66595
66781
|
let scheduleAction = null;
|
|
66596
66782
|
if (usesPullSchedule) {
|
|
66597
66783
|
bindTrafficSyncSchedule(tx, project.id, target.id, now);
|
|
66598
66784
|
scheduleAction = "upsert";
|
|
66599
66785
|
} else {
|
|
66600
|
-
const schedule = tx.select().from(schedules).where(
|
|
66601
|
-
|
|
66602
|
-
|
|
66786
|
+
const schedule = tx.select().from(schedules).where(and78(
|
|
66787
|
+
eq89(schedules.projectId, project.id),
|
|
66788
|
+
eq89(schedules.kind, SchedulableRunKinds["traffic-sync"])
|
|
66603
66789
|
)).get();
|
|
66604
66790
|
if (schedule) {
|
|
66605
|
-
tx.delete(schedules).where(
|
|
66791
|
+
tx.delete(schedules).where(eq89(schedules.id, schedule.id)).run();
|
|
66606
66792
|
scheduleAction = "delete";
|
|
66607
66793
|
}
|
|
66608
66794
|
}
|
|
66609
|
-
const source = tx.select().from(trafficSources).where(
|
|
66795
|
+
const source = tx.select().from(trafficSources).where(eq89(trafficSources.id, target.id)).get();
|
|
66610
66796
|
writeAuditLog(tx, {
|
|
66611
66797
|
projectId: project.id,
|
|
66612
66798
|
actor: "api",
|
|
@@ -66674,7 +66860,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66674
66860
|
}
|
|
66675
66861
|
const { bearerHash, credential } = authenticated;
|
|
66676
66862
|
const sourceId = credential.sourceId;
|
|
66677
|
-
const sourceRow2 = app.db.select().from(trafficSources).where(
|
|
66863
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(eq89(trafficSources.id, sourceId)).get();
|
|
66678
66864
|
if (!sourceRow2 || sourceRow2.sourceType !== TrafficSourceTypes.cloudflare || sourceRow2.status !== TrafficSourceStatuses.connected || !parseDirectPushCloudflareSourceConfig(sourceRow2.configJson) || !timingSafeEqualHex(bearerHash, sourceRow2.ingestTokenHash)) {
|
|
66679
66865
|
throw authRequired();
|
|
66680
66866
|
}
|
|
@@ -66752,7 +66938,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66752
66938
|
});
|
|
66753
66939
|
app.post("/projects/:name/traffic/sources/:id/sync", async (request) => {
|
|
66754
66940
|
const project = resolveProject(app.db, request.params.name);
|
|
66755
|
-
const initialSourceRow = app.db.select().from(trafficSources).where(
|
|
66941
|
+
const initialSourceRow = app.db.select().from(trafficSources).where(eq89(trafficSources.id, request.params.id)).get();
|
|
66756
66942
|
if (!initialSourceRow || initialSourceRow.projectId !== project.id) {
|
|
66757
66943
|
throw notFound("Traffic source", request.params.id);
|
|
66758
66944
|
}
|
|
@@ -66825,7 +67011,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66825
67011
|
});
|
|
66826
67012
|
const finishSync = (status, error, stopReason) => {
|
|
66827
67013
|
const queueDrain = drainSummary(stopReason);
|
|
66828
|
-
app.db.update(runs).set({ status, finishedAt: committedAt, syncSummary: queueDrain, ...error ? { error } : {} }).where(
|
|
67014
|
+
app.db.update(runs).set({ status, finishedAt: committedAt, syncSummary: queueDrain, ...error ? { error } : {} }).where(eq89(runs.id, runId)).run();
|
|
66829
67015
|
writeAuditLog(app.db, {
|
|
66830
67016
|
projectId: project.id,
|
|
66831
67017
|
actor: "api",
|
|
@@ -67000,7 +67186,7 @@ async function trafficRoutes(app, opts) {
|
|
|
67000
67186
|
now: (/* @__PURE__ */ new Date()).toISOString(),
|
|
67001
67187
|
ttlMs: CLOUDFLARE_QUEUE_SYNC_LEASE_TTL_MS
|
|
67002
67188
|
})) throw new Error("Queue sync lease was lost before acknowledgement");
|
|
67003
|
-
const latestSource = app.db.select().from(trafficSources).where(
|
|
67189
|
+
const latestSource = app.db.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
67004
67190
|
if (!latestSource || latestSource.projectId !== project.id || latestSource.status !== TrafficSourceStatuses.connected || latestSource.syncLeaseOwner !== leaseOwner || !isDeepStrictEqual(latestSource.configJson, sourceRow2.configJson)) {
|
|
67005
67191
|
throw validationError("Cloudflare Queue source was reconfigured before acknowledgement");
|
|
67006
67192
|
}
|
|
@@ -67089,13 +67275,13 @@ async function trafficRoutes(app, opts) {
|
|
|
67089
67275
|
error: safeError,
|
|
67090
67276
|
finishedAt: failedAt,
|
|
67091
67277
|
syncSummary: drainSummary(CloudflareQueueDrainStopReasons.error)
|
|
67092
|
-
}).where(
|
|
67093
|
-
const latestSource = tx.select().from(trafficSources).where(
|
|
67278
|
+
}).where(eq89(runs.id, runId)).run();
|
|
67279
|
+
const latestSource = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
67094
67280
|
if (latestSource && latestSource.status === TrafficSourceStatuses.connected && latestSource.syncLeaseOwner === leaseOwner && isDeepStrictEqual(latestSource.configJson, sourceRow2.configJson)) {
|
|
67095
|
-
tx.update(trafficSources).set({ lastError: safeError, updatedAt: failedAt }).where(
|
|
67096
|
-
|
|
67097
|
-
|
|
67098
|
-
|
|
67281
|
+
tx.update(trafficSources).set({ lastError: safeError, updatedAt: failedAt }).where(and78(
|
|
67282
|
+
eq89(trafficSources.id, sourceRow2.id),
|
|
67283
|
+
eq89(trafficSources.status, TrafficSourceStatuses.connected),
|
|
67284
|
+
eq89(trafficSources.syncLeaseOwner, leaseOwner)
|
|
67099
67285
|
)).run();
|
|
67100
67286
|
}
|
|
67101
67287
|
});
|
|
@@ -67114,7 +67300,7 @@ async function trafficRoutes(app, opts) {
|
|
|
67114
67300
|
`Sync for source type "${sourceRow2.sourceType}" is not implemented yet \u2014 only cloud-run, wordpress, and vercel are supported in v1.`
|
|
67115
67301
|
);
|
|
67116
67302
|
}
|
|
67117
|
-
const hasConnectedSibling = sourceRow2.status === TrafficSourceStatuses.error && app.db.select().from(trafficSources).where(
|
|
67303
|
+
const hasConnectedSibling = sourceRow2.status === TrafficSourceStatuses.error && app.db.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all().some((row) => row.id !== sourceRow2.id && row.status === TrafficSourceStatuses.connected);
|
|
67118
67304
|
if (sourceRow2.status === TrafficSourceStatuses.paused || sourceRow2.status === TrafficSourceStatuses.archived || hasConnectedSibling) {
|
|
67119
67305
|
throw validationError("Traffic source must be active before it can sync");
|
|
67120
67306
|
}
|
|
@@ -67151,10 +67337,10 @@ async function trafficRoutes(app, opts) {
|
|
|
67151
67337
|
const markFailed = (msg, errorCode2) => {
|
|
67152
67338
|
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
67153
67339
|
app.db.transaction((tx) => {
|
|
67154
|
-
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(
|
|
67155
|
-
const latestSource = tx.select().from(trafficSources).where(
|
|
67340
|
+
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(eq89(runs.id, runId)).run();
|
|
67341
|
+
const latestSource = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
67156
67342
|
if (latestSource?.status === TrafficSourceStatuses.connected && isSameTrafficSourceGeneration(latestSource, sourceRow2) && latestSource.syncLeaseOwner === syncLeaseOwner) {
|
|
67157
|
-
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(
|
|
67343
|
+
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(eq89(trafficSources.id, sourceRow2.id)).run();
|
|
67158
67344
|
}
|
|
67159
67345
|
});
|
|
67160
67346
|
try {
|
|
@@ -67248,7 +67434,7 @@ async function trafficRoutes(app, opts) {
|
|
|
67248
67434
|
}
|
|
67249
67435
|
const credential = credentialStore.getConnection(project.name);
|
|
67250
67436
|
if (!credential) {
|
|
67251
|
-
app.db.delete(runs).where(
|
|
67437
|
+
app.db.delete(runs).where(eq89(runs.id, runId)).run();
|
|
67252
67438
|
throw validationError(
|
|
67253
67439
|
`No WordPress credential found for project "${project.name}". Run "canonry traffic connect wordpress" first.`
|
|
67254
67440
|
);
|
|
@@ -67282,14 +67468,14 @@ async function trafficRoutes(app, opts) {
|
|
|
67282
67468
|
const windowEndIso2 = windowEnd.toISOString();
|
|
67283
67469
|
const reservedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
67284
67470
|
const reservedSource = app.db.transaction((tx) => {
|
|
67285
|
-
const latestSource = tx.select().from(trafficSources).where(
|
|
67471
|
+
const latestSource = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
67286
67472
|
if (!latestSource || !isAuthoritativeTrafficSource(tx, latestSource) || !isSameTrafficSourceGeneration(latestSource, sourceRow2) || latestSource.syncLeaseOwner !== syncLeaseOwner || latestSource.lastCursor || latestSource.wordpressPendingUntil) return void 0;
|
|
67287
67473
|
tx.update(trafficSources).set({
|
|
67288
67474
|
lastSyncedAt: windowStartIso2,
|
|
67289
67475
|
wordpressPendingUntil: windowEndIso2,
|
|
67290
67476
|
updatedAt: reservedAt
|
|
67291
|
-
}).where(
|
|
67292
|
-
return tx.select().from(trafficSources).where(
|
|
67477
|
+
}).where(eq89(trafficSources.id, sourceRow2.id)).run();
|
|
67478
|
+
return tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
67293
67479
|
}, { behavior: "immediate" });
|
|
67294
67480
|
if (!reservedSource) {
|
|
67295
67481
|
const message = "Traffic source changed while reserving the WordPress sync window; retry the sync.";
|
|
@@ -67349,12 +67535,12 @@ async function trafficRoutes(app, opts) {
|
|
|
67349
67535
|
auditAction = "traffic.vercel.synced";
|
|
67350
67536
|
const credentialStore = opts.vercelTrafficCredentialStore;
|
|
67351
67537
|
if (!credentialStore) {
|
|
67352
|
-
app.db.delete(runs).where(
|
|
67538
|
+
app.db.delete(runs).where(eq89(runs.id, runId)).run();
|
|
67353
67539
|
throw validationError("Vercel traffic credential storage is not configured for this deployment");
|
|
67354
67540
|
}
|
|
67355
67541
|
const credential = credentialStore.getConnection(project.name);
|
|
67356
67542
|
if (!credential) {
|
|
67357
|
-
app.db.delete(runs).where(
|
|
67543
|
+
app.db.delete(runs).where(eq89(runs.id, runId)).run();
|
|
67358
67544
|
throw validationError(
|
|
67359
67545
|
`No Vercel credential found for project "${project.name}". Run "canonry traffic connect vercel" first.`
|
|
67360
67546
|
);
|
|
@@ -67372,7 +67558,7 @@ async function trafficRoutes(app, opts) {
|
|
|
67372
67558
|
if (cappedStartMs > clampedStartMs) {
|
|
67373
67559
|
const previousSkip = sourceRow2.skippedThroughAt ? Date.parse(sourceRow2.skippedThroughAt) : Number.NaN;
|
|
67374
67560
|
const skippedThrough = Number.isFinite(previousSkip) ? new Date(Math.max(previousSkip, cappedStartMs)) : new Date(cappedStartMs);
|
|
67375
|
-
app.db.update(trafficSources).set({ skippedThroughAt: skippedThrough.toISOString() }).where(
|
|
67561
|
+
app.db.update(trafficSources).set({ skippedThroughAt: skippedThrough.toISOString() }).where(eq89(trafficSources.id, sourceRow2.id)).run();
|
|
67376
67562
|
request.log.warn(
|
|
67377
67563
|
{
|
|
67378
67564
|
sourceId: sourceRow2.id,
|
|
@@ -67466,15 +67652,15 @@ async function trafficRoutes(app, opts) {
|
|
|
67466
67652
|
throw providerError(`WordPress sync failed: ${msg}`);
|
|
67467
67653
|
}
|
|
67468
67654
|
const commitOutcome = app.db.transaction((tx) => {
|
|
67469
|
-
const latestRow = tx.select().from(trafficSources).where(
|
|
67470
|
-
const latestHasConnectedSibling = tx.select().from(trafficSources).where(
|
|
67655
|
+
const latestRow = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
67656
|
+
const latestHasConnectedSibling = tx.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all().some((row) => row.id !== sourceRow2.id && row.status === TrafficSourceStatuses.connected);
|
|
67471
67657
|
if (!latestRow || latestRow.status !== TrafficSourceStatuses.connected && latestRow.status !== TrafficSourceStatuses.error || latestHasConnectedSibling || latestRow.syncLeaseOwner !== syncLeaseOwner || !isSameTrafficSourceGeneration(latestRow, sourceRow2)) {
|
|
67472
67658
|
const abortedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
67473
67659
|
tx.update(runs).set({
|
|
67474
67660
|
status: RunStatuses.failed,
|
|
67475
67661
|
error: "Traffic source was deactivated or reconfigured during sync",
|
|
67476
67662
|
finishedAt: abortedAt
|
|
67477
|
-
}).where(
|
|
67663
|
+
}).where(eq89(runs.id, runId)).run();
|
|
67478
67664
|
return "source-inactive";
|
|
67479
67665
|
}
|
|
67480
67666
|
const previousIds = latestRow.lastEventIds ?? [];
|
|
@@ -67538,7 +67724,7 @@ async function trafficRoutes(app, opts) {
|
|
|
67538
67724
|
crawlerEventsHourly.status
|
|
67539
67725
|
],
|
|
67540
67726
|
set: {
|
|
67541
|
-
hits:
|
|
67727
|
+
hits: sql38`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
67542
67728
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
67543
67729
|
updatedAt: finishedAt
|
|
67544
67730
|
}
|
|
@@ -67569,7 +67755,7 @@ async function trafficRoutes(app, opts) {
|
|
|
67569
67755
|
crawlerVerificationManifestsHourly.manifestId
|
|
67570
67756
|
],
|
|
67571
67757
|
set: {
|
|
67572
|
-
hits:
|
|
67758
|
+
hits: sql38`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
67573
67759
|
manifestJson: bucket.verificationManifest,
|
|
67574
67760
|
updatedAt: finishedAt
|
|
67575
67761
|
}
|
|
@@ -67603,7 +67789,7 @@ async function trafficRoutes(app, opts) {
|
|
|
67603
67789
|
aiUserFetchEventsHourly.status
|
|
67604
67790
|
],
|
|
67605
67791
|
set: {
|
|
67606
|
-
hits:
|
|
67792
|
+
hits: sql38`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
67607
67793
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
67608
67794
|
updatedAt: finishedAt
|
|
67609
67795
|
}
|
|
@@ -67634,7 +67820,7 @@ async function trafficRoutes(app, opts) {
|
|
|
67634
67820
|
aiUserFetchVerificationManifestsHourly.manifestId
|
|
67635
67821
|
],
|
|
67636
67822
|
set: {
|
|
67637
|
-
hits:
|
|
67823
|
+
hits: sql38`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
67638
67824
|
manifestJson: bucket.verificationManifest,
|
|
67639
67825
|
updatedAt: finishedAt
|
|
67640
67826
|
}
|
|
@@ -67676,9 +67862,9 @@ async function trafficRoutes(app, opts) {
|
|
|
67676
67862
|
// colliding bucket adds its paid and organic arrivals to the ones
|
|
67677
67863
|
// already there rather than overwriting a single label.
|
|
67678
67864
|
set: {
|
|
67679
|
-
sessionsOrHits:
|
|
67680
|
-
paidSessionsOrHits:
|
|
67681
|
-
organicSessionsOrHits:
|
|
67865
|
+
sessionsOrHits: sql38`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
67866
|
+
paidSessionsOrHits: sql38`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
|
|
67867
|
+
organicSessionsOrHits: sql38`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
|
|
67682
67868
|
updatedAt: finishedAt
|
|
67683
67869
|
}
|
|
67684
67870
|
}).run();
|
|
@@ -67760,8 +67946,8 @@ async function trafficRoutes(app, opts) {
|
|
|
67760
67946
|
sourceUpdate.lastCursor = nextCursor ?? null;
|
|
67761
67947
|
sourceUpdate.wordpressPendingUntil = nextCursor ? windowEnd.toISOString() : null;
|
|
67762
67948
|
}
|
|
67763
|
-
tx.update(trafficSources).set(sourceUpdate).where(
|
|
67764
|
-
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
67949
|
+
tx.update(trafficSources).set(sourceUpdate).where(eq89(trafficSources.id, sourceRow2.id)).run();
|
|
67950
|
+
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq89(runs.id, runId)).run();
|
|
67765
67951
|
return "committed";
|
|
67766
67952
|
}, { behavior: "immediate" });
|
|
67767
67953
|
if (commitOutcome === "source-inactive") {
|
|
@@ -67830,7 +68016,7 @@ async function trafficRoutes(app, opts) {
|
|
|
67830
68016
|
});
|
|
67831
68017
|
app.post("/projects/:name/traffic/sources/:id/backfill", async (request) => withFeatureOutcome(app, { feature: "server_traffic", operation: "backfill" }, async (recordOutcome) => {
|
|
67832
68018
|
const project = resolveProject(app.db, request.params.name);
|
|
67833
|
-
const sourceRow2 = app.db.select().from(trafficSources).where(
|
|
68019
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(eq89(trafficSources.id, request.params.id)).get();
|
|
67834
68020
|
if (!sourceRow2 || sourceRow2.projectId !== project.id) {
|
|
67835
68021
|
throw notFound("Traffic source", request.params.id);
|
|
67836
68022
|
}
|
|
@@ -67955,7 +68141,7 @@ async function trafficRoutes(app, opts) {
|
|
|
67955
68141
|
}
|
|
67956
68142
|
try {
|
|
67957
68143
|
app.db.transaction((tx) => {
|
|
67958
|
-
const latestSource = tx.select().from(trafficSources).where(
|
|
68144
|
+
const latestSource = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
67959
68145
|
if (!latestSource || !isAuthoritativeTrafficSource(tx, latestSource) || !isSameTrafficSourceGeneration(latestSource, sourceRow2)) {
|
|
67960
68146
|
throw validationError("Traffic source must remain active and unchanged before it can backfill");
|
|
67961
68147
|
}
|
|
@@ -68003,10 +68189,10 @@ async function trafficRoutes(app, opts) {
|
|
|
68003
68189
|
function buildSourceDetail(projectId, row, since) {
|
|
68004
68190
|
const crawlerPathRows = app.db.select({
|
|
68005
68191
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
68006
|
-
hits:
|
|
68192
|
+
hits: sql38`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
68007
68193
|
}).from(crawlerEventsHourly).where(
|
|
68008
|
-
|
|
68009
|
-
|
|
68194
|
+
and78(
|
|
68195
|
+
eq89(crawlerEventsHourly.sourceId, row.id),
|
|
68010
68196
|
gte24(crawlerEventsHourly.tsHour, since)
|
|
68011
68197
|
)
|
|
68012
68198
|
).groupBy(crawlerEventsHourly.pathNormalized).all();
|
|
@@ -68014,41 +68200,41 @@ async function trafficRoutes(app, opts) {
|
|
|
68014
68200
|
crawlerPathRows.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
|
|
68015
68201
|
);
|
|
68016
68202
|
const crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
68017
|
-
const aiUserFetchTotals = app.db.select({ total:
|
|
68018
|
-
|
|
68019
|
-
|
|
68203
|
+
const aiUserFetchTotals = app.db.select({ total: sql38`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
68204
|
+
and78(
|
|
68205
|
+
eq89(aiUserFetchEventsHourly.sourceId, row.id),
|
|
68020
68206
|
gte24(aiUserFetchEventsHourly.tsHour, since)
|
|
68021
68207
|
)
|
|
68022
68208
|
).get();
|
|
68023
|
-
const likelyFakeTotals = app.db.select({ total:
|
|
68024
|
-
|
|
68025
|
-
|
|
68209
|
+
const likelyFakeTotals = app.db.select({ total: sql38`COALESCE(SUM(${aiUserFetchLikelyFakeHourly.hits}), 0)` }).from(aiUserFetchLikelyFakeHourly).where(
|
|
68210
|
+
and78(
|
|
68211
|
+
eq89(aiUserFetchLikelyFakeHourly.sourceId, row.id),
|
|
68026
68212
|
gte24(aiUserFetchLikelyFakeHourly.tsHour, since)
|
|
68027
68213
|
)
|
|
68028
68214
|
).get();
|
|
68029
68215
|
const aiTotals = app.db.select({
|
|
68030
|
-
total:
|
|
68031
|
-
nonhop:
|
|
68032
|
-
landed:
|
|
68216
|
+
total: sql38`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
68217
|
+
nonhop: sql38`COALESCE(SUM(CASE WHEN ${referralLandedCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`,
|
|
68218
|
+
landed: sql38`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`
|
|
68033
68219
|
}).from(aiReferralEventsHourly).where(
|
|
68034
|
-
|
|
68035
|
-
|
|
68220
|
+
and78(
|
|
68221
|
+
eq89(aiReferralEventsHourly.sourceId, row.id),
|
|
68036
68222
|
gte24(aiReferralEventsHourly.tsHour, since)
|
|
68037
68223
|
)
|
|
68038
68224
|
).get();
|
|
68039
|
-
const sampleTotals = app.db.select({ total:
|
|
68040
|
-
|
|
68041
|
-
|
|
68225
|
+
const sampleTotals = app.db.select({ total: sql38`COUNT(*)` }).from(rawEventSamples).where(
|
|
68226
|
+
and78(
|
|
68227
|
+
eq89(rawEventSamples.sourceId, row.id),
|
|
68042
68228
|
gte24(rawEventSamples.ts, since)
|
|
68043
68229
|
)
|
|
68044
68230
|
).get();
|
|
68045
68231
|
const latestRun = app.db.select().from(runs).where(
|
|
68046
|
-
|
|
68047
|
-
|
|
68048
|
-
|
|
68049
|
-
|
|
68232
|
+
and78(
|
|
68233
|
+
eq89(runs.projectId, projectId),
|
|
68234
|
+
eq89(runs.kind, RunKinds["traffic-sync"]),
|
|
68235
|
+
eq89(runs.sourceId, row.id)
|
|
68050
68236
|
)
|
|
68051
|
-
).orderBy(
|
|
68237
|
+
).orderBy(desc41(runs.startedAt)).limit(1).get();
|
|
68052
68238
|
return {
|
|
68053
68239
|
...rowToDto(row),
|
|
68054
68240
|
totals24h: {
|
|
@@ -68084,7 +68270,7 @@ async function trafficRoutes(app, opts) {
|
|
|
68084
68270
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
68085
68271
|
let updatedRow;
|
|
68086
68272
|
app.db.transaction((tx) => {
|
|
68087
|
-
const sourceRow2 = tx.select().from(trafficSources).where(
|
|
68273
|
+
const sourceRow2 = tx.select().from(trafficSources).where(and78(eq89(trafficSources.projectId, project.id), eq89(trafficSources.id, request.params.id))).get();
|
|
68088
68274
|
if (!sourceRow2) {
|
|
68089
68275
|
throw notFound("traffic source", request.params.id);
|
|
68090
68276
|
}
|
|
@@ -68098,7 +68284,7 @@ async function trafficRoutes(app, opts) {
|
|
|
68098
68284
|
`Traffic source "${sourceRow2.id}" is staged. Activate it explicitly instead of resetting it.`
|
|
68099
68285
|
);
|
|
68100
68286
|
}
|
|
68101
|
-
const hasConnectedSibling = tx.select().from(trafficSources).where(
|
|
68287
|
+
const hasConnectedSibling = tx.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all().some((row) => row.id !== sourceRow2.id && row.status === TrafficSourceStatuses.connected);
|
|
68102
68288
|
if (hasConnectedSibling) {
|
|
68103
68289
|
throw validationError(
|
|
68104
68290
|
`Traffic source "${sourceRow2.id}" is not authoritative. Activate it explicitly instead of resetting it.`
|
|
@@ -68115,7 +68301,7 @@ async function trafficRoutes(app, opts) {
|
|
|
68115
68301
|
status: TrafficSourceStatuses.connected,
|
|
68116
68302
|
lastError: null,
|
|
68117
68303
|
updatedAt: now
|
|
68118
|
-
}).where(
|
|
68304
|
+
}).where(eq89(trafficSources.id, sourceRow2.id)).run();
|
|
68119
68305
|
writeAuditLog(tx, auditFromRequest(request, {
|
|
68120
68306
|
projectId: project.id,
|
|
68121
68307
|
actor: "api",
|
|
@@ -68123,7 +68309,7 @@ async function trafficRoutes(app, opts) {
|
|
|
68123
68309
|
entityType: "traffic_source",
|
|
68124
68310
|
entityId: sourceRow2.id
|
|
68125
68311
|
}));
|
|
68126
|
-
updatedRow = tx.select().from(trafficSources).where(
|
|
68312
|
+
updatedRow = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
68127
68313
|
});
|
|
68128
68314
|
return buildSourceDetail(project.id, updatedRow, new Date(Date.now() - 24 * 60 * 6e4).toISOString());
|
|
68129
68315
|
}));
|
|
@@ -68135,7 +68321,7 @@ async function trafficRoutes(app, opts) {
|
|
|
68135
68321
|
}
|
|
68136
68322
|
let updatedRow;
|
|
68137
68323
|
app.db.transaction((tx) => {
|
|
68138
|
-
const sourceRow2 = tx.select().from(trafficSources).where(
|
|
68324
|
+
const sourceRow2 = tx.select().from(trafficSources).where(and78(eq89(trafficSources.projectId, project.id), eq89(trafficSources.id, request.params.id))).get();
|
|
68139
68325
|
if (!sourceRow2) {
|
|
68140
68326
|
throw notFound("traffic source", request.params.id);
|
|
68141
68327
|
}
|
|
@@ -68148,7 +68334,7 @@ async function trafficRoutes(app, opts) {
|
|
|
68148
68334
|
);
|
|
68149
68335
|
}
|
|
68150
68336
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
68151
|
-
tx.update(trafficSources).set({ skippedThroughAt: null, updatedAt: now }).where(
|
|
68337
|
+
tx.update(trafficSources).set({ skippedThroughAt: null, updatedAt: now }).where(eq89(trafficSources.id, sourceRow2.id)).run();
|
|
68152
68338
|
writeAuditLog(tx, auditFromRequest(request, {
|
|
68153
68339
|
projectId: project.id,
|
|
68154
68340
|
actor: "api",
|
|
@@ -68157,20 +68343,20 @@ async function trafficRoutes(app, opts) {
|
|
|
68157
68343
|
entityId: sourceRow2.id,
|
|
68158
68344
|
diff: { skippedThroughAt: sourceRow2.skippedThroughAt, sourceType: sourceRow2.sourceType }
|
|
68159
68345
|
}));
|
|
68160
|
-
updatedRow = tx.select().from(trafficSources).where(
|
|
68346
|
+
updatedRow = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
68161
68347
|
});
|
|
68162
68348
|
return buildSourceDetail(project.id, updatedRow, new Date(Date.now() - 24 * 60 * 6e4).toISOString());
|
|
68163
68349
|
});
|
|
68164
68350
|
app.get("/projects/:name/traffic/sources", async (request) => {
|
|
68165
68351
|
const project = resolveProject(app.db, request.params.name);
|
|
68166
|
-
const rows = app.db.select().from(trafficSources).where(
|
|
68352
|
+
const rows = app.db.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).orderBy(desc41(trafficSources.createdAt)).all();
|
|
68167
68353
|
const sources = rows.filter((row) => row.status !== TrafficSourceStatuses.archived).map(rowToDto);
|
|
68168
68354
|
const response = { sources };
|
|
68169
68355
|
return response;
|
|
68170
68356
|
});
|
|
68171
68357
|
app.get("/projects/:name/traffic/status", async (request) => {
|
|
68172
68358
|
const project = resolveProject(app.db, request.params.name);
|
|
68173
|
-
const rows = app.db.select().from(trafficSources).where(
|
|
68359
|
+
const rows = app.db.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).orderBy(desc41(trafficSources.createdAt)).all();
|
|
68174
68360
|
const since = new Date(Date.now() - 24 * 60 * 6e4).toISOString();
|
|
68175
68361
|
const sources = rows.filter((row) => row.status !== TrafficSourceStatuses.archived).map((row) => buildSourceDetail(project.id, row, since));
|
|
68176
68362
|
const response = { sources };
|
|
@@ -68180,7 +68366,7 @@ async function trafficRoutes(app, opts) {
|
|
|
68180
68366
|
"/projects/:name/traffic/sources/:id",
|
|
68181
68367
|
async (request) => {
|
|
68182
68368
|
const project = resolveProject(app.db, request.params.name);
|
|
68183
|
-
const row = app.db.select().from(trafficSources).where(
|
|
68369
|
+
const row = app.db.select().from(trafficSources).where(eq89(trafficSources.id, request.params.id)).get();
|
|
68184
68370
|
if (!row || row.projectId !== project.id) {
|
|
68185
68371
|
throw notFound("Traffic source", request.params.id);
|
|
68186
68372
|
}
|
|
@@ -68253,23 +68439,23 @@ async function trafficRoutes(app, opts) {
|
|
|
68253
68439
|
const seriesByBucket = /* @__PURE__ */ new Map();
|
|
68254
68440
|
if (kind === "all" || kind === TrafficEventKinds.crawler) {
|
|
68255
68441
|
const crawlerFilters = [
|
|
68256
|
-
|
|
68442
|
+
eq89(crawlerEventsHourly.projectId, project.id),
|
|
68257
68443
|
gte24(crawlerEventsHourly.tsHour, sinceIso),
|
|
68258
68444
|
lte25(crawlerEventsHourly.tsHour, untilIso)
|
|
68259
68445
|
];
|
|
68260
|
-
if (sourceIdParam) crawlerFilters.push(
|
|
68261
|
-
const crawlerWhere =
|
|
68446
|
+
if (sourceIdParam) crawlerFilters.push(eq89(crawlerEventsHourly.sourceId, sourceIdParam));
|
|
68447
|
+
const crawlerWhere = and78(...crawlerFilters);
|
|
68262
68448
|
const pathTotals = app.db.select({
|
|
68263
68449
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
68264
|
-
hits:
|
|
68265
|
-
rows:
|
|
68450
|
+
hits: sql38`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
68451
|
+
rows: sql38`COUNT(*)`
|
|
68266
68452
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerEventsHourly.pathNormalized).all();
|
|
68267
68453
|
crawlerSegments = segmentCrawlerHits(
|
|
68268
68454
|
pathTotals.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
|
|
68269
68455
|
);
|
|
68270
68456
|
crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
68271
68457
|
totalEventRows += pathTotals.reduce((sum, row) => sum + Number(row.rows), 0);
|
|
68272
|
-
const crawlerSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
68458
|
+
const crawlerSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql38`substr(${crawlerEventsHourly.tsHour}, 1, 10)` : crawlerEventsHourly.tsHour;
|
|
68273
68459
|
const distinctPaths = pathTotals.map((r) => ({ pathNormalized: r.pathNormalized }));
|
|
68274
68460
|
const contentPaths = [];
|
|
68275
68461
|
const infraPaths = [];
|
|
@@ -68281,12 +68467,12 @@ async function trafficRoutes(app, opts) {
|
|
|
68281
68467
|
const listed = useContentList ? contentPaths : infraPaths;
|
|
68282
68468
|
const IN_LIST_CAP = 2e4;
|
|
68283
68469
|
const listFitsInSql = listed.length <= IN_LIST_CAP;
|
|
68284
|
-
const contentExpr = listed.length === 0 ? useContentList ?
|
|
68470
|
+
const contentExpr = listed.length === 0 ? useContentList ? sql38`0` : sql38`${crawlerEventsHourly.hits}` : useContentList ? sql38`CASE WHEN ${inArray40(crawlerEventsHourly.pathNormalized, listed)} THEN ${crawlerEventsHourly.hits} ELSE 0 END` : sql38`CASE WHEN ${inArray40(crawlerEventsHourly.pathNormalized, listed)} THEN 0 ELSE ${crawlerEventsHourly.hits} END`;
|
|
68285
68471
|
if (listFitsInSql) {
|
|
68286
68472
|
const crawlerSeries = app.db.select({
|
|
68287
68473
|
bucket: crawlerSeriesBucket,
|
|
68288
|
-
hits:
|
|
68289
|
-
content:
|
|
68474
|
+
hits: sql38`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
68475
|
+
content: sql38`COALESCE(SUM(${contentExpr}), 0)`
|
|
68290
68476
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerSeriesBucket).all();
|
|
68291
68477
|
for (const row of crawlerSeries) {
|
|
68292
68478
|
const point = trafficSeriesPoint(seriesByBucket, String(row.bucket));
|
|
@@ -68298,7 +68484,7 @@ async function trafficRoutes(app, opts) {
|
|
|
68298
68484
|
const grouped = app.db.select({
|
|
68299
68485
|
bucket: crawlerSeriesBucket,
|
|
68300
68486
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
68301
|
-
hits:
|
|
68487
|
+
hits: sql38`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
68302
68488
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerSeriesBucket, crawlerEventsHourly.pathNormalized).all();
|
|
68303
68489
|
for (const row of grouped) {
|
|
68304
68490
|
const point = trafficSeriesPoint(seriesByBucket, String(row.bucket));
|
|
@@ -68308,7 +68494,7 @@ async function trafficRoutes(app, opts) {
|
|
|
68308
68494
|
if (useContentList ? inList : !inList) point.crawlerContentHits += hits;
|
|
68309
68495
|
}
|
|
68310
68496
|
}
|
|
68311
|
-
const rows = app.db.select().from(crawlerEventsHourly).where(crawlerWhere).orderBy(
|
|
68497
|
+
const rows = app.db.select().from(crawlerEventsHourly).where(crawlerWhere).orderBy(desc41(crawlerEventsHourly.tsHour)).limit(limit).all();
|
|
68312
68498
|
for (const r of rows) {
|
|
68313
68499
|
events.push({
|
|
68314
68500
|
kind: TrafficEventKinds.crawler,
|
|
@@ -68328,27 +68514,27 @@ async function trafficRoutes(app, opts) {
|
|
|
68328
68514
|
}
|
|
68329
68515
|
if (kind === "all" || kind === TrafficEventKinds["ai-user-fetch"]) {
|
|
68330
68516
|
const userFetchFilters = [
|
|
68331
|
-
|
|
68517
|
+
eq89(aiUserFetchEventsHourly.projectId, project.id),
|
|
68332
68518
|
gte24(aiUserFetchEventsHourly.tsHour, sinceIso),
|
|
68333
68519
|
lte25(aiUserFetchEventsHourly.tsHour, untilIso)
|
|
68334
68520
|
];
|
|
68335
|
-
if (sourceIdParam) userFetchFilters.push(
|
|
68336
|
-
const userFetchWhere =
|
|
68521
|
+
if (sourceIdParam) userFetchFilters.push(eq89(aiUserFetchEventsHourly.sourceId, sourceIdParam));
|
|
68522
|
+
const userFetchWhere = and78(...userFetchFilters);
|
|
68337
68523
|
const total = app.db.select({
|
|
68338
|
-
total:
|
|
68339
|
-
rows:
|
|
68524
|
+
total: sql38`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`,
|
|
68525
|
+
rows: sql38`COUNT(*)`
|
|
68340
68526
|
}).from(aiUserFetchEventsHourly).where(userFetchWhere).get();
|
|
68341
68527
|
aiUserFetchTotal = Number(total?.total ?? 0);
|
|
68342
68528
|
totalEventRows += Number(total?.rows ?? 0);
|
|
68343
|
-
const userFetchSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
68529
|
+
const userFetchSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql38`substr(${aiUserFetchEventsHourly.tsHour}, 1, 10)` : aiUserFetchEventsHourly.tsHour;
|
|
68344
68530
|
const userFetchSeries = app.db.select({
|
|
68345
68531
|
bucket: userFetchSeriesBucket,
|
|
68346
|
-
hits:
|
|
68532
|
+
hits: sql38`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`
|
|
68347
68533
|
}).from(aiUserFetchEventsHourly).where(userFetchWhere).groupBy(userFetchSeriesBucket).all();
|
|
68348
68534
|
for (const row of userFetchSeries) {
|
|
68349
68535
|
trafficSeriesPoint(seriesByBucket, row.bucket).aiUserFetchHits = Number(row.hits);
|
|
68350
68536
|
}
|
|
68351
|
-
const rows = app.db.select().from(aiUserFetchEventsHourly).where(userFetchWhere).orderBy(
|
|
68537
|
+
const rows = app.db.select().from(aiUserFetchEventsHourly).where(userFetchWhere).orderBy(desc41(aiUserFetchEventsHourly.tsHour)).limit(limit).all();
|
|
68352
68538
|
for (const r of rows) {
|
|
68353
68539
|
events.push({
|
|
68354
68540
|
kind: TrafficEventKinds["ai-user-fetch"],
|
|
@@ -68367,19 +68553,19 @@ async function trafficRoutes(app, opts) {
|
|
|
68367
68553
|
}
|
|
68368
68554
|
if (kind === "all" || kind === TrafficEventKinds["ai-referral"]) {
|
|
68369
68555
|
const aiFilters = [
|
|
68370
|
-
|
|
68556
|
+
eq89(aiReferralEventsHourly.projectId, project.id),
|
|
68371
68557
|
gte24(aiReferralEventsHourly.tsHour, sinceIso),
|
|
68372
68558
|
lte25(aiReferralEventsHourly.tsHour, untilIso)
|
|
68373
68559
|
];
|
|
68374
|
-
if (sourceIdParam) aiFilters.push(
|
|
68375
|
-
const aiWhere =
|
|
68560
|
+
if (sourceIdParam) aiFilters.push(eq89(aiReferralEventsHourly.sourceId, sourceIdParam));
|
|
68561
|
+
const aiWhere = and78(...aiFilters);
|
|
68376
68562
|
const total = app.db.select({
|
|
68377
|
-
total:
|
|
68378
|
-
nonhop:
|
|
68379
|
-
landed:
|
|
68380
|
-
paid:
|
|
68381
|
-
organic:
|
|
68382
|
-
rows:
|
|
68563
|
+
total: sql38`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
68564
|
+
nonhop: sql38`COALESCE(SUM(CASE WHEN ${referralLandedCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`,
|
|
68565
|
+
landed: sql38`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`,
|
|
68566
|
+
paid: sql38`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.paidSessionsOrHits} ELSE 0 END), 0)`,
|
|
68567
|
+
organic: sql38`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.organicSessionsOrHits} ELSE 0 END), 0)`,
|
|
68568
|
+
rows: sql38`COUNT(*)`
|
|
68383
68569
|
}).from(aiReferralEventsHourly).where(aiWhere).get();
|
|
68384
68570
|
aiReferralTotalHits = Number(total?.total ?? 0);
|
|
68385
68571
|
aiReferralNonHopHits = Number(total?.nonhop ?? 0);
|
|
@@ -68390,18 +68576,18 @@ async function trafficRoutes(app, opts) {
|
|
|
68390
68576
|
Number(total?.organic ?? 0)
|
|
68391
68577
|
);
|
|
68392
68578
|
totalEventRows += Number(total?.rows ?? 0);
|
|
68393
|
-
const referralSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
68579
|
+
const referralSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql38`substr(${aiReferralEventsHourly.tsHour}, 1, 10)` : aiReferralEventsHourly.tsHour;
|
|
68394
68580
|
const referralSeries = app.db.select({
|
|
68395
68581
|
bucket: referralSeriesBucket,
|
|
68396
|
-
hits:
|
|
68397
|
-
landed:
|
|
68582
|
+
hits: sql38`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
68583
|
+
landed: sql38`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`
|
|
68398
68584
|
}).from(aiReferralEventsHourly).where(aiWhere).groupBy(referralSeriesBucket).all();
|
|
68399
68585
|
for (const row of referralSeries) {
|
|
68400
68586
|
const point = trafficSeriesPoint(seriesByBucket, row.bucket);
|
|
68401
68587
|
point.aiReferralHits = Number(row.hits);
|
|
68402
68588
|
point.aiReferralLandedHits = Number(row.landed);
|
|
68403
68589
|
}
|
|
68404
|
-
const rows = app.db.select().from(aiReferralEventsHourly).where(aiWhere).orderBy(
|
|
68590
|
+
const rows = app.db.select().from(aiReferralEventsHourly).where(aiWhere).orderBy(desc41(aiReferralEventsHourly.tsHour)).limit(limit).all();
|
|
68405
68591
|
for (const r of rows) {
|
|
68406
68592
|
const counts2 = aiReferralClassCounts(r.sessionsOrHits, r.paidSessionsOrHits, r.organicSessionsOrHits);
|
|
68407
68593
|
events.push({
|
|
@@ -68447,11 +68633,11 @@ async function trafficRoutes(app, opts) {
|
|
|
68447
68633
|
windowEnd: untilIso,
|
|
68448
68634
|
series: (() => {
|
|
68449
68635
|
const points = completeTrafficSeries(since, until, granularity, seriesByBucket);
|
|
68450
|
-
const coverageScope = (col, srcCol) => sourceIdParam ?
|
|
68636
|
+
const coverageScope = (col, srcCol) => sourceIdParam ? and78(eq89(col, project.id), eq89(srcCol, sourceIdParam)) : eq89(col, project.id);
|
|
68451
68637
|
const firsts = [
|
|
68452
|
-
app.db.select({ v:
|
|
68453
|
-
app.db.select({ v:
|
|
68454
|
-
app.db.select({ v:
|
|
68638
|
+
app.db.select({ v: sql38`MIN(${crawlerEventsHourly.tsHour})` }).from(crawlerEventsHourly).where(coverageScope(crawlerEventsHourly.projectId, crawlerEventsHourly.sourceId)).get()?.v,
|
|
68639
|
+
app.db.select({ v: sql38`MIN(${aiUserFetchEventsHourly.tsHour})` }).from(aiUserFetchEventsHourly).where(coverageScope(aiUserFetchEventsHourly.projectId, aiUserFetchEventsHourly.sourceId)).get()?.v,
|
|
68640
|
+
app.db.select({ v: sql38`MIN(${aiReferralEventsHourly.tsHour})` }).from(aiReferralEventsHourly).where(coverageScope(aiReferralEventsHourly.projectId, aiReferralEventsHourly.sourceId)).get()?.v
|
|
68455
68641
|
].filter((v) => typeof v === "string" && v.length > 0);
|
|
68456
68642
|
const coverageStart = firsts.length ? firsts.slice().sort()[0] : null;
|
|
68457
68643
|
const bucketStartMs = (bucket) => Date.parse(
|
|
@@ -68525,7 +68711,7 @@ async function trafficRoutes(app, opts) {
|
|
|
68525
68711
|
}
|
|
68526
68712
|
|
|
68527
68713
|
// ../api-routes/src/doctor/checks/report-sweeps.ts
|
|
68528
|
-
import { and as
|
|
68714
|
+
import { and as and79, asc as asc23, eq as eq90, gte as gte25, lte as lte26 } from "drizzle-orm";
|
|
68529
68715
|
|
|
68530
68716
|
// ../api-routes/src/visibility-attribution.ts
|
|
68531
68717
|
function buildQueryAttribution2(projectQueries) {
|
|
@@ -68559,11 +68745,11 @@ function inspectSweep(ctx, run) {
|
|
|
68559
68745
|
answerMentioned: querySnapshots.answerMentioned,
|
|
68560
68746
|
answerText: querySnapshots.answerText,
|
|
68561
68747
|
measurementExecutionId: querySnapshots.measurementExecutionId
|
|
68562
|
-
}).from(querySnapshots).where(
|
|
68748
|
+
}).from(querySnapshots).where(eq90(querySnapshots.runId, run.id)).all();
|
|
68563
68749
|
if (snapshots.length === 0) return { detail: { ...base, complete: false, reason: "no-answers" } };
|
|
68564
68750
|
try {
|
|
68565
68751
|
if (run.measurementPlanVersionId) {
|
|
68566
|
-
const version = ctx.db.select().from(measurementPlanVersions).where(
|
|
68752
|
+
const version = ctx.db.select().from(measurementPlanVersions).where(and79(eq90(measurementPlanVersions.projectId, ctx.project.id), eq90(measurementPlanVersions.id, run.measurementPlanVersionId))).get();
|
|
68567
68753
|
if (!version) return { detail: { ...base, complete: false, reason: "missing-plan" } };
|
|
68568
68754
|
const plan = parseStoredMeasurementPlanAnyVersion(version.canonicalJson);
|
|
68569
68755
|
const manifest = measurementRunExpectedSlots(run, plan);
|
|
@@ -68595,10 +68781,10 @@ function inspectSweep(ctx, run) {
|
|
|
68595
68781
|
};
|
|
68596
68782
|
return unbound ? { detail } : { detail, coverage: slotCoverage(expected2, answered3) };
|
|
68597
68783
|
}
|
|
68598
|
-
const stored = ctx.db.select().from(simpleMeasurementDefinitions).where(
|
|
68784
|
+
const stored = ctx.db.select().from(simpleMeasurementDefinitions).where(eq90(simpleMeasurementDefinitions.runId, run.id)).get();
|
|
68599
68785
|
const frozen = stored ? simpleMeasurementDefinitionSchema.parse(stored.definition) : null;
|
|
68600
|
-
const project = ctx.db.select().from(projects).where(
|
|
68601
|
-
const basket = frozen ? frozen.queries.map((query2) => ({ id: query2.queryId, query: query2.queryText })) : ctx.db.select({ id: queries.id, query: queries.query }).from(queries).where(
|
|
68786
|
+
const project = ctx.db.select().from(projects).where(eq90(projects.id, ctx.project.id)).get();
|
|
68787
|
+
const basket = frozen ? frozen.queries.map((query2) => ({ id: query2.queryId, query: query2.queryText })) : ctx.db.select({ id: queries.id, query: queries.query }).from(queries).where(eq90(queries.projectId, project.id)).all();
|
|
68602
68788
|
const providers = frozen ? frozen.engines.map((engine) => engine.provider) : project.providers.length > 0 ? project.providers : [...new Set(snapshots.map((snapshot) => snapshot.provider))];
|
|
68603
68789
|
const attribution = buildQueryAttribution2(basket);
|
|
68604
68790
|
const expected = new Set(basket.flatMap((query2) => providers.map((provider) => JSON.stringify([query2.id, provider.trim().toLowerCase()]))));
|
|
@@ -68647,7 +68833,7 @@ var reportSweepsCheck = {
|
|
|
68647
68833
|
const graded = gradedReportMonths(reportMonths, now);
|
|
68648
68834
|
const months = reportMonths.map((month) => {
|
|
68649
68835
|
const bounds = calendarMonthBounds(month);
|
|
68650
|
-
const monthRuns = ctx.db.select().from(runs).where(
|
|
68836
|
+
const monthRuns = ctx.db.select().from(runs).where(and79(eq90(runs.projectId, ctx.project.id), eq90(runs.kind, RunKinds["answer-visibility"]), notProbeRun(), gte25(runs.createdAt, bounds.since), lte26(runs.createdAt, bounds.until))).orderBy(asc23(runs.createdAt), asc23(runs.id)).all();
|
|
68651
68837
|
const evidence = monthRuns.map((run) => inspectSweep(ctx, run));
|
|
68652
68838
|
return {
|
|
68653
68839
|
month,
|
|
@@ -68670,7 +68856,7 @@ var reportSweepsCheck = {
|
|
|
68670
68856
|
};
|
|
68671
68857
|
|
|
68672
68858
|
// ../api-routes/src/doctor/checks/report-models.ts
|
|
68673
|
-
import { and as
|
|
68859
|
+
import { and as and80, eq as eq91, gte as gte26, inArray as inArray41, lte as lte27 } from "drizzle-orm";
|
|
68674
68860
|
var reportModelsCheck = {
|
|
68675
68861
|
id: "report.models",
|
|
68676
68862
|
category: CheckCategories.providers,
|
|
@@ -68685,7 +68871,7 @@ var reportModelsCheck = {
|
|
|
68685
68871
|
const months = reportMonths.flatMap((month) => {
|
|
68686
68872
|
const bounds = calendarMonthBounds(month);
|
|
68687
68873
|
const previous = new Date(Date.parse(bounds.since) - 1).toISOString().slice(0, 7);
|
|
68688
|
-
const observations = ctx.db.select({ provider: querySnapshots.provider, model: querySnapshots.model, at: runs.createdAt }).from(querySnapshots).innerJoin(runs,
|
|
68874
|
+
const observations = ctx.db.select({ provider: querySnapshots.provider, model: querySnapshots.model, at: runs.createdAt }).from(querySnapshots).innerJoin(runs, eq91(runs.id, querySnapshots.runId)).where(and80(eq91(runs.projectId, ctx.project.id), eq91(runs.kind, RunKinds["answer-visibility"]), inArray41(runs.status, [RunStatuses.completed, RunStatuses.partial]), notProbeRun(), gte26(runs.createdAt, bounds.since), lte27(runs.createdAt, bounds.until))).all();
|
|
68689
68875
|
return readVisibilityContinuity(ctx.db, ctx.project.name, { from: previous, to: month }).map(({ frame, continuity: continuity2 }) => ({
|
|
68690
68876
|
month,
|
|
68691
68877
|
previousMonth: previous,
|
|
@@ -68726,7 +68912,7 @@ var reportModelsCheck = {
|
|
|
68726
68912
|
};
|
|
68727
68913
|
|
|
68728
68914
|
// ../api-routes/src/doctor/checks/report-daily-data.ts
|
|
68729
|
-
import { eq as
|
|
68915
|
+
import { eq as eq92 } from "drizzle-orm";
|
|
68730
68916
|
var REPORTING_LAG_DAYS = 3;
|
|
68731
68917
|
function dateSpan(dates) {
|
|
68732
68918
|
const sorted = [...dates].sort();
|
|
@@ -68735,31 +68921,31 @@ function dateSpan(dates) {
|
|
|
68735
68921
|
function latestSyncRange(ctx, source) {
|
|
68736
68922
|
const projectId = ctx.project.id;
|
|
68737
68923
|
if (source === "ga") {
|
|
68738
|
-
const rows2 = ctx.db.select({ date: gaDailyTotals.date, syncedAt: gaDailyTotals.syncedAt }).from(gaDailyTotals).where(
|
|
68924
|
+
const rows2 = ctx.db.select({ date: gaDailyTotals.date, syncedAt: gaDailyTotals.syncedAt }).from(gaDailyTotals).where(eq92(gaDailyTotals.projectId, projectId)).all();
|
|
68739
68925
|
const latest2 = rows2.reduce((max2, row) => max2 === null || row.syncedAt > max2 ? row.syncedAt : max2, null);
|
|
68740
|
-
const summary3 = ctx.db.select({ start: gaTrafficSummaries.periodStart, end: gaTrafficSummaries.periodEnd, syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(
|
|
68926
|
+
const summary3 = ctx.db.select({ start: gaTrafficSummaries.periodStart, end: gaTrafficSummaries.periodEnd, syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(eq92(gaTrafficSummaries.projectId, projectId)).get();
|
|
68741
68927
|
if (summary3 && (latest2 === null || summary3.syncedAt >= latest2)) return { start: summary3.start, end: summary3.end, syncedAt: summary3.syncedAt };
|
|
68742
68928
|
const span2 = dateSpan(rows2.filter((row) => row.syncedAt === latest2).map((row) => row.date));
|
|
68743
68929
|
return span2 && latest2 !== null ? { ...span2, syncedAt: latest2 } : null;
|
|
68744
68930
|
}
|
|
68745
|
-
const rows = ctx.db.select({ date: gscDailyTotals.date, createdAt: gscDailyTotals.createdAt }).from(gscDailyTotals).where(
|
|
68931
|
+
const rows = ctx.db.select({ date: gscDailyTotals.date, createdAt: gscDailyTotals.createdAt }).from(gscDailyTotals).where(eq92(gscDailyTotals.projectId, projectId)).all();
|
|
68746
68932
|
const latest = rows.reduce((max2, row) => max2 === null || row.createdAt > max2 ? row.createdAt : max2, null);
|
|
68747
68933
|
const span = dateSpan(rows.filter((row) => row.createdAt === latest).map((row) => row.date));
|
|
68748
68934
|
if (!span || latest === null) return null;
|
|
68749
|
-
const watermark = ctx.db.select({ through: gscDataWatermarks.syncedThroughDate, updatedAt: gscDataWatermarks.updatedAt }).from(gscDataWatermarks).where(
|
|
68935
|
+
const watermark = ctx.db.select({ through: gscDataWatermarks.syncedThroughDate, updatedAt: gscDataWatermarks.updatedAt }).from(gscDataWatermarks).where(eq92(gscDataWatermarks.projectId, projectId)).get();
|
|
68750
68936
|
return watermark?.through && watermark.updatedAt === latest && watermark.through > span.end ? { start: span.start, end: watermark.through, syncedAt: latest } : { ...span, syncedAt: latest };
|
|
68751
68937
|
}
|
|
68752
68938
|
function dailyCoverage(ctx, month, source) {
|
|
68753
68939
|
const project = ctx.project;
|
|
68754
68940
|
const connection = source === "ga" ? ctx.ga4CredentialStore?.getConnection(project.name) ?? ctx.googleConnectionStore?.getConnection(project.canonicalDomain, "ga4") : ctx.googleConnectionStore?.getConnection(project.canonicalDomain, "gsc");
|
|
68755
|
-
const stored = source === "ga" ? ctx.db.select({ date: gaDailyTotals.date, count: gaDailyTotals.sessions }).from(gaDailyTotals).where(
|
|
68941
|
+
const stored = source === "ga" ? ctx.db.select({ date: gaDailyTotals.date, count: gaDailyTotals.sessions }).from(gaDailyTotals).where(eq92(gaDailyTotals.projectId, project.id)).all() : ctx.db.select({ date: gscDailyTotals.date, count: gscDailyTotals.impressions }).from(gscDailyTotals).where(eq92(gscDailyTotals.projectId, project.id)).all();
|
|
68756
68942
|
if (!connection && stored.length === 0) return null;
|
|
68757
68943
|
const bounds = calendarMonthBounds(month);
|
|
68758
68944
|
const start = bounds.since.slice(0, 10);
|
|
68759
68945
|
const end = bounds.until.slice(0, 10);
|
|
68760
68946
|
const timeZone = source === "gsc" ? "America/Los_Angeles" : "UTC";
|
|
68761
68947
|
const matureThrough = isoDateDaysBeforeInTimeZone((/* @__PURE__ */ new Date()).toISOString(), REPORTING_LAG_DAYS, timeZone);
|
|
68762
|
-
const created = ctx.db.select({ date: projects.createdAt }).from(projects).where(
|
|
68948
|
+
const created = ctx.db.select({ date: projects.createdAt }).from(projects).where(eq92(projects.id, project.id)).get().date.slice(0, 10);
|
|
68763
68949
|
const connectedAt = connection?.createdAt?.slice(0, 10) ?? created;
|
|
68764
68950
|
const onboardingDate = [connectedAt, ...stored.map((row) => row.date)].sort()[0];
|
|
68765
68951
|
const byDate = new Map(stored.map((row) => [row.date, row.count]));
|
|
@@ -68841,7 +69027,7 @@ var reportDailyDataCheck = {
|
|
|
68841
69027
|
};
|
|
68842
69028
|
|
|
68843
69029
|
// ../api-routes/src/doctor/checks/referral-assessment.ts
|
|
68844
|
-
import { and as
|
|
69030
|
+
import { and as and81, eq as eq93, ne as ne16 } from "drizzle-orm";
|
|
68845
69031
|
var REFERRAL_ASSESSMENT_CHECKS = [{
|
|
68846
69032
|
id: "report.ai-referral-bursts",
|
|
68847
69033
|
category: CheckCategories.integrations,
|
|
@@ -68850,7 +69036,7 @@ var REFERRAL_ASSESSMENT_CHECKS = [{
|
|
|
68850
69036
|
notificationPolicy: CheckNotificationPolicies.silent,
|
|
68851
69037
|
run: (ctx) => {
|
|
68852
69038
|
if (!ctx.project) return { status: CheckStatuses.skipped, code: "report.ai-referral-bursts.no-project", summary: "Select a project to inspect referral evidence." };
|
|
68853
|
-
const configured = ctx.db.select({ id: trafficSources.id }).from(trafficSources).where(
|
|
69039
|
+
const configured = ctx.db.select({ id: trafficSources.id }).from(trafficSources).where(and81(eq93(trafficSources.projectId, ctx.project.id), ne16(trafficSources.status, TrafficSourceStatuses.archived))).get();
|
|
68854
69040
|
if (!configured) {
|
|
68855
69041
|
return { status: CheckStatuses.skipped, code: "report.ai-referral-bursts.not-configured", summary: "No non-archived server traffic source is configured for this project." };
|
|
68856
69042
|
}
|
|
@@ -69262,19 +69448,19 @@ var BACKLINKS_CHECKS = [
|
|
|
69262
69448
|
];
|
|
69263
69449
|
|
|
69264
69450
|
// ../api-routes/src/doctor/checks/bing-auth.ts
|
|
69265
|
-
import { and as
|
|
69451
|
+
import { and as and82, desc as desc42, eq as eq94, inArray as inArray42 } from "drizzle-orm";
|
|
69266
69452
|
var BING_CONNECTION_ENTITY = "bing_connection";
|
|
69267
69453
|
var BING_CONNECTED_ACTION = "bing.connected";
|
|
69268
69454
|
var BING_DISCONNECTED_ACTION = "bing.disconnected";
|
|
69269
69455
|
function lostBingConnection(ctx, project) {
|
|
69270
69456
|
const hasRow = (row) => row !== void 0;
|
|
69271
|
-
const storedData = hasRow(ctx.db.select({ id: bingCoverageSnapshots.id }).from(bingCoverageSnapshots).where(
|
|
69457
|
+
const storedData = hasRow(ctx.db.select({ id: bingCoverageSnapshots.id }).from(bingCoverageSnapshots).where(eq94(bingCoverageSnapshots.projectId, project.id)).limit(1).get()) || hasRow(ctx.db.select({ id: bingUrlInspections.id }).from(bingUrlInspections).where(eq94(bingUrlInspections.projectId, project.id)).limit(1).get()) || hasRow(ctx.db.select({ id: bingKeywordStats.id }).from(bingKeywordStats).where(eq94(bingKeywordStats.projectId, project.id)).limit(1).get()) || hasRow(ctx.db.select({ id: bingTrafficDaily.id }).from(bingTrafficDaily).where(eq94(bingTrafficDaily.projectId, project.id)).limit(1).get());
|
|
69272
69458
|
if (!storedData) return false;
|
|
69273
|
-
const lastChange = ctx.db.select({ action: auditLog.action }).from(auditLog).where(
|
|
69274
|
-
|
|
69275
|
-
|
|
69276
|
-
|
|
69277
|
-
)).orderBy(
|
|
69459
|
+
const lastChange = ctx.db.select({ action: auditLog.action }).from(auditLog).where(and82(
|
|
69460
|
+
eq94(auditLog.entityType, BING_CONNECTION_ENTITY),
|
|
69461
|
+
eq94(auditLog.entityId, project.canonicalDomain),
|
|
69462
|
+
inArray42(auditLog.action, [BING_CONNECTED_ACTION, BING_DISCONNECTED_ACTION])
|
|
69463
|
+
)).orderBy(desc42(auditLog.createdAt)).limit(1).get();
|
|
69278
69464
|
return lastChange?.action !== BING_DISCONNECTED_ACTION;
|
|
69279
69465
|
}
|
|
69280
69466
|
var BING_AUTH_CHECKS = [
|
|
@@ -69431,7 +69617,7 @@ var BING_AUTH_CHECKS = [
|
|
|
69431
69617
|
];
|
|
69432
69618
|
|
|
69433
69619
|
// ../api-routes/src/doctor/checks/content.ts
|
|
69434
|
-
import { eq as
|
|
69620
|
+
import { eq as eq95 } from "drizzle-orm";
|
|
69435
69621
|
var WINNABILITY_COVERAGE_WARN_THRESHOLD = 0.8;
|
|
69436
69622
|
var UNCLASSIFIED_DOMAIN_SAMPLE_LIMIT = 10;
|
|
69437
69623
|
function skippedNoProject2() {
|
|
@@ -69444,7 +69630,7 @@ function skippedNoProject2() {
|
|
|
69444
69630
|
}
|
|
69445
69631
|
function loadProject(ctx) {
|
|
69446
69632
|
if (!ctx.project) return null;
|
|
69447
|
-
return ctx.db.select().from(projects).where(
|
|
69633
|
+
return ctx.db.select().from(projects).where(eq95(projects.id, ctx.project.id)).get() ?? null;
|
|
69448
69634
|
}
|
|
69449
69635
|
var winnabilityCoverageCheck = {
|
|
69450
69636
|
id: "content.winnability.coverage",
|
|
@@ -69537,7 +69723,7 @@ var CONTENT_CHECK_BY_ID = Object.fromEntries(
|
|
|
69537
69723
|
);
|
|
69538
69724
|
|
|
69539
69725
|
// ../api-routes/src/doctor/checks/ads.ts
|
|
69540
|
-
import { eq as
|
|
69726
|
+
import { eq as eq96 } from "drizzle-orm";
|
|
69541
69727
|
var ADS_RECENT_SYNC_WARN_DAYS = 7;
|
|
69542
69728
|
var ADS_RECENT_SYNC_FAIL_DAYS = 30;
|
|
69543
69729
|
var adsConnectionCheck = {
|
|
@@ -69554,7 +69740,7 @@ var adsConnectionCheck = {
|
|
|
69554
69740
|
remediation: null
|
|
69555
69741
|
};
|
|
69556
69742
|
}
|
|
69557
|
-
const row = ctx.db.select().from(adsConnections).where(
|
|
69743
|
+
const row = ctx.db.select().from(adsConnections).where(eq96(adsConnections.projectId, ctx.project.id)).get();
|
|
69558
69744
|
if (!row) {
|
|
69559
69745
|
return {
|
|
69560
69746
|
status: CheckStatuses.skipped,
|
|
@@ -69604,7 +69790,7 @@ var adsRecentSyncCheck = {
|
|
|
69604
69790
|
remediation: null
|
|
69605
69791
|
};
|
|
69606
69792
|
}
|
|
69607
|
-
const row = ctx.db.select().from(adsConnections).where(
|
|
69793
|
+
const row = ctx.db.select().from(adsConnections).where(eq96(adsConnections.projectId, ctx.project.id)).get();
|
|
69608
69794
|
if (!row) {
|
|
69609
69795
|
return {
|
|
69610
69796
|
status: CheckStatuses.skipped,
|
|
@@ -69819,7 +70005,7 @@ var ga4ConnectionCheck = {
|
|
|
69819
70005
|
var GA_AUTH_CHECKS = [ga4ConnectionCheck];
|
|
69820
70006
|
|
|
69821
70007
|
// ../api-routes/src/doctor/checks/data-freshness.ts
|
|
69822
|
-
import { and as
|
|
70008
|
+
import { and as and83, desc as desc43, eq as eq97, inArray as inArray43 } from "drizzle-orm";
|
|
69823
70009
|
var GA_DATA_AGING_DAYS = 3;
|
|
69824
70010
|
var GA_DATA_STALE_DAYS = 5;
|
|
69825
70011
|
var GSC_DATA_AGING_DAYS = 5;
|
|
@@ -69849,7 +70035,7 @@ function skipped(code, summary3) {
|
|
|
69849
70035
|
return { status: CheckStatuses.skipped, code, summary: summary3, remediation: null };
|
|
69850
70036
|
}
|
|
69851
70037
|
function lastSyncAgeDays(ctx, kinds) {
|
|
69852
|
-
const row = ctx.db.select({ createdAt: runs.createdAt }).from(runs).where(
|
|
70038
|
+
const row = ctx.db.select({ createdAt: runs.createdAt }).from(runs).where(and83(eq97(runs.projectId, ctx.project.id), inArray43(runs.kind, kinds), eq97(runs.status, "completed"))).orderBy(desc43(runs.createdAt)).limit(1).get();
|
|
69853
70039
|
if (!row?.createdAt) return null;
|
|
69854
70040
|
const age = Math.floor((Date.now() - Date.parse(row.createdAt)) / DAY_MS5);
|
|
69855
70041
|
return Number.isFinite(age) ? age : null;
|
|
@@ -69951,7 +70137,7 @@ var gaRecentDataCheck = {
|
|
|
69951
70137
|
const connection = gaConnected(ctx);
|
|
69952
70138
|
if (connection === "store-unavailable") return skipped("ga.data.store-unavailable", "No GA4 credential store configured for this deployment.");
|
|
69953
70139
|
if (connection === "not-connected") return skipped("ga.data.not-connected", "GA4 is not connected for this project.");
|
|
69954
|
-
const newest = ctx.db.select({ date: gaDailyTotals.date }).from(gaDailyTotals).where(
|
|
70140
|
+
const newest = ctx.db.select({ date: gaDailyTotals.date }).from(gaDailyTotals).where(eq97(gaDailyTotals.projectId, ctx.project.id)).orderBy(desc43(gaDailyTotals.date)).limit(1).get();
|
|
69955
70141
|
return grade(ctx, GA_SPEC, newest?.date);
|
|
69956
70142
|
}
|
|
69957
70143
|
};
|
|
@@ -69966,16 +70152,16 @@ var gscRecentDataCheck = {
|
|
|
69966
70152
|
if (!ctx.googleConnectionStore.getConnection(ctx.project.canonicalDomain, "gsc")) {
|
|
69967
70153
|
return skipped("gsc.data.not-connected", "Search Console is not connected for this project.");
|
|
69968
70154
|
}
|
|
69969
|
-
const watermark = ctx.db.select({ dataThroughDate: gscDataWatermarks.dataThroughDate }).from(gscDataWatermarks).where(
|
|
70155
|
+
const watermark = ctx.db.select({ dataThroughDate: gscDataWatermarks.dataThroughDate }).from(gscDataWatermarks).where(eq97(gscDataWatermarks.projectId, ctx.project.id)).get();
|
|
69970
70156
|
if (watermark?.dataThroughDate) return grade(ctx, GSC_SPEC, watermark.dataThroughDate);
|
|
69971
|
-
const newest = ctx.db.select({ date: gscDailyTotals.date }).from(gscDailyTotals).where(
|
|
70157
|
+
const newest = ctx.db.select({ date: gscDailyTotals.date }).from(gscDailyTotals).where(eq97(gscDailyTotals.projectId, ctx.project.id)).orderBy(desc43(gscDailyTotals.date)).limit(1).get();
|
|
69972
70158
|
return grade(ctx, GSC_SPEC, newest?.date);
|
|
69973
70159
|
}
|
|
69974
70160
|
};
|
|
69975
70161
|
var DATA_FRESHNESS_CHECKS = [gaRecentDataCheck, gscRecentDataCheck];
|
|
69976
70162
|
|
|
69977
70163
|
// ../api-routes/src/doctor/checks/seo-refresh.ts
|
|
69978
|
-
import { and as
|
|
70164
|
+
import { and as and84, asc as asc24, desc as desc44, eq as eq98, gt as gt8, inArray as inArray44, sql as sql39 } from "drizzle-orm";
|
|
69979
70165
|
var GRACE_DAYS = 2;
|
|
69980
70166
|
var DAY_MS6 = 24 * 60 * 60 * 1e3;
|
|
69981
70167
|
var FIRING_SPAN_MS = 6e4;
|
|
@@ -69984,7 +70170,7 @@ function skipped2(code, summary3) {
|
|
|
69984
70170
|
return { status: CheckStatuses.skipped, code, summary: summary3, remediation: null };
|
|
69985
70171
|
}
|
|
69986
70172
|
function seoSyncRuns(projectId) {
|
|
69987
|
-
return
|
|
70173
|
+
return and84(eq98(runs.projectId, projectId), eq98(runs.kind, RunKinds["seo-sync"]));
|
|
69988
70174
|
}
|
|
69989
70175
|
function scheduleTiming(schedule) {
|
|
69990
70176
|
if (!schedule.recurrence) return { cronExpr: schedule.cronExpr, timezone: schedule.timezone, recurrence: null };
|
|
@@ -69992,22 +70178,22 @@ function scheduleTiming(schedule) {
|
|
|
69992
70178
|
return recurrence ? { cronExpr: schedule.cronExpr, timezone: schedule.timezone, recurrence } : null;
|
|
69993
70179
|
}
|
|
69994
70180
|
function latestFiring(ctx, projectId, enabledSince) {
|
|
69995
|
-
const scheduled =
|
|
69996
|
-
const newest = ctx.db.select({ createdAt: runs.createdAt }).from(runs).where(scheduled).orderBy(
|
|
70181
|
+
const scheduled = and84(seoSyncRuns(projectId), eq98(runs.trigger, RunTriggers.scheduled), gt8(runs.createdAt, enabledSince));
|
|
70182
|
+
const newest = ctx.db.select({ createdAt: runs.createdAt }).from(runs).where(scheduled).orderBy(desc44(runs.createdAt)).limit(1).get();
|
|
69997
70183
|
if (!newest) return [];
|
|
69998
70184
|
const newestMs = Date.parse(newest.createdAt);
|
|
69999
70185
|
const after = Number.isFinite(newestMs) ? new Date(newestMs - FIRING_SPAN_MS).toISOString() : newest.createdAt;
|
|
70000
|
-
return ctx.db.select({ id: runs.id, status: runs.status, error: runs.error, createdAt: runs.createdAt }).from(runs).where(
|
|
70186
|
+
return ctx.db.select({ id: runs.id, status: runs.status, error: runs.error, createdAt: runs.createdAt }).from(runs).where(and84(scheduled, gt8(runs.createdAt, after))).orderBy(asc24(runs.createdAt), asc24(runs.id)).all();
|
|
70001
70187
|
}
|
|
70002
70188
|
function lastCompleted(ctx, projectId) {
|
|
70003
|
-
const at =
|
|
70004
|
-
return ctx.db.select({ id: runs.id, at }).from(runs).where(
|
|
70189
|
+
const at = sql39`coalesce(${runs.finishedAt}, ${runs.createdAt})`;
|
|
70190
|
+
return ctx.db.select({ id: runs.id, at }).from(runs).where(and84(seoSyncRuns(projectId), eq98(runs.status, RunStatuses.completed))).orderBy(desc44(at)).limit(1).get() ?? null;
|
|
70005
70191
|
}
|
|
70006
70192
|
function isIncomplete(run) {
|
|
70007
70193
|
return run.status === RunStatuses.failed || run.status === RunStatuses.partial;
|
|
70008
70194
|
}
|
|
70009
70195
|
function runProfiles(ctx, firing) {
|
|
70010
|
-
const jobs = firing.length === 0 ? [] : ctx.db.select({ runId: seoJobs.runId, profileId: seoJobs.profileId }).from(seoJobs).where(
|
|
70196
|
+
const jobs = firing.length === 0 ? [] : ctx.db.select({ runId: seoJobs.runId, profileId: seoJobs.profileId }).from(seoJobs).where(inArray44(seoJobs.runId, firing.map((run) => run.id))).all();
|
|
70011
70197
|
const byRun = new Map(jobs.flatMap((job) => job.runId ? [[job.runId, job.profileId]] : []));
|
|
70012
70198
|
return new Map(firing.map((run) => [run.id, byRun.get(run.id) ?? skippedRefreshProfile(run.error)]));
|
|
70013
70199
|
}
|
|
@@ -70015,7 +70201,7 @@ function unsettledRuns(ctx, projectId, firing, profiles) {
|
|
|
70015
70201
|
const incomplete = firing.filter(isIncomplete);
|
|
70016
70202
|
const firedAt = firing[firing.length - 1]?.createdAt;
|
|
70017
70203
|
if (incomplete.length === 0 || !firedAt) return incomplete;
|
|
70018
|
-
const refreshed = new Set(ctx.db.select({ profileId: seoJobs.profileId }).from(runs).innerJoin(seoJobs,
|
|
70204
|
+
const refreshed = new Set(ctx.db.select({ profileId: seoJobs.profileId }).from(runs).innerJoin(seoJobs, eq98(seoJobs.runId, runs.id)).where(and84(seoSyncRuns(projectId), eq98(runs.status, RunStatuses.completed), gt8(runs.createdAt, firedAt))).all().map((row) => row.profileId));
|
|
70019
70205
|
return incomplete.filter((run) => {
|
|
70020
70206
|
const profile = profiles.get(run.id) ?? null;
|
|
70021
70207
|
return profile === null ? refreshed.size === 0 : !refreshed.has(profile);
|
|
@@ -70025,7 +70211,7 @@ var COUNT_LIST = new Intl.ListFormat("en", { style: "long", type: "conjunction"
|
|
|
70025
70211
|
function refreshReason(ctx, run) {
|
|
70026
70212
|
const stored = parseRunError(run.error);
|
|
70027
70213
|
if (stored) return formatRunErrorOneLine(stored);
|
|
70028
|
-
const tasks = currentSeoTasks(ctx.db.select({ id: seoTasks.id, retryOf: seoTasks.retryOf, status: seoTasks.status, error: seoTasks.error, request: seoTasks.request }).from(seoTasks).innerJoin(seoJobs,
|
|
70214
|
+
const tasks = currentSeoTasks(ctx.db.select({ id: seoTasks.id, retryOf: seoTasks.retryOf, status: seoTasks.status, error: seoTasks.error, request: seoTasks.request }).from(seoTasks).innerJoin(seoJobs, eq98(seoTasks.jobId, seoJobs.id)).where(eq98(seoJobs.runId, run.id)).all());
|
|
70029
70215
|
const failed = tasks.filter((task) => task.status === SeoRunStatuses.failed || task.status === SeoRunStatuses["reconciliation-required"]);
|
|
70030
70216
|
const partial = tasks.filter((task) => task.status === SeoRunStatuses.partial).length;
|
|
70031
70217
|
const cancelled = tasks.filter((task) => task.status === SeoRunStatuses.cancelled).length;
|
|
@@ -70045,7 +70231,7 @@ function refreshReason(ctx, run) {
|
|
|
70045
70231
|
return `Of ${tasks.length} ${unit}, ${COUNT_LIST.format(counts2)}.${errors ? ` ${errors}` : ""}`;
|
|
70046
70232
|
}
|
|
70047
70233
|
function gradeSeoRefresh(ctx, project) {
|
|
70048
|
-
const schedule = ctx.db.select().from(schedules).where(
|
|
70234
|
+
const schedule = ctx.db.select().from(schedules).where(and84(eq98(schedules.projectId, project.id), eq98(schedules.kind, SchedulableRunKinds["seo-sync"]))).get();
|
|
70049
70235
|
if (!schedule) return skipped2("seo.data.not-scheduled", "No SEO refresh schedule is set for this project.");
|
|
70050
70236
|
if (!schedule.enabled) return skipped2("seo.data.not-scheduled", "The SEO refresh schedule is disabled.");
|
|
70051
70237
|
const name = project.name;
|
|
@@ -70117,7 +70303,7 @@ function gradeSeoRefresh(ctx, project) {
|
|
|
70117
70303
|
};
|
|
70118
70304
|
}
|
|
70119
70305
|
if (now >= Date.parse(deadlineAt)) {
|
|
70120
|
-
const active = ctx.db.select({ id: runs.id, createdAt: runs.createdAt }).from(runs).where(
|
|
70306
|
+
const active = ctx.db.select({ id: runs.id, createdAt: runs.createdAt }).from(runs).where(and84(seoSyncRuns(project.id), inArray44(runs.status, [RunStatuses.queued, RunStatuses.running]))).orderBy(asc24(runs.createdAt)).limit(1).get();
|
|
70121
70307
|
const since = last ? `No SEO refresh has completed since ${day(last.at)}.` : "No SEO refresh has completed since the schedule was enabled.";
|
|
70122
70308
|
const missed = last ? `The scheduled refresh due ${day(dueAt)} did not complete within ${GRACE_DAYS} days.` : `Its first refresh, due ${day(dueAt)}, did not complete within ${GRACE_DAYS} days.`;
|
|
70123
70309
|
return {
|
|
@@ -70427,7 +70613,7 @@ var siteReachabilityCheck = {
|
|
|
70427
70613
|
var SITE_REACHABILITY_CHECKS = [siteReachabilityCheck];
|
|
70428
70614
|
|
|
70429
70615
|
// ../api-routes/src/doctor/checks/api-hosts.ts
|
|
70430
|
-
import { eq as
|
|
70616
|
+
import { eq as eq100 } from "drizzle-orm";
|
|
70431
70617
|
|
|
70432
70618
|
// ../api-routes/src/api-host-probe.ts
|
|
70433
70619
|
import dns4 from "dns/promises";
|
|
@@ -70533,7 +70719,7 @@ async function probeApiHost(host, options = {}) {
|
|
|
70533
70719
|
}
|
|
70534
70720
|
|
|
70535
70721
|
// ../api-routes/src/doctor/checks/integration-sync.ts
|
|
70536
|
-
import { and as
|
|
70722
|
+
import { and as and85, count as count7, desc as desc45, eq as eq99, gte as gte27, gt as gt9, inArray as inArray45, isNull as isNull14, notInArray as notInArray3, or as or19 } from "drizzle-orm";
|
|
70537
70723
|
var SYNC_FAILURE_WINDOW = 3;
|
|
70538
70724
|
var SUMMARY_ERROR_LIMIT = 300;
|
|
70539
70725
|
var FINISHED_STATUSES = [RunStatuses.completed, RunStatuses.partial, RunStatuses.failed];
|
|
@@ -70560,7 +70746,7 @@ function gscConnection(ctx, project) {
|
|
|
70560
70746
|
function gbpConnection(ctx, project) {
|
|
70561
70747
|
if (!ctx.googleConnectionStore) return { state: "store-unavailable" };
|
|
70562
70748
|
if (!ctx.googleConnectionStore.getConnection(project.canonicalDomain, "gbp")) return { state: "not-connected" };
|
|
70563
|
-
const selected = ctx.db.select({ locationName: gbpLocations.locationName }).from(gbpLocations).where(
|
|
70749
|
+
const selected = ctx.db.select({ locationName: gbpLocations.locationName }).from(gbpLocations).where(and85(eq99(gbpLocations.projectId, project.id), eq99(gbpLocations.selected, true))).limit(1).get();
|
|
70564
70750
|
return selected ? CONNECTED : { state: "no-selected-locations" };
|
|
70565
70751
|
}
|
|
70566
70752
|
function gaConnection(ctx, project) {
|
|
@@ -70633,16 +70819,16 @@ function latestErrorText(raw) {
|
|
|
70633
70819
|
return parsed ? formatRunErrorOneLine(parsed) : "no error was recorded";
|
|
70634
70820
|
}
|
|
70635
70821
|
function readSyncRunState(db, projectId, kind, options = {}) {
|
|
70636
|
-
const scope =
|
|
70637
|
-
|
|
70638
|
-
|
|
70639
|
-
...options.sourceId ? [
|
|
70822
|
+
const scope = and85(
|
|
70823
|
+
eq99(runs.projectId, projectId),
|
|
70824
|
+
eq99(runs.kind, kind),
|
|
70825
|
+
...options.sourceId ? [eq99(runs.sourceId, options.sourceId)] : [],
|
|
70640
70826
|
...options.nothingToSyncErrors?.length ? [or19(isNull14(runs.error), notInArray3(runs.error, [...options.nothingToSyncErrors]))] : []
|
|
70641
70827
|
);
|
|
70642
|
-
const graded = options.since ?
|
|
70643
|
-
const recent = db.select({ id: runs.id, status: runs.status, trigger: runs.trigger, error: runs.error, createdAt: runs.createdAt }).from(runs).where(
|
|
70644
|
-
const lastSuccess = db.select({ createdAt: runs.createdAt, finishedAt: runs.finishedAt }).from(runs).where(
|
|
70645
|
-
const failureCount = db.select({ count: count7() }).from(runs).where(
|
|
70828
|
+
const graded = options.since ? and85(scope, gte27(runs.createdAt, options.since)) : scope;
|
|
70829
|
+
const recent = db.select({ id: runs.id, status: runs.status, trigger: runs.trigger, error: runs.error, createdAt: runs.createdAt }).from(runs).where(and85(graded, inArray45(runs.status, FINISHED_STATUSES))).orderBy(desc45(runs.createdAt)).limit(SYNC_FAILURE_WINDOW).all();
|
|
70830
|
+
const lastSuccess = db.select({ createdAt: runs.createdAt, finishedAt: runs.finishedAt }).from(runs).where(and85(graded, inArray45(runs.status, SUCCESS_STATUSES))).orderBy(desc45(runs.createdAt)).limit(1).get();
|
|
70831
|
+
const failureCount = db.select({ count: count7() }).from(runs).where(and85(lastSuccess ? and85(graded, gt9(runs.createdAt, lastSuccess.createdAt)) : graded, eq99(runs.status, RunStatuses.failed))).get();
|
|
70646
70832
|
const failuresSinceSuccess = Number(failureCount?.count ?? 0);
|
|
70647
70833
|
const latest = recent[0];
|
|
70648
70834
|
return {
|
|
@@ -70739,7 +70925,7 @@ function connectionFor(prefix, ctx, project) {
|
|
|
70739
70925
|
if (prefix === "google-ads") return googleMarketingConnection("googleAds")(ctx);
|
|
70740
70926
|
if (prefix === "gtm") return googleMarketingConnection("gtm")(ctx);
|
|
70741
70927
|
if (prefix === "bing") return ctx.bingConnectionStore?.getConnection(project.canonicalDomain) ? { state: "connected", since: null } : { state: "not-connected" };
|
|
70742
|
-
return ctx.db.select({ id: adsConnections.id }).from(adsConnections).where(
|
|
70928
|
+
return ctx.db.select({ id: adsConnections.id }).from(adsConnections).where(eq100(adsConnections.projectId, project.id)).limit(1).get() ? { state: "connected", since: null } : { state: "not-connected" };
|
|
70743
70929
|
}
|
|
70744
70930
|
function syncCommand(prefix) {
|
|
70745
70931
|
if (prefix === "gsc") return "google sync";
|
|
@@ -70784,7 +70970,7 @@ function definition(prefix) {
|
|
|
70784
70970
|
var API_HOST_CHECKS = Object.keys(API_HOST_TABLE).map(definition);
|
|
70785
70971
|
|
|
70786
70972
|
// ../api-routes/src/doctor/checks/gbp-auth.ts
|
|
70787
|
-
import { and as
|
|
70973
|
+
import { and as and86, eq as eq101 } from "drizzle-orm";
|
|
70788
70974
|
var RECENT_SYNC_WARN_DAYS = 4;
|
|
70789
70975
|
var RECENT_SYNC_FAIL_DAYS = 30;
|
|
70790
70976
|
function skippedNoProject3() {
|
|
@@ -70804,7 +70990,7 @@ function storeUnavailable() {
|
|
|
70804
70990
|
};
|
|
70805
70991
|
}
|
|
70806
70992
|
function hasSelectedLocation(ctx, projectId) {
|
|
70807
|
-
return !!ctx.db.select({ id: gbpLocations.id }).from(gbpLocations).where(
|
|
70993
|
+
return !!ctx.db.select({ id: gbpLocations.id }).from(gbpLocations).where(and86(eq101(gbpLocations.projectId, projectId), eq101(gbpLocations.selected, true))).limit(1).get();
|
|
70808
70994
|
}
|
|
70809
70995
|
async function resolveGbpToken(ctx) {
|
|
70810
70996
|
if (!ctx.project) return { ok: false, output: skippedNoProject3() };
|
|
@@ -71064,7 +71250,7 @@ var recentSyncCheck = {
|
|
|
71064
71250
|
title: "GBP recent sync",
|
|
71065
71251
|
run: (ctx) => {
|
|
71066
71252
|
if (!ctx.project) return skippedNoProject3();
|
|
71067
|
-
const selected = ctx.db.select({ locationName: gbpLocations.locationName, syncedAt: gbpLocations.syncedAt }).from(gbpLocations).where(
|
|
71253
|
+
const selected = ctx.db.select({ locationName: gbpLocations.locationName, syncedAt: gbpLocations.syncedAt }).from(gbpLocations).where(and86(eq101(gbpLocations.projectId, ctx.project.id), eq101(gbpLocations.selected, true))).all();
|
|
71068
71254
|
if (selected.length === 0) {
|
|
71069
71255
|
return {
|
|
71070
71256
|
status: CheckStatuses.skipped,
|
|
@@ -71124,7 +71310,7 @@ var GBP_AUTH_CHECK_BY_ID = Object.fromEntries(
|
|
|
71124
71310
|
);
|
|
71125
71311
|
|
|
71126
71312
|
// ../api-routes/src/doctor/checks/places.ts
|
|
71127
|
-
import { eq as
|
|
71313
|
+
import { eq as eq102 } from "drizzle-orm";
|
|
71128
71314
|
var apiKeyCheck = {
|
|
71129
71315
|
id: "gbp.places.api-key",
|
|
71130
71316
|
category: CheckCategories.auth,
|
|
@@ -71169,7 +71355,7 @@ var apiKeyCheck = {
|
|
|
71169
71355
|
details: { tier: cfg.tier }
|
|
71170
71356
|
};
|
|
71171
71357
|
}
|
|
71172
|
-
const rows = ctx.db.select({ placeId: gbpLocations.placeId, selected: gbpLocations.selected }).from(gbpLocations).where(
|
|
71358
|
+
const rows = ctx.db.select({ placeId: gbpLocations.placeId, selected: gbpLocations.selected }).from(gbpLocations).where(eq102(gbpLocations.projectId, ctx.project.id)).all();
|
|
71173
71359
|
const selected = rows.filter((r) => r.selected);
|
|
71174
71360
|
const locationsWithPlaceId = selected.filter((r) => Boolean(r.placeId)).length;
|
|
71175
71361
|
const details = {
|
|
@@ -71202,7 +71388,7 @@ var PLACES_CHECK_BY_ID = Object.fromEntries(
|
|
|
71202
71388
|
);
|
|
71203
71389
|
|
|
71204
71390
|
// ../api-routes/src/doctor/checks/gbp-reviews.ts
|
|
71205
|
-
import { and as
|
|
71391
|
+
import { and as and87, desc as desc46, eq as eq103, gte as gte28, inArray as inArray46, sql as sql40 } from "drizzle-orm";
|
|
71206
71392
|
var REVIEW_EVENTS = /* @__PURE__ */ new Set(["review.negative", "review.rating-dropped"]);
|
|
71207
71393
|
var V4_ACCESS_REMEDIATION = "Business Profile reviews use the legacy My Business API (mybusiness.googleapis.com), which Google enables per Cloud project only on request; the API Library and `gcloud services enable` cannot turn it on. Open the enable link in the API access approval email as the approved account, or reply to the access-request thread asking Google to enable mybusiness.googleapis.com for your project number, then run `canonry gbp sync`.";
|
|
71208
71394
|
var reviewsAccessCheck = {
|
|
@@ -71230,7 +71416,7 @@ var reviewsAccessCheck = {
|
|
|
71230
71416
|
reviewsAccess: gbpLocations.reviewsAccess,
|
|
71231
71417
|
reviewsAccessReason: gbpLocations.reviewsAccessReason,
|
|
71232
71418
|
reviewsCheckedAt: gbpLocations.reviewsCheckedAt
|
|
71233
|
-
}).from(gbpLocations).where(
|
|
71419
|
+
}).from(gbpLocations).where(and87(eq103(gbpLocations.projectId, project.id), eq103(gbpLocations.selected, true))).all();
|
|
71234
71420
|
if (locations.length === 0) {
|
|
71235
71421
|
return {
|
|
71236
71422
|
status: CheckStatuses.skipped,
|
|
@@ -71250,7 +71436,7 @@ var reviewsAccessCheck = {
|
|
|
71250
71436
|
const places = ctx.getPlacesConfig?.();
|
|
71251
71437
|
const placesFallback = places !== void 0 && places.tier === "atmosphere" && Boolean(places.apiKey);
|
|
71252
71438
|
const v4Worked = new Set(
|
|
71253
|
-
ctx.db.select({ locationName: gbpReviewRatings.locationName }).from(gbpReviewRatings).where(
|
|
71439
|
+
ctx.db.select({ locationName: gbpReviewRatings.locationName }).from(gbpReviewRatings).where(and87(eq103(gbpReviewRatings.projectId, project.id), eq103(gbpReviewRatings.origin, "gbp"))).all().map((row) => row.locationName)
|
|
71254
71440
|
);
|
|
71255
71441
|
const coverageOf = (loc) => {
|
|
71256
71442
|
if (businessProfileReviewsCover(loc.reviewsAccess ?? null, v4Worked.has(loc.locationName))) return "full";
|
|
@@ -71284,7 +71470,7 @@ var reviewsAccessCheck = {
|
|
|
71284
71470
|
none > 0 ? `${none} have no review source` : null
|
|
71285
71471
|
].filter((part) => part !== null);
|
|
71286
71472
|
const summary3 = `Business Profile reviews are not available${why}. ${parts.join("; ")}.`;
|
|
71287
|
-
const subscribed = ctx.db.select({ config: notifications.config, enabled: notifications.enabled }).from(notifications).where(
|
|
71473
|
+
const subscribed = ctx.db.select({ config: notifications.config, enabled: notifications.enabled }).from(notifications).where(eq103(notifications.projectId, project.id)).all().some((n) => n.enabled && (n.config.events ?? []).some((event) => REVIEW_EVENTS.has(event)));
|
|
71288
71474
|
const placesHint = placesFallback ? "" : " For partial coverage meanwhile, set a Places API key (`GOOGLE_PLACES_API_KEY` or `places.apiKey`) with `places.tier: atmosphere`.";
|
|
71289
71475
|
if (none > 0 && subscribed) {
|
|
71290
71476
|
return {
|
|
@@ -71315,7 +71501,7 @@ var reviewDeliveryCheck = {
|
|
|
71315
71501
|
return { status: CheckStatuses.skipped, code: "gbp.reviews.delivery.no-project", summary: "Project context required.", remediation: null };
|
|
71316
71502
|
}
|
|
71317
71503
|
const since = new Date(Date.now() - UNDELIVERED_ATTENTION_DAYS * 864e5).toISOString();
|
|
71318
|
-
const lastTouched =
|
|
71504
|
+
const lastTouched = sql40`coalesce(${reviewWebhookDeliveries.attemptedAt}, ${reviewWebhookDeliveries.createdAt})`;
|
|
71319
71505
|
const rows = ctx.db.select({
|
|
71320
71506
|
id: reviewWebhookDeliveries.id,
|
|
71321
71507
|
notificationId: reviewWebhookDeliveries.notificationId,
|
|
@@ -71323,11 +71509,11 @@ var reviewDeliveryCheck = {
|
|
|
71323
71509
|
attempts: reviewWebhookDeliveries.attempts,
|
|
71324
71510
|
lastTouched,
|
|
71325
71511
|
payload: reviewWebhookDeliveries.payload
|
|
71326
|
-
}).from(reviewWebhookDeliveries).where(
|
|
71327
|
-
|
|
71328
|
-
|
|
71512
|
+
}).from(reviewWebhookDeliveries).where(and87(
|
|
71513
|
+
eq103(reviewWebhookDeliveries.projectId, ctx.project.id),
|
|
71514
|
+
inArray46(reviewWebhookDeliveries.state, [ReviewDeliveryStates.Uncertain, ReviewDeliveryStates.Failed]),
|
|
71329
71515
|
gte28(lastTouched, since)
|
|
71330
|
-
)).orderBy(
|
|
71516
|
+
)).orderBy(desc46(lastTouched)).all();
|
|
71331
71517
|
if (rows.length === 0) {
|
|
71332
71518
|
return {
|
|
71333
71519
|
status: CheckStatuses.ok,
|
|
@@ -71336,7 +71522,7 @@ var reviewDeliveryCheck = {
|
|
|
71336
71522
|
remediation: null
|
|
71337
71523
|
};
|
|
71338
71524
|
}
|
|
71339
|
-
const destinations = new Map(ctx.db.select({ id: notifications.id, config: notifications.config }).from(notifications).where(
|
|
71525
|
+
const destinations = new Map(ctx.db.select({ id: notifications.id, config: notifications.config }).from(notifications).where(eq103(notifications.projectId, ctx.project.id)).all().map((row) => [row.id, redactNotificationUrl(row.config.url).urlDisplay]));
|
|
71340
71526
|
const deliveries = rows.map((row) => {
|
|
71341
71527
|
const review = row.payload.reviews[0];
|
|
71342
71528
|
return {
|
|
@@ -71910,7 +72096,7 @@ var RUNTIME_STATE_CHECKS = [
|
|
|
71910
72096
|
];
|
|
71911
72097
|
|
|
71912
72098
|
// ../api-routes/src/doctor/checks/traffic-source.ts
|
|
71913
|
-
import { and as
|
|
72099
|
+
import { and as and88, desc as desc47, eq as eq104, gte as gte29, inArray as inArray47, isNotNull as isNotNull9, ne as ne17, sql as sql41 } from "drizzle-orm";
|
|
71914
72100
|
var RECENT_DATA_WARN_DAYS = 7;
|
|
71915
72101
|
var RECENT_DATA_FAIL_DAYS = 30;
|
|
71916
72102
|
function isCloudflareDirectPush(source) {
|
|
@@ -71949,8 +72135,8 @@ function skippedNoProject5() {
|
|
|
71949
72135
|
function loadProbes(ctx) {
|
|
71950
72136
|
if (!ctx.project) return [];
|
|
71951
72137
|
const rows = ctx.db.select().from(trafficSources).where(
|
|
71952
|
-
|
|
71953
|
-
|
|
72138
|
+
and88(
|
|
72139
|
+
eq104(trafficSources.projectId, ctx.project.id),
|
|
71954
72140
|
ne17(trafficSources.status, TrafficSourceStatuses.archived)
|
|
71955
72141
|
)
|
|
71956
72142
|
).all();
|
|
@@ -72087,28 +72273,28 @@ var recentDataCheck = {
|
|
|
72087
72273
|
const warnCutoff = new Date(now.getTime() - RECENT_DATA_WARN_DAYS * 24 * 60 * 6e4).toISOString();
|
|
72088
72274
|
const failCutoff = new Date(now.getTime() - RECENT_DATA_FAIL_DAYS * 24 * 60 * 6e4).toISOString();
|
|
72089
72275
|
const recentCrawlers = Number(
|
|
72090
|
-
ctx.db.select({ total:
|
|
72091
|
-
|
|
72092
|
-
|
|
72093
|
-
|
|
72276
|
+
ctx.db.select({ total: sql41`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
72277
|
+
and88(
|
|
72278
|
+
eq104(crawlerEventsHourly.projectId, ctx.project.id),
|
|
72279
|
+
inArray47(crawlerEventsHourly.sourceId, activeSourceIds),
|
|
72094
72280
|
gte29(crawlerEventsHourly.tsHour, warnCutoff)
|
|
72095
72281
|
)
|
|
72096
72282
|
).get()?.total ?? 0
|
|
72097
72283
|
);
|
|
72098
72284
|
const recentReferrals = Number(
|
|
72099
|
-
ctx.db.select({ total:
|
|
72100
|
-
|
|
72101
|
-
|
|
72102
|
-
|
|
72285
|
+
ctx.db.select({ total: sql41`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
72286
|
+
and88(
|
|
72287
|
+
eq104(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
72288
|
+
inArray47(aiReferralEventsHourly.sourceId, activeSourceIds),
|
|
72103
72289
|
gte29(aiReferralEventsHourly.tsHour, warnCutoff)
|
|
72104
72290
|
)
|
|
72105
72291
|
).get()?.total ?? 0
|
|
72106
72292
|
);
|
|
72107
72293
|
const recentUserFetches = Number(
|
|
72108
|
-
ctx.db.select({ total:
|
|
72109
|
-
|
|
72110
|
-
|
|
72111
|
-
|
|
72294
|
+
ctx.db.select({ total: sql41`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
72295
|
+
and88(
|
|
72296
|
+
eq104(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
72297
|
+
inArray47(aiUserFetchEventsHourly.sourceId, activeSourceIds),
|
|
72112
72298
|
gte29(aiUserFetchEventsHourly.tsHour, warnCutoff)
|
|
72113
72299
|
)
|
|
72114
72300
|
).get()?.total ?? 0
|
|
@@ -72131,28 +72317,28 @@ var recentDataCheck = {
|
|
|
72131
72317
|
};
|
|
72132
72318
|
}
|
|
72133
72319
|
const olderCrawlers = Number(
|
|
72134
|
-
ctx.db.select({ total:
|
|
72135
|
-
|
|
72136
|
-
|
|
72137
|
-
|
|
72320
|
+
ctx.db.select({ total: sql41`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
72321
|
+
and88(
|
|
72322
|
+
eq104(crawlerEventsHourly.projectId, ctx.project.id),
|
|
72323
|
+
inArray47(crawlerEventsHourly.sourceId, activeSourceIds),
|
|
72138
72324
|
gte29(crawlerEventsHourly.tsHour, failCutoff)
|
|
72139
72325
|
)
|
|
72140
72326
|
).get()?.total ?? 0
|
|
72141
72327
|
);
|
|
72142
72328
|
const olderReferrals = Number(
|
|
72143
|
-
ctx.db.select({ total:
|
|
72144
|
-
|
|
72145
|
-
|
|
72146
|
-
|
|
72329
|
+
ctx.db.select({ total: sql41`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
72330
|
+
and88(
|
|
72331
|
+
eq104(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
72332
|
+
inArray47(aiReferralEventsHourly.sourceId, activeSourceIds),
|
|
72147
72333
|
gte29(aiReferralEventsHourly.tsHour, failCutoff)
|
|
72148
72334
|
)
|
|
72149
72335
|
).get()?.total ?? 0
|
|
72150
72336
|
);
|
|
72151
72337
|
const olderUserFetches = Number(
|
|
72152
|
-
ctx.db.select({ total:
|
|
72153
|
-
|
|
72154
|
-
|
|
72155
|
-
|
|
72338
|
+
ctx.db.select({ total: sql41`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
72339
|
+
and88(
|
|
72340
|
+
eq104(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
72341
|
+
inArray47(aiUserFetchEventsHourly.sourceId, activeSourceIds),
|
|
72156
72342
|
gte29(aiUserFetchEventsHourly.tsHour, failCutoff)
|
|
72157
72343
|
)
|
|
72158
72344
|
).get()?.total ?? 0
|
|
@@ -72478,12 +72664,12 @@ var syncLagCheck = {
|
|
|
72478
72664
|
}
|
|
72479
72665
|
};
|
|
72480
72666
|
function lastQueueStopReason(ctx, sourceId) {
|
|
72481
|
-
const row = ctx.db.select({ syncSummary: runs.syncSummary }).from(runs).where(
|
|
72482
|
-
|
|
72483
|
-
|
|
72484
|
-
|
|
72667
|
+
const row = ctx.db.select({ syncSummary: runs.syncSummary }).from(runs).where(and88(
|
|
72668
|
+
eq104(runs.projectId, ctx.project.id),
|
|
72669
|
+
eq104(runs.kind, RunKinds["traffic-sync"]),
|
|
72670
|
+
eq104(runs.sourceId, sourceId),
|
|
72485
72671
|
isNotNull9(runs.syncSummary)
|
|
72486
|
-
)).orderBy(
|
|
72672
|
+
)).orderBy(desc47(runs.createdAt)).limit(1).get();
|
|
72487
72673
|
return row?.syncSummary?.type === "cloudflare-queue" ? row.syncSummary.stopReason : null;
|
|
72488
72674
|
}
|
|
72489
72675
|
var QUEUE_LIMIT_STOPS = [
|
|
@@ -73281,7 +73467,7 @@ async function doctorRoutes(app, opts) {
|
|
|
73281
73467
|
}
|
|
73282
73468
|
|
|
73283
73469
|
// ../api-routes/src/status.ts
|
|
73284
|
-
import { and as
|
|
73470
|
+
import { and as and89, desc as desc48, eq as eq105, isNull as isNull15 } from "drizzle-orm";
|
|
73285
73471
|
function ageDays(iso) {
|
|
73286
73472
|
if (!iso) return null;
|
|
73287
73473
|
const ms = Date.now() - Date.parse(iso);
|
|
@@ -73331,7 +73517,7 @@ function makeRow(integration, label, runKind, state, staleAfterDays, dataThrough
|
|
|
73331
73517
|
};
|
|
73332
73518
|
}
|
|
73333
73519
|
function latestDate2(db, table, projectId) {
|
|
73334
|
-
const row = db.select({ date: table.date }).from(table).where(
|
|
73520
|
+
const row = db.select({ date: table.date }).from(table).where(eq105(table.projectId, projectId)).orderBy(desc48(table.date)).limit(1).get();
|
|
73335
73521
|
return row?.date ?? null;
|
|
73336
73522
|
}
|
|
73337
73523
|
function projectRows(opts, project) {
|
|
@@ -73346,7 +73532,7 @@ function projectRows(opts, project) {
|
|
|
73346
73532
|
const rows = [];
|
|
73347
73533
|
const gsc = gscConnection(ctx, info);
|
|
73348
73534
|
if (connected(gsc)) {
|
|
73349
|
-
const watermarkRow = opts.db.select({ dataThroughDate: gscDataWatermarks.dataThroughDate }).from(gscDataWatermarks).where(
|
|
73535
|
+
const watermarkRow = opts.db.select({ dataThroughDate: gscDataWatermarks.dataThroughDate }).from(gscDataWatermarks).where(eq105(gscDataWatermarks.projectId, project.id)).limit(1).get();
|
|
73350
73536
|
const state = readSyncRunState(opts.db, project.id, RunKinds["gsc-sync"], { since: gsc.since });
|
|
73351
73537
|
const dataThrough = watermarkRow?.dataThroughDate ?? latestDate2(opts.db, gscDailyTotals, project.id);
|
|
73352
73538
|
rows.push(makeRow("gsc", "Search Console", RunKinds["gsc-sync"], state, SYNC_IDLE_DAYS, dataThrough, null, ["gsc.sync.recent-failures"], GSC_DATA_AGING_DAYS));
|
|
@@ -73362,12 +73548,12 @@ function projectRows(opts, project) {
|
|
|
73362
73548
|
rows.push(makeRow("bing", "Bing Webmaster Tools", RunKinds["bing-inspect-sitemap"], readSyncRunState(opts.db, project.id, RunKinds["bing-inspect-sitemap"]), null));
|
|
73363
73549
|
rows.push(makeRow("bing-search", "Bing search data", RunKinds["bing-sync"], readSyncRunState(opts.db, project.id, RunKinds["bing-sync"]), SYNC_IDLE_DAYS, latestDate2(opts.db, bingTrafficDaily, project.id), null, ["bing.sync.recent-failures"], BING_DATA_AGING_DAYS));
|
|
73364
73550
|
}
|
|
73365
|
-
const ads = opts.db.select({ id: adsConnections.id, lastSyncedAt: adsConnections.lastSyncedAt }).from(adsConnections).where(
|
|
73551
|
+
const ads = opts.db.select({ id: adsConnections.id, lastSyncedAt: adsConnections.lastSyncedAt }).from(adsConnections).where(eq105(adsConnections.projectId, project.id)).limit(1).get();
|
|
73366
73552
|
if (ads) rows.push(makeRow("ads", "OpenAI Ads", RunKinds["ads-sync"], readSyncRunState(opts.db, project.id, RunKinds["ads-sync"]), ADS_RECENT_SYNC_WARN_DAYS, null, null, ["ads.data.recent-sync"], ADS_RECENT_SYNC_WARN_DAYS, ads.lastSyncedAt));
|
|
73367
73553
|
const marketing = opts.getGoogleMarketingDoctorInput?.(ctx);
|
|
73368
73554
|
if (marketing?.googleAds) rows.push(makeRow("google-ads", "Google Ads", RunKinds["google-ads-sync"], readSyncRunState(opts.db, project.id, RunKinds["google-ads-sync"]), FRESH_SNAPSHOT_WARN_DAYS, null, null, ["google-ads.sync.recent-failures"], FRESH_SNAPSHOT_WARN_DAYS, marketing.googleAds.latestSnapshotAt));
|
|
73369
73555
|
if (marketing?.gtm) rows.push(makeRow("gtm", "Google Tag Manager", RunKinds["gtm-sync"], readSyncRunState(opts.db, project.id, RunKinds["gtm-sync"]), FRESH_SNAPSHOT_WARN_DAYS, null, null, ["gtm.sync.recent-failures"], FRESH_SNAPSHOT_WARN_DAYS, marketing.gtm.latestSnapshotAt));
|
|
73370
|
-
const sources = opts.db.select().from(trafficSources).where(
|
|
73556
|
+
const sources = opts.db.select().from(trafficSources).where(and89(eq105(trafficSources.projectId, project.id), isNull15(trafficSources.archivedAt))).all();
|
|
73371
73557
|
for (const source of sources) {
|
|
73372
73558
|
const row = makeRow("traffic", source.displayName, RunKinds["traffic-sync"], readSyncRunState(opts.db, project.id, RunKinds["traffic-sync"], { sourceId: source.id }), null, null, source.id, ["traffic.source.sync-lag", "traffic.source.recent-data"]);
|
|
73373
73559
|
const lag = source.lastSyncedAt ? Date.now() - Date.parse(source.lastSyncedAt) : Number.POSITIVE_INFINITY;
|
|
@@ -73379,7 +73565,7 @@ function projectRows(opts, project) {
|
|
|
73379
73565
|
}
|
|
73380
73566
|
function projectStatus(opts, project) {
|
|
73381
73567
|
const integrations = projectRows(opts, project);
|
|
73382
|
-
const healthRow = opts.db.select().from(doctorHealthState).where(
|
|
73568
|
+
const healthRow = opts.db.select().from(doctorHealthState).where(eq105(doctorHealthState.projectId, project.id)).limit(1).get();
|
|
73383
73569
|
const health = healthRow ? { status: healthRow.status, code: healthRow.code, summary: healthRow.summary, checkedAt: healthRow.checkedAt, notifiedAt: healthRow.notifiedAt } : null;
|
|
73384
73570
|
const rank = { ok: 0, "never-synced": 1, warning: 2, stale: 3, failing: 4 };
|
|
73385
73571
|
const healthState = health?.status === "fail" ? "failing" : health?.status === "warn" ? "warning" : "ok";
|
|
@@ -73399,7 +73585,7 @@ async function statusRoutes(app, opts) {
|
|
|
73399
73585
|
if (request.query.problems === "true") result.projects = result.projects.map((item2) => ({ ...item2, integrations: item2.integrations.filter((row) => row.state !== "ok") }));
|
|
73400
73586
|
return instanceStatusDtoSchema.parse(result);
|
|
73401
73587
|
}
|
|
73402
|
-
const projectsRows = (scopedId ? opts.db.select().from(projects).where(
|
|
73588
|
+
const projectsRows = (scopedId ? opts.db.select().from(projects).where(eq105(projects.id, scopedId)) : opts.db.select().from(projects)).all();
|
|
73403
73589
|
const output = { generatedAt: (/* @__PURE__ */ new Date()).toISOString(), ...opts.getMcpHealth ? { mcp: opts.getMcpHealth() } : {}, summary: { projects: projectsRows.length, ok: 0, failing: 0, warning: 0, stale: 0, neverSynced: 0 }, projects: projectsRows.map((project) => projectStatus(opts, project)) };
|
|
73404
73590
|
for (const project of output.projects) {
|
|
73405
73591
|
const key = project.state === "never-synced" ? "neverSynced" : project.state;
|
|
@@ -73412,7 +73598,7 @@ async function statusRoutes(app, opts) {
|
|
|
73412
73598
|
|
|
73413
73599
|
// ../api-routes/src/discovery/routes.ts
|
|
73414
73600
|
import crypto57 from "crypto";
|
|
73415
|
-
import { and as
|
|
73601
|
+
import { and as and90, desc as desc49, eq as eq106, gte as gte30, inArray as inArray48, isNull as isNull16, or as or20 } from "drizzle-orm";
|
|
73416
73602
|
var MAX_INFLIGHT_DISCOVERY_AGE_MS = 2 * 60 * 60 * 1e3;
|
|
73417
73603
|
async function discoveryRoutes(app, opts) {
|
|
73418
73604
|
app.post("/projects/:name/discover/run", async (request, reply) => {
|
|
@@ -73446,13 +73632,13 @@ async function discoveryRoutes(app, opts) {
|
|
|
73446
73632
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
73447
73633
|
const ageFloorIso = new Date(Date.now() - MAX_INFLIGHT_DISCOVERY_AGE_MS).toISOString();
|
|
73448
73634
|
const decision = app.db.transaction((tx) => {
|
|
73449
|
-
const existing = tx.select({ id: discoverySessions.id, runId: discoverySessions.runId }).from(discoverySessions).where(
|
|
73450
|
-
|
|
73451
|
-
|
|
73635
|
+
const existing = tx.select({ id: discoverySessions.id, runId: discoverySessions.runId }).from(discoverySessions).where(and90(
|
|
73636
|
+
eq106(discoverySessions.projectId, project.id),
|
|
73637
|
+
eq106(discoverySessions.icpDescription, icpDescription),
|
|
73452
73638
|
// Buyer is part of session identity: it changes the seed prompt's
|
|
73453
73639
|
// semantics, so a request with a different (or no) buyer must start
|
|
73454
73640
|
// its own session, never adopt another buyer's probes.
|
|
73455
|
-
parsed.data.buyerDescription == null ? isNull16(discoverySessions.buyerDescription) :
|
|
73641
|
+
parsed.data.buyerDescription == null ? isNull16(discoverySessions.buyerDescription) : eq106(discoverySessions.buyerDescription, parsed.data.buyerDescription),
|
|
73456
73642
|
// Locations are identity too: a different service-area subset seeds
|
|
73457
73643
|
// and probes a different geo, so it must never reuse another geo's
|
|
73458
73644
|
// session. resolveLocations is deterministic (project-config order),
|
|
@@ -73462,18 +73648,18 @@ async function discoveryRoutes(app, opts) {
|
|
|
73462
73648
|
// locations a NULL row's subset is unknowable, so it conservatively
|
|
73463
73649
|
// never reuses — a one-time, bounded (2h window) non-reuse after
|
|
73464
73650
|
// upgrade, never a wrong reuse.
|
|
73465
|
-
locations.length === 0 ? or20(isNull16(discoverySessions.locations),
|
|
73651
|
+
locations.length === 0 ? or20(isNull16(discoverySessions.locations), eq106(discoverySessions.locations, locations)) : eq106(discoverySessions.locations, locations),
|
|
73466
73652
|
// Seed provider set is identity: a different phrasing distribution
|
|
73467
73653
|
// must never reuse another set's session. Null (= Gemini-only
|
|
73468
73654
|
// default, including explicit ['gemini']) matches legacy rows.
|
|
73469
|
-
seedProviders == null ? isNull16(discoverySessions.seedProviders) :
|
|
73470
|
-
|
|
73655
|
+
seedProviders == null ? isNull16(discoverySessions.seedProviders) : eq106(discoverySessions.seedProviders, seedProviders),
|
|
73656
|
+
inArray48(discoverySessions.status, [
|
|
73471
73657
|
DiscoverySessionStatuses.queued,
|
|
73472
73658
|
DiscoverySessionStatuses.seeding,
|
|
73473
73659
|
DiscoverySessionStatuses.probing
|
|
73474
73660
|
]),
|
|
73475
73661
|
gte30(discoverySessions.createdAt, ageFloorIso)
|
|
73476
|
-
)).orderBy(
|
|
73662
|
+
)).orderBy(desc49(discoverySessions.createdAt)).get();
|
|
73477
73663
|
if (existing && existing.runId) {
|
|
73478
73664
|
return { reused: true, sessionId: existing.id, runId: existing.runId };
|
|
73479
73665
|
}
|
|
@@ -73542,7 +73728,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
73542
73728
|
const project = resolveProject(app.db, request.params.name);
|
|
73543
73729
|
const parsedLimit = parseInt(request.query.limit ?? "", 10);
|
|
73544
73730
|
const limit = Number.isNaN(parsedLimit) || parsedLimit <= 0 ? 50 : parsedLimit;
|
|
73545
|
-
const rows = app.db.select().from(discoverySessions).where(
|
|
73731
|
+
const rows = app.db.select().from(discoverySessions).where(eq106(discoverySessions.projectId, project.id)).orderBy(desc49(discoverySessions.createdAt)).limit(limit).all();
|
|
73546
73732
|
return reply.send(rows.map(serializeSession));
|
|
73547
73733
|
}
|
|
73548
73734
|
);
|
|
@@ -73550,11 +73736,11 @@ async function discoveryRoutes(app, opts) {
|
|
|
73550
73736
|
"/projects/:name/discover/sessions/:id",
|
|
73551
73737
|
async (request, reply) => {
|
|
73552
73738
|
const project = resolveProject(app.db, request.params.name);
|
|
73553
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
73739
|
+
const session = app.db.select().from(discoverySessions).where(eq106(discoverySessions.id, request.params.id)).get();
|
|
73554
73740
|
if (!session || session.projectId !== project.id) {
|
|
73555
73741
|
throw notFound("Discovery session", request.params.id);
|
|
73556
73742
|
}
|
|
73557
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
73743
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq106(discoveryProbes.sessionId, session.id)).all();
|
|
73558
73744
|
const detail = {
|
|
73559
73745
|
...serializeSession(session),
|
|
73560
73746
|
probes: probeRows.map(serializeProbe)
|
|
@@ -73566,7 +73752,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
73566
73752
|
"/projects/:name/discover/sessions/:id/harvest",
|
|
73567
73753
|
async (request, reply) => {
|
|
73568
73754
|
const project = resolveProject(app.db, request.params.name);
|
|
73569
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
73755
|
+
const session = app.db.select().from(discoverySessions).where(eq106(discoverySessions.id, request.params.id)).get();
|
|
73570
73756
|
if (!session || session.projectId !== project.id) {
|
|
73571
73757
|
throw notFound("Discovery session", request.params.id);
|
|
73572
73758
|
}
|
|
@@ -73574,7 +73760,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
73574
73760
|
const minProbeHits = Number.isNaN(parsedFloor) || parsedFloor < 1 ? 1 : parsedFloor;
|
|
73575
73761
|
const applyAnchor = request.query.anchor !== "false";
|
|
73576
73762
|
const provider = session.seedProvider ?? "gemini";
|
|
73577
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
73763
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq106(discoveryProbes.sessionId, session.id)).all();
|
|
73578
73764
|
const extract = opts.harvestSearchQueries;
|
|
73579
73765
|
const probesWithQueries = probeRows.map((row) => {
|
|
73580
73766
|
if (!extract || !row.rawResponse) return { searchQueries: [] };
|
|
@@ -73585,7 +73771,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
73585
73771
|
return { searchQueries: [] };
|
|
73586
73772
|
}
|
|
73587
73773
|
});
|
|
73588
|
-
const trackedQueries = app.db.select({ query: queries.query }).from(queries).where(
|
|
73774
|
+
const trackedQueries = app.db.select({ query: queries.query }).from(queries).where(eq106(queries.projectId, project.id)).all().map((r) => r.query);
|
|
73589
73775
|
const anchorTerms = buildHarvestAnchorTerms(
|
|
73590
73776
|
[session.icpDescription ?? "", ...trackedQueries],
|
|
73591
73777
|
effectiveDomains(project)
|
|
@@ -73632,11 +73818,11 @@ async function discoveryRoutes(app, opts) {
|
|
|
73632
73818
|
"/projects/:name/discover/sessions/:id/promote",
|
|
73633
73819
|
async (request, reply) => {
|
|
73634
73820
|
const project = resolveProject(app.db, request.params.name);
|
|
73635
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
73821
|
+
const session = app.db.select().from(discoverySessions).where(eq106(discoverySessions.id, request.params.id)).get();
|
|
73636
73822
|
if (!session || session.projectId !== project.id) {
|
|
73637
73823
|
throw notFound("Discovery session", request.params.id);
|
|
73638
73824
|
}
|
|
73639
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
73825
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq106(discoveryProbes.sessionId, session.id)).all();
|
|
73640
73826
|
const cited = /* @__PURE__ */ new Set();
|
|
73641
73827
|
const aspirational = /* @__PURE__ */ new Set();
|
|
73642
73828
|
const wasted = /* @__PURE__ */ new Set();
|
|
@@ -73671,7 +73857,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
73671
73857
|
);
|
|
73672
73858
|
app.post("/projects/:name/discover/sessions/:id/promote", async (request, reply) => {
|
|
73673
73859
|
const project = resolveProject(app.db, request.params.name);
|
|
73674
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
73860
|
+
const session = app.db.select().from(discoverySessions).where(eq106(discoverySessions.id, request.params.id)).get();
|
|
73675
73861
|
if (!session || session.projectId !== project.id) {
|
|
73676
73862
|
throw notFound("Discovery session", request.params.id);
|
|
73677
73863
|
}
|
|
@@ -73694,7 +73880,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
73694
73880
|
const bucketSet = new Set(buckets);
|
|
73695
73881
|
const includeCompetitors = parsed.data.includeCompetitors ?? true;
|
|
73696
73882
|
const competitorTypes = parsed.data.competitorTypes ?? DEFAULT_DISCOVERY_PROMOTE_COMPETITOR_TYPES;
|
|
73697
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
73883
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq106(discoveryProbes.sessionId, session.id)).all();
|
|
73698
73884
|
const candidateQueries = /* @__PURE__ */ new Set();
|
|
73699
73885
|
for (const probe of probeRows) {
|
|
73700
73886
|
if (!probe.bucket) continue;
|
|
@@ -73702,7 +73888,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
73702
73888
|
if (bucket.success && bucketSet.has(bucket.data)) candidateQueries.add(probe.query);
|
|
73703
73889
|
}
|
|
73704
73890
|
const existingQueries = new Set(
|
|
73705
|
-
app.db.select({ query: queries.query }).from(queries).where(
|
|
73891
|
+
app.db.select({ query: queries.query }).from(queries).where(eq106(queries.projectId, project.id)).all().map((r) => r.query.toLowerCase())
|
|
73706
73892
|
);
|
|
73707
73893
|
const promotedQueries = [];
|
|
73708
73894
|
const skippedQueries = [];
|
|
@@ -73916,7 +74102,7 @@ function planCompetitorPromotion(input) {
|
|
|
73916
74102
|
|
|
73917
74103
|
// ../api-routes/src/discovery/orchestrate.ts
|
|
73918
74104
|
import crypto58 from "crypto";
|
|
73919
|
-
import { eq as
|
|
74105
|
+
import { eq as eq107 } from "drizzle-orm";
|
|
73920
74106
|
var DEFAULT_MAX_PROBES = 100;
|
|
73921
74107
|
var ABSOLUTE_MAX_PROBES = 500;
|
|
73922
74108
|
function classifyProbeBucket(input) {
|
|
@@ -74002,7 +74188,7 @@ async function executeDiscovery(opts) {
|
|
|
74002
74188
|
status: DiscoverySessionStatuses.seeding,
|
|
74003
74189
|
dedupThreshold,
|
|
74004
74190
|
startedAt
|
|
74005
|
-
}).where(
|
|
74191
|
+
}).where(eq107(discoverySessions.id, opts.sessionId)).run();
|
|
74006
74192
|
const seedResult = await opts.deps.seed({
|
|
74007
74193
|
project: opts.project,
|
|
74008
74194
|
icpDescription: opts.icpDescription,
|
|
@@ -74063,7 +74249,7 @@ async function executeDiscovery(opts) {
|
|
|
74063
74249
|
dedupBandPairFraction: dedupStats.bandPairFraction,
|
|
74064
74250
|
dedupPairsTotal: dedupStats.pairsTotal,
|
|
74065
74251
|
warning
|
|
74066
|
-
}).where(
|
|
74252
|
+
}).where(eq107(discoverySessions.id, opts.sessionId)).run();
|
|
74067
74253
|
const probeLocation = opts.locations?.[0];
|
|
74068
74254
|
const probeResults = await mapWithConcurrency(
|
|
74069
74255
|
probedCanonicals,
|
|
@@ -74117,7 +74303,7 @@ async function executeDiscovery(opts) {
|
|
|
74117
74303
|
wastedCount: buckets["wasted-surface"],
|
|
74118
74304
|
competitorMap,
|
|
74119
74305
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
74120
|
-
}).where(
|
|
74306
|
+
}).where(eq107(discoverySessions.id, opts.sessionId)).run();
|
|
74121
74307
|
upsertDomainClassifications(opts.db, opts.project.id, opts.sessionId, competitorMap);
|
|
74122
74308
|
return {
|
|
74123
74309
|
buckets,
|
|
@@ -74157,7 +74343,7 @@ function markSessionFailed(db, sessionId, error) {
|
|
|
74157
74343
|
status: DiscoverySessionStatuses.failed,
|
|
74158
74344
|
error,
|
|
74159
74345
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
74160
|
-
}).where(
|
|
74346
|
+
}).where(eq107(discoverySessions.id, sessionId)).run();
|
|
74161
74347
|
}
|
|
74162
74348
|
function dedupeStrings(input) {
|
|
74163
74349
|
const seen2 = /* @__PURE__ */ new Set();
|
|
@@ -74175,12 +74361,12 @@ function dedupeStrings(input) {
|
|
|
74175
74361
|
|
|
74176
74362
|
// ../api-routes/src/technical-aeo.ts
|
|
74177
74363
|
import crypto59 from "crypto";
|
|
74178
|
-
import { and as
|
|
74364
|
+
import { and as and92, asc as asc26, count as count8, desc as desc50, eq as eq109, gte as gte31, inArray as inArray49, isNotNull as isNotNull10, isNull as isNull17, lt as lt17, or as or21, sql as sql43 } from "drizzle-orm";
|
|
74179
74365
|
import { alias } from "drizzle-orm/sqlite-core";
|
|
74180
74366
|
|
|
74181
74367
|
// ../api-routes/src/technical-aeo-issues.ts
|
|
74182
74368
|
import { createHash as createHash10 } from "crypto";
|
|
74183
|
-
import { and as
|
|
74369
|
+
import { and as and91, asc as asc25, eq as eq108, gt as gt10, sql as sql42 } from "drizzle-orm";
|
|
74184
74370
|
var ADVISORY_CODES = /* @__PURE__ */ new Set([
|
|
74185
74371
|
"entity-consistency.title.short",
|
|
74186
74372
|
"site.duplicate-title",
|
|
@@ -74325,12 +74511,12 @@ function readSiteAuditIssues(input) {
|
|
|
74325
74511
|
}
|
|
74326
74512
|
}
|
|
74327
74513
|
if (snapshot.detailsAvailable && snapshot.attemptId) {
|
|
74328
|
-
const scope =
|
|
74329
|
-
|
|
74330
|
-
|
|
74331
|
-
|
|
74514
|
+
const scope = and91(
|
|
74515
|
+
eq108(siteCrawlPages.projectId, project.id),
|
|
74516
|
+
eq108(siteCrawlPages.runId, snapshot.runId),
|
|
74517
|
+
eq108(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
74332
74518
|
);
|
|
74333
|
-
const size = db.select({ bytes:
|
|
74519
|
+
const size = db.select({ bytes: sql42`coalesce(sum(length(cast(${siteCrawlPages.auditFields} as blob)) + length(cast(${siteCrawlPages.url} as blob))), 0)` }).from(siteCrawlPages).where(scope).get()?.bytes ?? 0;
|
|
74334
74520
|
if (size + crossBytes > MAX_EVIDENCE_BYTES) {
|
|
74335
74521
|
throw validationError("Stored audit exceeds the 128 MiB issue-evidence read budget");
|
|
74336
74522
|
}
|
|
@@ -74341,7 +74527,7 @@ function readSiteAuditIssues(input) {
|
|
|
74341
74527
|
url: siteCrawlPages.url,
|
|
74342
74528
|
auditScore: siteCrawlPages.auditScore,
|
|
74343
74529
|
auditFields: siteCrawlPages.auditFields
|
|
74344
|
-
}).from(siteCrawlPages).where(
|
|
74530
|
+
}).from(siteCrawlPages).where(and91(scope, after === void 0 ? void 0 : gt10(siteCrawlPages.nodeKey, after))).orderBy(asc25(siteCrawlPages.nodeKey)).limit(200).all();
|
|
74345
74531
|
for (const page2 of pages) {
|
|
74346
74532
|
evidenceHash.update(JSON.stringify(page2));
|
|
74347
74533
|
const fields = record(page2.auditFields);
|
|
@@ -74478,10 +74664,10 @@ var FETCHED_SITE_CRAWL_STATES = /* @__PURE__ */ new Set([
|
|
|
74478
74664
|
var SURFACEABLE_STATUSES = [RunStatuses.completed, RunStatuses.partial];
|
|
74479
74665
|
function surfaceableSiteAuditFilters(projectId) {
|
|
74480
74666
|
return [
|
|
74481
|
-
|
|
74482
|
-
|
|
74483
|
-
|
|
74484
|
-
|
|
74667
|
+
eq109(siteAuditSnapshots.projectId, projectId),
|
|
74668
|
+
eq109(runs.projectId, projectId),
|
|
74669
|
+
eq109(runs.kind, RunKinds["site-audit"]),
|
|
74670
|
+
inArray49(runs.status, SURFACEABLE_STATUSES),
|
|
74485
74671
|
notProbeRun()
|
|
74486
74672
|
];
|
|
74487
74673
|
}
|
|
@@ -74721,13 +74907,13 @@ function parseLinkKind(value) {
|
|
|
74721
74907
|
return allowed.data;
|
|
74722
74908
|
}
|
|
74723
74909
|
function linkKindFilter(linkKind) {
|
|
74724
|
-
if (linkKind === SiteHealthLinkKinds.content) return
|
|
74725
|
-
if (linkKind === SiteHealthLinkKinds.template) return
|
|
74910
|
+
if (linkKind === SiteHealthLinkKinds.content) return eq109(siteCrawlEdges.isTemplate, false);
|
|
74911
|
+
if (linkKind === SiteHealthLinkKinds.template) return eq109(siteCrawlEdges.isTemplate, true);
|
|
74726
74912
|
return void 0;
|
|
74727
74913
|
}
|
|
74728
74914
|
var graphSourceNode = alias(siteCrawlGraphNodes, "site_crawl_graph_source_node");
|
|
74729
74915
|
var graphTargetNode = alias(siteCrawlGraphNodes, "site_crawl_graph_target_node");
|
|
74730
|
-
var PAGE_CHANGE_PREDICATE =
|
|
74916
|
+
var PAGE_CHANGE_PREDICATE = sql43`(
|
|
74731
74917
|
current.url IS NOT previous.url
|
|
74732
74918
|
OR current.final_url IS NOT previous.final_url
|
|
74733
74919
|
OR current.path IS NOT previous.path
|
|
@@ -74749,7 +74935,7 @@ var PAGE_CHANGE_PREDICATE = sql42`(
|
|
|
74749
74935
|
OR current.link_score_raw IS NOT previous.link_score_raw
|
|
74750
74936
|
OR current.link_score_normalized IS NOT previous.link_score_normalized
|
|
74751
74937
|
)`;
|
|
74752
|
-
var LINK_CHANGE_PREDICATE =
|
|
74938
|
+
var LINK_CHANGE_PREDICATE = sql43`(
|
|
74753
74939
|
current.source_node_key IS NOT previous.source_node_key
|
|
74754
74940
|
OR current.source_url IS NOT previous.source_url
|
|
74755
74941
|
OR current.target_node_key IS NOT previous.target_node_key
|
|
@@ -74826,20 +75012,20 @@ function changedFields(before, after, fields) {
|
|
|
74826
75012
|
}
|
|
74827
75013
|
async function technicalAeoRoutes(app, opts) {
|
|
74828
75014
|
const visibleCrawlFilters = (projectId) => [
|
|
74829
|
-
|
|
74830
|
-
|
|
74831
|
-
|
|
74832
|
-
|
|
75015
|
+
eq109(siteCrawlSnapshots.projectId, projectId),
|
|
75016
|
+
eq109(runs.projectId, projectId),
|
|
75017
|
+
eq109(runs.kind, RunKinds["site-audit"]),
|
|
75018
|
+
inArray49(runs.status, SURFACEABLE_STATUSES),
|
|
74833
75019
|
notProbeRun()
|
|
74834
75020
|
];
|
|
74835
75021
|
const availableScanDatesFor = (projectId, requestedDate) => {
|
|
74836
75022
|
if (!requestedDate) return void 0;
|
|
74837
|
-
const date =
|
|
75023
|
+
const date = sql43`substr(${siteCrawlSnapshots.createdAt}, 1, 10)`;
|
|
74838
75024
|
const filters = visibleCrawlFilters(projectId);
|
|
74839
|
-
const allDates =
|
|
74840
|
-
const matchingDates =
|
|
74841
|
-
const readDates = (where) => app.db.select({ date }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
74842
|
-
const countDates = (where) => app.db.select({ value:
|
|
75025
|
+
const allDates = and92(...filters);
|
|
75026
|
+
const matchingDates = and92(...filters, eq109(sql43`substr(${siteCrawlSnapshots.createdAt}, 6, 5)`, requestedDate.slice(5)));
|
|
75027
|
+
const readDates = (where) => app.db.select({ date }).from(siteCrawlSnapshots).innerJoin(runs, eq109(siteCrawlSnapshots.runId, runs.id)).where(where).groupBy(date).orderBy(desc50(date)).limit(10).all().map((row) => row.date);
|
|
75028
|
+
const countDates = (where) => app.db.select({ value: sql43`count(distinct ${date})` }).from(siteCrawlSnapshots).innerJoin(runs, eq109(siteCrawlSnapshots.runId, runs.id)).where(where).get()?.value ?? 0;
|
|
74843
75029
|
return {
|
|
74844
75030
|
recentDates: readDates(allDates),
|
|
74845
75031
|
totalDates: countDates(allDates),
|
|
@@ -74855,34 +75041,34 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
74855
75041
|
nextDate2.setUTCDate(nextDate2.getUTCDate() + 1);
|
|
74856
75042
|
filters.push(gte31(siteCrawlSnapshots.createdAt, requestedDate), lt17(siteCrawlSnapshots.createdAt, nextDate2.toISOString()));
|
|
74857
75043
|
}
|
|
74858
|
-
const selectionFilters = scope.scoredOnly ? [...filters,
|
|
74859
|
-
const selected = app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
74860
|
-
|
|
74861
|
-
|
|
74862
|
-
|
|
74863
|
-
|
|
74864
|
-
|
|
75044
|
+
const selectionFilters = scope.scoredOnly ? [...filters, sql43`exists (select 1 from ${siteAuditSnapshots} where ${siteAuditSnapshots.projectId} = ${siteCrawlSnapshots.projectId} and ${siteAuditSnapshots.runId} = ${siteCrawlSnapshots.runId})`] : filters;
|
|
75045
|
+
const selected = app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq109(siteCrawlSnapshots.runId, runs.id)).where(and92(...selectionFilters, runId ? eq109(siteCrawlSnapshots.runId, runId) : void 0)).orderBy(
|
|
75046
|
+
desc50(sql43`substr(${siteCrawlSnapshots.createdAt}, 1, 10)`),
|
|
75047
|
+
desc50(siteCrawlSnapshots.complete),
|
|
75048
|
+
desc50(siteCrawlSnapshots.pagesFetched),
|
|
75049
|
+
desc50(siteCrawlSnapshots.createdAt),
|
|
75050
|
+
desc50(siteCrawlSnapshots.runId)
|
|
74865
75051
|
).limit(1).get();
|
|
74866
75052
|
if (!selected) return void 0;
|
|
74867
75053
|
const date = selected.snapshot.createdAt.slice(0, 10);
|
|
74868
75054
|
const nextDate = /* @__PURE__ */ new Date(`${date}T00:00:00.000Z`);
|
|
74869
75055
|
nextDate.setUTCDate(nextDate.getUTCDate() + 1);
|
|
74870
|
-
const sameDate =
|
|
75056
|
+
const sameDate = and92(
|
|
74871
75057
|
...filters,
|
|
74872
75058
|
gte31(siteCrawlSnapshots.createdAt, date),
|
|
74873
75059
|
lt17(siteCrawlSnapshots.createdAt, nextDate.toISOString())
|
|
74874
75060
|
);
|
|
74875
|
-
const sameDateRunCount = app.db.select({ value: count8() }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
75061
|
+
const sameDateRunCount = app.db.select({ value: count8() }).from(siteCrawlSnapshots).innerJoin(runs, eq109(siteCrawlSnapshots.runId, runs.id)).where(sameDate).get()?.value ?? 0;
|
|
74876
75062
|
const candidates = app.db.select({
|
|
74877
75063
|
runId: siteCrawlSnapshots.runId,
|
|
74878
75064
|
createdAt: siteCrawlSnapshots.createdAt,
|
|
74879
75065
|
complete: siteCrawlSnapshots.complete,
|
|
74880
75066
|
pages: siteCrawlSnapshots.pagesFetched
|
|
74881
|
-
}).from(siteCrawlSnapshots).innerJoin(runs,
|
|
74882
|
-
|
|
74883
|
-
|
|
74884
|
-
|
|
74885
|
-
|
|
75067
|
+
}).from(siteCrawlSnapshots).innerJoin(runs, eq109(siteCrawlSnapshots.runId, runs.id)).where(sameDate).orderBy(
|
|
75068
|
+
desc50(siteCrawlSnapshots.complete),
|
|
75069
|
+
desc50(siteCrawlSnapshots.pagesFetched),
|
|
75070
|
+
desc50(siteCrawlSnapshots.createdAt),
|
|
75071
|
+
desc50(siteCrawlSnapshots.runId)
|
|
74886
75072
|
).limit(10).all();
|
|
74887
75073
|
const runSelection = {
|
|
74888
75074
|
reason: runId ? "explicit-run" : scope.scoredOnly ? "latest-scored-scan" : requestedDate ? selected.snapshot.complete ? "requested-date-complete" : "requested-date-most-pages" : selected.snapshot.complete ? "latest-date-complete" : "latest-date-most-pages",
|
|
@@ -74899,13 +75085,13 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
74899
75085
|
...surfaceableSiteAuditFilters(projectId),
|
|
74900
75086
|
...recordedBefore ? [lt17(siteAuditSnapshots.createdAt, recordedBefore)] : []
|
|
74901
75087
|
];
|
|
74902
|
-
const complete =
|
|
74903
|
-
const selectAudit = (pinnedRunId) => app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs,
|
|
74904
|
-
|
|
74905
|
-
|
|
74906
|
-
|
|
74907
|
-
|
|
74908
|
-
|
|
75088
|
+
const complete = sql43`case when ${runs.status} = ${RunStatuses.completed} then 1 else 0 end`;
|
|
75089
|
+
const selectAudit = (pinnedRunId) => app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs, eq109(siteAuditSnapshots.runId, runs.id)).where(and92(...filters, pinnedRunId ? eq109(siteAuditSnapshots.runId, pinnedRunId) : void 0)).orderBy(
|
|
75090
|
+
desc50(sql43`substr(${siteAuditSnapshots.createdAt}, 1, 10)`),
|
|
75091
|
+
desc50(complete),
|
|
75092
|
+
desc50(siteAuditSnapshots.pagesAudited),
|
|
75093
|
+
desc50(siteAuditSnapshots.createdAt),
|
|
75094
|
+
desc50(siteAuditSnapshots.runId)
|
|
74909
75095
|
).limit(1).get();
|
|
74910
75096
|
const auditOfCrawl = (target) => {
|
|
74911
75097
|
const selected2 = target ? selectAudit(target.snapshot.runId) : void 0;
|
|
@@ -74921,22 +75107,22 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
74921
75107
|
const date = selected.snap.createdAt.slice(0, 10);
|
|
74922
75108
|
const nextDate = /* @__PURE__ */ new Date(`${date}T00:00:00.000Z`);
|
|
74923
75109
|
nextDate.setUTCDate(nextDate.getUTCDate() + 1);
|
|
74924
|
-
const sameDate =
|
|
75110
|
+
const sameDate = and92(
|
|
74925
75111
|
...filters,
|
|
74926
75112
|
gte31(siteAuditSnapshots.createdAt, date),
|
|
74927
75113
|
lt17(siteAuditSnapshots.createdAt, nextDate.toISOString())
|
|
74928
75114
|
);
|
|
74929
|
-
const sameDateRunCount = app.db.select({ value: count8() }).from(siteAuditSnapshots).innerJoin(runs,
|
|
75115
|
+
const sameDateRunCount = app.db.select({ value: count8() }).from(siteAuditSnapshots).innerJoin(runs, eq109(siteAuditSnapshots.runId, runs.id)).where(sameDate).get()?.value ?? 0;
|
|
74930
75116
|
const candidates = app.db.select({
|
|
74931
75117
|
runId: siteAuditSnapshots.runId,
|
|
74932
75118
|
createdAt: siteAuditSnapshots.createdAt,
|
|
74933
75119
|
complete,
|
|
74934
75120
|
pages: siteAuditSnapshots.pagesAudited
|
|
74935
|
-
}).from(siteAuditSnapshots).innerJoin(runs,
|
|
74936
|
-
|
|
74937
|
-
|
|
74938
|
-
|
|
74939
|
-
|
|
75121
|
+
}).from(siteAuditSnapshots).innerJoin(runs, eq109(siteAuditSnapshots.runId, runs.id)).where(sameDate).orderBy(
|
|
75122
|
+
desc50(complete),
|
|
75123
|
+
desc50(siteAuditSnapshots.pagesAudited),
|
|
75124
|
+
desc50(siteAuditSnapshots.createdAt),
|
|
75125
|
+
desc50(siteAuditSnapshots.runId)
|
|
74940
75126
|
).limit(10).all().map((candidate) => ({ ...candidate, complete: Boolean(candidate.complete) }));
|
|
74941
75127
|
const runSelection = {
|
|
74942
75128
|
// A crawl exists here only when none stored a scorecard: this score is a fallback.
|
|
@@ -74951,9 +75137,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
74951
75137
|
};
|
|
74952
75138
|
const detailScopeFor = (projectId, snapshot) => snapshot.detailsAvailable && snapshot.attemptId ? { projectId, runId: snapshot.runId, attemptId: snapshot.attemptId } : null;
|
|
74953
75139
|
const crawlReasonGroups = (where) => {
|
|
74954
|
-
const canonicalOther =
|
|
75140
|
+
const canonicalOther = sql43`case when ${siteCrawlPages.canonicalNodeKey} is not null
|
|
74955
75141
|
and ${siteCrawlPages.canonicalNodeKey} <> ${siteCrawlPages.nodeKey} then 1 else 0 end`;
|
|
74956
|
-
const canonicalReason =
|
|
75142
|
+
const canonicalReason = sql43`exists (select 1 from json_each(${siteCrawlPages.indexabilityReasons})
|
|
74957
75143
|
where value = ${SiteCrawlIndexabilityReasons.canonicalToOther})`;
|
|
74958
75144
|
const groups = app.db.select({
|
|
74959
75145
|
fetchState: siteCrawlPages.fetchState,
|
|
@@ -74962,7 +75148,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
74962
75148
|
canonicalOther,
|
|
74963
75149
|
canonicalReason,
|
|
74964
75150
|
pages: count8(),
|
|
74965
|
-
exampleUrl:
|
|
75151
|
+
exampleUrl: sql43`min(${siteCrawlPages.url})`
|
|
74966
75152
|
}).from(siteCrawlPages).where(where).groupBy(
|
|
74967
75153
|
siteCrawlPages.fetchState,
|
|
74968
75154
|
siteCrawlPages.indexabilityState,
|
|
@@ -74998,19 +75184,19 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
74998
75184
|
};
|
|
74999
75185
|
const inventorySummaryFor = (scope) => {
|
|
75000
75186
|
if (!scope) return null;
|
|
75001
|
-
const inScope =
|
|
75002
|
-
|
|
75003
|
-
|
|
75004
|
-
|
|
75187
|
+
const inScope = and92(
|
|
75188
|
+
eq109(siteCrawlPages.projectId, scope.projectId),
|
|
75189
|
+
eq109(siteCrawlPages.runId, scope.runId),
|
|
75190
|
+
eq109(siteCrawlPages.attemptId, scope.attemptId)
|
|
75005
75191
|
);
|
|
75006
75192
|
const groups = crawlReasonGroups(inScope);
|
|
75007
75193
|
const total = groups.reduce((sum, group) => sum + group.pages, 0);
|
|
75008
75194
|
const eligible = groups.reduce((sum, group) => sum + (group.inventoryEligible ? group.pages : 0), 0);
|
|
75009
|
-
const urlVersions = app.db.select({ value: count8() }).from(siteCrawlPages).where(
|
|
75010
|
-
const urlVersionsNotFetched = app.db.select({ value: siteCrawlAttempts.urlVersionsSkipped }).from(siteCrawlAttempts).where(
|
|
75011
|
-
|
|
75012
|
-
|
|
75013
|
-
|
|
75195
|
+
const urlVersions = app.db.select({ value: count8() }).from(siteCrawlPages).where(and92(inScope, isNotNull10(siteCrawlPages.urlVersionOf))).get()?.value ?? 0;
|
|
75196
|
+
const urlVersionsNotFetched = app.db.select({ value: siteCrawlAttempts.urlVersionsSkipped }).from(siteCrawlAttempts).where(and92(
|
|
75197
|
+
eq109(siteCrawlAttempts.projectId, scope.projectId),
|
|
75198
|
+
eq109(siteCrawlAttempts.runId, scope.runId),
|
|
75199
|
+
eq109(siteCrawlAttempts.id, scope.attemptId)
|
|
75014
75200
|
)).get()?.value ?? 0;
|
|
75015
75201
|
return {
|
|
75016
75202
|
scope: "selected-snapshot",
|
|
@@ -75030,34 +75216,34 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75030
75216
|
const NO_CRAWL_COMPLETENESS = { complete: false, termination: null };
|
|
75031
75217
|
const pageInScope = (scope, selector) => {
|
|
75032
75218
|
if (!selector.nodeKey && !selector.url) return null;
|
|
75033
|
-
return app.db.select().from(siteCrawlPages).where(
|
|
75034
|
-
|
|
75035
|
-
|
|
75036
|
-
|
|
75037
|
-
selector.nodeKey ?
|
|
75038
|
-
)).orderBy(
|
|
75039
|
-
};
|
|
75040
|
-
const rootPageInScope = (scope, rootUrl) => pageInScope(scope, { url: rootUrl }) ?? app.db.select().from(siteCrawlPages).where(
|
|
75041
|
-
|
|
75042
|
-
|
|
75043
|
-
|
|
75044
|
-
|
|
75045
|
-
)).orderBy(
|
|
75046
|
-
const isSurfaceableAuditRun = (projectId, runId) => Boolean(app.db.select({ id: runs.id }).from(runs).where(
|
|
75047
|
-
|
|
75048
|
-
|
|
75049
|
-
|
|
75050
|
-
|
|
75219
|
+
return app.db.select().from(siteCrawlPages).where(and92(
|
|
75220
|
+
eq109(siteCrawlPages.projectId, scope.projectId),
|
|
75221
|
+
eq109(siteCrawlPages.runId, scope.runId),
|
|
75222
|
+
eq109(siteCrawlPages.attemptId, scope.attemptId),
|
|
75223
|
+
selector.nodeKey ? eq109(siteCrawlPages.nodeKey, selector.nodeKey) : eq109(siteCrawlPages.url, selector.url)
|
|
75224
|
+
)).orderBy(asc26(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
75225
|
+
};
|
|
75226
|
+
const rootPageInScope = (scope, rootUrl) => pageInScope(scope, { url: rootUrl }) ?? app.db.select().from(siteCrawlPages).where(and92(
|
|
75227
|
+
eq109(siteCrawlPages.projectId, scope.projectId),
|
|
75228
|
+
eq109(siteCrawlPages.runId, scope.runId),
|
|
75229
|
+
eq109(siteCrawlPages.attemptId, scope.attemptId),
|
|
75230
|
+
eq109(siteCrawlPages.depth, 0)
|
|
75231
|
+
)).orderBy(asc26(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
75232
|
+
const isSurfaceableAuditRun = (projectId, runId) => Boolean(app.db.select({ id: runs.id }).from(runs).where(and92(
|
|
75233
|
+
eq109(runs.id, runId),
|
|
75234
|
+
eq109(runs.projectId, projectId),
|
|
75235
|
+
eq109(runs.kind, RunKinds["site-audit"]),
|
|
75236
|
+
inArray49(runs.status, SURFACEABLE_STATUSES),
|
|
75051
75237
|
notProbeRun()
|
|
75052
75238
|
)).limit(1).get());
|
|
75053
75239
|
const assertKnownAuditRun = (projectId, runId) => {
|
|
75054
75240
|
if (runId && !isSurfaceableAuditRun(projectId, runId)) throw notFound("Site crawl run", runId);
|
|
75055
75241
|
};
|
|
75056
|
-
const hasLegacyAudit = (projectId) => Boolean(app.db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).innerJoin(runs,
|
|
75057
|
-
|
|
75058
|
-
|
|
75059
|
-
|
|
75060
|
-
|
|
75242
|
+
const hasLegacyAudit = (projectId) => Boolean(app.db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).innerJoin(runs, eq109(siteAuditSnapshots.runId, runs.id)).where(and92(
|
|
75243
|
+
eq109(siteAuditSnapshots.projectId, projectId),
|
|
75244
|
+
eq109(runs.projectId, projectId),
|
|
75245
|
+
eq109(runs.kind, RunKinds["site-audit"]),
|
|
75246
|
+
inArray49(runs.status, SURFACEABLE_STATUSES),
|
|
75061
75247
|
notProbeRun()
|
|
75062
75248
|
)).limit(1).get());
|
|
75063
75249
|
const emptyCrawlSummary = (projectName, legacyAuditAvailable) => ({
|
|
@@ -75158,14 +75344,14 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75158
75344
|
};
|
|
75159
75345
|
}
|
|
75160
75346
|
const statusFilter = request.query.status === "success" || request.query.status === "error" ? request.query.status : null;
|
|
75161
|
-
const conds = [
|
|
75162
|
-
if (statusFilter) conds.push(
|
|
75163
|
-
const where =
|
|
75347
|
+
const conds = [eq109(siteAuditPages.projectId, project.id), eq109(siteAuditPages.runId, latest.snap.runId)];
|
|
75348
|
+
if (statusFilter) conds.push(eq109(siteAuditPages.status, statusFilter));
|
|
75349
|
+
const where = and92(...conds);
|
|
75164
75350
|
const totalRow = app.db.select({ value: count8() }).from(siteAuditPages).where(where).get();
|
|
75165
75351
|
const total = totalRow?.value ?? 0;
|
|
75166
75352
|
const limit = parsePositiveInt(request.query.limit, 100, 500);
|
|
75167
75353
|
const offset = parsePositiveInt(request.query.offset, 0, Number.MAX_SAFE_INTEGER);
|
|
75168
|
-
const orderBy = request.query.sort === "score-desc" ?
|
|
75354
|
+
const orderBy = request.query.sort === "score-desc" ? desc50(siteAuditPages.overallScore) : request.query.sort === "url" ? asc26(siteAuditPages.url) : asc26(siteAuditPages.overallScore);
|
|
75169
75355
|
const rows = app.db.select().from(siteAuditPages).where(where).orderBy(orderBy).limit(limit).offset(offset).all();
|
|
75170
75356
|
const pages = rows.map((row) => ({
|
|
75171
75357
|
url: row.url,
|
|
@@ -75184,7 +75370,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75184
75370
|
auditedAt: siteAuditSnapshots.auditedAt,
|
|
75185
75371
|
aggregateScore: siteAuditSnapshots.aggregateScore,
|
|
75186
75372
|
pagesAudited: siteAuditSnapshots.pagesAudited
|
|
75187
|
-
}).from(siteAuditSnapshots).innerJoin(runs,
|
|
75373
|
+
}).from(siteAuditSnapshots).innerJoin(runs, eq109(siteAuditSnapshots.runId, runs.id)).where(and92(...surfaceableSiteAuditFilters(project.id))).orderBy(desc50(siteAuditSnapshots.createdAt)).limit(limit).all();
|
|
75188
75374
|
return { project: project.name, points: rows.reverse() };
|
|
75189
75375
|
});
|
|
75190
75376
|
app.get("/projects/:name/technical-aeo/crawl", async (request) => {
|
|
@@ -75284,10 +75470,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75284
75470
|
{ projectId: project.id, runId: snapshot.runId, attemptId: snapshot.attemptId },
|
|
75285
75471
|
snapshot.rootUrl
|
|
75286
75472
|
)?.nodeKey ?? null;
|
|
75287
|
-
const persistedLayout = app.db.select().from(siteCrawlGraphLayouts).where(
|
|
75288
|
-
|
|
75289
|
-
|
|
75290
|
-
|
|
75473
|
+
const persistedLayout = app.db.select().from(siteCrawlGraphLayouts).where(and92(
|
|
75474
|
+
eq109(siteCrawlGraphLayouts.projectId, project.id),
|
|
75475
|
+
eq109(siteCrawlGraphLayouts.runId, snapshot.runId),
|
|
75476
|
+
eq109(siteCrawlGraphLayouts.attemptId, snapshot.attemptId)
|
|
75291
75477
|
)).limit(1).get();
|
|
75292
75478
|
if (!persistedLayout) {
|
|
75293
75479
|
return {
|
|
@@ -75340,9 +75526,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75340
75526
|
const maxNodes = parseBoundedLimit(request.query.maxNodes, SITE_CRAWL_GRAPH_DEFAULT_MAX_NODES, SITE_CRAWL_GRAPH_MAX_NODES);
|
|
75341
75527
|
const maxEdges = parseBoundedLimit(request.query.maxEdges, SITE_CRAWL_GRAPH_DEFAULT_MAX_EDGES, SITE_CRAWL_GRAPH_MAX_EDGES);
|
|
75342
75528
|
const graphScope = [
|
|
75343
|
-
|
|
75344
|
-
|
|
75345
|
-
|
|
75529
|
+
eq109(siteCrawlGraphNodes.projectId, project.id),
|
|
75530
|
+
eq109(siteCrawlGraphNodes.runId, snapshot.runId),
|
|
75531
|
+
eq109(siteCrawlGraphNodes.attemptId, snapshot.attemptId)
|
|
75346
75532
|
];
|
|
75347
75533
|
const nodeRows = app.db.select({
|
|
75348
75534
|
nodeKey: siteCrawlPages.nodeKey,
|
|
@@ -75361,18 +75547,18 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75361
75547
|
linkScoreNormalized: siteCrawlPages.linkScoreNormalized,
|
|
75362
75548
|
x: siteCrawlGraphNodes.x,
|
|
75363
75549
|
y: siteCrawlGraphNodes.y
|
|
75364
|
-
}).from(siteCrawlGraphNodes).innerJoin(siteCrawlPages,
|
|
75365
|
-
|
|
75366
|
-
|
|
75367
|
-
|
|
75368
|
-
|
|
75369
|
-
)).where(
|
|
75550
|
+
}).from(siteCrawlGraphNodes).innerJoin(siteCrawlPages, and92(
|
|
75551
|
+
eq109(siteCrawlPages.projectId, siteCrawlGraphNodes.projectId),
|
|
75552
|
+
eq109(siteCrawlPages.runId, siteCrawlGraphNodes.runId),
|
|
75553
|
+
eq109(siteCrawlPages.attemptId, siteCrawlGraphNodes.attemptId),
|
|
75554
|
+
eq109(siteCrawlPages.nodeKey, siteCrawlGraphNodes.nodeKey)
|
|
75555
|
+
)).where(and92(...graphScope, lt17(siteCrawlGraphNodes.sampleRank, maxNodes))).orderBy(asc26(siteCrawlGraphNodes.sampleRank)).all();
|
|
75370
75556
|
const nodes = nodeRows.map(({ indexabilityReasons, canonicalNodeKey, ...row }) => ({
|
|
75371
75557
|
...row,
|
|
75372
75558
|
healthState: deriveSiteHealthState({ ...row, indexabilityReasons, canonicalNodeKey }),
|
|
75373
75559
|
healthReason: deriveSiteHealthReason({ ...row, indexabilityReasons, canonicalNodeKey })
|
|
75374
75560
|
}));
|
|
75375
|
-
const graphLinkKindFilter = linkKind === SiteHealthLinkKinds.all ? void 0 :
|
|
75561
|
+
const graphLinkKindFilter = linkKind === SiteHealthLinkKinds.all ? void 0 : eq109(siteCrawlGraphEdges.isTemplate, linkKind === SiteHealthLinkKinds.template);
|
|
75376
75562
|
const edges = app.db.select({
|
|
75377
75563
|
edgeKey: siteCrawlGraphEdges.edgeKey,
|
|
75378
75564
|
sourceNodeKey: siteCrawlGraphEdges.sourceNodeKey,
|
|
@@ -75380,25 +75566,25 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75380
75566
|
followable: siteCrawlGraphEdges.followable,
|
|
75381
75567
|
occurrences: siteCrawlGraphEdges.occurrences,
|
|
75382
75568
|
isTemplate: siteCrawlGraphEdges.isTemplate
|
|
75383
|
-
}).from(siteCrawlGraphEdges).innerJoin(graphSourceNode,
|
|
75384
|
-
|
|
75385
|
-
|
|
75386
|
-
|
|
75387
|
-
|
|
75569
|
+
}).from(siteCrawlGraphEdges).innerJoin(graphSourceNode, and92(
|
|
75570
|
+
eq109(graphSourceNode.projectId, siteCrawlGraphEdges.projectId),
|
|
75571
|
+
eq109(graphSourceNode.runId, siteCrawlGraphEdges.runId),
|
|
75572
|
+
eq109(graphSourceNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
75573
|
+
eq109(graphSourceNode.nodeKey, siteCrawlGraphEdges.sourceNodeKey),
|
|
75388
75574
|
lt17(graphSourceNode.sampleRank, maxNodes)
|
|
75389
|
-
)).innerJoin(graphTargetNode,
|
|
75390
|
-
|
|
75391
|
-
|
|
75392
|
-
|
|
75393
|
-
|
|
75575
|
+
)).innerJoin(graphTargetNode, and92(
|
|
75576
|
+
eq109(graphTargetNode.projectId, siteCrawlGraphEdges.projectId),
|
|
75577
|
+
eq109(graphTargetNode.runId, siteCrawlGraphEdges.runId),
|
|
75578
|
+
eq109(graphTargetNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
75579
|
+
eq109(graphTargetNode.nodeKey, siteCrawlGraphEdges.targetNodeKey),
|
|
75394
75580
|
lt17(graphTargetNode.sampleRank, maxNodes)
|
|
75395
|
-
)).where(
|
|
75396
|
-
|
|
75397
|
-
|
|
75398
|
-
|
|
75581
|
+
)).where(and92(
|
|
75582
|
+
eq109(siteCrawlGraphEdges.projectId, project.id),
|
|
75583
|
+
eq109(siteCrawlGraphEdges.runId, snapshot.runId),
|
|
75584
|
+
eq109(siteCrawlGraphEdges.attemptId, snapshot.attemptId),
|
|
75399
75585
|
lt17(siteCrawlGraphEdges.sampleRank, maxEdges),
|
|
75400
75586
|
graphLinkKindFilter
|
|
75401
|
-
)).orderBy(
|
|
75587
|
+
)).orderBy(asc26(siteCrawlGraphEdges.sampleRank)).all();
|
|
75402
75588
|
const totalNodes = persistedLayout.totalNodes;
|
|
75403
75589
|
const totalEdges = persistedLayout.totalEdges;
|
|
75404
75590
|
const totalTemplateEdges = persistedLayout.totalTemplateEdges;
|
|
@@ -75490,17 +75676,17 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75490
75676
|
let frontier = [focus.nodeKey];
|
|
75491
75677
|
let queryTruncated = false;
|
|
75492
75678
|
for (let distance = 0; distance < hops && frontier.length > 0; distance += 1) {
|
|
75493
|
-
const candidates = app.db.select().from(siteCrawlEdges).where(
|
|
75494
|
-
|
|
75495
|
-
|
|
75496
|
-
|
|
75497
|
-
|
|
75679
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and92(
|
|
75680
|
+
eq109(siteCrawlEdges.projectId, scope.projectId),
|
|
75681
|
+
eq109(siteCrawlEdges.runId, scope.runId),
|
|
75682
|
+
eq109(siteCrawlEdges.attemptId, scope.attemptId),
|
|
75683
|
+
eq109(siteCrawlEdges.internal, true),
|
|
75498
75684
|
isNotNull10(siteCrawlEdges.targetNodeKey),
|
|
75499
75685
|
or21(
|
|
75500
|
-
|
|
75501
|
-
|
|
75686
|
+
inArray49(siteCrawlEdges.sourceNodeKey, frontier),
|
|
75687
|
+
inArray49(siteCrawlEdges.targetNodeKey, frontier)
|
|
75502
75688
|
)
|
|
75503
|
-
)).orderBy(
|
|
75689
|
+
)).orderBy(asc26(siteCrawlEdges.edgeKey)).limit(maxEdges + maxNodes + 1).all();
|
|
75504
75690
|
if (candidates.length > maxEdges + maxNodes) queryTruncated = true;
|
|
75505
75691
|
const next = /* @__PURE__ */ new Set();
|
|
75506
75692
|
for (const edge of candidates.slice(0, maxEdges + maxNodes)) {
|
|
@@ -75531,22 +75717,22 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75531
75717
|
frontier = [...next].sort();
|
|
75532
75718
|
}
|
|
75533
75719
|
const nodeKeys = [...distances.keys()];
|
|
75534
|
-
const pageRows = nodeKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(
|
|
75535
|
-
|
|
75536
|
-
|
|
75537
|
-
|
|
75538
|
-
|
|
75720
|
+
const pageRows = nodeKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and92(
|
|
75721
|
+
eq109(siteCrawlPages.projectId, scope.projectId),
|
|
75722
|
+
eq109(siteCrawlPages.runId, scope.runId),
|
|
75723
|
+
eq109(siteCrawlPages.attemptId, scope.attemptId),
|
|
75724
|
+
inArray49(siteCrawlPages.nodeKey, nodeKeys)
|
|
75539
75725
|
)).all();
|
|
75540
75726
|
const persistedNodeKeys = new Set(pageRows.map((row) => row.nodeKey));
|
|
75541
75727
|
const outermostNodeKeys = pageRows.filter((row) => distances.get(row.nodeKey) === hops).map((row) => row.nodeKey);
|
|
75542
|
-
const omittedOutermostEdge = outermostNodeKeys.length === 0 ? void 0 : app.db.select({ edgeKey: siteCrawlEdges.edgeKey }).from(siteCrawlEdges).where(
|
|
75543
|
-
|
|
75544
|
-
|
|
75545
|
-
|
|
75546
|
-
|
|
75728
|
+
const omittedOutermostEdge = outermostNodeKeys.length === 0 ? void 0 : app.db.select({ edgeKey: siteCrawlEdges.edgeKey }).from(siteCrawlEdges).where(and92(
|
|
75729
|
+
eq109(siteCrawlEdges.projectId, scope.projectId),
|
|
75730
|
+
eq109(siteCrawlEdges.runId, scope.runId),
|
|
75731
|
+
eq109(siteCrawlEdges.attemptId, scope.attemptId),
|
|
75732
|
+
eq109(siteCrawlEdges.internal, true),
|
|
75547
75733
|
isNotNull10(siteCrawlEdges.targetNodeKey),
|
|
75548
|
-
|
|
75549
|
-
|
|
75734
|
+
inArray49(siteCrawlEdges.sourceNodeKey, outermostNodeKeys),
|
|
75735
|
+
inArray49(siteCrawlEdges.targetNodeKey, outermostNodeKeys)
|
|
75550
75736
|
)).limit(1).get();
|
|
75551
75737
|
if (omittedOutermostEdge && !seenEdgeKeys.has(omittedOutermostEdge.edgeKey)) {
|
|
75552
75738
|
omittedEdgeKeys.add(omittedOutermostEdge.edgeKey);
|
|
@@ -75639,23 +75825,23 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75639
75825
|
let found = fromPage.nodeKey === toPage.nodeKey;
|
|
75640
75826
|
let truncated = false;
|
|
75641
75827
|
for (let depth = 0; depth < maxDepth && frontier.length > 0 && !found; depth += 1) {
|
|
75642
|
-
const candidates = app.db.select().from(siteCrawlEdges).where(
|
|
75643
|
-
|
|
75644
|
-
|
|
75645
|
-
|
|
75646
|
-
|
|
75647
|
-
|
|
75648
|
-
|
|
75828
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and92(
|
|
75829
|
+
eq109(siteCrawlEdges.projectId, scope.projectId),
|
|
75830
|
+
eq109(siteCrawlEdges.runId, scope.runId),
|
|
75831
|
+
eq109(siteCrawlEdges.attemptId, scope.attemptId),
|
|
75832
|
+
eq109(siteCrawlEdges.internal, true),
|
|
75833
|
+
eq109(siteCrawlEdges.followable, true),
|
|
75834
|
+
eq109(siteCrawlEdges.relation, "anchor"),
|
|
75649
75835
|
isNotNull10(siteCrawlEdges.targetNodeKey),
|
|
75650
|
-
|
|
75651
|
-
)).orderBy(
|
|
75836
|
+
inArray49(siteCrawlEdges.sourceNodeKey, frontier)
|
|
75837
|
+
)).orderBy(asc26(siteCrawlEdges.edgeKey)).limit(SITE_HEALTH_PATH_MAX_VISITED_NODES + 1).all();
|
|
75652
75838
|
if (candidates.length > SITE_HEALTH_PATH_MAX_VISITED_NODES) truncated = true;
|
|
75653
75839
|
const candidateTargetKeys = [...new Set(candidates.map((edge) => edge.targetNodeKey).filter((nodeKey) => nodeKey != null))];
|
|
75654
|
-
const persistedTargetKeys = new Set(candidateTargetKeys.length === 0 ? [] : app.db.select({ nodeKey: siteCrawlPages.nodeKey }).from(siteCrawlPages).where(
|
|
75655
|
-
|
|
75656
|
-
|
|
75657
|
-
|
|
75658
|
-
|
|
75840
|
+
const persistedTargetKeys = new Set(candidateTargetKeys.length === 0 ? [] : app.db.select({ nodeKey: siteCrawlPages.nodeKey }).from(siteCrawlPages).where(and92(
|
|
75841
|
+
eq109(siteCrawlPages.projectId, scope.projectId),
|
|
75842
|
+
eq109(siteCrawlPages.runId, scope.runId),
|
|
75843
|
+
eq109(siteCrawlPages.attemptId, scope.attemptId),
|
|
75844
|
+
inArray49(siteCrawlPages.nodeKey, candidateTargetKeys)
|
|
75659
75845
|
)).all().map((row) => row.nodeKey));
|
|
75660
75846
|
const next = /* @__PURE__ */ new Set();
|
|
75661
75847
|
for (const edge of candidates.slice(0, SITE_HEALTH_PATH_MAX_VISITED_NODES)) {
|
|
@@ -75696,11 +75882,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75696
75882
|
pathEdges.unshift(step.edge);
|
|
75697
75883
|
pathKeys.unshift(step.nodeKey);
|
|
75698
75884
|
}
|
|
75699
|
-
const pathRows = app.db.select().from(siteCrawlPages).where(
|
|
75700
|
-
|
|
75701
|
-
|
|
75702
|
-
|
|
75703
|
-
|
|
75885
|
+
const pathRows = app.db.select().from(siteCrawlPages).where(and92(
|
|
75886
|
+
eq109(siteCrawlPages.projectId, scope.projectId),
|
|
75887
|
+
eq109(siteCrawlPages.runId, scope.runId),
|
|
75888
|
+
eq109(siteCrawlPages.attemptId, scope.attemptId),
|
|
75889
|
+
inArray49(siteCrawlPages.nodeKey, pathKeys)
|
|
75704
75890
|
)).all();
|
|
75705
75891
|
const pageByKey = new Map(pathRows.map((row) => [row.nodeKey, row]));
|
|
75706
75892
|
const pathDetection = templateDetectionOf(snapshot.templateDetection);
|
|
@@ -75728,25 +75914,25 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75728
75914
|
throw validationError("change must be all, added, removed, or changed");
|
|
75729
75915
|
}
|
|
75730
75916
|
const completeSnapshotFilters = [
|
|
75731
|
-
|
|
75732
|
-
|
|
75733
|
-
|
|
75734
|
-
|
|
75735
|
-
|
|
75917
|
+
eq109(siteCrawlSnapshots.projectId, project.id),
|
|
75918
|
+
eq109(runs.projectId, project.id),
|
|
75919
|
+
eq109(runs.kind, RunKinds["site-audit"]),
|
|
75920
|
+
eq109(runs.status, RunStatuses.completed),
|
|
75921
|
+
eq109(siteCrawlSnapshots.complete, true),
|
|
75736
75922
|
notProbeRun()
|
|
75737
75923
|
];
|
|
75738
|
-
const afterTarget = request.query.toRunId ? resolveCrawl(project.id, request.query.toRunId) : app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
75924
|
+
const afterTarget = request.query.toRunId ? resolveCrawl(project.id, request.query.toRunId) : app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq109(siteCrawlSnapshots.runId, runs.id)).where(and92(...completeSnapshotFilters)).orderBy(desc50(siteCrawlSnapshots.createdAt), desc50(siteCrawlSnapshots.runId)).limit(1).get();
|
|
75739
75925
|
if (request.query.toRunId && !afterTarget) throw notFound("Site crawl run", request.query.toRunId);
|
|
75740
|
-
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
75926
|
+
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq109(siteCrawlSnapshots.runId, runs.id)).where(and92(
|
|
75741
75927
|
...completeSnapshotFilters,
|
|
75742
75928
|
or21(
|
|
75743
75929
|
lt17(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
75744
|
-
|
|
75745
|
-
|
|
75930
|
+
and92(
|
|
75931
|
+
eq109(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
75746
75932
|
lt17(siteCrawlSnapshots.runId, afterTarget.snapshot.runId)
|
|
75747
75933
|
)
|
|
75748
75934
|
)
|
|
75749
|
-
)).orderBy(
|
|
75935
|
+
)).orderBy(desc50(siteCrawlSnapshots.createdAt), desc50(siteCrawlSnapshots.runId)).limit(1).get() : void 0;
|
|
75750
75936
|
if (request.query.fromRunId && !beforeTarget) throw notFound("Site crawl run", request.query.fromRunId);
|
|
75751
75937
|
if (!afterTarget) {
|
|
75752
75938
|
const latestCrawl = resolveCrawl(project.id);
|
|
@@ -75830,9 +76016,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75830
76016
|
const summary4 = { pages: zero2(), links: zero2() };
|
|
75831
76017
|
if (scopeFilter !== "links") {
|
|
75832
76018
|
if (includeAdded || includeChanged) {
|
|
75833
|
-
const addedExpression = includeAdded ?
|
|
75834
|
-
const changedExpression = includeChanged ?
|
|
75835
|
-
const [counts2 = { added: 0, changed: 0 }] = app.db.all(
|
|
76019
|
+
const addedExpression = includeAdded ? sql43`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql43`0`;
|
|
76020
|
+
const changedExpression = includeChanged ? sql43`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${PAGE_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql43`0`;
|
|
76021
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql43`
|
|
75836
76022
|
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
75837
76023
|
FROM site_crawl_pages AS current
|
|
75838
76024
|
LEFT JOIN site_crawl_pages AS previous
|
|
@@ -75848,7 +76034,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75848
76034
|
summary4.pages.changed = Number(counts2.changed);
|
|
75849
76035
|
}
|
|
75850
76036
|
if (includeRemoved) {
|
|
75851
|
-
const [counts2 = { removed: 0 }] = app.db.all(
|
|
76037
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql43`
|
|
75852
76038
|
SELECT COUNT(*) AS removed
|
|
75853
76039
|
FROM site_crawl_pages AS previous
|
|
75854
76040
|
LEFT JOIN site_crawl_pages AS current
|
|
@@ -75866,9 +76052,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75866
76052
|
}
|
|
75867
76053
|
if (scopeFilter !== "pages") {
|
|
75868
76054
|
if (includeAdded || includeChanged) {
|
|
75869
|
-
const addedExpression = includeAdded ?
|
|
75870
|
-
const changedExpression = includeChanged ?
|
|
75871
|
-
const [counts2 = { added: 0, changed: 0 }] = app.db.all(
|
|
76055
|
+
const addedExpression = includeAdded ? sql43`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql43`0`;
|
|
76056
|
+
const changedExpression = includeChanged ? sql43`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${LINK_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql43`0`;
|
|
76057
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql43`
|
|
75872
76058
|
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
75873
76059
|
FROM site_crawl_edges AS current
|
|
75874
76060
|
LEFT JOIN site_crawl_edges AS previous
|
|
@@ -75886,7 +76072,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75886
76072
|
summary4.links.changed = Number(counts2.changed);
|
|
75887
76073
|
}
|
|
75888
76074
|
if (includeRemoved) {
|
|
75889
|
-
const [counts2 = { removed: 0 }] = app.db.all(
|
|
76075
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql43`
|
|
75890
76076
|
SELECT COUNT(*) AS removed
|
|
75891
76077
|
FROM site_crawl_edges AS previous
|
|
75892
76078
|
LEFT JOIN site_crawl_edges AS current
|
|
@@ -75916,48 +76102,48 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75916
76102
|
const includeRemoved = changeFilter === "all" || changeFilter === "removed";
|
|
75917
76103
|
const table = entity === "page" ? "site_crawl_pages" : "site_crawl_edges";
|
|
75918
76104
|
const keyColumn = entity === "page" ? "node_key" : "edge_key";
|
|
75919
|
-
const internalCurrent = entity === "link" ?
|
|
75920
|
-
const internalPreviousJoin = entity === "link" ?
|
|
75921
|
-
const internalPrevious = entity === "link" ?
|
|
75922
|
-
const internalCurrentJoin = entity === "link" ?
|
|
76105
|
+
const internalCurrent = entity === "link" ? sql43`AND current.internal = 1` : sql43``;
|
|
76106
|
+
const internalPreviousJoin = entity === "link" ? sql43`AND previous.internal = 1` : sql43``;
|
|
76107
|
+
const internalPrevious = entity === "link" ? sql43`AND previous.internal = 1` : sql43``;
|
|
76108
|
+
const internalCurrentJoin = entity === "link" ? sql43`AND current.internal = 1` : sql43``;
|
|
75923
76109
|
const difference3 = entity === "page" ? PAGE_CHANGE_PREDICATE : LINK_CHANGE_PREDICATE;
|
|
75924
|
-
const afterKind = changeFilter === "added" ?
|
|
75925
|
-
const afterRows = includeAfter ? app.db.all(
|
|
76110
|
+
const afterKind = changeFilter === "added" ? sql43`previous.id IS NULL` : changeFilter === "changed" ? sql43`previous.id IS NOT NULL AND ${difference3}` : sql43`previous.id IS NULL OR (previous.id IS NOT NULL AND ${difference3})`;
|
|
76111
|
+
const afterRows = includeAfter ? app.db.all(sql43`
|
|
75926
76112
|
SELECT ${entity} AS entity,
|
|
75927
76113
|
CASE WHEN previous.id IS NULL THEN 'added' ELSE 'changed' END AS change,
|
|
75928
|
-
current.${
|
|
75929
|
-
FROM ${
|
|
75930
|
-
LEFT JOIN ${
|
|
76114
|
+
current.${sql43.raw(keyColumn)} AS key
|
|
76115
|
+
FROM ${sql43.raw(table)} AS current
|
|
76116
|
+
LEFT JOIN ${sql43.raw(table)} AS previous
|
|
75931
76117
|
ON previous.project_id = ${beforeScope.projectId}
|
|
75932
76118
|
AND previous.run_id = ${beforeScope.runId}
|
|
75933
76119
|
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
75934
76120
|
${internalPreviousJoin}
|
|
75935
|
-
AND previous.${
|
|
76121
|
+
AND previous.${sql43.raw(keyColumn)} = current.${sql43.raw(keyColumn)}
|
|
75936
76122
|
WHERE current.project_id = ${afterScope.projectId}
|
|
75937
76123
|
AND current.run_id = ${afterScope.runId}
|
|
75938
76124
|
AND current.attempt_id = ${afterScope.attemptId}
|
|
75939
76125
|
${internalCurrent}
|
|
75940
|
-
AND current.${
|
|
76126
|
+
AND current.${sql43.raw(keyColumn)} > ${afterKey}
|
|
75941
76127
|
AND (${afterKind})
|
|
75942
|
-
ORDER BY current.${
|
|
76128
|
+
ORDER BY current.${sql43.raw(keyColumn)} ASC
|
|
75943
76129
|
LIMIT ${wanted2}
|
|
75944
76130
|
`) : [];
|
|
75945
|
-
const removedRows = includeRemoved ? app.db.all(
|
|
75946
|
-
SELECT ${entity} AS entity, 'removed' AS change, previous.${
|
|
75947
|
-
FROM ${
|
|
75948
|
-
LEFT JOIN ${
|
|
76131
|
+
const removedRows = includeRemoved ? app.db.all(sql43`
|
|
76132
|
+
SELECT ${entity} AS entity, 'removed' AS change, previous.${sql43.raw(keyColumn)} AS key
|
|
76133
|
+
FROM ${sql43.raw(table)} AS previous
|
|
76134
|
+
LEFT JOIN ${sql43.raw(table)} AS current
|
|
75949
76135
|
ON current.project_id = ${afterScope.projectId}
|
|
75950
76136
|
AND current.run_id = ${afterScope.runId}
|
|
75951
76137
|
AND current.attempt_id = ${afterScope.attemptId}
|
|
75952
76138
|
${internalCurrentJoin}
|
|
75953
|
-
AND current.${
|
|
76139
|
+
AND current.${sql43.raw(keyColumn)} = previous.${sql43.raw(keyColumn)}
|
|
75954
76140
|
WHERE previous.project_id = ${beforeScope.projectId}
|
|
75955
76141
|
AND previous.run_id = ${beforeScope.runId}
|
|
75956
76142
|
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
75957
76143
|
${internalPrevious}
|
|
75958
|
-
AND previous.${
|
|
76144
|
+
AND previous.${sql43.raw(keyColumn)} > ${afterKey}
|
|
75959
76145
|
AND current.id IS NULL
|
|
75960
|
-
ORDER BY previous.${
|
|
76146
|
+
ORDER BY previous.${sql43.raw(keyColumn)} ASC
|
|
75961
76147
|
LIMIT ${wanted2}
|
|
75962
76148
|
`) : [];
|
|
75963
76149
|
const merged = [];
|
|
@@ -75987,18 +76173,18 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75987
76173
|
const visibleKeys = keyRows.slice(0, limit);
|
|
75988
76174
|
const pageKeys = visibleKeys.filter((row) => row.entity === "page").map((row) => row.key);
|
|
75989
76175
|
const linkKeys = visibleKeys.filter((row) => row.entity === "link").map((row) => row.key);
|
|
75990
|
-
const pageRowsFor = (scope) => pageKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(
|
|
75991
|
-
|
|
75992
|
-
|
|
75993
|
-
|
|
75994
|
-
|
|
76176
|
+
const pageRowsFor = (scope) => pageKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and92(
|
|
76177
|
+
eq109(siteCrawlPages.projectId, scope.projectId),
|
|
76178
|
+
eq109(siteCrawlPages.runId, scope.runId),
|
|
76179
|
+
eq109(siteCrawlPages.attemptId, scope.attemptId),
|
|
76180
|
+
inArray49(siteCrawlPages.nodeKey, pageKeys)
|
|
75995
76181
|
)).all();
|
|
75996
|
-
const edgeRowsFor = (scope) => linkKeys.length === 0 ? [] : app.db.select().from(siteCrawlEdges).where(
|
|
75997
|
-
|
|
75998
|
-
|
|
75999
|
-
|
|
76000
|
-
|
|
76001
|
-
|
|
76182
|
+
const edgeRowsFor = (scope) => linkKeys.length === 0 ? [] : app.db.select().from(siteCrawlEdges).where(and92(
|
|
76183
|
+
eq109(siteCrawlEdges.projectId, scope.projectId),
|
|
76184
|
+
eq109(siteCrawlEdges.runId, scope.runId),
|
|
76185
|
+
eq109(siteCrawlEdges.attemptId, scope.attemptId),
|
|
76186
|
+
eq109(siteCrawlEdges.internal, true),
|
|
76187
|
+
inArray49(siteCrawlEdges.edgeKey, linkKeys)
|
|
76002
76188
|
)).all();
|
|
76003
76189
|
const beforePages = new Map(pageRowsFor(beforeScope).map((row) => [row.nodeKey, mapCrawlPage(row)]));
|
|
76004
76190
|
const afterPages = new Map(pageRowsFor(afterScope).map((row) => [row.nodeKey, mapCrawlPage(row)]));
|
|
@@ -76171,16 +76357,16 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76171
76357
|
}
|
|
76172
76358
|
const inventorySummary = inventorySummaryFor(detailScopeFor(project.id, snapshot));
|
|
76173
76359
|
const filters = [
|
|
76174
|
-
|
|
76175
|
-
|
|
76176
|
-
|
|
76360
|
+
eq109(siteCrawlPages.projectId, project.id),
|
|
76361
|
+
eq109(siteCrawlPages.runId, snapshot.runId),
|
|
76362
|
+
eq109(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
76177
76363
|
];
|
|
76178
|
-
if (request.query.nodeKey) filters.push(
|
|
76364
|
+
if (request.query.nodeKey) filters.push(eq109(siteCrawlPages.nodeKey, request.query.nodeKey));
|
|
76179
76365
|
const inventoryEligible = parseBoolean2(request.query.inventoryEligible);
|
|
76180
|
-
if (inventoryEligible != null) filters.push(
|
|
76181
|
-
if (request.query.fetchState) filters.push(
|
|
76182
|
-
if (request.query.indexabilityState) filters.push(
|
|
76183
|
-
if (request.query.auditState) filters.push(
|
|
76366
|
+
if (inventoryEligible != null) filters.push(eq109(siteCrawlPages.inventoryEligible, inventoryEligible));
|
|
76367
|
+
if (request.query.fetchState) filters.push(eq109(siteCrawlPages.fetchState, request.query.fetchState));
|
|
76368
|
+
if (request.query.indexabilityState) filters.push(eq109(siteCrawlPages.indexabilityState, request.query.indexabilityState));
|
|
76369
|
+
if (request.query.auditState) filters.push(eq109(siteCrawlPages.auditState, request.query.auditState));
|
|
76184
76370
|
const urlVersions = parseUrlVersionFilter(request.query.urlVersions);
|
|
76185
76371
|
if (urlVersions === SiteCrawlUrlVersionFilters.exclude) filters.push(isNull17(siteCrawlPages.urlVersionOf));
|
|
76186
76372
|
if (urlVersions === SiteCrawlUrlVersionFilters.only) filters.push(isNotNull10(siteCrawlPages.urlVersionOf));
|
|
@@ -76203,17 +76389,17 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76203
76389
|
sort
|
|
76204
76390
|
})).digest("base64url");
|
|
76205
76391
|
const offset = decodeCrawlPageCursor(request.query.cursor, selectionKey);
|
|
76206
|
-
const orderBy = sort === "score-desc" ? [
|
|
76392
|
+
const orderBy = sort === "score-desc" ? [desc50(siteCrawlPages.auditScore), asc26(siteCrawlPages.nodeKey)] : sort === "score-asc" ? [asc26(siteCrawlPages.auditScore), asc26(siteCrawlPages.nodeKey)] : sort === "path" ? [asc26(siteCrawlPages.path), asc26(siteCrawlPages.nodeKey)] : [asc26(siteCrawlPages.url), asc26(siteCrawlPages.nodeKey)];
|
|
76207
76393
|
let healthStateFilter = null;
|
|
76208
76394
|
if (healthState) {
|
|
76209
|
-
const legacyRow = app.db.select({ id: siteCrawlPages.id }).from(siteCrawlPages).where(
|
|
76210
|
-
|
|
76211
|
-
|
|
76212
|
-
|
|
76395
|
+
const legacyRow = app.db.select({ id: siteCrawlPages.id }).from(siteCrawlPages).where(and92(
|
|
76396
|
+
eq109(siteCrawlPages.projectId, project.id),
|
|
76397
|
+
eq109(siteCrawlPages.runId, snapshot.runId),
|
|
76398
|
+
eq109(siteCrawlPages.attemptId, snapshot.attemptId),
|
|
76213
76399
|
isNull17(siteCrawlPages.healthState)
|
|
76214
76400
|
)).limit(1).get();
|
|
76215
76401
|
healthStateFilter = legacyRow ? "unavailable-legacy-scan" : "applied";
|
|
76216
|
-
if (healthStateFilter === "applied") filters.push(
|
|
76402
|
+
if (healthStateFilter === "applied") filters.push(eq109(siteCrawlPages.healthState, healthState));
|
|
76217
76403
|
}
|
|
76218
76404
|
if (healthStateFilter === "unavailable-legacy-scan") {
|
|
76219
76405
|
return {
|
|
@@ -76230,7 +76416,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76230
76416
|
pages: []
|
|
76231
76417
|
};
|
|
76232
76418
|
}
|
|
76233
|
-
const where =
|
|
76419
|
+
const where = and92(...filters);
|
|
76234
76420
|
const total = app.db.select({ value: count8() }).from(siteCrawlPages).where(where).get()?.value ?? 0;
|
|
76235
76421
|
const rows = app.db.select().from(siteCrawlPages).where(where).orderBy(...orderBy).limit(limit).offset(offset).all();
|
|
76236
76422
|
const nextOffset = offset + rows.length;
|
|
@@ -76265,10 +76451,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76265
76451
|
url: siteCrawlPages.url,
|
|
76266
76452
|
inventoryEligible: siteCrawlPages.inventoryEligible,
|
|
76267
76453
|
fetchState: siteCrawlPages.fetchState
|
|
76268
|
-
}).from(siteCrawlPages).where(
|
|
76269
|
-
|
|
76270
|
-
|
|
76271
|
-
|
|
76454
|
+
}).from(siteCrawlPages).where(and92(
|
|
76455
|
+
eq109(siteCrawlPages.projectId, project.id),
|
|
76456
|
+
eq109(siteCrawlPages.runId, snapshot.runId),
|
|
76457
|
+
eq109(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
76272
76458
|
)).limit(MAX_STRUCTURE_SOURCE_ROWS + 1).all();
|
|
76273
76459
|
if (sourceRows.length > MAX_STRUCTURE_SOURCE_ROWS) {
|
|
76274
76460
|
throw validationError(`Persisted crawl exceeds the ${MAX_STRUCTURE_SOURCE_ROWS}-page structure limit`);
|
|
@@ -76350,21 +76536,21 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76350
76536
|
};
|
|
76351
76537
|
}
|
|
76352
76538
|
const filters = [
|
|
76353
|
-
|
|
76354
|
-
|
|
76355
|
-
|
|
76356
|
-
|
|
76539
|
+
eq109(siteCrawlEdges.projectId, project.id),
|
|
76540
|
+
eq109(siteCrawlEdges.runId, snapshot.runId),
|
|
76541
|
+
eq109(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
76542
|
+
eq109(siteCrawlEdges.internal, true),
|
|
76357
76543
|
linkKindFilter(linkKind)
|
|
76358
76544
|
];
|
|
76359
|
-
if (request.query.sourceUrl) filters.push(
|
|
76360
|
-
if (request.query.targetUrl) filters.push(
|
|
76545
|
+
if (request.query.sourceUrl) filters.push(eq109(siteCrawlEdges.sourceUrl, request.query.sourceUrl));
|
|
76546
|
+
if (request.query.targetUrl) filters.push(eq109(siteCrawlEdges.targetUrl, request.query.targetUrl));
|
|
76361
76547
|
const followable = parseBoolean2(request.query.followable);
|
|
76362
|
-
if (followable != null) filters.push(
|
|
76363
|
-
const where =
|
|
76548
|
+
if (followable != null) filters.push(eq109(siteCrawlEdges.followable, followable));
|
|
76549
|
+
const where = and92(...filters);
|
|
76364
76550
|
const total = app.db.select({ value: count8() }).from(siteCrawlEdges).where(where).get()?.value ?? 0;
|
|
76365
76551
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
76366
76552
|
const offset = decodeCursor(request.query.cursor);
|
|
76367
|
-
const rows = app.db.select().from(siteCrawlEdges).where(where).orderBy(
|
|
76553
|
+
const rows = app.db.select().from(siteCrawlEdges).where(where).orderBy(asc26(siteCrawlEdges.edgeKey)).limit(limit).offset(offset).all();
|
|
76368
76554
|
const nextOffset = offset + rows.length;
|
|
76369
76555
|
return {
|
|
76370
76556
|
project: project.name,
|
|
@@ -76421,17 +76607,17 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76421
76607
|
};
|
|
76422
76608
|
}
|
|
76423
76609
|
const scope = [
|
|
76424
|
-
|
|
76425
|
-
|
|
76426
|
-
|
|
76427
|
-
|
|
76610
|
+
eq109(siteCrawlEdges.projectId, project.id),
|
|
76611
|
+
eq109(siteCrawlEdges.runId, snapshot.runId),
|
|
76612
|
+
eq109(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
76613
|
+
eq109(siteCrawlEdges.internal, true),
|
|
76428
76614
|
linkKindFilter(linkKind)
|
|
76429
76615
|
];
|
|
76430
|
-
const inboundMatch = request.query.nodeKey && request.query.url ? or21(
|
|
76431
|
-
const outboundMatch = request.query.nodeKey && request.query.url ? or21(
|
|
76616
|
+
const inboundMatch = request.query.nodeKey && request.query.url ? or21(eq109(siteCrawlEdges.targetNodeKey, request.query.nodeKey), eq109(siteCrawlEdges.targetUrl, request.query.url)) : request.query.nodeKey ? eq109(siteCrawlEdges.targetNodeKey, request.query.nodeKey) : eq109(siteCrawlEdges.targetUrl, request.query.url);
|
|
76617
|
+
const outboundMatch = request.query.nodeKey && request.query.url ? or21(eq109(siteCrawlEdges.sourceNodeKey, request.query.nodeKey), eq109(siteCrawlEdges.sourceUrl, request.query.url)) : request.query.nodeKey ? eq109(siteCrawlEdges.sourceNodeKey, request.query.nodeKey) : eq109(siteCrawlEdges.sourceUrl, request.query.url);
|
|
76432
76618
|
const limit = parseBoundedLimit(request.query.limit, 50, 100);
|
|
76433
|
-
const inboundRows = app.db.select().from(siteCrawlEdges).where(
|
|
76434
|
-
const outboundRows = app.db.select().from(siteCrawlEdges).where(
|
|
76619
|
+
const inboundRows = app.db.select().from(siteCrawlEdges).where(and92(...scope, inboundMatch)).orderBy(asc26(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
76620
|
+
const outboundRows = app.db.select().from(siteCrawlEdges).where(and92(...scope, outboundMatch)).orderBy(asc26(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
76435
76621
|
return {
|
|
76436
76622
|
project: project.name,
|
|
76437
76623
|
hasCrawlData: true,
|
|
@@ -76465,11 +76651,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76465
76651
|
if (!snapshot.attemptId) {
|
|
76466
76652
|
return { project: project.name, runId: snapshot.runId, state: "partial", checkDeadLinks: true, checked: snapshot.deadLinksChecked, found: snapshot.deadLinksFound, unverified: snapshot.deadLinksUnverified, total: 0, nextCursor: null, deadLinks: [], runSelection: target.runSelection };
|
|
76467
76653
|
}
|
|
76468
|
-
const where =
|
|
76469
|
-
|
|
76470
|
-
|
|
76471
|
-
|
|
76472
|
-
|
|
76654
|
+
const where = and92(
|
|
76655
|
+
eq109(siteCrawlFindings.projectId, project.id),
|
|
76656
|
+
eq109(siteCrawlFindings.runId, snapshot.runId),
|
|
76657
|
+
eq109(siteCrawlFindings.attemptId, snapshot.attemptId),
|
|
76658
|
+
eq109(siteCrawlFindings.findingType, "dead-link"),
|
|
76473
76659
|
// Belt and braces on the client-facing surface: a row without a status
|
|
76474
76660
|
// code is not evidence of a broken link, and this response is where such
|
|
76475
76661
|
// a row would be read as one. Migration 140 removed every stored row that
|
|
@@ -76479,12 +76665,12 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76479
76665
|
// a fabricated broken link shown to a client is not worth the saved
|
|
76480
76666
|
// predicate. Applied to `total` as well as the rows, or the count would
|
|
76481
76667
|
// promise a page the list cannot fill.
|
|
76482
|
-
isNotNull10(
|
|
76668
|
+
isNotNull10(sql43`json_extract(${siteCrawlFindings.evidence}, '$.statusCode')`)
|
|
76483
76669
|
);
|
|
76484
76670
|
const total = app.db.select({ value: count8() }).from(siteCrawlFindings).where(where).get()?.value ?? 0;
|
|
76485
76671
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
76486
76672
|
const offset = decodeCursor(request.query.cursor);
|
|
76487
|
-
const rows = app.db.select().from(siteCrawlFindings).where(where).orderBy(
|
|
76673
|
+
const rows = app.db.select().from(siteCrawlFindings).where(where).orderBy(asc26(siteCrawlFindings.findingKey)).limit(limit).offset(offset).all();
|
|
76488
76674
|
const nextOffset = offset + rows.length;
|
|
76489
76675
|
const deadLinks = rows.map((row) => ({
|
|
76490
76676
|
findingKey: row.findingKey,
|
|
@@ -76519,19 +76705,19 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76519
76705
|
createdAt: runs.createdAt,
|
|
76520
76706
|
startedAt: runs.startedAt,
|
|
76521
76707
|
finishedAt: runs.finishedAt
|
|
76522
|
-
}).from(runs).where(
|
|
76523
|
-
|
|
76524
|
-
|
|
76708
|
+
}).from(runs).where(and92(
|
|
76709
|
+
eq109(runs.projectId, project.id),
|
|
76710
|
+
eq109(runs.kind, RunKinds["site-audit"]),
|
|
76525
76711
|
notProbeRun()
|
|
76526
|
-
)).orderBy(
|
|
76712
|
+
)).orderBy(desc50(runs.createdAt), desc50(runs.id)).limit(limit).all();
|
|
76527
76713
|
const runIds = rows.map((row) => row.runId);
|
|
76528
|
-
const crawlRunIds = new Set(runIds.length === 0 ? [] : app.db.select({ runId: siteCrawlSnapshots.runId }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
76529
|
-
|
|
76530
|
-
|
|
76531
|
-
|
|
76532
|
-
|
|
76714
|
+
const crawlRunIds = new Set(runIds.length === 0 ? [] : app.db.select({ runId: siteCrawlSnapshots.runId }).from(siteCrawlSnapshots).innerJoin(runs, eq109(siteCrawlSnapshots.runId, runs.id)).where(and92(
|
|
76715
|
+
eq109(siteCrawlSnapshots.projectId, project.id),
|
|
76716
|
+
eq109(runs.projectId, project.id),
|
|
76717
|
+
eq109(runs.kind, RunKinds["site-audit"]),
|
|
76718
|
+
inArray49(runs.status, SURFACEABLE_STATUSES),
|
|
76533
76719
|
notProbeRun(),
|
|
76534
|
-
|
|
76720
|
+
inArray49(siteCrawlSnapshots.runId, runIds)
|
|
76535
76721
|
)).all().map((row) => row.runId));
|
|
76536
76722
|
return {
|
|
76537
76723
|
project: project.name,
|
|
@@ -76545,21 +76731,21 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76545
76731
|
});
|
|
76546
76732
|
app.get("/projects/:name/technical-aeo/runs/:runId/progress", async (request) => {
|
|
76547
76733
|
const project = resolveProject(app.db, request.params.name);
|
|
76548
|
-
const run = app.db.select().from(runs).where(
|
|
76549
|
-
|
|
76550
|
-
|
|
76551
|
-
|
|
76734
|
+
const run = app.db.select().from(runs).where(and92(
|
|
76735
|
+
eq109(runs.id, request.params.runId),
|
|
76736
|
+
eq109(runs.projectId, project.id),
|
|
76737
|
+
eq109(runs.kind, RunKinds["site-audit"]),
|
|
76552
76738
|
notProbeRun()
|
|
76553
76739
|
)).get();
|
|
76554
76740
|
if (!run) throw notFound("Site audit run", request.params.runId);
|
|
76555
|
-
const attempt = app.db.select().from(siteCrawlAttempts).where(
|
|
76556
|
-
|
|
76557
|
-
|
|
76558
|
-
)).orderBy(
|
|
76559
|
-
const persistedLayout = attempt ? app.db.select().from(siteCrawlGraphLayouts).where(
|
|
76560
|
-
|
|
76561
|
-
|
|
76562
|
-
|
|
76741
|
+
const attempt = app.db.select().from(siteCrawlAttempts).where(and92(
|
|
76742
|
+
eq109(siteCrawlAttempts.projectId, project.id),
|
|
76743
|
+
eq109(siteCrawlAttempts.runId, run.id)
|
|
76744
|
+
)).orderBy(desc50(siteCrawlAttempts.attemptNumber), desc50(siteCrawlAttempts.updatedAt)).limit(1).get();
|
|
76745
|
+
const persistedLayout = attempt ? app.db.select().from(siteCrawlGraphLayouts).where(and92(
|
|
76746
|
+
eq109(siteCrawlGraphLayouts.projectId, project.id),
|
|
76747
|
+
eq109(siteCrawlGraphLayouts.runId, run.id),
|
|
76748
|
+
eq109(siteCrawlGraphLayouts.attemptId, attempt.id)
|
|
76563
76749
|
)).get() : void 0;
|
|
76564
76750
|
const layoutState = persistedLayout?.state === "ready" || persistedLayout?.state === "unavailable" ? persistedLayout.state : run.status === RunStatuses.completed || run.status === RunStatuses.partial ? "unavailable" : "pending";
|
|
76565
76751
|
return {
|
|
@@ -76590,19 +76776,19 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76590
76776
|
};
|
|
76591
76777
|
});
|
|
76592
76778
|
app.get("/projects/:name/technical-aeo/runs/:runId/page-health-preview", async (request) => app.db.transaction((tx) => {
|
|
76593
|
-
const project = tx.select().from(projects).where(
|
|
76779
|
+
const project = tx.select().from(projects).where(eq109(projects.name, request.params.name)).get();
|
|
76594
76780
|
if (!project) throw notFound("Project", request.params.name);
|
|
76595
|
-
const run = tx.select().from(runs).where(
|
|
76596
|
-
|
|
76597
|
-
|
|
76598
|
-
|
|
76781
|
+
const run = tx.select().from(runs).where(and92(
|
|
76782
|
+
eq109(runs.id, request.params.runId),
|
|
76783
|
+
eq109(runs.projectId, project.id),
|
|
76784
|
+
eq109(runs.kind, RunKinds["site-audit"]),
|
|
76599
76785
|
notProbeRun()
|
|
76600
76786
|
)).get();
|
|
76601
76787
|
if (!run) throw notFound("Site audit run", request.params.runId);
|
|
76602
|
-
const attempt = tx.select().from(siteCrawlAttempts).where(
|
|
76603
|
-
|
|
76604
|
-
|
|
76605
|
-
)).orderBy(
|
|
76788
|
+
const attempt = tx.select().from(siteCrawlAttempts).where(and92(
|
|
76789
|
+
eq109(siteCrawlAttempts.projectId, project.id),
|
|
76790
|
+
eq109(siteCrawlAttempts.runId, run.id)
|
|
76791
|
+
)).orderBy(desc50(siteCrawlAttempts.attemptNumber), desc50(siteCrawlAttempts.updatedAt)).limit(1).get();
|
|
76606
76792
|
const state = run.status === RunStatuses.queued ? "waiting" : run.status === RunStatuses.running ? "collecting" : "terminal";
|
|
76607
76793
|
const base = {
|
|
76608
76794
|
project: project.name,
|
|
@@ -76615,11 +76801,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76615
76801
|
if (!attempt) {
|
|
76616
76802
|
return { ...base, pagesAudited: 0, examples: [] };
|
|
76617
76803
|
}
|
|
76618
|
-
const auditedWhere =
|
|
76619
|
-
|
|
76620
|
-
|
|
76621
|
-
|
|
76622
|
-
|
|
76804
|
+
const auditedWhere = and92(
|
|
76805
|
+
eq109(siteCrawlPages.projectId, project.id),
|
|
76806
|
+
eq109(siteCrawlPages.runId, run.id),
|
|
76807
|
+
eq109(siteCrawlPages.attemptId, attempt.id),
|
|
76808
|
+
eq109(siteCrawlPages.auditState, "success")
|
|
76623
76809
|
);
|
|
76624
76810
|
const pagesAudited = tx.select({ value: count8() }).from(siteCrawlPages).where(auditedWhere).get()?.value ?? 0;
|
|
76625
76811
|
if (state !== "collecting") {
|
|
@@ -76630,11 +76816,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76630
76816
|
url: siteCrawlPages.url,
|
|
76631
76817
|
auditScore: siteCrawlPages.auditScore,
|
|
76632
76818
|
auditFields: siteCrawlPages.auditFields
|
|
76633
|
-
}).from(siteCrawlPages).where(
|
|
76819
|
+
}).from(siteCrawlPages).where(and92(
|
|
76634
76820
|
auditedWhere,
|
|
76635
76821
|
isNotNull10(siteCrawlPages.auditScore),
|
|
76636
76822
|
lt17(siteCrawlPages.auditScore, 70)
|
|
76637
|
-
)).orderBy(
|
|
76823
|
+
)).orderBy(asc26(siteCrawlPages.auditScore), asc26(siteCrawlPages.nodeKey)).limit(LIVE_PAGE_HEALTH_CANDIDATE_LIMIT).all();
|
|
76638
76824
|
const examples = candidates.flatMap((candidate) => {
|
|
76639
76825
|
if (candidate.auditScore == null) return [];
|
|
76640
76826
|
const checksNeedingAttention = countLivePageHealthChecks(candidate.auditFields);
|
|
@@ -76666,13 +76852,13 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76666
76852
|
status: runs.status,
|
|
76667
76853
|
identityKey: siteCrawlRunRequests.identityKey,
|
|
76668
76854
|
effectiveOptions: siteCrawlRunRequests.effectiveOptions
|
|
76669
|
-
}).from(runs).leftJoin(siteCrawlRunRequests,
|
|
76670
|
-
|
|
76671
|
-
|
|
76672
|
-
)).where(
|
|
76673
|
-
|
|
76674
|
-
|
|
76675
|
-
|
|
76855
|
+
}).from(runs).leftJoin(siteCrawlRunRequests, and92(
|
|
76856
|
+
eq109(siteCrawlRunRequests.projectId, runs.projectId),
|
|
76857
|
+
eq109(siteCrawlRunRequests.runId, runs.id)
|
|
76858
|
+
)).where(and92(
|
|
76859
|
+
eq109(runs.projectId, project.id),
|
|
76860
|
+
eq109(runs.kind, RunKinds["site-audit"]),
|
|
76861
|
+
inArray49(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
76676
76862
|
)).get();
|
|
76677
76863
|
if (existing) {
|
|
76678
76864
|
if (existing.identityKey === identityKey) {
|
|
@@ -76726,7 +76912,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76726
76912
|
}
|
|
76727
76913
|
|
|
76728
76914
|
// ../api-routes/src/trends.ts
|
|
76729
|
-
import { and as
|
|
76915
|
+
import { and as and93, asc as asc27, eq as eq110 } from "drizzle-orm";
|
|
76730
76916
|
function parseSurface(value) {
|
|
76731
76917
|
const parsed = trendSurfaceSchema.safeParse(value);
|
|
76732
76918
|
if (!parsed.success) {
|
|
@@ -76782,8 +76968,8 @@ async function trendsRoutes(app) {
|
|
|
76782
76968
|
function readSurface(app, projectId, surface) {
|
|
76783
76969
|
switch (surface) {
|
|
76784
76970
|
case TrendSurfaces["index-coverage"]: {
|
|
76785
|
-
const google = app.db.select({ date: gscCoverageSnapshots.date, indexed: gscCoverageSnapshots.indexed, notIndexed: gscCoverageSnapshots.notIndexed }).from(gscCoverageSnapshots).where(
|
|
76786
|
-
const bing = app.db.select({ date: bingCoverageSnapshots.date, indexed: bingCoverageSnapshots.indexed, notIndexed: bingCoverageSnapshots.notIndexed }).from(bingCoverageSnapshots).where(
|
|
76971
|
+
const google = app.db.select({ date: gscCoverageSnapshots.date, indexed: gscCoverageSnapshots.indexed, notIndexed: gscCoverageSnapshots.notIndexed }).from(gscCoverageSnapshots).where(eq110(gscCoverageSnapshots.projectId, projectId)).orderBy(asc27(gscCoverageSnapshots.date)).all();
|
|
76972
|
+
const bing = app.db.select({ date: bingCoverageSnapshots.date, indexed: bingCoverageSnapshots.indexed, notIndexed: bingCoverageSnapshots.notIndexed }).from(bingCoverageSnapshots).where(eq110(bingCoverageSnapshots.projectId, projectId)).orderBy(asc27(bingCoverageSnapshots.date)).all();
|
|
76787
76973
|
const coverage = (key, label, rows) => ({
|
|
76788
76974
|
key,
|
|
76789
76975
|
label,
|
|
@@ -76795,7 +76981,7 @@ function readSurface(app, projectId, surface) {
|
|
|
76795
76981
|
return [coverage("google", "Google", google), coverage("bing", "Bing", bing)];
|
|
76796
76982
|
}
|
|
76797
76983
|
case TrendSurfaces["site-health"]: {
|
|
76798
|
-
const rows = app.db.select({ auditedAt: siteAuditSnapshots.auditedAt, aggregateScore: siteAuditSnapshots.aggregateScore, pagesAudited: siteAuditSnapshots.pagesAudited }).from(siteAuditSnapshots).innerJoin(runs,
|
|
76984
|
+
const rows = app.db.select({ auditedAt: siteAuditSnapshots.auditedAt, aggregateScore: siteAuditSnapshots.aggregateScore, pagesAudited: siteAuditSnapshots.pagesAudited }).from(siteAuditSnapshots).innerJoin(runs, eq110(siteAuditSnapshots.runId, runs.id)).where(and93(...surfaceableSiteAuditFilters(projectId))).orderBy(asc27(siteAuditSnapshots.auditedAt)).all();
|
|
76799
76985
|
return [{
|
|
76800
76986
|
key: "site-health",
|
|
76801
76987
|
label: "Site Health",
|
|
@@ -76807,7 +76993,7 @@ function readSurface(app, projectId, surface) {
|
|
|
76807
76993
|
}
|
|
76808
76994
|
case TrendSurfaces.backlinks: {
|
|
76809
76995
|
const totals = computeFilteredReleaseTotals(app.db, projectId, BacklinkSources.commoncrawl);
|
|
76810
|
-
const rows = app.db.select({ release: backlinkSummaries.release, totalLinkingDomains: backlinkSummaries.totalLinkingDomains, queriedAt: backlinkSummaries.queriedAt }).from(backlinkSummaries).where(
|
|
76996
|
+
const rows = app.db.select({ release: backlinkSummaries.release, totalLinkingDomains: backlinkSummaries.totalLinkingDomains, queriedAt: backlinkSummaries.queriedAt }).from(backlinkSummaries).where(and93(eq110(backlinkSummaries.projectId, projectId), eq110(backlinkSummaries.source, BacklinkSources.commoncrawl))).all().sort((a, b) => compareReleaseIds(a.release, b.release)).map((row) => {
|
|
76811
76997
|
const filtered = totals.get(row.release);
|
|
76812
76998
|
const known = (filtered?.domainRows ?? 0) > 0 || row.totalLinkingDomains === 0;
|
|
76813
76999
|
const windowEnd = releaseWindowEndMonth(row.release);
|
|
@@ -76837,7 +77023,7 @@ function readSurface(app, projectId, surface) {
|
|
|
76837
77023
|
// ../api-routes/src/research.ts
|
|
76838
77024
|
import crypto60 from "crypto";
|
|
76839
77025
|
import { z as z7 } from "zod";
|
|
76840
|
-
import { and as
|
|
77026
|
+
import { and as and94, count as count9, desc as desc51, eq as eq111, gte as gte32, lt as lt18, or as or22, sql as sql44 } from "drizzle-orm";
|
|
76841
77027
|
var sameLocation = (a, b) => a.label === b.label && a.city === b.city && a.region === b.region && a.country === b.country && a.timezone === b.timezone;
|
|
76842
77028
|
var BATCH_RECEIPT_PREFIX = "__canonry_research_batch__:";
|
|
76843
77029
|
async function researchRoutes(app, opts) {
|
|
@@ -76856,7 +77042,7 @@ async function researchRoutes(app, opts) {
|
|
|
76856
77042
|
const input = parsed.data;
|
|
76857
77043
|
if (input.idempotencyKey?.startsWith(BATCH_RECEIPT_PREFIX)) throw validationError("This idempotency key prefix is reserved for internal research batch receipts.");
|
|
76858
77044
|
const requestHash2 = directResearchRequestHash(input);
|
|
76859
|
-
const existingReceipt = input.idempotencyKey ? app.db.select().from(researchRuns).where(
|
|
77045
|
+
const existingReceipt = input.idempotencyKey ? app.db.select().from(researchRuns).where(and94(eq111(researchRuns.projectId, project.id), eq111(researchRuns.idempotencyKey, input.idempotencyKey))).get() : void 0;
|
|
76860
77046
|
if (existingReceipt) {
|
|
76861
77047
|
assertSameDirectResearchRequest(existingReceipt, input, requestHash2);
|
|
76862
77048
|
const result2 = getDetail(app, project.id, existingReceipt.id);
|
|
@@ -76891,7 +77077,7 @@ async function researchRoutes(app, opts) {
|
|
|
76891
77077
|
const initiatedBy = researchPrincipal(request);
|
|
76892
77078
|
const decision = app.db.transaction((tx) => {
|
|
76893
77079
|
if (input.idempotencyKey) {
|
|
76894
|
-
const existing = tx.select().from(researchRuns).where(
|
|
77080
|
+
const existing = tx.select().from(researchRuns).where(and94(eq111(researchRuns.projectId, project.id), eq111(researchRuns.idempotencyKey, input.idempotencyKey))).get();
|
|
76895
77081
|
if (existing) {
|
|
76896
77082
|
assertSameDirectResearchRequest(existing, input, requestHash2);
|
|
76897
77083
|
return { reused: true, id: existing.id, shouldDispatch: existing.status === ResearchRunStatuses.queued };
|
|
@@ -76899,11 +77085,11 @@ async function researchRoutes(app, opts) {
|
|
|
76899
77085
|
}
|
|
76900
77086
|
if (initiatedBy?.limited) {
|
|
76901
77087
|
const { start, end, date } = utcDayBounds(now);
|
|
76902
|
-
const used = tx.select({ value: count9() }).from(researchRuns).where(
|
|
76903
|
-
|
|
77088
|
+
const used = tx.select({ value: count9() }).from(researchRuns).where(and94(
|
|
77089
|
+
eq111(researchRuns.projectId, project.id),
|
|
76904
77090
|
gte32(researchRuns.createdAt, start),
|
|
76905
77091
|
lt18(researchRuns.createdAt, end),
|
|
76906
|
-
|
|
77092
|
+
sql44`(json_extract(${researchRuns.initiatedBy}, '$.role') = ${UserRoles.viewer} OR json_extract(${researchRuns.initiatedBy}, '$.limited') = 1)`
|
|
76907
77093
|
)).get()?.value ?? 0;
|
|
76908
77094
|
if (used >= viewerDailyRunLimit) {
|
|
76909
77095
|
throw researchDailyLimitExceeded(project.name, viewerDailyRunLimit, date);
|
|
@@ -76931,15 +77117,15 @@ async function researchRoutes(app, opts) {
|
|
|
76931
77117
|
if (input.idempotencyKey.startsWith(BATCH_RECEIPT_PREFIX)) throw validationError("This idempotency key prefix is reserved for internal research batch receipts.");
|
|
76932
77118
|
const requestHash2 = researchBatchRequestHash(input);
|
|
76933
77119
|
const receiptKeys = input.runs.map((_run, index) => batchChildReceiptKey(input.idempotencyKey, index));
|
|
76934
|
-
const firstReceipt = app.db.select().from(researchRuns).where(
|
|
76935
|
-
|
|
76936
|
-
|
|
77120
|
+
const firstReceipt = app.db.select().from(researchRuns).where(and94(
|
|
77121
|
+
eq111(researchRuns.projectId, project.id),
|
|
77122
|
+
eq111(researchRuns.idempotencyKey, receiptKeys[0])
|
|
76937
77123
|
)).get();
|
|
76938
77124
|
if (firstReceipt) {
|
|
76939
77125
|
if (firstReceipt.requestHash !== requestHash2) throw alreadyExists("Research batch idempotency key", input.idempotencyKey);
|
|
76940
|
-
const saved = receiptKeys.map((key) => app.db.select().from(researchRuns).where(
|
|
76941
|
-
|
|
76942
|
-
|
|
77126
|
+
const saved = receiptKeys.map((key) => app.db.select().from(researchRuns).where(and94(
|
|
77127
|
+
eq111(researchRuns.projectId, project.id),
|
|
77128
|
+
eq111(researchRuns.idempotencyKey, key)
|
|
76943
77129
|
)).get());
|
|
76944
77130
|
if (saved.some((row) => !row || row.requestHash !== requestHash2)) throw alreadyExists("Research batch idempotency key", input.idempotencyKey);
|
|
76945
77131
|
const runs3 = saved.map((row) => getDetail(app, project.id, row.id));
|
|
@@ -76955,11 +77141,11 @@ async function researchRoutes(app, opts) {
|
|
|
76955
77141
|
const decision = app.db.transaction((tx) => {
|
|
76956
77142
|
if (initiatedBy?.limited) {
|
|
76957
77143
|
const { start, end, date } = utcDayBounds(now);
|
|
76958
|
-
const used = tx.select({ value: count9() }).from(researchRuns).where(
|
|
76959
|
-
|
|
77144
|
+
const used = tx.select({ value: count9() }).from(researchRuns).where(and94(
|
|
77145
|
+
eq111(researchRuns.projectId, project.id),
|
|
76960
77146
|
gte32(researchRuns.createdAt, start),
|
|
76961
77147
|
lt18(researchRuns.createdAt, end),
|
|
76962
|
-
|
|
77148
|
+
sql44`(json_extract(${researchRuns.initiatedBy}, '$.role') = ${UserRoles.viewer} OR json_extract(${researchRuns.initiatedBy}, '$.limited') = 1)`
|
|
76963
77149
|
)).get()?.value ?? 0;
|
|
76964
77150
|
if (used + prepared.length > viewerDailyRunLimit) throw researchDailyLimitExceeded(project.name, viewerDailyRunLimit, date);
|
|
76965
77151
|
}
|
|
@@ -77010,10 +77196,10 @@ async function researchRoutes(app, opts) {
|
|
|
77010
77196
|
const requested = Number.parseInt(request.query.limit ?? "", 10);
|
|
77011
77197
|
const limit = Number.isInteger(requested) && requested > 0 ? Math.min(requested, 100) : 20;
|
|
77012
77198
|
const cursor = request.query.cursor ? parseResearchCursor(request.query.cursor, project.id) : null;
|
|
77013
|
-
const rows = app.db.select().from(researchRuns).where(
|
|
77014
|
-
|
|
77015
|
-
cursor ? or22(lt18(researchRuns.createdAt, cursor.createdAt),
|
|
77016
|
-
)).orderBy(
|
|
77199
|
+
const rows = app.db.select().from(researchRuns).where(and94(
|
|
77200
|
+
eq111(researchRuns.projectId, project.id),
|
|
77201
|
+
cursor ? or22(lt18(researchRuns.createdAt, cursor.createdAt), and94(eq111(researchRuns.createdAt, cursor.createdAt), lt18(researchRuns.id, cursor.id))) : void 0
|
|
77202
|
+
)).orderBy(desc51(researchRuns.createdAt), desc51(researchRuns.id)).limit(limit + 1).all();
|
|
77017
77203
|
const runs2 = rows.slice(0, limit).map(serializeRun);
|
|
77018
77204
|
const last = runs2.at(-1);
|
|
77019
77205
|
const nextCursor = rows.length > limit && last ? Buffer.from(JSON.stringify({ projectId: project.id, createdAt: last.createdAt, id: last.id })).toString("base64url") : null;
|
|
@@ -77162,9 +77348,9 @@ function resolveStoredResearchTemplate(stored, selection, idempotencyKey) {
|
|
|
77162
77348
|
return stored;
|
|
77163
77349
|
}
|
|
77164
77350
|
function resolveResearchTemplate(app, projectId, selection, scope, location) {
|
|
77165
|
-
const savedTemplate = app.db.select().from(measurementQueryTemplates).where(
|
|
77166
|
-
|
|
77167
|
-
|
|
77351
|
+
const savedTemplate = app.db.select().from(measurementQueryTemplates).where(and94(
|
|
77352
|
+
eq111(measurementQueryTemplates.projectId, projectId),
|
|
77353
|
+
eq111(measurementQueryTemplates.id, selection.templateId)
|
|
77168
77354
|
)).get();
|
|
77169
77355
|
if (!savedTemplate) throw validationError("Unknown or stale research template. Reload the selected template and try again.");
|
|
77170
77356
|
const templateVersion = savedTemplate.updatedAt;
|
|
@@ -77173,7 +77359,7 @@ function resolveResearchTemplate(app, projectId, selection, scope, location) {
|
|
|
77173
77359
|
}
|
|
77174
77360
|
const bindingLocation = selection.bindingLocation === void 0 ? location : selection.bindingLocation;
|
|
77175
77361
|
if (bindingLocation) {
|
|
77176
|
-
const project = app.db.select({ locations: projects.locations }).from(projects).where(
|
|
77362
|
+
const project = app.db.select({ locations: projects.locations }).from(projects).where(eq111(projects.id, projectId)).get();
|
|
77177
77363
|
if (!project.locations.some((item2) => sameLocation(item2, bindingLocation))) throw validationError("Research template binding location must match a configured project location.");
|
|
77178
77364
|
}
|
|
77179
77365
|
const { bindings, output } = expandResearchTemplate(savedTemplate, scope, bindingLocation);
|
|
@@ -77186,9 +77372,9 @@ function classifyResearchQueries(project, plan, queries2) {
|
|
|
77186
77372
|
return queries2.map((query2) => classifier?.classify(query2) ?? null);
|
|
77187
77373
|
}
|
|
77188
77374
|
function getDetail(app, projectId, id) {
|
|
77189
|
-
const row = app.db.select().from(researchRuns).where(
|
|
77375
|
+
const row = app.db.select().from(researchRuns).where(and94(eq111(researchRuns.id, id), eq111(researchRuns.projectId, projectId))).get();
|
|
77190
77376
|
if (!row) throw notFound("Research run", id);
|
|
77191
|
-
const queries2 = app.db.select().from(researchRunQueries).where(
|
|
77377
|
+
const queries2 = app.db.select().from(researchRunQueries).where(eq111(researchRunQueries.researchRunId, id)).orderBy(researchRunQueries.position).all().map(serializeQuery);
|
|
77192
77378
|
return { ...serializeRun(row), queries: queries2 };
|
|
77193
77379
|
}
|
|
77194
77380
|
function serializeRun(row) {
|
|
@@ -77243,7 +77429,7 @@ function utcDayBounds(now) {
|
|
|
77243
77429
|
|
|
77244
77430
|
// ../api-routes/src/simple-measurement-definitions.ts
|
|
77245
77431
|
import crypto61 from "crypto";
|
|
77246
|
-
import { and as
|
|
77432
|
+
import { and as and95, eq as eq112 } from "drizzle-orm";
|
|
77247
77433
|
function captureSimpleMeasurementDefinition(db, input) {
|
|
77248
77434
|
return db.transaction((tx) => {
|
|
77249
77435
|
const run = tx.select({
|
|
@@ -77251,7 +77437,7 @@ function captureSimpleMeasurementDefinition(db, input) {
|
|
|
77251
77437
|
trigger: runs.trigger,
|
|
77252
77438
|
status: runs.status,
|
|
77253
77439
|
measurementPlanVersionId: runs.measurementPlanVersionId
|
|
77254
|
-
}).from(runs).where(
|
|
77440
|
+
}).from(runs).where(and95(eq112(runs.projectId, input.projectId), eq112(runs.id, input.runId))).get();
|
|
77255
77441
|
if (!run) throw notFound("Run", input.runId);
|
|
77256
77442
|
if (run.kind !== RunKinds["answer-visibility"] || run.trigger === RunTriggers.probe || run.measurementPlanVersionId !== null) {
|
|
77257
77443
|
return null;
|
|
@@ -77272,7 +77458,7 @@ function captureSimpleMeasurementDefinition(db, input) {
|
|
|
77272
77458
|
throw validationError("Qualified aliases must be a sorted subset of the captured aliases.");
|
|
77273
77459
|
}
|
|
77274
77460
|
}
|
|
77275
|
-
const existing = tx.select().from(simpleMeasurementDefinitions).where(
|
|
77461
|
+
const existing = tx.select().from(simpleMeasurementDefinitions).where(and95(eq112(simpleMeasurementDefinitions.projectId, input.projectId), eq112(simpleMeasurementDefinitions.runId, input.runId))).get();
|
|
77276
77462
|
if (existing) {
|
|
77277
77463
|
const frozen = simpleMeasurementDefinitionSchema.parse(existing.definition);
|
|
77278
77464
|
const { qualifiedAliases: _liveQualifiedAliases, ...liveIdentity } = definition2.identity;
|
|
@@ -77298,15 +77484,15 @@ function captureSimpleMeasurementDefinition(db, input) {
|
|
|
77298
77484
|
if (run.status !== RunStatuses.running) {
|
|
77299
77485
|
throw validationError("Only a running run can capture a new measurement definition.");
|
|
77300
77486
|
}
|
|
77301
|
-
if (tx.select({ id: querySnapshots.id }).from(querySnapshots).where(
|
|
77487
|
+
if (tx.select({ id: querySnapshots.id }).from(querySnapshots).where(eq112(querySnapshots.runId, input.runId)).limit(1).get()) {
|
|
77302
77488
|
throw validationError("This run already has stored answers. Its measurement definition cannot be inferred afterward.");
|
|
77303
77489
|
}
|
|
77304
|
-
const projectQueryIds = new Set(tx.select({ id: queries.id }).from(queries).where(
|
|
77490
|
+
const projectQueryIds = new Set(tx.select({ id: queries.id }).from(queries).where(eq112(queries.projectId, input.projectId)).all().map((query2) => query2.id));
|
|
77305
77491
|
if (definition2.queries.some((query2) => !projectQueryIds.has(query2.queryId))) {
|
|
77306
77492
|
throw validationError("Every captured query must belong to the run project.");
|
|
77307
77493
|
}
|
|
77308
77494
|
if (definition2.competitors !== void 0) {
|
|
77309
|
-
const liveCompetitorDomains = new Set(tx.select({ domain: competitors.domain }).from(competitors).where(
|
|
77495
|
+
const liveCompetitorDomains = new Set(tx.select({ domain: competitors.domain }).from(competitors).where(eq112(competitors.projectId, input.projectId)).orderBy(competitors.domain).all().map((competitor) => competitor.domain));
|
|
77310
77496
|
const capturedDomains = new Set(definition2.competitors.map((competitor) => competitor.domain));
|
|
77311
77497
|
if (liveCompetitorDomains.size !== capturedDomains.size || [...liveCompetitorDomains].some((domain) => !capturedDomains.has(domain))) {
|
|
77312
77498
|
throw validationError("Captured competitors must exactly match the project competitors dispatched for this run.");
|
|
@@ -77583,6 +77769,10 @@ async function apiRoutes(app, opts) {
|
|
|
77583
77769
|
onSyncRequested: opts.onBingSyncRequested,
|
|
77584
77770
|
claimScheduledSync: opts.claimScheduledSync
|
|
77585
77771
|
});
|
|
77772
|
+
await api.register(sitemapRoutes, {
|
|
77773
|
+
googleConnectionStore: opts.googleConnectionStore,
|
|
77774
|
+
bingConnectionStore: opts.bingConnectionStore
|
|
77775
|
+
});
|
|
77586
77776
|
await api.register(googleRoutes, {
|
|
77587
77777
|
getGoogleAuthConfig: opts.getGoogleAuthConfig,
|
|
77588
77778
|
googleConnectionStore: opts.googleConnectionStore,
|
|
@@ -78046,6 +78236,7 @@ export {
|
|
|
78046
78236
|
getAuthUrl,
|
|
78047
78237
|
exchangeCode,
|
|
78048
78238
|
refreshAccessToken,
|
|
78239
|
+
listSitemaps,
|
|
78049
78240
|
fetchSearchAnalytics,
|
|
78050
78241
|
inspectUrl,
|
|
78051
78242
|
EgressRefusedError,
|