@canonry/canonry 7.88.0 → 7.90.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +16 -4
- package/assets/assets/{AgentSetupPage-DV06oPTd.js → AgentSetupPage-oSoM5flN.js} +6 -6
- package/assets/assets/AuditHistoryPanel-YrEggsyA.js +1 -0
- package/assets/assets/{BacklinksPage-Dw0wgYlg.js → BacklinksPage-vXUxxplH.js} +1 -1
- package/assets/assets/{ChartPrimitives-4D5wgAn5.js → ChartPrimitives-Cs2iavxU.js} +1 -1
- package/assets/assets/{Hint-CfeZmZig.js → Hint-uoVNfr0U.js} +1 -1
- package/assets/assets/{HistoryPage-DsDx15am.js → HistoryPage-VJ_JQ7R3.js} +1 -1
- package/assets/assets/{InfoTooltip-DrBIPuSU.js → InfoTooltip-CNLlIWeP.js} +1 -1
- package/assets/assets/MeasurementPropertyPage-BZKiS1V0.js +4 -0
- package/assets/assets/ProjectPage-p-rDlzSq.js +11 -0
- package/assets/assets/{RunsPage-BggqKbOG.js → RunsPage-CjkPCc9Z.js} +1 -1
- package/assets/assets/{SettingsPage-B7ns_ECx.js → SettingsPage-BNHMlx6C.js} +1 -1
- package/assets/assets/SiteHealthSection-BZhulwSy.js +2 -0
- package/assets/assets/{ToneBadge-B8hKcr8p.js → ToneBadge-B3rD6bDF.js} +6 -6
- package/assets/assets/TrafficPage-Dr8NTQpA.js +40 -0
- package/assets/assets/TrafficSourceDetailPage-C79eQilP.js +1 -0
- package/assets/assets/{account-context-pzV-boTi.js → account-context-GstYbqpX.js} +1 -1
- package/assets/assets/agent-trust-copy-Ca4ys8J2.js +1 -0
- package/assets/assets/{card-Mu_5i5fM.js → card-CgdGerYK.js} +1 -1
- package/assets/assets/{format-helpers-C8QYorwm.js → format-helpers-ByAY4dv0.js} +2 -2
- package/assets/assets/{index-BOUrrC9J.css → index-Bhpq6fUo.css} +1 -1
- package/assets/assets/index-BrHemYsG.js +60 -0
- package/assets/assets/query-client-DjRrRIsq.js +1 -0
- package/assets/assets/react-query-B8JD_K7g.js +1 -0
- package/assets/assets/{react-sigma_core.esm.min-BfnfPCis.js → react-sigma_core.esm.min-OHqBWE8n.js} +1 -1
- package/assets/assets/server-traffic-Bt9bexaP.js +1 -0
- package/assets/assets/{service-B85wjZQP.js → service-SgtD5EOS.js} +1 -1
- package/assets/assets/{vendor-lucide-CoTHJwai.js → vendor-lucide-Bo69m9KC.js} +1 -1
- package/assets/assets/{vendor-markdown-cQgSk2i0.js → vendor-markdown-BiRP51B0.js} +1 -1
- package/assets/assets/{vendor-radix-wS8sWIWE.js → vendor-radix-PQeN8gOe.js} +1 -1
- package/assets/assets/{vendor-recharts-AnNqECZW.js → vendor-recharts-CY6tH-r6.js} +1 -1
- package/assets/assets/{vendor-tanstack-q5SvZtcv.js → vendor-tanstack-tBjrfzKu.js} +1 -1
- package/assets/index.html +15 -15
- package/dist/{ads-4TAIMWZC.js → ads-6BAITIBY.js} +3 -3
- package/dist/{agent-C7NKP2TO.js → agent-BE3D65U4.js} +7 -7
- package/dist/{agent-ask-EPH5FDM6.js → agent-ask-TT6BAV2T.js} +3 -3
- package/dist/{agent-conversations-LPBBVRKK.js → agent-conversations-GJDAWU36.js} +3 -3
- package/dist/{agent-journeys-MBORINJA.js → agent-journeys-UEL27GNY.js} +3 -3
- package/dist/{agent-memory-NJIZO75R.js → agent-memory-PB74OP46.js} +3 -3
- package/dist/{agent-providers-ZQVUOCX6.js → agent-providers-5WFMNRRK.js} +3 -3
- package/dist/{agent-transcript-67HFY5GT.js → agent-transcript-IAGB3JXG.js} +3 -3
- package/dist/{analytics-TUO6BVB6.js → analytics-5BVE7KUN.js} +3 -3
- package/dist/{apply-V7A5O7YS.js → apply-YJJWQ4JX.js} +3 -3
- package/dist/{backfill-PGKYR6NA.js → backfill-6OLGZIEV.js} +10 -10
- package/dist/{backlinks-5GGPQNR4.js → backlinks-4JK26OIO.js} +3 -3
- package/dist/{bing-PQOLIW2X.js → bing-NZYYPB2M.js} +43 -8
- package/dist/{bootstrap-IH3XX6PI.js → bootstrap-6KHK63W3.js} +10 -10
- package/dist/{cdp-B3PFQ5RB.js → cdp-PTXWVX6T.js} +7 -7
- package/dist/{chunk-EL4HRIID.js → chunk-2UULABPI.js} +7 -3
- package/dist/{chunk-PZLEKWYA.js → chunk-7ASKZHS4.js} +1 -1
- package/dist/{chunk-R7K24KY7.js → chunk-7VJJXLRP.js} +3 -3
- package/dist/{chunk-4A4X7VXE.js → chunk-AFYKUX2X.js} +2 -2
- package/dist/{chunk-EYSQMZ6L.js → chunk-C2XWWIJZ.js} +413 -376
- package/dist/{chunk-2KL2XHS6.js → chunk-DF4B6OGX.js} +4 -4
- package/dist/{chunk-OKLDBQYW.js → chunk-DZO6CATO.js} +4 -4
- package/dist/{chunk-Q2SJRZA3.js → chunk-ETIIPSWL.js} +117 -6
- package/dist/{chunk-5WWEZ26Y.js → chunk-F442MSX2.js} +2 -2
- package/dist/{chunk-EGUJNHAZ.js → chunk-F6WNKKGG.js} +33 -7
- package/dist/{chunk-TL52ELES.js → chunk-GKX5F7NX.js} +1 -1
- package/dist/{chunk-LBZYPZAA.js → chunk-HUQDD26S.js} +7 -7
- package/dist/{chunk-N6AWKBLY.js → chunk-I7DAEE3L.js} +5 -5
- package/dist/{chunk-FNQFM3TK.js → chunk-L2MDMZFT.js} +3 -3
- package/dist/{chunk-DILCH7RL.js → chunk-LQWMNDZR.js} +1 -1
- package/dist/{chunk-FLJVGHTQ.js → chunk-LTZNUDSI.js} +3 -3
- package/dist/{chunk-PBOIN52J.js → chunk-N7LRPMWR.js} +2 -2
- package/dist/{chunk-E4DUE2PW.js → chunk-NKGQCCUS.js} +6 -6
- package/dist/{chunk-IR3MYKIA.js → chunk-NPX5SL54.js} +1 -1
- package/dist/{chunk-5KDJJO2N.js → chunk-OOVWMWMT.js} +1 -1
- package/dist/{chunk-DVWAA3ES.js → chunk-OVUFIG22.js} +89 -21
- package/dist/{chunk-FL76YWYN.js → chunk-PID3C4B7.js} +1 -1
- package/dist/{chunk-D354CQ6T.js → chunk-Q6EZEVXE.js} +1 -1
- package/dist/{chunk-UVXQQZB5.js → chunk-QLVTMPFW.js} +2 -2
- package/dist/{chunk-4T2VZPDG.js → chunk-QWNDNZJ3.js} +2 -2
- package/dist/{chunk-4VMHN5LL.js → chunk-SDXMG4VX.js} +1159 -899
- package/dist/{chunk-CYQCO5NY.js → chunk-TA4L7U2M.js} +1 -1
- package/dist/{chunk-MCQK432C.js → chunk-UG7TZJIG.js} +1 -1
- package/dist/{chunk-ZQGWB7VJ.js → chunk-USHY3SH6.js} +62 -1
- package/dist/{chunk-O7OZVZWR.js → chunk-WNDBACO7.js} +43 -13
- package/dist/{chunk-MEN27RAJ.js → chunk-WYMBWCG5.js} +3 -1
- package/dist/{chunk-BDLC3DFM.js → chunk-XCEA5OMI.js} +2 -2
- package/dist/{chunk-6DIDDYG6.js → chunk-YB2GAK32.js} +5 -5
- package/dist/{chunk-TAQHHXBC.js → chunk-YPJCZD5H.js} +4 -4
- package/dist/{chunk-IPOYH4WE.js → chunk-ZV6PMFF4.js} +1 -1
- package/dist/{citations-J3IGPX4L.js → citations-WX4VO6ZI.js} +3 -3
- package/dist/{cli-catalog-2PCRGKPH.js → cli-catalog-BTRNR2UP.js} +3 -3
- package/dist/{cli-commands-4MUJZCZO.js → cli-commands-5ISZA43S.js} +7 -7
- package/dist/cli.js +13 -13
- package/dist/{competitor-X7Q2AZGL.js → competitor-HZRZWCEN.js} +3 -3
- package/dist/{content-HTR3PJPX.js → content-NOZDHI3M.js} +3 -3
- package/dist/{dashboard-LSO57ZEE.js → dashboard-TZLWHXP4.js} +6 -6
- package/dist/{demo-P25N4SKK.js → demo-2PYBQD47.js} +1 -1
- package/dist/{demo-server-6Z5XSHFS.js → demo-server-JEQNGMMV.js} +12 -12
- package/dist/{discover-KE6TQPOC.js → discover-A2XFXHOC.js} +3 -3
- package/dist/{discover-eval-XKOYGCRW.js → discover-eval-A6SBUYXF.js} +3 -3
- package/dist/{doctor-RL2J54I3.js → doctor-GZACELPF.js} +3 -3
- package/dist/{evidence-CYJ4AKUZ.js → evidence-RVFMUWIL.js} +3 -3
- package/dist/{export-cmd-2YM3PXCW.js → export-cmd-X2TFZGNJ.js} +3 -3
- package/dist/{ga-XAB66MJO.js → ga-TOT3V67A.js} +3 -3
- package/dist/{gbp-CDGBLRVV.js → gbp-KP4YZU6S.js} +3 -3
- package/dist/{google-XHCUKQ3J.js → google-GCFSTV5N.js} +4 -4
- package/dist/{google-marketing-ZR6NTWEY.js → google-marketing-ANS65KQN.js} +1 -1
- package/dist/{health-cmd-VS3EIFJD.js → health-cmd-UKMNLU6D.js} +3 -3
- package/dist/{history-YIGYYNZJ.js → history-TWGYSAZZ.js} +3 -3
- package/dist/index.d.ts +402 -44
- package/dist/index.js +25 -25
- package/dist/{init-ST3BXHQF.js → init-K4623VOY.js} +12 -12
- package/dist/{insights-KZS6TYYL.js → insights-7IOC7WIR.js} +3 -3
- package/dist/intelligence-service-42Q3OVI5.js +16 -0
- package/dist/{journeys-USOPWGW5.js → journeys-NWKT45PT.js} +3 -3
- package/dist/{keys-XY7LSM3S.js → keys-ZZYZUHUK.js} +3 -3
- package/dist/{keyword-PKBFHHF5.js → keyword-F6EIX5KM.js} +3 -3
- package/dist/{logs-PROW5TQJ.js → logs-B45FXS6E.js} +3 -3
- package/dist/{mcp-OQGC62QU.js → mcp-WPNCNWA5.js} +4 -4
- package/dist/mcp.js +9 -9
- package/dist/{measurement-plan-AI4NNBTO.js → measurement-plan-QT5HJ3KC.js} +5 -5
- package/dist/{notify-6HRPNJZ5.js → notify-Z3LIF3SO.js} +3 -3
- package/dist/{organic-evidence-IXCSUGGY.js → organic-evidence-R5TM7XUB.js} +3 -3
- package/dist/{overview-2WS3CFHK.js → overview-6TZHJLQN.js} +4 -4
- package/dist/{project-ENUZG62V.js → project-VMPVOQ2A.js} +3 -3
- package/dist/{query-WE3ZWTIB.js → query-EXSH5QP7.js} +3 -3
- package/dist/{research-DZSD5UZ2.js → research-6R6BEL4X.js} +3 -3
- package/dist/{results-export-JMUMYNUE.js → results-export-ZJRL3ECU.js} +3 -3
- package/dist/{run-QXOO7CRK.js → run-C4VXAJE3.js} +4 -4
- package/dist/{schedule-3RJKGUDN.js → schedule-KC6ETWMO.js} +3 -3
- package/dist/{search-SGS4H7I4.js → search-SUR6BEQD.js} +3 -3
- package/dist/{serve-T24QKEUY.js → serve-TVODHNFU.js} +26 -26
- package/dist/{session-HPH2A4AZ.js → session-D42KUKCZ.js} +13 -13
- package/dist/{settings-K44RK2TW.js → settings-LTCG3D5B.js} +3 -3
- package/dist/sitemaps-YMTZLF5S.js +46 -0
- package/dist/{skills-HGPPGBXG.js → skills-THMG3L64.js} +2 -2
- package/dist/{snapshot-FEHHQRY3.js → snapshot-ZEVB3EE4.js} +3 -3
- package/dist/{src-4PAYI7GE.js → src-HPVSTXVF.js} +6 -2
- package/dist/{status-L5HVFWTS.js → status-LPCPTWKO.js} +4 -4
- package/dist/{technical-aeo-Z43UCW6E.js → technical-aeo-J6LNT45B.js} +3 -3
- package/dist/{telemetry-D47SSSG7.js → telemetry-BPY34GDV.js} +4 -4
- package/dist/{tools-WCWUY2ZJ.js → tools-K3G43ESI.js} +11 -11
- package/dist/{traffic-5KH62P43.js → traffic-BOIFAPAR.js} +5 -5
- package/dist/{trends-RMAXZKRO.js → trends-A2MDRSQQ.js} +3 -3
- package/dist/{users-WT54XNEI.js → users-VJOQODJK.js} +3 -3
- package/dist/{visibility-stats-HU62U5MA.js → visibility-stats-YNMJYASA.js} +3 -3
- package/dist/{wordpress-BDAKQ4MU.js → wordpress-BWG6D7VH.js} +3 -3
- package/package.json +1 -1
- package/assets/assets/AuditHistoryPanel-C1JnxBFK.js +0 -1
- package/assets/assets/MeasurementPropertyPage-Bkxt3QcY.js +0 -4
- package/assets/assets/ProjectPage-CzrD1lpf.js +0 -11
- package/assets/assets/SiteHealthSection-B2YibqCs.js +0 -2
- package/assets/assets/TrafficPage-DRTbgzKI.js +0 -40
- package/assets/assets/TrafficSourceDetailPage-sgLE2gdJ.js +0 -1
- package/assets/assets/agent-trust-copy-DBtiOpFu.js +0 -1
- package/assets/assets/extract-error-message-D_ptq0tW.js +0 -1
- package/assets/assets/index-fN5qDR8H.js +0 -60
- package/assets/assets/query-client-CdG4DC3B.js +0 -1
- package/assets/assets/react-query-CxeygwoL.js +0 -1
- package/dist/intelligence-service-LCTP5BC7.js +0 -16
|
@@ -15,14 +15,14 @@ import {
|
|
|
15
15
|
normalizeCloudflareWorkerEvent,
|
|
16
16
|
sessionizeJourneyRequests,
|
|
17
17
|
verifyRequestSignature
|
|
18
|
-
} from "./chunk-
|
|
18
|
+
} from "./chunk-NPX5SL54.js";
|
|
19
19
|
import {
|
|
20
20
|
currentOutcomeAttribution,
|
|
21
21
|
getRequestContext,
|
|
22
22
|
registerRequestContext,
|
|
23
23
|
requestAuditIdentity,
|
|
24
24
|
sanitizeRequestContext
|
|
25
|
-
} from "./chunk-
|
|
25
|
+
} from "./chunk-ZV6PMFF4.js";
|
|
26
26
|
import {
|
|
27
27
|
RawEventSampleEventTypes,
|
|
28
28
|
SentimentIdempotencyConflict,
|
|
@@ -106,6 +106,7 @@ import {
|
|
|
106
106
|
gscDataWatermarks,
|
|
107
107
|
gscQueryDailyTotals,
|
|
108
108
|
gscSearchData,
|
|
109
|
+
gscSitemaps,
|
|
109
110
|
gscUrlInspections,
|
|
110
111
|
gtmConnections,
|
|
111
112
|
gtmRawSnapshots,
|
|
@@ -171,6 +172,7 @@ import {
|
|
|
171
172
|
siteCrawlPages,
|
|
172
173
|
siteCrawlRunRequests,
|
|
173
174
|
siteCrawlSnapshots,
|
|
175
|
+
sitemapListSyncs,
|
|
174
176
|
trafficEventReceipts,
|
|
175
177
|
trafficSources,
|
|
176
178
|
unrecognizedAgentEventsHourly,
|
|
@@ -178,7 +180,7 @@ import {
|
|
|
178
180
|
userSessions,
|
|
179
181
|
users,
|
|
180
182
|
writeNegativeReviewMaxStars
|
|
181
|
-
} from "./chunk-
|
|
183
|
+
} from "./chunk-USHY3SH6.js";
|
|
182
184
|
import {
|
|
183
185
|
ADS_ACTIVATE_SCOPE,
|
|
184
186
|
ADS_ACTIVATION_ABSOLUTE_MAX_ENTITIES,
|
|
@@ -534,6 +536,7 @@ import {
|
|
|
534
536
|
bingSitemapsResponseDtoSchema,
|
|
535
537
|
bingSitesResponseDtoSchema,
|
|
536
538
|
bingStatusDtoSchema,
|
|
539
|
+
bingSubmitSitemapResponseDtoSchema,
|
|
537
540
|
bingTrafficResponseDtoSchema,
|
|
538
541
|
bingUrlChildrenResponseDtoSchema,
|
|
539
542
|
bingUrlInspectionDtoSchema,
|
|
@@ -1190,6 +1193,7 @@ import {
|
|
|
1190
1193
|
siteHealthStateSchema,
|
|
1191
1194
|
siteHealthSubgraphResponseSchema,
|
|
1192
1195
|
siteHealthTemplateDetectionSchema,
|
|
1196
|
+
sitemapCoverageResponseDtoSchema,
|
|
1193
1197
|
snapshotDiffResponseSchema,
|
|
1194
1198
|
snapshotListResponseSchema,
|
|
1195
1199
|
snapshotReportSchema,
|
|
@@ -1270,7 +1274,7 @@ import {
|
|
|
1270
1274
|
wordpressSchemaDeployResultDtoSchema,
|
|
1271
1275
|
wordpressSchemaStatusResultDtoSchema,
|
|
1272
1276
|
wordpressStatusDtoSchema
|
|
1273
|
-
} from "./chunk-
|
|
1277
|
+
} from "./chunk-ETIIPSWL.js";
|
|
1274
1278
|
|
|
1275
1279
|
// ../api-routes/src/user-session.ts
|
|
1276
1280
|
import crypto2 from "crypto";
|
|
@@ -29668,8 +29672,8 @@ async function analyticsRoutes(app) {
|
|
|
29668
29672
|
}
|
|
29669
29673
|
const earliest = new Date(projectRuns[0].createdAt);
|
|
29670
29674
|
const latest = new Date(projectRuns[projectRuns.length - 1].createdAt);
|
|
29671
|
-
const
|
|
29672
|
-
const bucketSize = bucketSizeForSpan(
|
|
29675
|
+
const spanDays2 = Math.max(1, Math.ceil((latest.getTime() - earliest.getTime()) / 864e5));
|
|
29676
|
+
const bucketSize = bucketSizeForSpan(spanDays2);
|
|
29673
29677
|
const buckets = computeBuckets(
|
|
29674
29678
|
allSnapshots,
|
|
29675
29679
|
projectRuns,
|
|
@@ -30213,10 +30217,10 @@ function bumpCategory(counts2, category, domain) {
|
|
|
30213
30217
|
}
|
|
30214
30218
|
domains.set(domain, (domains.get(domain) ?? 0) + 1);
|
|
30215
30219
|
}
|
|
30216
|
-
function bucketSizeForSpan(
|
|
30217
|
-
if (
|
|
30218
|
-
if (
|
|
30219
|
-
if (
|
|
30220
|
+
function bucketSizeForSpan(spanDays2) {
|
|
30221
|
+
if (spanDays2 <= 14) return 1;
|
|
30222
|
+
if (spanDays2 <= 60) return 7;
|
|
30223
|
+
if (spanDays2 <= 180) return 14;
|
|
30220
30224
|
return 30;
|
|
30221
30225
|
}
|
|
30222
30226
|
function computeProviderMetric(snapshots) {
|
|
@@ -35635,6 +35639,8 @@ var SCHEMA_TABLE = {
|
|
|
35635
35639
|
BingSiteResponseDto: bingSiteResponseDtoSchema,
|
|
35636
35640
|
BingSitemapsResponseDto: bingSitemapsResponseDtoSchema,
|
|
35637
35641
|
BingTrafficResponseDto: bingTrafficResponseDtoSchema,
|
|
35642
|
+
BingSubmitSitemapResponseDto: bingSubmitSitemapResponseDtoSchema,
|
|
35643
|
+
SitemapCoverageResponseDto: sitemapCoverageResponseDtoSchema,
|
|
35638
35644
|
BingUrlChildrenResponseDto: bingUrlChildrenResponseDtoSchema,
|
|
35639
35645
|
BingUrlTrafficResponseDto: bingUrlTrafficResponseDtoSchema,
|
|
35640
35646
|
BingCoverageSnapshotDto: bingCoverageSnapshotDtoSchema,
|
|
@@ -41807,12 +41813,43 @@ var routeCatalog = [
|
|
|
41807
41813
|
404: errorResponse("Project not found.")
|
|
41808
41814
|
}
|
|
41809
41815
|
},
|
|
41816
|
+
{
|
|
41817
|
+
method: "post",
|
|
41818
|
+
path: "/api/v1/projects/{name}/bing/sitemaps",
|
|
41819
|
+
summary: "Ask Bing to read a sitemap (adds a new one, rereads a listed one)",
|
|
41820
|
+
description: "Calls Bing `SubmitFeed` for the selected site. Bing lists a new sitemap once it has read it, so it shows in `/bing/sitemaps` and `/sitemaps` after the next `bing-sync`.",
|
|
41821
|
+
tags: ["bing"],
|
|
41822
|
+
parameters: [nameParameter],
|
|
41823
|
+
requestBody: {
|
|
41824
|
+
required: true,
|
|
41825
|
+
content: { "application/json": { schema: { type: "object", required: ["sitemapUrl"], properties: { sitemapUrl: stringSchema } } } }
|
|
41826
|
+
},
|
|
41827
|
+
responses: {
|
|
41828
|
+
200: jsonResponse("Bing accepted the sitemap.", "BingSubmitSitemapResponseDto"),
|
|
41829
|
+
400: errorResponse("Invalid sitemap URL, or Bing is not connected or has no site selected for this project."),
|
|
41830
|
+
404: errorResponse("Project not found."),
|
|
41831
|
+
429: errorResponse("Bing is throttling this key."),
|
|
41832
|
+
502: errorResponse("Bing refused the sitemap.")
|
|
41833
|
+
}
|
|
41834
|
+
},
|
|
41835
|
+
{
|
|
41836
|
+
method: "get",
|
|
41837
|
+
path: "/api/v1/projects/{name}/sitemaps",
|
|
41838
|
+
summary: "Compare the site's sitemaps with what Google and Bing list",
|
|
41839
|
+
description: "Stored data only. `site.sitemaps` are the sitemap files the latest Site Health scan read pages from. `google` and `bing` hold each engine's stored list (null when the engine is not connected or its list was never read) and `missing`: sitemaps the site declares, or someone submitted to the other engine, that this engine lacks. A file inside a missing index is left out; Google lists only sitemaps inside the connected property.",
|
|
41840
|
+
tags: ["sitemaps"],
|
|
41841
|
+
parameters: [nameParameter],
|
|
41842
|
+
responses: {
|
|
41843
|
+
200: jsonResponse("Sitemap coverage returned.", "SitemapCoverageResponseDto"),
|
|
41844
|
+
404: errorResponse("Project not found.")
|
|
41845
|
+
}
|
|
41846
|
+
},
|
|
41810
41847
|
{
|
|
41811
41848
|
method: "get",
|
|
41812
41849
|
path: "/api/v1/projects/{name}/bing/traffic",
|
|
41813
|
-
summary: "Get stored daily Bing site clicks and impressions",
|
|
41850
|
+
summary: "Get stored daily Bing site clicks and impressions, with trend fits and a period comparison",
|
|
41814
41851
|
tags: ["bing"],
|
|
41815
|
-
parameters: [nameParameter, { name: "startDate", in: "query", description: "First day (YYYY-MM-DD). Wins over days.", schema: stringSchema }, { name: "endDate", in: "query", description: "Last day (YYYY-MM-DD).", schema: stringSchema }, { name: "days", in: "query", description: "Span in days ending at endDate, or at the newest stored day (default 28).", schema: stringSchema }],
|
|
41852
|
+
parameters: [nameParameter, { name: "startDate", in: "query", description: "First day (YYYY-MM-DD). Wins over days.", schema: stringSchema }, { name: "endDate", in: "query", description: "Last day (YYYY-MM-DD).", schema: stringSchema }, { name: "days", in: "query", description: "Span in days ending at endDate, or at the newest stored day (default 28).", schema: stringSchema }, { name: "window", in: "query", description: "Window ending at endDate or the newest stored day: 7d, 30d, 90d or all. Not allowed with startDate or days.", schema: { type: "string", enum: ["7d", "30d", "90d", "all"] } }],
|
|
41816
41853
|
responses: {
|
|
41817
41854
|
200: jsonResponse("Stored Bing traffic returned.", "BingTrafficResponseDto"),
|
|
41818
41855
|
400: errorResponse("Invalid request, or Bing is not connected or has no site selected for this project."),
|
|
@@ -41824,7 +41861,7 @@ var routeCatalog = [
|
|
|
41824
41861
|
path: "/api/v1/projects/{name}/bing/queries",
|
|
41825
41862
|
summary: "Get stored Bing query stats (weekly buckets summed over the window)",
|
|
41826
41863
|
tags: ["bing"],
|
|
41827
|
-
parameters: [nameParameter, { name: "startDate", in: "query", description: "First day (YYYY-MM-DD). Wins over days.", schema: stringSchema }, { name: "endDate", in: "query", description: "Last day (YYYY-MM-DD).", schema: stringSchema }, { name: "days", in: "query", description: "Span in days ending at endDate, or at the newest stored day (default 28).", schema: stringSchema }, { name: "query", in: "query", description: "
|
|
41864
|
+
parameters: [nameParameter, { name: "startDate", in: "query", description: "First day (YYYY-MM-DD). Wins over days.", schema: stringSchema }, { name: "endDate", in: "query", description: "Last day (YYYY-MM-DD).", schema: stringSchema }, { name: "days", in: "query", description: "Span in days ending at endDate, or at the newest stored day (default 28).", schema: stringSchema }, { name: "window", in: "query", description: "Window ending at endDate or the newest stored day: 7d, 30d, 90d or all. Not allowed with startDate or days.", schema: { type: "string", enum: ["7d", "30d", "90d", "all"] } }, { name: "query", in: "query", description: "Filter on the query: a substring, or the whole query with match=exact.", schema: stringSchema }, { name: "match", in: "query", description: "contains (default) or exact.", schema: { type: "string", enum: ["contains", "exact"] } }, { name: "groupBy", in: "query", description: "query (default) or week.", schema: stringSchema }, limitQueryParameter, offsetQueryParameter],
|
|
41828
41865
|
responses: {
|
|
41829
41866
|
200: jsonResponse("Stored Bing query stats returned.", "BingQueryTotalsResponseDto"),
|
|
41830
41867
|
400: errorResponse("Invalid request, or Bing is not connected or has no site selected for this project."),
|
|
@@ -41836,7 +41873,7 @@ var routeCatalog = [
|
|
|
41836
41873
|
path: "/api/v1/projects/{name}/bing/pages",
|
|
41837
41874
|
summary: "Get stored Bing page stats (weekly buckets summed over the window)",
|
|
41838
41875
|
tags: ["bing"],
|
|
41839
|
-
parameters: [nameParameter, { name: "startDate", in: "query", description: "First day (YYYY-MM-DD). Wins over days.", schema: stringSchema }, { name: "endDate", in: "query", description: "Last day (YYYY-MM-DD).", schema: stringSchema }, { name: "days", in: "query", description: "Span in days ending at endDate, or at the newest stored day (default 28).", schema: stringSchema }, { name: "page", in: "query", description: "
|
|
41876
|
+
parameters: [nameParameter, { name: "startDate", in: "query", description: "First day (YYYY-MM-DD). Wins over days.", schema: stringSchema }, { name: "endDate", in: "query", description: "Last day (YYYY-MM-DD).", schema: stringSchema }, { name: "days", in: "query", description: "Span in days ending at endDate, or at the newest stored day (default 28).", schema: stringSchema }, { name: "window", in: "query", description: "Window ending at endDate or the newest stored day: 7d, 30d, 90d or all. Not allowed with startDate or days.", schema: { type: "string", enum: ["7d", "30d", "90d", "all"] } }, { name: "page", in: "query", description: "Filter on the page URL: a substring, or the whole URL with match=exact.", schema: stringSchema }, { name: "match", in: "query", description: "contains (default) or exact.", schema: { type: "string", enum: ["contains", "exact"] } }, { name: "groupBy", in: "query", description: "page (default) or week.", schema: stringSchema }, limitQueryParameter, offsetQueryParameter],
|
|
41840
41877
|
responses: {
|
|
41841
41878
|
200: jsonResponse("Stored Bing page stats returned.", "BingPageTotalsResponseDto"),
|
|
41842
41879
|
400: errorResponse("Invalid request, or Bing is not connected or has no site selected for this project."),
|
|
@@ -47437,8 +47474,8 @@ function computeGscPeriodComparison(daily, bounds) {
|
|
|
47437
47474
|
const firstMs = calendarDateMs(first);
|
|
47438
47475
|
const lastMs = calendarDateMs(last);
|
|
47439
47476
|
if (firstMs === null || lastMs === null || firstMs > lastMs) return null;
|
|
47440
|
-
const
|
|
47441
|
-
const periodDays = Math.floor(
|
|
47477
|
+
const spanDays2 = Math.floor((lastMs - firstMs) / DAY_MS3) + 1;
|
|
47478
|
+
const periodDays = Math.floor(spanDays2 / 2);
|
|
47442
47479
|
if (periodDays < 1) return null;
|
|
47443
47480
|
const byDate = new Map(daily.map((row) => [row.date, row]));
|
|
47444
47481
|
const trailingStart = calendarDateAt(lastMs - (periodDays - 1) * DAY_MS3);
|
|
@@ -57892,6 +57929,15 @@ async function getChildrenUrlInfo(apiKey, siteUrl, url, page2 = 0, filterPropert
|
|
|
57892
57929
|
async function getFeeds(apiKey, siteUrl) {
|
|
57893
57930
|
return siteList(apiKey, "GetFeeds", siteUrl);
|
|
57894
57931
|
}
|
|
57932
|
+
async function submitFeed(apiKey, siteUrl, feedUrl) {
|
|
57933
|
+
validateApiKey(apiKey);
|
|
57934
|
+
validateSiteUrl2(siteUrl);
|
|
57935
|
+
validateUrl2(feedUrl);
|
|
57936
|
+
await bingFetch(apiKey, "SubmitFeed", {
|
|
57937
|
+
method: "POST",
|
|
57938
|
+
body: { siteUrl, feedUrl }
|
|
57939
|
+
});
|
|
57940
|
+
}
|
|
57895
57941
|
async function getFeedDetails(apiKey, siteUrl, feedUrl) {
|
|
57896
57942
|
validateUrl2(feedUrl);
|
|
57897
57943
|
return siteList(apiKey, "GetFeedDetails", siteUrl, { feedUrl });
|
|
@@ -58061,32 +58107,47 @@ function requireHttpUrl(value, name) {
|
|
|
58061
58107
|
}
|
|
58062
58108
|
return raw;
|
|
58063
58109
|
}
|
|
58110
|
+
var BING_WINDOW_DAYS = { "7d": 7, "30d": 30, "90d": 90, all: null };
|
|
58064
58111
|
function resolveBingWindow(query2, latest, defaultDays = BING_DEFAULT_WINDOW_DAYS) {
|
|
58065
58112
|
assertForwardRange(query2.startDate, query2.endDate);
|
|
58113
|
+
if (query2.window !== void 0 && query2.window !== "") {
|
|
58114
|
+
if (!(query2.window in BING_WINDOW_DAYS)) throw validationError('"window" must be 7d, 30d, 90d or all');
|
|
58115
|
+
if (query2.startDate || query2.days !== void 0 && query2.days !== "") {
|
|
58116
|
+
throw validationError('"window" cannot be combined with "startDate" or "days"');
|
|
58117
|
+
}
|
|
58118
|
+
}
|
|
58066
58119
|
if (query2.startDate) return { startDate: query2.startDate, endDate: query2.endDate ?? null };
|
|
58067
|
-
const days = parseDays(query2.days, defaultDays);
|
|
58068
58120
|
const anchor = query2.endDate ?? latest;
|
|
58121
|
+
if (query2.window !== void 0 && query2.window !== "") {
|
|
58122
|
+
const windowDays2 = BING_WINDOW_DAYS[query2.window] ?? null;
|
|
58123
|
+
if (!anchor) return { startDate: null, endDate: null };
|
|
58124
|
+
return { startDate: windowDays2 === null ? null : shiftDay(anchor, -(windowDays2 - 1)), endDate: anchor };
|
|
58125
|
+
}
|
|
58126
|
+
const days = parseDays(query2.days, defaultDays);
|
|
58069
58127
|
if (!anchor) return { startDate: null, endDate: null };
|
|
58070
58128
|
return { startDate: shiftDay(anchor, -(days - 1)), endDate: anchor };
|
|
58071
58129
|
}
|
|
58130
|
+
function spanDays(startDate, endDate) {
|
|
58131
|
+
return Math.round((Date.parse(`${endDate}T00:00:00Z`) - Date.parse(`${startDate}T00:00:00Z`)) / 864e5) + 1;
|
|
58132
|
+
}
|
|
58072
58133
|
function ctrOf(clicks, impressions) {
|
|
58073
58134
|
return impressions > 0 ? clicks / impressions : 0;
|
|
58074
58135
|
}
|
|
58075
58136
|
function round(value) {
|
|
58076
58137
|
return value === null ? null : Math.round(value * 100) / 100;
|
|
58077
58138
|
}
|
|
58078
|
-
function bingReadError(err, what) {
|
|
58139
|
+
function bingReadError(err, what, verb = "read") {
|
|
58079
58140
|
if (!(err instanceof BingApiError)) return err;
|
|
58080
58141
|
if (err.isThrottle || err.status === 429) {
|
|
58081
58142
|
return quotaExceeded("Bing Webmaster API", { reason: "bing-throttled", detail: err.message });
|
|
58082
58143
|
}
|
|
58083
58144
|
if (err.bingErrorCode === 14) {
|
|
58084
|
-
return providerError(`Bing refused to
|
|
58145
|
+
return providerError(`Bing refused to ${verb} ${what}: this API key is not allowed to (owner-only).`, { bingErrorCode: 14 });
|
|
58085
58146
|
}
|
|
58086
58147
|
if (err.status === 400 && err.bingErrorCode === null) {
|
|
58087
58148
|
return validationError(err.message);
|
|
58088
58149
|
}
|
|
58089
|
-
return providerError(`Bing could not
|
|
58150
|
+
return providerError(`Bing could not ${verb} ${what}: ${err.message}`, { bingErrorCode: err.bingErrorCode, httpStatus: err.status });
|
|
58090
58151
|
}
|
|
58091
58152
|
async function liveRead(what, read) {
|
|
58092
58153
|
try {
|
|
@@ -58141,49 +58202,110 @@ async function bingDataRoutes(app, opts) {
|
|
|
58141
58202
|
opts.onSyncRequested?.(runId, project.id);
|
|
58142
58203
|
return app.db.select().from(runs).where(eq79(runs.id, runId)).get();
|
|
58143
58204
|
});
|
|
58205
|
+
app.post("/projects/:name/bing/sitemaps", async (request) => {
|
|
58206
|
+
const project = resolveProject(app.db, request.params.name);
|
|
58207
|
+
const sitemapUrl = requireHttpUrl(request.body?.sitemapUrl, "sitemapUrl");
|
|
58208
|
+
const { apiKey, siteUrl } = opts.requireSite(project.canonicalDomain);
|
|
58209
|
+
try {
|
|
58210
|
+
await submitFeed(apiKey, siteUrl, sitemapUrl);
|
|
58211
|
+
} catch (err) {
|
|
58212
|
+
throw bingReadError(err, "the sitemap", "accept");
|
|
58213
|
+
}
|
|
58214
|
+
return { sitemapUrl, accepted: true };
|
|
58215
|
+
});
|
|
58144
58216
|
app.get("/projects/:name/bing/traffic", async (request) => {
|
|
58145
58217
|
const project = resolveProject(app.db, request.params.name);
|
|
58146
58218
|
const bounds = app.db.select({ latest: sql32`MAX(${bingTrafficDaily.date})`, earliest: sql32`MIN(${bingTrafficDaily.date})` }).from(bingTrafficDaily).where(eq79(bingTrafficDaily.projectId, project.id)).get();
|
|
58147
58219
|
const latest = bounds?.latest ?? null;
|
|
58220
|
+
const earliest = bounds?.earliest ?? null;
|
|
58148
58221
|
const window2 = resolveBingWindow(request.query, latest);
|
|
58222
|
+
const priorStart = window2.startDate && window2.endDate ? shiftDay(window2.startDate, -spanDays(window2.startDate, window2.endDate)) : null;
|
|
58223
|
+
const priorWindowIsSynced = priorStart !== null && earliest !== null && earliest <= priorStart;
|
|
58224
|
+
const fetchStart = priorWindowIsSynced ? priorStart : window2.startDate;
|
|
58149
58225
|
const conditions = [eq79(bingTrafficDaily.projectId, project.id)];
|
|
58150
|
-
if (
|
|
58226
|
+
if (fetchStart) conditions.push(gte21(bingTrafficDaily.date, fetchStart));
|
|
58151
58227
|
if (window2.endDate) conditions.push(lte21(bingTrafficDaily.date, window2.endDate));
|
|
58152
|
-
const
|
|
58228
|
+
const fetched = latest === null && !request.query.startDate ? [] : app.db.select().from(bingTrafficDaily).where(and67(...conditions)).orderBy(asc18(bingTrafficDaily.date)).all();
|
|
58229
|
+
const rows = window2.startDate ? fetched.filter((row) => row.date >= window2.startDate) : fetched;
|
|
58153
58230
|
const clicks = rows.reduce((sum, row) => sum + row.clicks, 0);
|
|
58154
58231
|
const impressions = rows.reduce((sum, row) => sum + row.impressions, 0);
|
|
58232
|
+
const byDate = new Map(rows.map((row) => [row.date, row]));
|
|
58233
|
+
const syncedStart = window2.startDate && earliest && window2.startDate > earliest ? window2.startDate : earliest ?? window2.startDate;
|
|
58234
|
+
const denseDates = rows.length === 0 ? [] : calendarDateRange(syncedStart ?? rows[0].date, window2.endDate ?? rows[rows.length - 1].date);
|
|
58235
|
+
const dense = (pick) => denseDates.map((date) => pick(byDate.get(date) ?? { clicks: 0, impressions: 0 }));
|
|
58236
|
+
const endsWithinData = latest !== null && window2.endDate !== null && window2.endDate <= latest;
|
|
58237
|
+
const comparisonRows = fetched.map((row) => ({ date: row.date, clicks: row.clicks, impressions: row.impressions, position: null, fromPropertyTotals: true }));
|
|
58238
|
+
const periodComparison = !endsWithinData || rows.length === 0 ? null : priorWindowIsSynced ? computeGscPeriodComparison(comparisonRows, { startDate: priorStart, endDate: window2.endDate, basis: "prior-window" }) : computeGscPeriodComparison(comparisonRows, { startDate: syncedStart ?? rows[0].date, endDate: window2.endDate, basis: "split-window" });
|
|
58155
58239
|
return {
|
|
58156
58240
|
rows: rows.map((row) => ({ date: row.date, clicks: row.clicks, impressions: row.impressions, ctr: ctrOf(row.clicks, row.impressions) })),
|
|
58157
58241
|
totals: { clicks, impressions, ctr: ctrOf(clicks, impressions) },
|
|
58158
58242
|
window: window2,
|
|
58159
58243
|
latestAvailableDate: latest,
|
|
58160
|
-
earliestAvailableDate:
|
|
58244
|
+
earliestAvailableDate: earliest,
|
|
58245
|
+
trends: {
|
|
58246
|
+
clicks: linearTrend(dense((row) => row.clicks)),
|
|
58247
|
+
impressions: linearTrend(dense((row) => row.impressions)),
|
|
58248
|
+
ctr: linearTrend(dense((row) => row.impressions > 0 ? row.clicks / row.impressions : null))
|
|
58249
|
+
},
|
|
58250
|
+
periodComparison
|
|
58161
58251
|
};
|
|
58162
58252
|
});
|
|
58163
58253
|
function weeklyList(table, keyColumn, projectId, query2, keyGroup) {
|
|
58164
|
-
const
|
|
58254
|
+
const bounds = app.db.select({ latest: sql32`MAX(${table.date})`, earliest: sql32`MIN(${table.date})` }).from(table).where(eq79(table.projectId, projectId)).get();
|
|
58255
|
+
const latest = bounds?.latest ?? null;
|
|
58165
58256
|
const window2 = resolveBingWindow(query2, latest);
|
|
58166
58257
|
const groupBy = query2.groupBy ?? keyGroup;
|
|
58167
58258
|
if (groupBy !== keyGroup && groupBy !== "week") {
|
|
58168
58259
|
throw validationError(`"groupBy" must be ${keyGroup} or week`);
|
|
58169
58260
|
}
|
|
58170
|
-
const
|
|
58171
|
-
if (
|
|
58172
|
-
if (window2.endDate) conditions.push(lte21(table.date, window2.endDate));
|
|
58261
|
+
const match = query2.match ?? "contains";
|
|
58262
|
+
if (match !== "contains" && match !== "exact") throw validationError('"match" must be contains or exact');
|
|
58173
58263
|
const search = query2.search?.trim();
|
|
58174
|
-
|
|
58175
|
-
|
|
58264
|
+
const filters = [eq79(table.projectId, projectId)];
|
|
58265
|
+
if (search) {
|
|
58266
|
+
filters.push(match === "exact" ? eq79(keyColumn, search) : sql32`${keyColumn} LIKE ${"%" + escapeLikePattern(search) + "%"} ESCAPE '\\'`);
|
|
58267
|
+
}
|
|
58268
|
+
const between = (startDate, endDate) => {
|
|
58269
|
+
const conditions = [...filters];
|
|
58270
|
+
if (startDate) conditions.push(gte21(table.date, startDate));
|
|
58271
|
+
if (endDate) conditions.push(lte21(table.date, endDate));
|
|
58272
|
+
return and67(...conditions);
|
|
58273
|
+
};
|
|
58274
|
+
const where = between(window2.startDate, window2.endDate);
|
|
58176
58275
|
const empty = latest === null && !query2.startDate;
|
|
58276
|
+
const sumOver = (condition) => totalsShape(
|
|
58277
|
+
app.db.select(weeklyAggregates(table)).from(table).where(condition).get() ?? { clicks: 0, impressions: 0, avgImpressionPosition: null, avgClickPosition: null, weeks: 0 }
|
|
58278
|
+
);
|
|
58279
|
+
const windowTotals = empty ? null : sumOver(where);
|
|
58280
|
+
const totals = windowTotals && windowTotals.weeks > 0 ? windowTotals : null;
|
|
58281
|
+
let comparison = null;
|
|
58282
|
+
if (totals && window2.startDate && window2.endDate && bounds?.earliest) {
|
|
58283
|
+
const days = spanDays(window2.startDate, window2.endDate);
|
|
58284
|
+
const priorStart = shiftDay(window2.startDate, -days);
|
|
58285
|
+
if (bounds.earliest <= shiftDay(priorStart, 6)) {
|
|
58286
|
+
const prior = sumOver(between(priorStart, shiftDay(window2.startDate, -1)));
|
|
58287
|
+
comparison = {
|
|
58288
|
+
days,
|
|
58289
|
+
prior,
|
|
58290
|
+
difference: {
|
|
58291
|
+
clicks: totals.clicks - prior.clicks,
|
|
58292
|
+
impressions: totals.impressions - prior.impressions,
|
|
58293
|
+
ctr: roundRatio(totals.ctr - prior.ctr, RatioUnits.fraction),
|
|
58294
|
+
avgImpressionPosition: totals.avgImpressionPosition !== null && prior.avgImpressionPosition !== null ? round(totals.avgImpressionPosition - prior.avgImpressionPosition) : null
|
|
58295
|
+
}
|
|
58296
|
+
};
|
|
58297
|
+
}
|
|
58298
|
+
}
|
|
58177
58299
|
if (groupBy === "week") {
|
|
58178
58300
|
const weekly = empty ? [] : app.db.select({ date: table.date, ...weeklyAggregates(table) }).from(table).where(where).groupBy(table.date).orderBy(asc18(table.date)).all().map((row) => ({ date: row.date, ...totalsShape(row) }));
|
|
58179
|
-
return { groupBy: "week", keyed: [], weekly, totalMatching: weekly.length, truncated: false, window: window2, latestAvailableDate: latest };
|
|
58301
|
+
return { groupBy: "week", keyed: [], weekly, totalMatching: weekly.length, truncated: false, window: window2, latestAvailableDate: latest, totals, comparison };
|
|
58180
58302
|
}
|
|
58181
58303
|
const limit = parseLimit(query2.limit);
|
|
58182
58304
|
const offset = parseOffset(query2.offset);
|
|
58183
58305
|
const aggregates = weeklyAggregates(table);
|
|
58184
58306
|
const keyed = empty ? [] : app.db.select({ key: keyColumn, ...aggregates }).from(table).where(where).groupBy(keyColumn).orderBy(desc34(aggregates.clicks), desc34(aggregates.impressions), asc18(keyColumn)).limit(limit).offset(offset).all().map((row) => ({ key: row.key, ...totalsShape(row) }));
|
|
58185
58307
|
const totalMatching = empty ? 0 : Number(app.db.select({ total: sql32`COUNT(DISTINCT ${keyColumn})` }).from(table).where(where).get()?.total ?? 0);
|
|
58186
|
-
return { groupBy: keyGroup, keyed, weekly: [], totalMatching, truncated: offset + keyed.length < totalMatching, window: window2, latestAvailableDate: latest };
|
|
58308
|
+
return { groupBy: keyGroup, keyed, weekly: [], totalMatching, truncated: offset + keyed.length < totalMatching, window: window2, latestAvailableDate: latest, totals, comparison };
|
|
58187
58309
|
}
|
|
58188
58310
|
app.get("/projects/:name/bing/queries", async (request) => {
|
|
58189
58311
|
const project = resolveProject(app.db, request.params.name);
|
|
@@ -58195,7 +58317,9 @@ async function bingDataRoutes(app, opts) {
|
|
|
58195
58317
|
totalMatching: result.totalMatching,
|
|
58196
58318
|
truncated: result.truncated,
|
|
58197
58319
|
window: result.window,
|
|
58198
|
-
latestAvailableDate: result.latestAvailableDate
|
|
58320
|
+
latestAvailableDate: result.latestAvailableDate,
|
|
58321
|
+
totals: result.totals,
|
|
58322
|
+
comparison: result.comparison
|
|
58199
58323
|
};
|
|
58200
58324
|
});
|
|
58201
58325
|
app.get("/projects/:name/bing/pages", async (request) => {
|
|
@@ -58208,7 +58332,9 @@ async function bingDataRoutes(app, opts) {
|
|
|
58208
58332
|
totalMatching: result.totalMatching,
|
|
58209
58333
|
truncated: result.truncated,
|
|
58210
58334
|
window: result.window,
|
|
58211
|
-
latestAvailableDate: result.latestAvailableDate
|
|
58335
|
+
latestAvailableDate: result.latestAvailableDate,
|
|
58336
|
+
totals: result.totals,
|
|
58337
|
+
comparison: result.comparison
|
|
58212
58338
|
};
|
|
58213
58339
|
});
|
|
58214
58340
|
app.get("/projects/:name/bing/crawl-stats", async (request) => {
|
|
@@ -58966,18 +59092,147 @@ async function bingRoutes(app, opts) {
|
|
|
58966
59092
|
});
|
|
58967
59093
|
}
|
|
58968
59094
|
|
|
59095
|
+
// ../api-routes/src/sitemaps.ts
|
|
59096
|
+
import { and as and69, asc as asc19, desc as desc36, eq as eq81, inArray as inArray39, sql as sql33 } from "drizzle-orm";
|
|
59097
|
+
|
|
59098
|
+
// ../api-routes/src/sitemap-coverage.ts
|
|
59099
|
+
function sitemapKey(url) {
|
|
59100
|
+
try {
|
|
59101
|
+
const parsed = new URL(url);
|
|
59102
|
+
return `${parsed.protocol}//${parsed.host.toLowerCase()}${parsed.pathname}${parsed.search}`;
|
|
59103
|
+
} catch {
|
|
59104
|
+
return url;
|
|
59105
|
+
}
|
|
59106
|
+
}
|
|
59107
|
+
function googleSitemapState(row) {
|
|
59108
|
+
if (row.isPending) return { state: "pending", statusLabel: "Processing" };
|
|
59109
|
+
if ((row.errors ?? 0) > 0) return { state: "errors", statusLabel: "Errors" };
|
|
59110
|
+
if ((row.warnings ?? 0) > 0) return { state: "warnings", statusLabel: "Warnings" };
|
|
59111
|
+
return { state: "ok", statusLabel: "Success" };
|
|
59112
|
+
}
|
|
59113
|
+
function bingSitemapState(status) {
|
|
59114
|
+
const word = status.trim().toLowerCase();
|
|
59115
|
+
if (word === "success") return "ok";
|
|
59116
|
+
if (word === "pending" || word === "processing") return "pending";
|
|
59117
|
+
if (word.includes("error") || word.includes("fail")) return "errors";
|
|
59118
|
+
if (word.includes("warn")) return "warnings";
|
|
59119
|
+
return "other";
|
|
59120
|
+
}
|
|
59121
|
+
function missingFor(list, other, site, parents, accepts) {
|
|
59122
|
+
const have = new Set(list.sitemaps.map((row) => sitemapKey(row.url)));
|
|
59123
|
+
const candidates = /* @__PURE__ */ new Map();
|
|
59124
|
+
const add = (url, source) => {
|
|
59125
|
+
const key = sitemapKey(url);
|
|
59126
|
+
if (have.has(key) || !accepts(url)) return;
|
|
59127
|
+
const entry = candidates.get(key) ?? { url, foundIn: /* @__PURE__ */ new Set() };
|
|
59128
|
+
entry.foundIn.add(source);
|
|
59129
|
+
candidates.set(key, entry);
|
|
59130
|
+
};
|
|
59131
|
+
for (const url of site) add(url, "site");
|
|
59132
|
+
for (const row of other?.list.sitemaps ?? []) if (row.parentUrl === null && row.submittedAt !== null) add(row.url, other.name);
|
|
59133
|
+
return [...candidates.entries()].filter(([key]) => {
|
|
59134
|
+
const parent2 = parents.get(key);
|
|
59135
|
+
return parent2 === void 0 || !candidates.has(parent2);
|
|
59136
|
+
}).map(([, entry]) => ({ url: entry.url, foundIn: ["site", "google", "bing"].filter((source) => entry.foundIn.has(source)) })).sort((a, b) => a.url.localeCompare(b.url));
|
|
59137
|
+
}
|
|
59138
|
+
function buildSitemapCoverage(input) {
|
|
59139
|
+
const parents = /* @__PURE__ */ new Map();
|
|
59140
|
+
for (const list of [input.google, input.bing]) {
|
|
59141
|
+
for (const row of list?.sitemaps ?? []) if (row.parentUrl !== null) parents.set(sitemapKey(row.url), sitemapKey(row.parentUrl));
|
|
59142
|
+
}
|
|
59143
|
+
const engine = (name, list, otherName, other) => list === null ? null : {
|
|
59144
|
+
syncedAt: list.syncedAt,
|
|
59145
|
+
sitemaps: list.sitemaps,
|
|
59146
|
+
missing: missingFor(list, other ? { name: otherName, list: other } : null, input.site.sitemaps, parents, input.accepts?.[name] ?? (() => true))
|
|
59147
|
+
};
|
|
59148
|
+
return {
|
|
59149
|
+
site: input.site,
|
|
59150
|
+
google: engine("google", input.google, "bing", input.bing),
|
|
59151
|
+
bing: engine("bing", input.bing, "google", input.google)
|
|
59152
|
+
};
|
|
59153
|
+
}
|
|
59154
|
+
|
|
59155
|
+
// ../api-routes/src/sitemaps.ts
|
|
59156
|
+
async function sitemapRoutes(app, opts) {
|
|
59157
|
+
app.get("/projects/:name/sitemaps", async (request) => {
|
|
59158
|
+
const project = resolveProject(app.db, request.params.name);
|
|
59159
|
+
const scan = app.db.select({ runId: siteCrawlSnapshots.runId, attemptId: siteCrawlSnapshots.attemptId, createdAt: siteCrawlSnapshots.createdAt }).from(siteCrawlSnapshots).innerJoin(runs, eq81(siteCrawlSnapshots.runId, runs.id)).where(and69(
|
|
59160
|
+
eq81(siteCrawlSnapshots.projectId, project.id),
|
|
59161
|
+
eq81(runs.projectId, project.id),
|
|
59162
|
+
eq81(runs.kind, RunKinds["site-audit"]),
|
|
59163
|
+
inArray39(runs.status, [RunStatuses.completed, RunStatuses.partial]),
|
|
59164
|
+
notProbeRun()
|
|
59165
|
+
)).orderBy(desc36(siteCrawlSnapshots.createdAt), desc36(siteCrawlSnapshots.runId)).limit(1).get();
|
|
59166
|
+
const siteSitemaps = scan ? app.db.all(sql33`
|
|
59167
|
+
select distinct source.value as url
|
|
59168
|
+
from ${siteCrawlPages}, json_each(${siteCrawlPages.sitemapMetadata}, '$.sources') as source
|
|
59169
|
+
where ${siteCrawlPages.projectId} = ${project.id}
|
|
59170
|
+
and ${siteCrawlPages.runId} = ${scan.runId}
|
|
59171
|
+
and (${scan.attemptId} is null or ${siteCrawlPages.attemptId} = ${scan.attemptId})
|
|
59172
|
+
and source.type = 'text'
|
|
59173
|
+
order by url`).map((row) => row.url) : [];
|
|
59174
|
+
const listedAt = new Map(app.db.select({ engine: sitemapListSyncs.engine, syncedAt: sitemapListSyncs.syncedAt }).from(sitemapListSyncs).where(eq81(sitemapListSyncs.projectId, project.id)).all().map((row) => [row.engine, row.syncedAt]));
|
|
59175
|
+
const googleRows = app.db.select().from(gscSitemaps).where(eq81(gscSitemaps.projectId, project.id)).orderBy(asc19(gscSitemaps.url)).all();
|
|
59176
|
+
const google = listedAt.has("google") ? {
|
|
59177
|
+
syncedAt: listedAt.get("google"),
|
|
59178
|
+
sitemaps: googleRows.map((row) => ({
|
|
59179
|
+
url: row.url,
|
|
59180
|
+
parentUrl: row.parentUrl,
|
|
59181
|
+
isIndex: row.isIndex,
|
|
59182
|
+
type: row.type,
|
|
59183
|
+
...googleSitemapState(row),
|
|
59184
|
+
warnings: row.warnings,
|
|
59185
|
+
errors: row.errors,
|
|
59186
|
+
urlCount: row.urlCount,
|
|
59187
|
+
submittedAt: row.submittedAt,
|
|
59188
|
+
lastReadAt: row.lastDownloadedAt
|
|
59189
|
+
}))
|
|
59190
|
+
} : null;
|
|
59191
|
+
const bingRows = app.db.select().from(bingSitemaps).where(eq81(bingSitemaps.projectId, project.id)).orderBy(asc19(bingSitemaps.url)).all();
|
|
59192
|
+
const bing = listedAt.has("bing") ? {
|
|
59193
|
+
syncedAt: listedAt.get("bing"),
|
|
59194
|
+
sitemaps: bingRows.map((row) => ({
|
|
59195
|
+
url: row.url,
|
|
59196
|
+
parentUrl: row.parentUrl,
|
|
59197
|
+
isIndex: /index/i.test(row.type),
|
|
59198
|
+
type: row.type,
|
|
59199
|
+
state: bingSitemapState(row.status),
|
|
59200
|
+
statusLabel: row.status,
|
|
59201
|
+
warnings: null,
|
|
59202
|
+
errors: null,
|
|
59203
|
+
urlCount: row.urlCount,
|
|
59204
|
+
submittedAt: row.submittedAt,
|
|
59205
|
+
lastReadAt: row.lastCrawledAt
|
|
59206
|
+
}))
|
|
59207
|
+
} : null;
|
|
59208
|
+
const propertyId = opts.googleConnectionStore?.getConnection(project.canonicalDomain, "gsc")?.propertyId ?? null;
|
|
59209
|
+
const bingSite = opts.bingConnectionStore?.getConnection(project.canonicalDomain)?.siteUrl ?? null;
|
|
59210
|
+
return buildSitemapCoverage({
|
|
59211
|
+
site: { scannedAt: scan?.createdAt ?? null, runId: scan?.runId ?? null, sitemaps: siteSitemaps },
|
|
59212
|
+
google,
|
|
59213
|
+
bing,
|
|
59214
|
+
accepts: {
|
|
59215
|
+
// Google rejects a sitemap outside the property, so it is never "missing" there.
|
|
59216
|
+
google: (url) => propertyId === null || isSitemapOwnedByProperty(url, propertyId, project.canonicalDomain),
|
|
59217
|
+
// Bing takes sitemaps on its site's host and subdomains.
|
|
59218
|
+
bing: (url) => bingSite === null || hostMatchesDomain(url, bingSite)
|
|
59219
|
+
}
|
|
59220
|
+
});
|
|
59221
|
+
});
|
|
59222
|
+
}
|
|
59223
|
+
|
|
58969
59224
|
// ../api-routes/src/cdp.ts
|
|
58970
59225
|
import fs from "fs";
|
|
58971
59226
|
import path from "path";
|
|
58972
59227
|
import os from "os";
|
|
58973
|
-
import { eq as
|
|
59228
|
+
import { eq as eq82, and as and70 } from "drizzle-orm";
|
|
58974
59229
|
function getScreenshotDir() {
|
|
58975
59230
|
return path.join(os.homedir(), ".canonry", "screenshots");
|
|
58976
59231
|
}
|
|
58977
59232
|
async function cdpRoutes(app, opts) {
|
|
58978
59233
|
app.get("/screenshots/:snapshotId", async (request, reply) => {
|
|
58979
59234
|
const { snapshotId } = request.params;
|
|
58980
|
-
const snapshot = app.db.select({ screenshotPath: querySnapshots.screenshotPath, projectId: runs.projectId }).from(querySnapshots).innerJoin(runs,
|
|
59235
|
+
const snapshot = app.db.select({ screenshotPath: querySnapshots.screenshotPath, projectId: runs.projectId }).from(querySnapshots).innerJoin(runs, eq82(querySnapshots.runId, runs.id)).where(eq82(querySnapshots.id, snapshotId)).get();
|
|
58981
59236
|
if (!snapshot?.screenshotPath) {
|
|
58982
59237
|
const err = notFound("Screenshot", snapshotId);
|
|
58983
59238
|
return reply.code(err.statusCode).send(err.toJSON());
|
|
@@ -59058,7 +59313,7 @@ async function cdpRoutes(app, opts) {
|
|
|
59058
59313
|
async (request, reply) => {
|
|
59059
59314
|
const project = resolveProject(app.db, request.params.name);
|
|
59060
59315
|
const { runId } = request.params;
|
|
59061
|
-
const run = app.db.select().from(runs).where(
|
|
59316
|
+
const run = app.db.select().from(runs).where(and70(eq82(runs.id, runId), eq82(runs.projectId, project.id))).get();
|
|
59062
59317
|
if (!run) {
|
|
59063
59318
|
const err = notFound("Run", runId);
|
|
59064
59319
|
return reply.code(err.statusCode).send(err.toJSON());
|
|
@@ -59071,8 +59326,8 @@ async function cdpRoutes(app, opts) {
|
|
|
59071
59326
|
citedDomains: querySnapshots.citedDomains,
|
|
59072
59327
|
screenshotPath: querySnapshots.screenshotPath,
|
|
59073
59328
|
rawResponse: querySnapshots.rawResponse
|
|
59074
|
-
}).from(querySnapshots).where(
|
|
59075
|
-
const queryRows2 = app.db.select({ id: queries.id, query: queries.query }).from(queries).where(
|
|
59329
|
+
}).from(querySnapshots).where(eq82(querySnapshots.runId, runId)).all());
|
|
59330
|
+
const queryRows2 = app.db.select({ id: queries.id, query: queries.query }).from(queries).where(eq82(queries.projectId, project.id)).all();
|
|
59076
59331
|
const queryMap = new Map(queryRows2.map((q) => [q.id, q.query]));
|
|
59077
59332
|
const byQuery = /* @__PURE__ */ new Map();
|
|
59078
59333
|
for (const snap of snapshots) {
|
|
@@ -59155,7 +59410,7 @@ async function cdpRoutes(app, opts) {
|
|
|
59155
59410
|
|
|
59156
59411
|
// ../api-routes/src/ga.ts
|
|
59157
59412
|
import crypto51 from "crypto";
|
|
59158
|
-
import { eq as
|
|
59413
|
+
import { eq as eq84, desc as desc38, and as and72, sql as sql35 } from "drizzle-orm";
|
|
59159
59414
|
|
|
59160
59415
|
// ../api-routes/src/ga-session-history.ts
|
|
59161
59416
|
function buildSessionHistory(landingPageDays, dailyTotals) {
|
|
@@ -59320,7 +59575,7 @@ function findBiggestMover(current, prior) {
|
|
|
59320
59575
|
|
|
59321
59576
|
// ../api-routes/src/ga-search-landing-pages.ts
|
|
59322
59577
|
import crypto50 from "crypto";
|
|
59323
|
-
import { and as
|
|
59578
|
+
import { and as and71, asc as asc20, count as count5, desc as desc37, eq as eq83, gte as gte22, lte as lte22, sql as sql34 } from "drizzle-orm";
|
|
59324
59579
|
function persistSearchLandingSnapshot(db, input) {
|
|
59325
59580
|
db.transaction((tx) => {
|
|
59326
59581
|
deleteSearchLandingRows(tx, input.projectId);
|
|
@@ -59384,7 +59639,7 @@ function persistSearchLandingSnapshot(db, input) {
|
|
|
59384
59639
|
}
|
|
59385
59640
|
function persistSearchLandingFailure(db, projectId, propertyId, status, message, attemptedAt) {
|
|
59386
59641
|
db.transaction((tx) => {
|
|
59387
|
-
const storedProperties = tx.selectDistinct({ propertyId: gaSearchLandingWindows.propertyId }).from(gaSearchLandingWindows).where(
|
|
59642
|
+
const storedProperties = tx.selectDistinct({ propertyId: gaSearchLandingWindows.propertyId }).from(gaSearchLandingWindows).where(eq83(gaSearchLandingWindows.projectId, projectId)).all();
|
|
59388
59643
|
const otherProperty = storedProperties.some((stored) => stored.propertyId !== propertyId);
|
|
59389
59644
|
if (otherProperty) deleteSearchLandingRows(tx, projectId);
|
|
59390
59645
|
upsertSearchLandingState(tx, projectId, {
|
|
@@ -59396,8 +59651,8 @@ function persistSearchLandingFailure(db, projectId, propertyId, status, message,
|
|
|
59396
59651
|
});
|
|
59397
59652
|
}
|
|
59398
59653
|
function deleteSearchLandingRows(db, projectId) {
|
|
59399
|
-
db.delete(gaSearchLandingPages).where(
|
|
59400
|
-
db.delete(gaSearchLandingWindows).where(
|
|
59654
|
+
db.delete(gaSearchLandingPages).where(eq83(gaSearchLandingPages.projectId, projectId)).run();
|
|
59655
|
+
db.delete(gaSearchLandingWindows).where(eq83(gaSearchLandingWindows.projectId, projectId)).run();
|
|
59401
59656
|
}
|
|
59402
59657
|
function upsertSearchLandingState(db, projectId, fields, updatedAt) {
|
|
59403
59658
|
db.insert(gaMeasurementSyncStates).values({ projectId, ...fields, updatedAt }).onConflictDoUpdate({
|
|
@@ -59429,13 +59684,13 @@ function buildGaSearchLandingPages(db, projectName, query2, connections) {
|
|
|
59429
59684
|
throw validationError('"offset" must be a non-negative integer');
|
|
59430
59685
|
}
|
|
59431
59686
|
const project = resolveProject(db, projectName);
|
|
59432
|
-
const storedWindow = db.select().from(gaSearchLandingWindows).where(
|
|
59687
|
+
const storedWindow = db.select().from(gaSearchLandingWindows).where(and71(eq83(gaSearchLandingWindows.projectId, project.id), eq83(gaSearchLandingWindows.windowKey, window2))).get();
|
|
59433
59688
|
const otherProperty = storedWindow !== void 0 && storedWindow.propertyId !== resolveCurrentGa4PropertyId(connections, project.name, project.canonicalDomain);
|
|
59434
59689
|
const stored = otherProperty ? void 0 : storedWindow;
|
|
59435
|
-
const state = otherProperty ? void 0 : db.select().from(gaMeasurementSyncStates).where(
|
|
59436
|
-
const pageScope =
|
|
59690
|
+
const state = otherProperty ? void 0 : db.select().from(gaMeasurementSyncStates).where(eq83(gaMeasurementSyncStates.projectId, project.id)).get();
|
|
59691
|
+
const pageScope = and71(eq83(gaSearchLandingPages.projectId, project.id), eq83(gaSearchLandingPages.windowKey, window2));
|
|
59437
59692
|
const totalRows = otherProperty ? 0 : db.select({ value: count5() }).from(gaSearchLandingPages).where(pageScope).get()?.value ?? 0;
|
|
59438
|
-
const rows = otherProperty ? [] : db.select().from(gaSearchLandingPages).where(pageScope).orderBy(
|
|
59693
|
+
const rows = otherProperty ? [] : db.select().from(gaSearchLandingPages).where(pageScope).orderBy(desc37(gaSearchLandingPages.clicks), desc37(gaSearchLandingPages.impressions), asc20(gaSearchLandingPages.landingPage)).limit(limit).offset(offset).all();
|
|
59439
59694
|
const total = stored ? toWireMetrics({
|
|
59440
59695
|
clicks: stored.totalClicks,
|
|
59441
59696
|
impressions: stored.totalImpressions,
|
|
@@ -59472,8 +59727,8 @@ function buildGaSearchLandingPages(db, projectName, query2, connections) {
|
|
|
59472
59727
|
function searchConsoleLinkFor(db, projectId, total, windowStart, windowEnd) {
|
|
59473
59728
|
const stored = db.select({
|
|
59474
59729
|
days: count5(),
|
|
59475
|
-
impressions:
|
|
59476
|
-
}).from(gscDailyTotals).where(
|
|
59730
|
+
impressions: sql34`sum(${gscDailyTotals.impressions})`
|
|
59731
|
+
}).from(gscDailyTotals).where(and71(eq83(gscDailyTotals.projectId, projectId), gte22(gscDailyTotals.date, windowStart), lte22(gscDailyTotals.date, windowEnd))).get();
|
|
59477
59732
|
const searchConsoleImpressions = stored && stored.days > 0 ? Number(stored.impressions ?? 0) : null;
|
|
59478
59733
|
if (gaSearchLandingHasSearchData(total)) return { state: "has-data", searchConsoleImpressions };
|
|
59479
59734
|
if (searchConsoleImpressions === null) return { state: "unknown", searchConsoleImpressions };
|
|
@@ -59498,8 +59753,8 @@ function withSessionShares(rows) {
|
|
|
59498
59753
|
}
|
|
59499
59754
|
function dateRangeConditions(column, range) {
|
|
59500
59755
|
const conditions = [];
|
|
59501
|
-
if (range.startDate) conditions.push(
|
|
59502
|
-
if (range.endDate) conditions.push(
|
|
59756
|
+
if (range.startDate) conditions.push(sql35`${column} >= ${range.startDate}`);
|
|
59757
|
+
if (range.endDate) conditions.push(sql35`${column} <= ${range.endDate}`);
|
|
59503
59758
|
return conditions;
|
|
59504
59759
|
}
|
|
59505
59760
|
var SOCIAL_CHANNEL_GROUPS2 = /* @__PURE__ */ new Set(["Organic Social", "Paid Social"]);
|
|
@@ -59557,10 +59812,10 @@ function keepWinningDimensionRows(rows, groupKey) {
|
|
|
59557
59812
|
function persistAcquisitionMeasurement(db, input) {
|
|
59558
59813
|
const { startDate, endDate } = input.report;
|
|
59559
59814
|
db.transaction((tx) => {
|
|
59560
|
-
tx.delete(gaAcquisitionDaily).where(
|
|
59561
|
-
|
|
59562
|
-
|
|
59563
|
-
|
|
59815
|
+
tx.delete(gaAcquisitionDaily).where(and72(
|
|
59816
|
+
eq84(gaAcquisitionDaily.projectId, input.projectId),
|
|
59817
|
+
sql35`${gaAcquisitionDaily.date} >= ${startDate}`,
|
|
59818
|
+
sql35`${gaAcquisitionDaily.date} <= ${endDate}`
|
|
59564
59819
|
)).run();
|
|
59565
59820
|
for (const row of input.report.rows) {
|
|
59566
59821
|
tx.insert(gaAcquisitionDaily).values({
|
|
@@ -59599,10 +59854,10 @@ function persistAcquisitionMeasurement(db, input) {
|
|
|
59599
59854
|
function persistLeadMeasurement(db, input) {
|
|
59600
59855
|
const { startDate, endDate } = input.report;
|
|
59601
59856
|
db.transaction((tx) => {
|
|
59602
|
-
tx.delete(gaLeadEventsDaily).where(
|
|
59603
|
-
|
|
59604
|
-
|
|
59605
|
-
|
|
59857
|
+
tx.delete(gaLeadEventsDaily).where(and72(
|
|
59858
|
+
eq84(gaLeadEventsDaily.projectId, input.projectId),
|
|
59859
|
+
sql35`${gaLeadEventsDaily.date} >= ${startDate}`,
|
|
59860
|
+
sql35`${gaLeadEventsDaily.date} <= ${endDate}`
|
|
59606
59861
|
)).run();
|
|
59607
59862
|
for (const row of input.report.rows) {
|
|
59608
59863
|
tx.insert(gaLeadEventsDaily).values({
|
|
@@ -59646,7 +59901,7 @@ function persistLeadMeasurement(db, input) {
|
|
|
59646
59901
|
}
|
|
59647
59902
|
function resetLeadMeasurement(db, projectId, updatedAt) {
|
|
59648
59903
|
db.transaction((tx) => {
|
|
59649
|
-
tx.delete(gaLeadEventsDaily).where(
|
|
59904
|
+
tx.delete(gaLeadEventsDaily).where(eq84(gaLeadEventsDaily.projectId, projectId)).run();
|
|
59650
59905
|
tx.insert(gaMeasurementSyncStates).values({ projectId, leadStatus: "never-synced", leadError: null, leadSyncedAt: null, leadAttributionScope: null, leadEventNames: null, updatedAt }).onConflictDoUpdate({
|
|
59651
59906
|
target: gaMeasurementSyncStates.projectId,
|
|
59652
59907
|
set: { leadStatus: "never-synced", leadError: null, leadSyncedAt: null, leadAttributionScope: null, leadEventNames: null, updatedAt }
|
|
@@ -59988,7 +60243,7 @@ async function ga4Routes(app, opts) {
|
|
|
59988
60243
|
if (!connected2) {
|
|
59989
60244
|
return { connected: false, propertyId: null, clientEmail: null, authMethod: null, lastSyncedAt: null };
|
|
59990
60245
|
}
|
|
59991
|
-
const latestSync = app.db.select({ syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(
|
|
60246
|
+
const latestSync = app.db.select({ syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(eq84(gaTrafficSummaries.projectId, project.id)).orderBy(desc38(gaTrafficSummaries.syncedAt)).limit(1).get();
|
|
59992
60247
|
return {
|
|
59993
60248
|
connected: true,
|
|
59994
60249
|
propertyId: saConn?.propertyId ?? oauthConn?.propertyId ?? null,
|
|
@@ -60011,7 +60266,7 @@ async function ga4Routes(app, opts) {
|
|
|
60011
60266
|
const syncAi = !only || only === "ai";
|
|
60012
60267
|
const syncSocial = !only || only === "social";
|
|
60013
60268
|
const syncSearchLanding = !only || only === "search-landing";
|
|
60014
|
-
const measurementState = app.db.select().from(gaMeasurementSyncStates).where(
|
|
60269
|
+
const measurementState = app.db.select().from(gaMeasurementSyncStates).where(eq84(gaMeasurementSyncStates.projectId, project.id)).get();
|
|
60015
60270
|
const acquisitionDays = measurementState?.acquisitionSyncedAt != null ? days : GA4_MAX_SYNC_DAYS;
|
|
60016
60271
|
let leadDays = GA4_MAX_SYNC_DAYS;
|
|
60017
60272
|
let leadEventNames = [];
|
|
@@ -60083,10 +60338,10 @@ async function ga4Routes(app, opts) {
|
|
|
60083
60338
|
app.db.transaction((tx) => {
|
|
60084
60339
|
if (syncTraffic) {
|
|
60085
60340
|
tx.delete(gaTrafficSnapshots).where(
|
|
60086
|
-
|
|
60087
|
-
|
|
60088
|
-
|
|
60089
|
-
|
|
60341
|
+
and72(
|
|
60342
|
+
eq84(gaTrafficSnapshots.projectId, project.id),
|
|
60343
|
+
sql35`${gaTrafficSnapshots.date} >= ${summary3.periodStart}`,
|
|
60344
|
+
sql35`${gaTrafficSnapshots.date} <= ${summary3.periodEnd}`
|
|
60090
60345
|
)
|
|
60091
60346
|
).run();
|
|
60092
60347
|
for (const row of rows) {
|
|
@@ -60106,10 +60361,10 @@ async function ga4Routes(app, opts) {
|
|
|
60106
60361
|
}
|
|
60107
60362
|
}
|
|
60108
60363
|
tx.delete(gaDailyTotals).where(
|
|
60109
|
-
|
|
60110
|
-
|
|
60111
|
-
|
|
60112
|
-
|
|
60364
|
+
and72(
|
|
60365
|
+
eq84(gaDailyTotals.projectId, project.id),
|
|
60366
|
+
sql35`${gaDailyTotals.date} >= ${dailyTotals.startDate}`,
|
|
60367
|
+
sql35`${gaDailyTotals.date} <= ${dailyTotals.endDate}`
|
|
60113
60368
|
)
|
|
60114
60369
|
).run();
|
|
60115
60370
|
for (const row of dailyTotals.rows) {
|
|
@@ -60137,7 +60392,7 @@ async function ga4Routes(app, opts) {
|
|
|
60137
60392
|
createdAt: now
|
|
60138
60393
|
}).run();
|
|
60139
60394
|
}
|
|
60140
|
-
tx.delete(gaWindowMetrics).where(
|
|
60395
|
+
tx.delete(gaWindowMetrics).where(eq84(gaWindowMetrics.projectId, project.id)).run();
|
|
60141
60396
|
for (const windowMetric of windowMetrics) {
|
|
60142
60397
|
for (const [period2, totals] of [
|
|
60143
60398
|
[GaOverviewPeriods.current, windowMetric.current],
|
|
@@ -60168,10 +60423,10 @@ async function ga4Routes(app, opts) {
|
|
|
60168
60423
|
}
|
|
60169
60424
|
if (syncAi) {
|
|
60170
60425
|
tx.delete(gaAiReferrals).where(
|
|
60171
|
-
|
|
60172
|
-
|
|
60173
|
-
|
|
60174
|
-
|
|
60426
|
+
and72(
|
|
60427
|
+
eq84(gaAiReferrals.projectId, project.id),
|
|
60428
|
+
sql35`${gaAiReferrals.date} >= ${summary3.periodStart}`,
|
|
60429
|
+
sql35`${gaAiReferrals.date} <= ${summary3.periodEnd}`
|
|
60175
60430
|
)
|
|
60176
60431
|
).run();
|
|
60177
60432
|
for (const row of aiReferrals) {
|
|
@@ -60200,10 +60455,10 @@ async function ga4Routes(app, opts) {
|
|
|
60200
60455
|
}
|
|
60201
60456
|
if (syncSocial) {
|
|
60202
60457
|
tx.delete(gaSocialReferrals).where(
|
|
60203
|
-
|
|
60204
|
-
|
|
60205
|
-
|
|
60206
|
-
|
|
60458
|
+
and72(
|
|
60459
|
+
eq84(gaSocialReferrals.projectId, project.id),
|
|
60460
|
+
sql35`${gaSocialReferrals.date} >= ${summary3.periodStart}`,
|
|
60461
|
+
sql35`${gaSocialReferrals.date} <= ${summary3.periodEnd}`
|
|
60207
60462
|
)
|
|
60208
60463
|
).run();
|
|
60209
60464
|
for (const row of socialReferrals) {
|
|
@@ -60222,7 +60477,7 @@ async function ga4Routes(app, opts) {
|
|
|
60222
60477
|
}
|
|
60223
60478
|
}
|
|
60224
60479
|
if (syncSummary) {
|
|
60225
|
-
tx.delete(gaTrafficSummaries).where(
|
|
60480
|
+
tx.delete(gaTrafficSummaries).where(eq84(gaTrafficSummaries.projectId, project.id)).run();
|
|
60226
60481
|
tx.insert(gaTrafficSummaries).values({
|
|
60227
60482
|
id: crypto51.randomUUID(),
|
|
60228
60483
|
projectId: project.id,
|
|
@@ -60234,7 +60489,7 @@ async function ga4Routes(app, opts) {
|
|
|
60234
60489
|
syncedAt: now,
|
|
60235
60490
|
syncRunId: runId
|
|
60236
60491
|
}).run();
|
|
60237
|
-
tx.delete(gaTrafficWindowSummaries).where(
|
|
60492
|
+
tx.delete(gaTrafficWindowSummaries).where(eq84(gaTrafficWindowSummaries.projectId, project.id)).run();
|
|
60238
60493
|
for (const ws of windowSummaries) {
|
|
60239
60494
|
tx.insert(gaTrafficWindowSummaries).values({
|
|
60240
60495
|
id: crypto51.randomUUID(),
|
|
@@ -60364,7 +60619,7 @@ async function ga4Routes(app, opts) {
|
|
|
60364
60619
|
return recordSearchLandingFailure("error", error);
|
|
60365
60620
|
}
|
|
60366
60621
|
})();
|
|
60367
|
-
app.db.update(runs).set({ status: RunStatuses.completed, finishedAt: measurementNow }).where(
|
|
60622
|
+
app.db.update(runs).set({ status: RunStatuses.completed, finishedAt: measurementNow }).where(eq84(runs.id, runId)).run();
|
|
60368
60623
|
const syncedComponents = only ? [
|
|
60369
60624
|
...syncTraffic ? ["traffic"] : [],
|
|
60370
60625
|
...syncSummary ? ["summary"] : [],
|
|
@@ -60410,7 +60665,7 @@ async function ga4Routes(app, opts) {
|
|
|
60410
60665
|
} catch (e) {
|
|
60411
60666
|
const msg = describeError(e);
|
|
60412
60667
|
gaLog("error", "sync.fetch-failed", { projectId: project.id, runId, error: msg });
|
|
60413
|
-
app.db.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
60668
|
+
app.db.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq84(runs.id, runId)).run();
|
|
60414
60669
|
outcome2.report({ status: OutcomeStatuses.failed, ...ga4SyncFailure(e, credentialsResolved, opts, project) });
|
|
60415
60670
|
throw e;
|
|
60416
60671
|
}
|
|
@@ -60428,9 +60683,9 @@ async function ga4Routes(app, opts) {
|
|
|
60428
60683
|
totalDirectSessions: gaTrafficWindowSummaries.totalDirectSessions,
|
|
60429
60684
|
totalUsers: gaTrafficWindowSummaries.totalUsers
|
|
60430
60685
|
}).from(gaTrafficWindowSummaries).where(
|
|
60431
|
-
|
|
60432
|
-
|
|
60433
|
-
|
|
60686
|
+
and72(
|
|
60687
|
+
eq84(gaTrafficWindowSummaries.projectId, project.id),
|
|
60688
|
+
eq84(gaTrafficWindowSummaries.windowKey, range.window)
|
|
60434
60689
|
)
|
|
60435
60690
|
).get() : null;
|
|
60436
60691
|
const projectSummaryRow = app.db.select({
|
|
@@ -60439,11 +60694,11 @@ async function ga4Routes(app, opts) {
|
|
|
60439
60694
|
totalSessions: gaTrafficSummaries.totalSessions,
|
|
60440
60695
|
totalOrganicSessions: gaTrafficSummaries.totalOrganicSessions,
|
|
60441
60696
|
totalUsers: gaTrafficSummaries.totalUsers
|
|
60442
|
-
}).from(gaTrafficSummaries).where(
|
|
60697
|
+
}).from(gaTrafficSummaries).where(eq84(gaTrafficSummaries.projectId, project.id)).get();
|
|
60443
60698
|
const retainedBounds = [
|
|
60444
|
-
app.db.select({ min:
|
|
60445
|
-
app.db.select({ min:
|
|
60446
|
-
app.db.select({ min:
|
|
60699
|
+
app.db.select({ min: sql35`MIN(${gaTrafficSnapshots.date})`, max: sql35`MAX(${gaTrafficSnapshots.date})` }).from(gaTrafficSnapshots).where(eq84(gaTrafficSnapshots.projectId, project.id)).get(),
|
|
60700
|
+
app.db.select({ min: sql35`MIN(${gaAiReferrals.date})`, max: sql35`MAX(${gaAiReferrals.date})` }).from(gaAiReferrals).where(eq84(gaAiReferrals.projectId, project.id)).get(),
|
|
60701
|
+
app.db.select({ min: sql35`MIN(${gaSocialReferrals.date})`, max: sql35`MAX(${gaSocialReferrals.date})` }).from(gaSocialReferrals).where(eq84(gaSocialReferrals.projectId, project.id)).get()
|
|
60447
60702
|
];
|
|
60448
60703
|
const retainedDates = retainedBounds.flatMap((bound) => [bound?.min, bound?.max]).filter((date) => Boolean(date));
|
|
60449
60704
|
const projectSummaryCoversAll = Boolean(
|
|
@@ -60452,15 +60707,15 @@ async function ga4Routes(app, opts) {
|
|
|
60452
60707
|
const denominatorRow = windowSummaryRow ?? (range.window === "all" && projectSummaryCoversAll ? projectSummaryRow : null);
|
|
60453
60708
|
const denominatorPeriod = denominatorRow?.periodStart && denominatorRow.periodEnd ? { startDate: denominatorRow.periodStart, endDate: denominatorRow.periodEnd } : null;
|
|
60454
60709
|
const measuredRange = denominatorPeriod ? { ...range, ...denominatorPeriod } : range;
|
|
60455
|
-
const snapshotConditions = [
|
|
60456
|
-
const aiConditions = [
|
|
60457
|
-
const socialConditions = [
|
|
60710
|
+
const snapshotConditions = [eq84(gaTrafficSnapshots.projectId, project.id), ...dateRangeConditions(gaTrafficSnapshots.date, measuredRange)];
|
|
60711
|
+
const aiConditions = [eq84(gaAiReferrals.projectId, project.id), ...dateRangeConditions(gaAiReferrals.date, measuredRange)];
|
|
60712
|
+
const socialConditions = [eq84(gaSocialReferrals.projectId, project.id), ...dateRangeConditions(gaSocialReferrals.date, measuredRange)];
|
|
60458
60713
|
const snapshotTotalsRow = !denominatorRow ? (() => {
|
|
60459
60714
|
const summed = app.db.select({
|
|
60460
|
-
totalSessions:
|
|
60461
|
-
totalOrganicSessions:
|
|
60462
|
-
totalUsers:
|
|
60463
|
-
}).from(gaTrafficSnapshots).where(
|
|
60715
|
+
totalSessions: sql35`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)`,
|
|
60716
|
+
totalOrganicSessions: sql35`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)`,
|
|
60717
|
+
totalUsers: sql35`COALESCE(SUM(${gaTrafficSnapshots.users}), 0)`
|
|
60718
|
+
}).from(gaTrafficSnapshots).where(and72(...snapshotConditions)).get();
|
|
60464
60719
|
return {
|
|
60465
60720
|
totalSessions: summed?.totalSessions ?? 0,
|
|
60466
60721
|
totalOrganicSessions: summed?.totalOrganicSessions ?? 0,
|
|
@@ -60472,35 +60727,35 @@ async function ga4Routes(app, opts) {
|
|
|
60472
60727
|
})() : null;
|
|
60473
60728
|
const summaryRow = denominatorRow ?? snapshotTotalsRow;
|
|
60474
60729
|
const directTotalRow = windowSummaryRow ? { totalDirectSessions: windowSummaryRow.totalDirectSessions } : app.db.select({
|
|
60475
|
-
totalDirectSessions:
|
|
60476
|
-
}).from(gaTrafficSnapshots).where(
|
|
60730
|
+
totalDirectSessions: sql35`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)`
|
|
60731
|
+
}).from(gaTrafficSnapshots).where(and72(...snapshotConditions)).get();
|
|
60477
60732
|
const rows = app.db.select({
|
|
60478
|
-
landingPage:
|
|
60479
|
-
sessions:
|
|
60480
|
-
organicSessions:
|
|
60481
|
-
directSessions:
|
|
60482
|
-
users:
|
|
60483
|
-
}).from(gaTrafficSnapshots).where(
|
|
60733
|
+
landingPage: sql35`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`,
|
|
60734
|
+
sessions: sql35`SUM(${gaTrafficSnapshots.sessions})`,
|
|
60735
|
+
organicSessions: sql35`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
60736
|
+
directSessions: sql35`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)`,
|
|
60737
|
+
users: sql35`SUM(${gaTrafficSnapshots.users})`
|
|
60738
|
+
}).from(gaTrafficSnapshots).where(and72(...snapshotConditions)).groupBy(sql35`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`).orderBy(sql35`SUM(${gaTrafficSnapshots.sessions}) DESC`).limit(limit).all();
|
|
60484
60739
|
const aiReferralRows = app.db.select({
|
|
60485
60740
|
source: gaAiReferrals.source,
|
|
60486
60741
|
medium: gaAiReferrals.medium,
|
|
60487
60742
|
trafficClass: gaAiReferrals.trafficClass,
|
|
60488
60743
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
60489
|
-
sessions:
|
|
60490
|
-
}).from(gaAiReferrals).where(
|
|
60744
|
+
sessions: sql35`SUM(${gaAiReferrals.sessions})`
|
|
60745
|
+
}).from(gaAiReferrals).where(and72(...aiConditions)).groupBy(gaAiReferrals.source, gaAiReferrals.medium, gaAiReferrals.trafficClass, gaAiReferrals.sourceDimension).all();
|
|
60491
60746
|
const aiReferralLandingPageRows = app.db.select({
|
|
60492
60747
|
source: gaAiReferrals.source,
|
|
60493
60748
|
medium: gaAiReferrals.medium,
|
|
60494
60749
|
trafficClass: gaAiReferrals.trafficClass,
|
|
60495
60750
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
60496
|
-
landingPage:
|
|
60497
|
-
sessions:
|
|
60498
|
-
}).from(gaAiReferrals).where(
|
|
60751
|
+
landingPage: sql35`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`,
|
|
60752
|
+
sessions: sql35`SUM(${gaAiReferrals.sessions})`
|
|
60753
|
+
}).from(gaAiReferrals).where(and72(...aiConditions)).groupBy(
|
|
60499
60754
|
gaAiReferrals.source,
|
|
60500
60755
|
gaAiReferrals.medium,
|
|
60501
60756
|
gaAiReferrals.trafficClass,
|
|
60502
60757
|
gaAiReferrals.sourceDimension,
|
|
60503
|
-
|
|
60758
|
+
sql35`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`
|
|
60504
60759
|
).all();
|
|
60505
60760
|
const aiRowsForTotals = app.db.select({
|
|
60506
60761
|
date: gaAiReferrals.date,
|
|
@@ -60510,7 +60765,7 @@ async function ga4Routes(app, opts) {
|
|
|
60510
60765
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
60511
60766
|
channelGroup: gaAiReferrals.channelGroup,
|
|
60512
60767
|
sessions: gaAiReferrals.sessions
|
|
60513
|
-
}).from(gaAiReferrals).where(
|
|
60768
|
+
}).from(gaAiReferrals).where(and72(...aiConditions)).all();
|
|
60514
60769
|
const aiReferrals = keepWinningDimensionRows(aiReferralRows, (r) => r.source);
|
|
60515
60770
|
const aiReferralLandingPages = keepWinningDimensionRows(
|
|
60516
60771
|
aiReferralLandingPageRows,
|
|
@@ -60526,17 +60781,17 @@ async function ga4Routes(app, opts) {
|
|
|
60526
60781
|
// is GA's COUNT DISTINCT at that grain, so summing over the window's
|
|
60527
60782
|
// dates counts a returning visitor once per day they returned. Same
|
|
60528
60783
|
// conclusion the AI-referral surfaces reached in 4.135.0.
|
|
60529
|
-
sessions:
|
|
60530
|
-
}).from(gaSocialReferrals).where(
|
|
60784
|
+
sessions: sql35`SUM(${gaSocialReferrals.sessions})`
|
|
60785
|
+
}).from(gaSocialReferrals).where(and72(...socialConditions)).groupBy(gaSocialReferrals.source, gaSocialReferrals.medium, gaSocialReferrals.channelGroup).orderBy(sql35`SUM(${gaSocialReferrals.sessions}) DESC`).all();
|
|
60531
60786
|
const socialDailyRows = app.db.select({
|
|
60532
60787
|
date: gaSocialReferrals.date,
|
|
60533
60788
|
source: gaSocialReferrals.source,
|
|
60534
|
-
sessions:
|
|
60535
|
-
}).from(gaSocialReferrals).where(
|
|
60789
|
+
sessions: sql35`SUM(${gaSocialReferrals.sessions})`
|
|
60790
|
+
}).from(gaSocialReferrals).where(and72(...socialConditions)).groupBy(gaSocialReferrals.date, gaSocialReferrals.source).all();
|
|
60536
60791
|
const socialTotals = app.db.select({
|
|
60537
|
-
sessions:
|
|
60538
|
-
}).from(gaSocialReferrals).where(
|
|
60539
|
-
const latestSync = app.db.select({ syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(
|
|
60792
|
+
sessions: sql35`SUM(${gaSocialReferrals.sessions})`
|
|
60793
|
+
}).from(gaSocialReferrals).where(and72(...socialConditions)).get();
|
|
60794
|
+
const latestSync = app.db.select({ syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(eq84(gaTrafficSummaries.projectId, project.id)).orderBy(desc38(gaTrafficSummaries.syncedAt)).limit(1).get();
|
|
60540
60795
|
const total = summaryRow?.totalSessions ?? 0;
|
|
60541
60796
|
const totalDirectSessions = directTotalRow?.totalDirectSessions ?? 0;
|
|
60542
60797
|
const totalOrganicSessions = summaryRow?.totalOrganicSessions ?? 0;
|
|
@@ -60659,26 +60914,26 @@ async function ga4Routes(app, opts) {
|
|
|
60659
60914
|
const project = resolveProject(app.db, request.params.name);
|
|
60660
60915
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
60661
60916
|
const range = resolveDateRange(request.query);
|
|
60662
|
-
const conditions = [
|
|
60917
|
+
const conditions = [eq84(gaAiReferrals.projectId, project.id), ...dateRangeConditions(gaAiReferrals.date, range)];
|
|
60663
60918
|
const rows = app.db.select({
|
|
60664
60919
|
date: gaAiReferrals.date,
|
|
60665
60920
|
source: gaAiReferrals.source,
|
|
60666
60921
|
medium: gaAiReferrals.medium,
|
|
60667
60922
|
trafficClass: gaAiReferrals.trafficClass,
|
|
60668
|
-
landingPage:
|
|
60923
|
+
landingPage: sql35`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`,
|
|
60669
60924
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
60670
60925
|
// No `users`: ga-ai-referral-aggregation states the rule for this
|
|
60671
60926
|
// table, that there is no grain at which the stored users column may be
|
|
60672
60927
|
// added up. ga4AiReferralDtoSchema.users was deprecated for this in
|
|
60673
60928
|
// 4.135.0; the history endpoint kept emitting one.
|
|
60674
|
-
sessions:
|
|
60675
|
-
}).from(gaAiReferrals).where(
|
|
60929
|
+
sessions: sql35`SUM(${gaAiReferrals.sessions})`
|
|
60930
|
+
}).from(gaAiReferrals).where(and72(...conditions)).groupBy(
|
|
60676
60931
|
gaAiReferrals.date,
|
|
60677
60932
|
gaAiReferrals.source,
|
|
60678
60933
|
gaAiReferrals.medium,
|
|
60679
60934
|
gaAiReferrals.trafficClass,
|
|
60680
60935
|
gaAiReferrals.sourceDimension,
|
|
60681
|
-
|
|
60936
|
+
sql35`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`
|
|
60682
60937
|
).orderBy(gaAiReferrals.date).all();
|
|
60683
60938
|
return rows;
|
|
60684
60939
|
});
|
|
@@ -60686,7 +60941,7 @@ async function ga4Routes(app, opts) {
|
|
|
60686
60941
|
const project = resolveProject(app.db, request.params.name);
|
|
60687
60942
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
60688
60943
|
const range = resolveDateRange(request.query);
|
|
60689
|
-
const conditions = [
|
|
60944
|
+
const conditions = [eq84(gaAiReferrals.projectId, project.id), ...dateRangeConditions(gaAiReferrals.date, range)];
|
|
60690
60945
|
const rows = app.db.select({
|
|
60691
60946
|
date: gaAiReferrals.date,
|
|
60692
60947
|
source: gaAiReferrals.source,
|
|
@@ -60695,14 +60950,14 @@ async function ga4Routes(app, opts) {
|
|
|
60695
60950
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
60696
60951
|
channelGroup: gaAiReferrals.channelGroup,
|
|
60697
60952
|
sessions: gaAiReferrals.sessions
|
|
60698
|
-
}).from(gaAiReferrals).where(
|
|
60953
|
+
}).from(gaAiReferrals).where(and72(...conditions)).all();
|
|
60699
60954
|
return buildAiReferralDailySeries(rows);
|
|
60700
60955
|
});
|
|
60701
60956
|
app.get("/projects/:name/ga/social-referral-history", async (request, _reply) => {
|
|
60702
60957
|
const project = resolveProject(app.db, request.params.name);
|
|
60703
60958
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
60704
60959
|
const range = resolveDateRange(request.query);
|
|
60705
|
-
const conditions = [
|
|
60960
|
+
const conditions = [eq84(gaSocialReferrals.projectId, project.id), ...dateRangeConditions(gaSocialReferrals.date, range)];
|
|
60706
60961
|
const rows = app.db.select({
|
|
60707
60962
|
date: gaSocialReferrals.date,
|
|
60708
60963
|
source: gaSocialReferrals.source,
|
|
@@ -60710,7 +60965,7 @@ async function ga4Routes(app, opts) {
|
|
|
60710
60965
|
channelGroup: gaSocialReferrals.channelGroup,
|
|
60711
60966
|
sessions: gaSocialReferrals.sessions,
|
|
60712
60967
|
users: gaSocialReferrals.users
|
|
60713
|
-
}).from(gaSocialReferrals).where(
|
|
60968
|
+
}).from(gaSocialReferrals).where(and72(...conditions)).orderBy(gaSocialReferrals.date).all();
|
|
60714
60969
|
return rows;
|
|
60715
60970
|
});
|
|
60716
60971
|
app.get("/projects/:name/ga/social-referral-trend", async (request, _reply) => {
|
|
@@ -60723,10 +60978,10 @@ async function ga4Routes(app, opts) {
|
|
|
60723
60978
|
d.setDate(d.getDate() - n);
|
|
60724
60979
|
return fmt(d);
|
|
60725
60980
|
};
|
|
60726
|
-
const sumSocial = (from, to) => app.db.select({ sessions:
|
|
60727
|
-
|
|
60728
|
-
|
|
60729
|
-
|
|
60981
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql35`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and72(
|
|
60982
|
+
eq84(gaSocialReferrals.projectId, project.id),
|
|
60983
|
+
sql35`${gaSocialReferrals.date} >= ${from}`,
|
|
60984
|
+
sql35`${gaSocialReferrals.date} < ${to}`
|
|
60730
60985
|
)).get();
|
|
60731
60986
|
const current7d = sumSocial(daysAgo(7), fmt(today));
|
|
60732
60987
|
const prev7d = sumSocial(daysAgo(14), daysAgo(7));
|
|
@@ -60734,19 +60989,19 @@ async function ga4Routes(app, opts) {
|
|
|
60734
60989
|
const prev30d = sumSocial(daysAgo(60), daysAgo(30));
|
|
60735
60990
|
const sourceCurrent = app.db.select({
|
|
60736
60991
|
source: gaSocialReferrals.source,
|
|
60737
|
-
sessions:
|
|
60738
|
-
}).from(gaSocialReferrals).where(
|
|
60739
|
-
|
|
60740
|
-
|
|
60741
|
-
|
|
60992
|
+
sessions: sql35`SUM(${gaSocialReferrals.sessions})`
|
|
60993
|
+
}).from(gaSocialReferrals).where(and72(
|
|
60994
|
+
eq84(gaSocialReferrals.projectId, project.id),
|
|
60995
|
+
sql35`${gaSocialReferrals.date} >= ${daysAgo(7)}`,
|
|
60996
|
+
sql35`${gaSocialReferrals.date} < ${fmt(today)}`
|
|
60742
60997
|
)).groupBy(gaSocialReferrals.source).all();
|
|
60743
60998
|
const sourcePrev = app.db.select({
|
|
60744
60999
|
source: gaSocialReferrals.source,
|
|
60745
|
-
sessions:
|
|
60746
|
-
}).from(gaSocialReferrals).where(
|
|
60747
|
-
|
|
60748
|
-
|
|
60749
|
-
|
|
61000
|
+
sessions: sql35`SUM(${gaSocialReferrals.sessions})`
|
|
61001
|
+
}).from(gaSocialReferrals).where(and72(
|
|
61002
|
+
eq84(gaSocialReferrals.projectId, project.id),
|
|
61003
|
+
sql35`${gaSocialReferrals.date} >= ${daysAgo(14)}`,
|
|
61004
|
+
sql35`${gaSocialReferrals.date} < ${daysAgo(7)}`
|
|
60750
61005
|
)).groupBy(gaSocialReferrals.source).all();
|
|
60751
61006
|
const response = {
|
|
60752
61007
|
socialSessions7d: current7d?.sessions ?? 0,
|
|
@@ -60769,16 +61024,16 @@ async function ga4Routes(app, opts) {
|
|
|
60769
61024
|
d.setDate(d.getDate() - n);
|
|
60770
61025
|
return fmt(d);
|
|
60771
61026
|
};
|
|
60772
|
-
const sumTotal = (from, to) => app.db.select({ sessions:
|
|
60773
|
-
const sumOrganic = (from, to) => app.db.select({ sessions:
|
|
60774
|
-
const sumDirect = (from, to) => app.db.select({ sessions:
|
|
60775
|
-
const sumAi = (from, to) => app.db.select({ sessions:
|
|
60776
|
-
|
|
60777
|
-
|
|
60778
|
-
|
|
60779
|
-
|
|
61027
|
+
const sumTotal = (from, to) => app.db.select({ sessions: sql35`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)` }).from(gaTrafficSnapshots).where(and72(eq84(gaTrafficSnapshots.projectId, project.id), sql35`${gaTrafficSnapshots.date} >= ${from}`, sql35`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
61028
|
+
const sumOrganic = (from, to) => app.db.select({ sessions: sql35`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)` }).from(gaTrafficSnapshots).where(and72(eq84(gaTrafficSnapshots.projectId, project.id), sql35`${gaTrafficSnapshots.date} >= ${from}`, sql35`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
61029
|
+
const sumDirect = (from, to) => app.db.select({ sessions: sql35`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)` }).from(gaTrafficSnapshots).where(and72(eq84(gaTrafficSnapshots.projectId, project.id), sql35`${gaTrafficSnapshots.date} >= ${from}`, sql35`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
61030
|
+
const sumAi = (from, to) => app.db.select({ sessions: sql35`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and72(
|
|
61031
|
+
eq84(gaAiReferrals.projectId, project.id),
|
|
61032
|
+
sql35`${gaAiReferrals.date} >= ${from}`,
|
|
61033
|
+
sql35`${gaAiReferrals.date} < ${to}`,
|
|
61034
|
+
eq84(gaAiReferrals.sourceDimension, "session")
|
|
60780
61035
|
)).get();
|
|
60781
|
-
const sumSocial = (from, to) => app.db.select({ sessions:
|
|
61036
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql35`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and72(eq84(gaSocialReferrals.projectId, project.id), sql35`${gaSocialReferrals.date} >= ${from}`, sql35`${gaSocialReferrals.date} < ${to}`)).get();
|
|
60782
61037
|
const todayStr = fmt(today);
|
|
60783
61038
|
const buildTrend = (sum) => {
|
|
60784
61039
|
const c7 = sum(daysAgo(7), todayStr)?.sessions ?? 0;
|
|
@@ -60787,20 +61042,20 @@ async function ga4Routes(app, opts) {
|
|
|
60787
61042
|
const p30 = sum(daysAgo(60), daysAgo(30))?.sessions ?? 0;
|
|
60788
61043
|
return { sessions7d: c7, sessionsPrev7d: p7, trend7dPct: deltaPercent(c7, p7), sessions30d: c30, sessionsPrev30d: p30, trend30dPct: deltaPercent(c30, p30) };
|
|
60789
61044
|
};
|
|
60790
|
-
const aiSourceCurrent = app.db.select({ source: gaAiReferrals.source, sessions:
|
|
60791
|
-
|
|
60792
|
-
|
|
60793
|
-
|
|
60794
|
-
|
|
61045
|
+
const aiSourceCurrent = app.db.select({ source: gaAiReferrals.source, sessions: sql35`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and72(
|
|
61046
|
+
eq84(gaAiReferrals.projectId, project.id),
|
|
61047
|
+
sql35`${gaAiReferrals.date} >= ${daysAgo(7)}`,
|
|
61048
|
+
sql35`${gaAiReferrals.date} < ${todayStr}`,
|
|
61049
|
+
eq84(gaAiReferrals.sourceDimension, "session")
|
|
60795
61050
|
)).groupBy(gaAiReferrals.source).all();
|
|
60796
|
-
const aiSourcePrev = app.db.select({ source: gaAiReferrals.source, sessions:
|
|
60797
|
-
|
|
60798
|
-
|
|
60799
|
-
|
|
60800
|
-
|
|
61051
|
+
const aiSourcePrev = app.db.select({ source: gaAiReferrals.source, sessions: sql35`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and72(
|
|
61052
|
+
eq84(gaAiReferrals.projectId, project.id),
|
|
61053
|
+
sql35`${gaAiReferrals.date} >= ${daysAgo(14)}`,
|
|
61054
|
+
sql35`${gaAiReferrals.date} < ${daysAgo(7)}`,
|
|
61055
|
+
eq84(gaAiReferrals.sourceDimension, "session")
|
|
60801
61056
|
)).groupBy(gaAiReferrals.source).all();
|
|
60802
|
-
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions:
|
|
60803
|
-
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions:
|
|
61057
|
+
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions: sql35`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and72(eq84(gaSocialReferrals.projectId, project.id), sql35`${gaSocialReferrals.date} >= ${daysAgo(7)}`, sql35`${gaSocialReferrals.date} < ${todayStr}`)).groupBy(gaSocialReferrals.source).all();
|
|
61058
|
+
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions: sql35`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and72(eq84(gaSocialReferrals.projectId, project.id), sql35`${gaSocialReferrals.date} >= ${daysAgo(14)}`, sql35`${gaSocialReferrals.date} < ${daysAgo(7)}`)).groupBy(gaSocialReferrals.source).all();
|
|
60804
61059
|
const response = {
|
|
60805
61060
|
total: buildTrend(sumTotal),
|
|
60806
61061
|
organic: buildTrend(sumOrganic),
|
|
@@ -60816,16 +61071,16 @@ async function ga4Routes(app, opts) {
|
|
|
60816
61071
|
const project = resolveProject(app.db, request.params.name);
|
|
60817
61072
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
60818
61073
|
const window2 = parseGaOverviewWindow(request.query.window);
|
|
60819
|
-
const periodRows = app.db.select().from(gaWindowMetrics).where(
|
|
61074
|
+
const periodRows = app.db.select().from(gaWindowMetrics).where(and72(eq84(gaWindowMetrics.projectId, project.id), eq84(gaWindowMetrics.windowKey, window2))).all();
|
|
60820
61075
|
const current = periodRows.find((row) => row.period === GaOverviewPeriods.current) ?? null;
|
|
60821
61076
|
const previous = periodRows.find((row) => row.period === GaOverviewPeriods.previous) ?? null;
|
|
60822
61077
|
const fallback = gaOverviewComparisonRanges(window2);
|
|
60823
61078
|
const from = previous?.startDate ?? fallback.previous.startDate;
|
|
60824
61079
|
const to = current?.endDate ?? fallback.current.endDate;
|
|
60825
|
-
const daily = app.db.select().from(gaDailyTotals).where(
|
|
60826
|
-
|
|
60827
|
-
|
|
60828
|
-
|
|
61080
|
+
const daily = app.db.select().from(gaDailyTotals).where(and72(
|
|
61081
|
+
eq84(gaDailyTotals.projectId, project.id),
|
|
61082
|
+
sql35`${gaDailyTotals.date} >= ${from}`,
|
|
61083
|
+
sql35`${gaDailyTotals.date} <= ${to}`
|
|
60829
61084
|
)).all();
|
|
60830
61085
|
return buildGaOverview({ window: window2, current, previous, daily, fallback });
|
|
60831
61086
|
});
|
|
@@ -60833,15 +61088,15 @@ async function ga4Routes(app, opts) {
|
|
|
60833
61088
|
const project = resolveProject(app.db, request.params.name);
|
|
60834
61089
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
60835
61090
|
const range = resolveDateRange(request.query);
|
|
60836
|
-
const conditions = [
|
|
61091
|
+
const conditions = [eq84(gaTrafficSnapshots.projectId, project.id), ...dateRangeConditions(gaTrafficSnapshots.date, range)];
|
|
60837
61092
|
const rows = app.db.select({
|
|
60838
61093
|
date: gaTrafficSnapshots.date,
|
|
60839
|
-
sessions:
|
|
60840
|
-
organicSessions:
|
|
60841
|
-
users:
|
|
60842
|
-
}).from(gaTrafficSnapshots).where(
|
|
60843
|
-
const totalConditions = [
|
|
60844
|
-
const totals = app.db.select({ date: gaDailyTotals.date, users: gaDailyTotals.users }).from(gaDailyTotals).where(
|
|
61094
|
+
sessions: sql35`SUM(${gaTrafficSnapshots.sessions})`,
|
|
61095
|
+
organicSessions: sql35`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
61096
|
+
users: sql35`SUM(${gaTrafficSnapshots.users})`
|
|
61097
|
+
}).from(gaTrafficSnapshots).where(and72(...conditions)).groupBy(gaTrafficSnapshots.date).orderBy(gaTrafficSnapshots.date).all();
|
|
61098
|
+
const totalConditions = [eq84(gaDailyTotals.projectId, project.id), ...dateRangeConditions(gaDailyTotals.date, range)];
|
|
61099
|
+
const totals = app.db.select({ date: gaDailyTotals.date, users: gaDailyTotals.users }).from(gaDailyTotals).where(and72(...totalConditions)).all();
|
|
60845
61100
|
return buildSessionHistory(
|
|
60846
61101
|
rows.map((r) => ({
|
|
60847
61102
|
date: r.date,
|
|
@@ -60856,11 +61111,11 @@ async function ga4Routes(app, opts) {
|
|
|
60856
61111
|
const project = resolveProject(app.db, request.params.name);
|
|
60857
61112
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
60858
61113
|
const trafficPages = app.db.select({
|
|
60859
|
-
landingPage:
|
|
60860
|
-
sessions:
|
|
60861
|
-
organicSessions:
|
|
60862
|
-
users:
|
|
60863
|
-
}).from(gaTrafficSnapshots).where(
|
|
61114
|
+
landingPage: sql35`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`,
|
|
61115
|
+
sessions: sql35`SUM(${gaTrafficSnapshots.sessions})`,
|
|
61116
|
+
organicSessions: sql35`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
61117
|
+
users: sql35`SUM(${gaTrafficSnapshots.users})`
|
|
61118
|
+
}).from(gaTrafficSnapshots).where(eq84(gaTrafficSnapshots.projectId, project.id)).groupBy(sql35`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`).orderBy(sql35`SUM(${gaTrafficSnapshots.sessions}) DESC`).all();
|
|
60864
61119
|
return {
|
|
60865
61120
|
pages: trafficPages.map((r) => ({
|
|
60866
61121
|
landingPage: r.landingPage,
|
|
@@ -60873,7 +61128,7 @@ async function ga4Routes(app, opts) {
|
|
|
60873
61128
|
}
|
|
60874
61129
|
|
|
60875
61130
|
// ../api-routes/src/ga-acquisition.ts
|
|
60876
|
-
import { and as
|
|
61131
|
+
import { and as and73, asc as asc21, count as count6, desc as desc39, eq as eq85, gte as gte23, lte as lte23, sql as sql36 } from "drizzle-orm";
|
|
60877
61132
|
async function gaAcquisitionRoutes(app) {
|
|
60878
61133
|
app.get("/projects/:name/ga/acquisition", async (request) => {
|
|
60879
61134
|
const project = resolveProject(app.db, request.params.name);
|
|
@@ -60887,7 +61142,7 @@ async function gaAcquisitionRoutes(app) {
|
|
|
60887
61142
|
const groupBy = query2.groupBy === void 0 ? [...gaAcquisitionDimensionSchema.options] : [...new Set(query2.groupBy.split(",").map((key) => gaAcquisitionDimensionSchema.parse(key.trim())))];
|
|
60888
61143
|
const limit = query2.limit ?? 100;
|
|
60889
61144
|
const offset = query2.offset ?? 0;
|
|
60890
|
-
const landingPage =
|
|
61145
|
+
const landingPage = sql36`coalesce(${gaAcquisitionDaily.landingPageNormalized}, ${gaAcquisitionDaily.landingPage})`;
|
|
60891
61146
|
const dimensions = {
|
|
60892
61147
|
date: gaAcquisitionDaily.date,
|
|
60893
61148
|
channelGroup: gaAcquisitionDaily.channelGroup,
|
|
@@ -60896,36 +61151,36 @@ async function gaAcquisitionRoutes(app) {
|
|
|
60896
61151
|
hostName: gaAcquisitionDaily.hostName,
|
|
60897
61152
|
landingPage
|
|
60898
61153
|
};
|
|
60899
|
-
const windowFilters = [
|
|
61154
|
+
const windowFilters = [eq85(gaAcquisitionDaily.projectId, project.id)];
|
|
60900
61155
|
if (range.startDate) windowFilters.push(gte23(gaAcquisitionDaily.date, range.startDate));
|
|
60901
61156
|
if (range.endDate) windowFilters.push(lte23(gaAcquisitionDaily.date, range.endDate));
|
|
60902
61157
|
const filters = [...windowFilters];
|
|
60903
|
-
if (query2.channel !== void 0) filters.push(
|
|
60904
|
-
if (query2.hostName !== void 0) filters.push(
|
|
60905
|
-
if (query2.source !== void 0) filters.push(
|
|
60906
|
-
if (query2.medium !== void 0) filters.push(
|
|
61158
|
+
if (query2.channel !== void 0) filters.push(sql36`lower(${gaAcquisitionDaily.channelGroup}) = lower(${query2.channel})`);
|
|
61159
|
+
if (query2.hostName !== void 0) filters.push(sql36`lower(${gaAcquisitionDaily.hostName}) = lower(${query2.hostName})`);
|
|
61160
|
+
if (query2.source !== void 0) filters.push(eq85(gaAcquisitionDaily.source, query2.source));
|
|
61161
|
+
if (query2.medium !== void 0) filters.push(eq85(gaAcquisitionDaily.medium, query2.medium));
|
|
60907
61162
|
const selectedLandingPage = query2.landingPage === void 0 ? void 0 : normalizeUrlPath(query2.landingPage) ?? query2.landingPage;
|
|
60908
|
-
if (selectedLandingPage !== void 0) filters.push(
|
|
61163
|
+
if (selectedLandingPage !== void 0) filters.push(sql36`${landingPage} = ${selectedLandingPage}`);
|
|
60909
61164
|
const pathPrefix = normalizePathPrefix(query2.pathPrefix);
|
|
60910
61165
|
if (pathPrefix && pathPrefix !== "/") {
|
|
60911
|
-
const pathname =
|
|
61166
|
+
const pathname = sql36`case when instr(${landingPage}, '?') = 0 then ${landingPage} else substr(${landingPage}, 1, instr(${landingPage}, '?') - 1) end`;
|
|
60912
61167
|
filters.push(literalPathPrefixCondition(pathname, pathPrefix));
|
|
60913
61168
|
}
|
|
60914
|
-
const where =
|
|
60915
|
-
const sessions =
|
|
61169
|
+
const where = and73(...filters);
|
|
61170
|
+
const sessions = sql36`coalesce(sum(${gaAcquisitionDaily.sessions}), 0)`.mapWith(Number);
|
|
60916
61171
|
const selected = Object.fromEntries(groupBy.map((key) => [key, dimensions[key]]));
|
|
60917
61172
|
const grouped = app.db.select({ ...selected, sessions }).from(gaAcquisitionDaily).where(where).groupBy(...groupBy.map((key) => dimensions[key]));
|
|
60918
61173
|
const totalRows = app.db.select({ total: count6() }).from(grouped.as("grouped_acquisition")).get().total;
|
|
60919
61174
|
const totalSessions = app.db.select({ total: sessions }).from(gaAcquisitionDaily).where(where).get().total;
|
|
60920
61175
|
const order = [
|
|
60921
|
-
...groupBy.includes(GaAcquisitionDimensions.date) ? [
|
|
60922
|
-
|
|
60923
|
-
...groupBy.filter((key) => key !== GaAcquisitionDimensions.date).map((key) =>
|
|
61176
|
+
...groupBy.includes(GaAcquisitionDimensions.date) ? [desc39(gaAcquisitionDaily.date)] : [],
|
|
61177
|
+
desc39(sessions),
|
|
61178
|
+
...groupBy.filter((key) => key !== GaAcquisitionDimensions.date).map((key) => asc21(dimensions[key]))
|
|
60924
61179
|
];
|
|
60925
61180
|
const rows = grouped.orderBy(...order).limit(limit).offset(offset).all();
|
|
60926
|
-
const coverage = app.db.select({ earliestDate:
|
|
60927
|
-
const channelGroups = app.db.selectDistinct({ channel: gaAcquisitionDaily.channelGroup }).from(gaAcquisitionDaily).where(
|
|
60928
|
-
const state = app.db.select().from(gaMeasurementSyncStates).where(
|
|
61181
|
+
const coverage = app.db.select({ earliestDate: sql36`min(${gaAcquisitionDaily.date})`, latestDate: sql36`max(${gaAcquisitionDaily.date})` }).from(gaAcquisitionDaily).where(eq85(gaAcquisitionDaily.projectId, project.id)).get();
|
|
61182
|
+
const channelGroups = app.db.selectDistinct({ channel: gaAcquisitionDaily.channelGroup }).from(gaAcquisitionDaily).where(and73(...windowFilters)).orderBy(asc21(gaAcquisitionDaily.channelGroup)).all().map((row) => row.channel);
|
|
61183
|
+
const state = app.db.select().from(gaMeasurementSyncStates).where(eq85(gaMeasurementSyncStates.projectId, project.id)).get();
|
|
60929
61184
|
const response = {
|
|
60930
61185
|
source: "ga4-acquisition",
|
|
60931
61186
|
status: state?.acquisitionStatus ?? "never-synced",
|
|
@@ -62657,7 +62912,7 @@ async function wordpressRoutes(app, opts) {
|
|
|
62657
62912
|
|
|
62658
62913
|
// ../api-routes/src/backlinks.ts
|
|
62659
62914
|
import crypto53 from "crypto";
|
|
62660
|
-
import { and as
|
|
62915
|
+
import { and as and76, asc as asc22, desc as desc40, eq as eq87, sql as sql37 } from "drizzle-orm";
|
|
62661
62916
|
|
|
62662
62917
|
// ../integration-commoncrawl/src/constants.ts
|
|
62663
62918
|
import os2 from "os";
|
|
@@ -62994,7 +63249,7 @@ async function queryBacklinks(opts) {
|
|
|
62994
63249
|
const reversed = opts.targets.map(reverseDomain);
|
|
62995
63250
|
const targetList = reversed.map(quote).join(", ");
|
|
62996
63251
|
const limitClause = opts.limitPerTarget ? `QUALIFY row_number() OVER (PARTITION BY t.target_rev_domain ORDER BY v.num_hosts DESC) <= ${Math.floor(opts.limitPerTarget)}` : "";
|
|
62997
|
-
const
|
|
63252
|
+
const sql45 = `
|
|
62998
63253
|
WITH vertices AS (
|
|
62999
63254
|
SELECT * FROM read_csv(
|
|
63000
63255
|
${quote(opts.vertexPath)},
|
|
@@ -63030,7 +63285,7 @@ async function queryBacklinks(opts) {
|
|
|
63030
63285
|
const conn = await instance.connect();
|
|
63031
63286
|
let rows;
|
|
63032
63287
|
try {
|
|
63033
|
-
const reader = await conn.runAndReadAll(
|
|
63288
|
+
const reader = await conn.runAndReadAll(sql45);
|
|
63034
63289
|
rows = reader.getRowObjects();
|
|
63035
63290
|
} finally {
|
|
63036
63291
|
conn.disconnectSync?.();
|
|
@@ -63107,7 +63362,7 @@ function pruneCachedRelease(release, opts = {}) {
|
|
|
63107
63362
|
}
|
|
63108
63363
|
|
|
63109
63364
|
// ../api-routes/src/backlinks-filter.ts
|
|
63110
|
-
import { and as
|
|
63365
|
+
import { and as and74, ne as ne15, notLike } from "drizzle-orm";
|
|
63111
63366
|
var BACKLINK_FILTER_PATTERNS = [
|
|
63112
63367
|
"*.google.com",
|
|
63113
63368
|
"*.googleusercontent.com",
|
|
@@ -63130,23 +63385,23 @@ function backlinkCrawlerExclusionClause() {
|
|
|
63130
63385
|
conditions.push(ne15(backlinkDomains.linkingDomain, pattern));
|
|
63131
63386
|
}
|
|
63132
63387
|
}
|
|
63133
|
-
const combined =
|
|
63388
|
+
const combined = and74(...conditions);
|
|
63134
63389
|
if (!combined) throw new Error("BACKLINK_FILTER_PATTERNS is unexpectedly empty");
|
|
63135
63390
|
return combined;
|
|
63136
63391
|
}
|
|
63137
63392
|
|
|
63138
63393
|
// ../api-routes/src/backlinks-source-state.ts
|
|
63139
|
-
import { and as
|
|
63394
|
+
import { and as and75, eq as eq86, isNotNull as isNotNull8, or as or17 } from "drizzle-orm";
|
|
63140
63395
|
function readCommonCrawlSourceState(db, projectId) {
|
|
63141
63396
|
const completedReleases = db.select({ status: ccReleaseSyncs.status }).from(ccReleaseSyncs).where(or17(
|
|
63142
|
-
|
|
63397
|
+
eq86(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready),
|
|
63143
63398
|
isNotNull8(ccReleaseSyncs.queryFinishedAt)
|
|
63144
63399
|
)).all();
|
|
63145
63400
|
const hasReadyRelease = completedReleases.length > 0;
|
|
63146
63401
|
const hasCurrentReadyRelease = completedReleases.some((release) => release.status === CcReleaseSyncStatuses.ready);
|
|
63147
|
-
const hasProjectData = db.select({ id: backlinkSummaries.id }).from(backlinkSummaries).where(
|
|
63148
|
-
|
|
63149
|
-
|
|
63402
|
+
const hasProjectData = db.select({ id: backlinkSummaries.id }).from(backlinkSummaries).where(and75(
|
|
63403
|
+
eq86(backlinkSummaries.projectId, projectId),
|
|
63404
|
+
eq86(backlinkSummaries.source, BacklinkSources.commoncrawl)
|
|
63150
63405
|
)).limit(1).get() !== void 0;
|
|
63151
63406
|
return { hasReadyRelease, hasCurrentReadyRelease, hasProjectData, connected: hasReadyRelease && hasProjectData };
|
|
63152
63407
|
}
|
|
@@ -63217,11 +63472,11 @@ function mapRunRow(row) {
|
|
|
63217
63472
|
}
|
|
63218
63473
|
function latestSummaryForProject(db, projectId, source, release) {
|
|
63219
63474
|
const conditions = [
|
|
63220
|
-
|
|
63221
|
-
|
|
63475
|
+
eq87(backlinkSummaries.projectId, projectId),
|
|
63476
|
+
eq87(backlinkSummaries.source, source)
|
|
63222
63477
|
];
|
|
63223
|
-
if (release) conditions.push(
|
|
63224
|
-
const rows = db.select().from(backlinkSummaries).where(
|
|
63478
|
+
if (release) conditions.push(eq87(backlinkSummaries.release, release));
|
|
63479
|
+
const rows = db.select().from(backlinkSummaries).where(and76(...conditions)).orderBy(desc40(backlinkSummaries.queriedAt)).all();
|
|
63225
63480
|
return rows.reduce((latest, row) => {
|
|
63226
63481
|
if (!latest) return row;
|
|
63227
63482
|
const newer = isValidReleaseId(row.release) && isValidReleaseId(latest.release) && compareReleaseIds(row.release, latest.release) > 0;
|
|
@@ -63234,21 +63489,21 @@ function parseExcludeCrawlers(value) {
|
|
|
63234
63489
|
return lower === "1" || lower === "true" || lower === "yes";
|
|
63235
63490
|
}
|
|
63236
63491
|
function computeFilteredSummary(db, base) {
|
|
63237
|
-
const baseDomainCondition =
|
|
63238
|
-
|
|
63239
|
-
|
|
63240
|
-
|
|
63492
|
+
const baseDomainCondition = and76(
|
|
63493
|
+
eq87(backlinkDomains.projectId, base.projectId),
|
|
63494
|
+
eq87(backlinkDomains.source, base.source),
|
|
63495
|
+
eq87(backlinkDomains.release, base.release)
|
|
63241
63496
|
);
|
|
63242
|
-
const filteredCondition =
|
|
63497
|
+
const filteredCondition = and76(baseDomainCondition, backlinkCrawlerExclusionClause());
|
|
63243
63498
|
const unfilteredAgg = db.select({
|
|
63244
|
-
count:
|
|
63245
|
-
total:
|
|
63499
|
+
count: sql37`count(*)`,
|
|
63500
|
+
total: sql37`coalesce(sum(${backlinkDomains.numHosts}), 0)`
|
|
63246
63501
|
}).from(backlinkDomains).where(baseDomainCondition).get();
|
|
63247
63502
|
const filteredAgg = db.select({
|
|
63248
|
-
count:
|
|
63249
|
-
total:
|
|
63503
|
+
count: sql37`count(*)`,
|
|
63504
|
+
total: sql37`coalesce(sum(${backlinkDomains.numHosts}), 0)`
|
|
63250
63505
|
}).from(backlinkDomains).where(filteredCondition).get();
|
|
63251
|
-
const top10Rows = db.select({ numHosts: backlinkDomains.numHosts }).from(backlinkDomains).where(filteredCondition).orderBy(
|
|
63506
|
+
const top10Rows = db.select({ numHosts: backlinkDomains.numHosts }).from(backlinkDomains).where(filteredCondition).orderBy(desc40(backlinkDomains.numHosts)).limit(10).all();
|
|
63252
63507
|
const totalLinkingDomains = Number(filteredAgg?.count ?? 0);
|
|
63253
63508
|
const totalHosts = Number(filteredAgg?.total ?? 0);
|
|
63254
63509
|
const unfilteredLinkingDomains = Number(unfilteredAgg?.count ?? 0);
|
|
@@ -63269,8 +63524,8 @@ function computeFilteredSummary(db, base) {
|
|
|
63269
63524
|
};
|
|
63270
63525
|
}
|
|
63271
63526
|
function computeFilteredReleaseTotals(db, projectId, source) {
|
|
63272
|
-
const kept =
|
|
63273
|
-
const rows = db.all(
|
|
63527
|
+
const kept = sql37`case when ${backlinkCrawlerExclusionClause()} then 1 else 0 end`;
|
|
63528
|
+
const rows = db.all(sql37`
|
|
63274
63529
|
SELECT release,
|
|
63275
63530
|
COUNT(*) AS domain_rows,
|
|
63276
63531
|
SUM(kept) AS linking_domains,
|
|
@@ -63280,7 +63535,7 @@ function computeFilteredReleaseTotals(db, projectId, source) {
|
|
|
63280
63535
|
SELECT ${backlinkDomains.release} AS release, ${backlinkDomains.numHosts} AS num_hosts, ${kept} AS kept,
|
|
63281
63536
|
ROW_NUMBER() OVER (PARTITION BY ${backlinkDomains.release}, ${kept} ORDER BY ${backlinkDomains.numHosts} DESC) AS kept_rank
|
|
63282
63537
|
FROM ${backlinkDomains}
|
|
63283
|
-
WHERE ${
|
|
63538
|
+
WHERE ${and76(eq87(backlinkDomains.projectId, projectId), eq87(backlinkDomains.source, source))}
|
|
63284
63539
|
)
|
|
63285
63540
|
GROUP BY release`);
|
|
63286
63541
|
return new Map(rows.map((row) => {
|
|
@@ -63294,13 +63549,13 @@ function computeFilteredReleaseTotals(db, projectId, source) {
|
|
|
63294
63549
|
}));
|
|
63295
63550
|
}
|
|
63296
63551
|
function buildSourceAvailability(db, projectId, source, connected2, excludeCrawlers) {
|
|
63297
|
-
const summary3 = db.select().from(backlinkSummaries).where(
|
|
63552
|
+
const summary3 = db.select().from(backlinkSummaries).where(and76(eq87(backlinkSummaries.projectId, projectId), eq87(backlinkSummaries.source, source))).orderBy(desc40(backlinkSummaries.queriedAt)).limit(1).get();
|
|
63298
63553
|
let totalLinkingDomains = summary3?.totalLinkingDomains ?? 0;
|
|
63299
63554
|
if (summary3 && excludeCrawlers) {
|
|
63300
|
-
const filtered = db.select({ count:
|
|
63301
|
-
|
|
63302
|
-
|
|
63303
|
-
|
|
63555
|
+
const filtered = db.select({ count: sql37`count(*)` }).from(backlinkDomains).where(and76(
|
|
63556
|
+
eq87(backlinkDomains.projectId, projectId),
|
|
63557
|
+
eq87(backlinkDomains.source, source),
|
|
63558
|
+
eq87(backlinkDomains.release, summary3.release),
|
|
63304
63559
|
backlinkCrawlerExclusionClause()
|
|
63305
63560
|
)).get();
|
|
63306
63561
|
totalLinkingDomains = Number(filtered?.count ?? 0);
|
|
@@ -63370,7 +63625,7 @@ async function backlinksRoutes(app, opts) {
|
|
|
63370
63625
|
"@duckdb/node-api is not installed. Run `canonry backlinks install` to enable the backlinks feature."
|
|
63371
63626
|
);
|
|
63372
63627
|
}
|
|
63373
|
-
const existing = app.db.select().from(ccReleaseSyncs).where(
|
|
63628
|
+
const existing = app.db.select().from(ccReleaseSyncs).where(eq87(ccReleaseSyncs.release, release)).get();
|
|
63374
63629
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
63375
63630
|
if (existing) {
|
|
63376
63631
|
if (NON_TERMINAL_SYNC_STATUSES.has(existing.status)) {
|
|
@@ -63381,9 +63636,9 @@ async function backlinksRoutes(app, opts) {
|
|
|
63381
63636
|
phaseDetail: null,
|
|
63382
63637
|
error: null,
|
|
63383
63638
|
updatedAt: now
|
|
63384
|
-
}).where(
|
|
63639
|
+
}).where(eq87(ccReleaseSyncs.id, existing.id)).run();
|
|
63385
63640
|
opts.onReleaseSyncRequested(existing.id, release);
|
|
63386
|
-
const refreshed = app.db.select().from(ccReleaseSyncs).where(
|
|
63641
|
+
const refreshed = app.db.select().from(ccReleaseSyncs).where(eq87(ccReleaseSyncs.id, existing.id)).get();
|
|
63387
63642
|
return reply.status(200).send(mapSyncRow(refreshed));
|
|
63388
63643
|
}
|
|
63389
63644
|
const id = crypto53.randomUUID();
|
|
@@ -63395,15 +63650,15 @@ async function backlinksRoutes(app, opts) {
|
|
|
63395
63650
|
updatedAt: now
|
|
63396
63651
|
}).run();
|
|
63397
63652
|
opts.onReleaseSyncRequested(id, release);
|
|
63398
|
-
const inserted = app.db.select().from(ccReleaseSyncs).where(
|
|
63653
|
+
const inserted = app.db.select().from(ccReleaseSyncs).where(eq87(ccReleaseSyncs.id, id)).get();
|
|
63399
63654
|
return reply.status(201).send(mapSyncRow(inserted));
|
|
63400
63655
|
});
|
|
63401
63656
|
app.get("/backlinks/syncs/latest", async (_request, reply) => {
|
|
63402
|
-
const row = app.db.select().from(ccReleaseSyncs).orderBy(
|
|
63657
|
+
const row = app.db.select().from(ccReleaseSyncs).orderBy(desc40(ccReleaseSyncs.updatedAt)).limit(1).get();
|
|
63403
63658
|
return reply.send(row ? mapSyncRow(row) : null);
|
|
63404
63659
|
});
|
|
63405
63660
|
app.get("/backlinks/syncs", async (_request, reply) => {
|
|
63406
|
-
const rows = app.db.select().from(ccReleaseSyncs).orderBy(
|
|
63661
|
+
const rows = app.db.select().from(ccReleaseSyncs).orderBy(desc40(ccReleaseSyncs.updatedAt)).all();
|
|
63407
63662
|
return reply.send(rows.map(mapSyncRow));
|
|
63408
63663
|
});
|
|
63409
63664
|
app.get("/backlinks/releases", async (_request, reply) => {
|
|
@@ -63453,7 +63708,7 @@ async function backlinksRoutes(app, opts) {
|
|
|
63453
63708
|
createdAt: now
|
|
63454
63709
|
}).run();
|
|
63455
63710
|
opts.onBacklinkExtractRequested(runId, project.id, release);
|
|
63456
|
-
const run = app.db.select().from(runs).where(
|
|
63711
|
+
const run = app.db.select().from(runs).where(eq87(runs.id, runId)).get();
|
|
63457
63712
|
return reply.status(201).send(mapRunRow(run));
|
|
63458
63713
|
});
|
|
63459
63714
|
app.get(
|
|
@@ -63479,18 +63734,18 @@ async function backlinksRoutes(app, opts) {
|
|
|
63479
63734
|
const limit = Math.min(Math.max(parseInt(request.query.limit ?? "50", 10) || 50, 1), 500);
|
|
63480
63735
|
const offset = Math.max(parseInt(request.query.offset ?? "0", 10) || 0, 0);
|
|
63481
63736
|
const excludeCrawlers = parseExcludeCrawlers(request.query.excludeCrawlers);
|
|
63482
|
-
const baseDomainCondition =
|
|
63483
|
-
|
|
63484
|
-
|
|
63485
|
-
|
|
63737
|
+
const baseDomainCondition = and76(
|
|
63738
|
+
eq87(backlinkDomains.projectId, project.id),
|
|
63739
|
+
eq87(backlinkDomains.source, source),
|
|
63740
|
+
eq87(backlinkDomains.release, targetRelease)
|
|
63486
63741
|
);
|
|
63487
|
-
const domainCondition = excludeCrawlers ?
|
|
63488
|
-
const totalRow = app.db.select({ count:
|
|
63742
|
+
const domainCondition = excludeCrawlers ? and76(baseDomainCondition, backlinkCrawlerExclusionClause()) : baseDomainCondition;
|
|
63743
|
+
const totalRow = app.db.select({ count: sql37`count(*)` }).from(backlinkDomains).where(domainCondition).get();
|
|
63489
63744
|
const rows = app.db.select({
|
|
63490
63745
|
linkingDomain: backlinkDomains.linkingDomain,
|
|
63491
63746
|
numHosts: backlinkDomains.numHosts,
|
|
63492
63747
|
source: backlinkDomains.source
|
|
63493
|
-
}).from(backlinkDomains).where(domainCondition).orderBy(
|
|
63748
|
+
}).from(backlinkDomains).where(domainCondition).orderBy(desc40(backlinkDomains.numHosts)).limit(limit).offset(offset).all();
|
|
63494
63749
|
let summary3 = null;
|
|
63495
63750
|
if (summaryRow) {
|
|
63496
63751
|
summary3 = excludeCrawlers ? computeFilteredSummary(app.db, summaryRow) : mapSummaryRow(summaryRow);
|
|
@@ -63508,7 +63763,7 @@ async function backlinksRoutes(app, opts) {
|
|
|
63508
63763
|
async (request, reply) => {
|
|
63509
63764
|
const project = resolveProject(app.db, request.params.name);
|
|
63510
63765
|
const source = parseSourceParam(request.query.source);
|
|
63511
|
-
const rows = app.db.select().from(backlinkSummaries).where(
|
|
63766
|
+
const rows = app.db.select().from(backlinkSummaries).where(and76(eq87(backlinkSummaries.projectId, project.id), eq87(backlinkSummaries.source, source))).orderBy(asc22(backlinkSummaries.queriedAt)).all();
|
|
63512
63767
|
const response = rows.map((r) => ({
|
|
63513
63768
|
release: r.release,
|
|
63514
63769
|
totalLinkingDomains: r.totalLinkingDomains,
|
|
@@ -63535,7 +63790,7 @@ async function backlinksRoutes(app, opts) {
|
|
|
63535
63790
|
import crypto55 from "crypto";
|
|
63536
63791
|
import { isIP as isIP3 } from "net";
|
|
63537
63792
|
import { isDeepStrictEqual } from "util";
|
|
63538
|
-
import { and as
|
|
63793
|
+
import { and as and78, desc as desc41, eq as eq89, gt as gt7, gte as gte24, inArray as inArray40, lt as lt16, lte as lte25, sql as sql38 } from "drizzle-orm";
|
|
63539
63794
|
|
|
63540
63795
|
// ../api-routes/src/traffic-limits.ts
|
|
63541
63796
|
var VERCEL_MAX_SYNC_WINDOW_MS = 24 * 60 * 6e4;
|
|
@@ -64878,7 +65133,7 @@ async function ackCloudflareQueueMessages(client, options) {
|
|
|
64878
65133
|
}
|
|
64879
65134
|
|
|
64880
65135
|
// ../api-routes/src/traffic-sync-lease.ts
|
|
64881
|
-
import { and as
|
|
65136
|
+
import { and as and77, eq as eq88, isNull as isNull13, lte as lte24, or as or18 } from "drizzle-orm";
|
|
64882
65137
|
function leaseExpiresAt(now, ttlMs) {
|
|
64883
65138
|
const nowMs = Date.parse(now);
|
|
64884
65139
|
if (!Number.isFinite(nowMs)) throw new RangeError("Traffic sync lease now must be an ISO instant");
|
|
@@ -64891,12 +65146,12 @@ function tryClaimTrafficSyncLease(input) {
|
|
|
64891
65146
|
const changed = tx.update(trafficSources).set({
|
|
64892
65147
|
syncLeaseOwner: input.owner,
|
|
64893
65148
|
syncLeaseExpiresAt: expiresAt
|
|
64894
|
-
}).where(
|
|
64895
|
-
|
|
65149
|
+
}).where(and77(
|
|
65150
|
+
eq88(trafficSources.id, input.sourceId),
|
|
64896
65151
|
or18(
|
|
64897
65152
|
isNull13(trafficSources.syncLeaseOwner),
|
|
64898
65153
|
lte24(trafficSources.syncLeaseExpiresAt, input.now),
|
|
64899
|
-
|
|
65154
|
+
eq88(trafficSources.syncLeaseOwner, input.owner)
|
|
64900
65155
|
)
|
|
64901
65156
|
)).run();
|
|
64902
65157
|
return changed.changes === 1;
|
|
@@ -64907,9 +65162,9 @@ function releaseTrafficSyncLease(input) {
|
|
|
64907
65162
|
const changed = tx.update(trafficSources).set({
|
|
64908
65163
|
syncLeaseOwner: null,
|
|
64909
65164
|
syncLeaseExpiresAt: null
|
|
64910
|
-
}).where(
|
|
64911
|
-
|
|
64912
|
-
|
|
65165
|
+
}).where(and77(
|
|
65166
|
+
eq88(trafficSources.id, input.sourceId),
|
|
65167
|
+
eq88(trafficSources.syncLeaseOwner, input.owner)
|
|
64913
65168
|
)).run();
|
|
64914
65169
|
return changed.changes === 1;
|
|
64915
65170
|
}, { behavior: "immediate" });
|
|
@@ -64976,14 +65231,14 @@ async function loadQueueReceiptPolicy(db, sourceId, readRetentionSeconds, warn2)
|
|
|
64976
65231
|
if (liveTtlMs === null) return { ttlMsAt: () => CLOUDFLARE_QUEUE_RECEIPT_TTL_MS };
|
|
64977
65232
|
const ttlMs = liveTtlMs;
|
|
64978
65233
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
64979
|
-
const alive =
|
|
64980
|
-
const lifetimeMs =
|
|
64981
|
-
const shortest = db.select({ ms:
|
|
65234
|
+
const alive = and78(eq89(trafficEventReceipts.sourceId, sourceId), gt7(trafficEventReceipts.expiresAt, now));
|
|
65235
|
+
const lifetimeMs = sql38`(julianday(${trafficEventReceipts.expiresAt}) - julianday(${trafficEventReceipts.receivedAt})) * ${DAY_MS4}`;
|
|
65236
|
+
const shortest = db.select({ ms: sql38`min(${lifetimeMs})` }).from(trafficEventReceipts).where(alive).get()?.ms ?? null;
|
|
64982
65237
|
if (shortest !== null && shortest < ttlMs - 1e3) {
|
|
64983
|
-
const extended =
|
|
64984
|
-
db.update(trafficEventReceipts).set({ expiresAt: extended }).where(
|
|
65238
|
+
const extended = sql38`strftime('%Y-%m-%dT%H:%M:%fZ', julianday(${trafficEventReceipts.receivedAt}) + ${ttlMs / DAY_MS4})`;
|
|
65239
|
+
db.update(trafficEventReceipts).set({ expiresAt: extended }).where(and78(alive, lt16(trafficEventReceipts.expiresAt, extended))).run();
|
|
64985
65240
|
}
|
|
64986
|
-
const floorExpiry = db.select({ at:
|
|
65241
|
+
const floorExpiry = db.select({ at: sql38`max(${trafficEventReceipts.expiresAt})` }).from(trafficEventReceipts).where(and78(alive, sql38`${lifetimeMs} < ${CLOUDFLARE_QUEUE_RECEIPT_TTL_MS - 1e3}`)).get()?.at ?? null;
|
|
64987
65242
|
const floorMs = floorExpiry ? Date.parse(floorExpiry) : 0;
|
|
64988
65243
|
return { ttlMsAt: (receivedAt) => Math.max(ttlMs, floorMs - Date.parse(receivedAt)) };
|
|
64989
65244
|
}
|
|
@@ -65038,7 +65293,7 @@ function cloudflareIngestRateLimitKey(request, store, db) {
|
|
|
65038
65293
|
const authenticated = authenticateCloudflareIngest(request, store);
|
|
65039
65294
|
const params = request.params;
|
|
65040
65295
|
if (authenticated && typeof params.name === "string" && authenticated.credential.projectName === params.name) {
|
|
65041
|
-
const source = db.select().from(trafficSources).where(
|
|
65296
|
+
const source = db.select().from(trafficSources).where(eq89(trafficSources.id, authenticated.credential.sourceId)).get();
|
|
65042
65297
|
if (source && source.sourceType === TrafficSourceTypes.cloudflare && source.status === TrafficSourceStatuses.connected && parseDirectPushCloudflareSourceConfig(source.configJson) && timingSafeEqualHex(authenticated.bearerHash, source.ingestTokenHash)) {
|
|
65043
65298
|
return `cloudflare-source:${authenticated.credential.sourceId}`;
|
|
65044
65299
|
}
|
|
@@ -65192,7 +65447,7 @@ function crawlerVerificationProvenance(db, projectId, events) {
|
|
|
65192
65447
|
manifestId: crawlerVerificationManifestsHourly.manifestId,
|
|
65193
65448
|
manifest: crawlerVerificationManifestsHourly.manifestJson,
|
|
65194
65449
|
hits: crawlerVerificationManifestsHourly.hits
|
|
65195
|
-
}).from(crawlerVerificationManifestsHourly).where(
|
|
65450
|
+
}).from(crawlerVerificationManifestsHourly).where(sql38`(
|
|
65196
65451
|
${crawlerVerificationManifestsHourly.projectId},
|
|
65197
65452
|
${crawlerVerificationManifestsHourly.sourceId},
|
|
65198
65453
|
${crawlerVerificationManifestsHourly.tsHour},
|
|
@@ -65226,7 +65481,7 @@ function aiUserFetchVerificationProvenance(db, projectId, events) {
|
|
|
65226
65481
|
manifestId: aiUserFetchVerificationManifestsHourly.manifestId,
|
|
65227
65482
|
manifest: aiUserFetchVerificationManifestsHourly.manifestJson,
|
|
65228
65483
|
hits: aiUserFetchVerificationManifestsHourly.hits
|
|
65229
|
-
}).from(aiUserFetchVerificationManifestsHourly).where(
|
|
65484
|
+
}).from(aiUserFetchVerificationManifestsHourly).where(sql38`(
|
|
65230
65485
|
${aiUserFetchVerificationManifestsHourly.projectId},
|
|
65231
65486
|
${aiUserFetchVerificationManifestsHourly.sourceId},
|
|
65232
65487
|
${aiUserFetchVerificationManifestsHourly.tsHour},
|
|
@@ -65281,7 +65536,7 @@ async function defaultResolveAccessToken(record2) {
|
|
|
65281
65536
|
);
|
|
65282
65537
|
}
|
|
65283
65538
|
function hasConnectedTrafficSourceSibling(db, projectId, sourceId) {
|
|
65284
|
-
return db.select().from(trafficSources).where(
|
|
65539
|
+
return db.select().from(trafficSources).where(eq89(trafficSources.projectId, projectId)).all().some((row) => row.id !== sourceId && row.status === TrafficSourceStatuses.connected);
|
|
65285
65540
|
}
|
|
65286
65541
|
function isAuthoritativeTrafficSource(db, source) {
|
|
65287
65542
|
return (source.status === TrafficSourceStatuses.connected || source.status === TrafficSourceStatuses.error) && !hasConnectedTrafficSourceSibling(db, source.projectId, source.id);
|
|
@@ -65308,13 +65563,13 @@ function wordpressBaseUrlIdentity(value) {
|
|
|
65308
65563
|
}
|
|
65309
65564
|
}
|
|
65310
65565
|
function bindTrafficSyncSchedule(tx, projectId, sourceId, now, createIfMissing = true) {
|
|
65311
|
-
const schedule = tx.select().from(schedules).where(
|
|
65312
|
-
|
|
65313
|
-
|
|
65566
|
+
const schedule = tx.select().from(schedules).where(and78(
|
|
65567
|
+
eq89(schedules.projectId, projectId),
|
|
65568
|
+
eq89(schedules.kind, SchedulableRunKinds["traffic-sync"])
|
|
65314
65569
|
)).get();
|
|
65315
65570
|
if (schedule) {
|
|
65316
65571
|
if (schedule.sourceId === sourceId) return { changed: false, created: false };
|
|
65317
|
-
tx.update(schedules).set({ sourceId, updatedAt: nextScheduleUpdatedAt(schedule.updatedAt, Date.parse(now)) }).where(
|
|
65572
|
+
tx.update(schedules).set({ sourceId, updatedAt: nextScheduleUpdatedAt(schedule.updatedAt, Date.parse(now)) }).where(eq89(schedules.id, schedule.id)).run();
|
|
65318
65573
|
return { changed: true, created: false };
|
|
65319
65574
|
}
|
|
65320
65575
|
if (!createIfMissing) return { changed: false, created: false };
|
|
@@ -65356,7 +65611,7 @@ async function runBackfillTask(options) {
|
|
|
65356
65611
|
};
|
|
65357
65612
|
const renewLease2 = setInterval(() => {
|
|
65358
65613
|
try {
|
|
65359
|
-
const latest = app.db.select().from(trafficSources).where(
|
|
65614
|
+
const latest = app.db.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
65360
65615
|
if (!latest || !isAuthoritativeTrafficSource(app.db, latest) || !isSameTrafficSourceGeneration(latest, sourceRow2)) {
|
|
65361
65616
|
clearInterval(renewLease2);
|
|
65362
65617
|
stopped.abort();
|
|
@@ -65404,10 +65659,10 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65404
65659
|
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
65405
65660
|
try {
|
|
65406
65661
|
app.db.transaction((tx) => {
|
|
65407
|
-
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(
|
|
65408
|
-
const latestSource = tx.select().from(trafficSources).where(
|
|
65662
|
+
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(eq89(runs.id, runId)).run();
|
|
65663
|
+
const latestSource = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
65409
65664
|
if (!preserveSourceState && latestSource && isAuthoritativeTrafficSource(tx, latestSource) && isSameBackfillSource(latestSource, sourceRow2, leaseOwner)) {
|
|
65410
|
-
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(
|
|
65665
|
+
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(eq89(trafficSources.id, sourceRow2.id)).run();
|
|
65411
65666
|
}
|
|
65412
65667
|
});
|
|
65413
65668
|
} catch {
|
|
@@ -65439,7 +65694,7 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65439
65694
|
let emptyOutcome;
|
|
65440
65695
|
try {
|
|
65441
65696
|
const completed = app.db.transaction((tx) => {
|
|
65442
|
-
const latestSource = tx.select().from(trafficSources).where(
|
|
65697
|
+
const latestSource = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
65443
65698
|
const stillAuthoritative = latestSource && isAuthoritativeTrafficSource(tx, latestSource) && isSameBackfillSource(latestSource, sourceRow2, leaseOwner);
|
|
65444
65699
|
if (stillAuthoritative) {
|
|
65445
65700
|
enforceRawEventSampleRetention(tx, sourceRow2.id, finishedAt2);
|
|
@@ -65448,7 +65703,7 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65448
65703
|
status: RunStatuses.failed,
|
|
65449
65704
|
error: "Traffic source was deactivated or reconfigured during backfill",
|
|
65450
65705
|
finishedAt: finishedAt2
|
|
65451
|
-
}).where(
|
|
65706
|
+
}).where(eq89(runs.id, runId)).run();
|
|
65452
65707
|
return Boolean(stillAuthoritative);
|
|
65453
65708
|
});
|
|
65454
65709
|
emptyOutcome = completed ? { status: OutcomeStatuses.skipped, reasonCode: OutcomeReasonCodes.NO_DATA } : { status: OutcomeStatuses.failed, reasonCode: OutcomeReasonCodes.NOT_CONNECTED };
|
|
@@ -65480,13 +65735,13 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65480
65735
|
const nextLastSyncedAt = Math.max(currentLastSyncedMs, windowEnd.getTime()) === windowEnd.getTime() ? windowEndIso : sourceRow2.lastSyncedAt;
|
|
65481
65736
|
try {
|
|
65482
65737
|
const commitOutcome = app.db.transaction((tx) => {
|
|
65483
|
-
const latestSource = tx.select().from(trafficSources).where(
|
|
65738
|
+
const latestSource = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
65484
65739
|
if (latestSource?.sourceType === TrafficSourceTypes.wordpress) {
|
|
65485
65740
|
tx.update(runs).set({
|
|
65486
65741
|
status: RunStatuses.failed,
|
|
65487
65742
|
error: "Generic WordPress replace backfill is unavailable because retained coverage is unproven.",
|
|
65488
65743
|
finishedAt
|
|
65489
|
-
}).where(
|
|
65744
|
+
}).where(eq89(runs.id, runId)).run();
|
|
65490
65745
|
return "wordpress-backfill-unsupported";
|
|
65491
65746
|
}
|
|
65492
65747
|
if (!latestSource || !isAuthoritativeTrafficSource(tx, latestSource) || !isSameBackfillSource(latestSource, sourceRow2, leaseOwner)) {
|
|
@@ -65494,7 +65749,7 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65494
65749
|
status: RunStatuses.failed,
|
|
65495
65750
|
error: "Traffic source was deactivated or reconfigured during backfill",
|
|
65496
65751
|
finishedAt
|
|
65497
|
-
}).where(
|
|
65752
|
+
}).where(eq89(runs.id, runId)).run();
|
|
65498
65753
|
return "source-inactive";
|
|
65499
65754
|
}
|
|
65500
65755
|
const rawSampleCutoff = enforceRawEventSampleRetention(
|
|
@@ -65503,37 +65758,37 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65503
65758
|
finishedAt
|
|
65504
65759
|
);
|
|
65505
65760
|
tx.delete(crawlerEventsHourly).where(
|
|
65506
|
-
|
|
65507
|
-
|
|
65761
|
+
and78(
|
|
65762
|
+
eq89(crawlerEventsHourly.sourceId, sourceRow2.id),
|
|
65508
65763
|
gte24(crawlerEventsHourly.tsHour, windowStartIso),
|
|
65509
65764
|
lte25(crawlerEventsHourly.tsHour, windowEndIso)
|
|
65510
65765
|
)
|
|
65511
65766
|
).run();
|
|
65512
65767
|
tx.delete(aiUserFetchEventsHourly).where(
|
|
65513
|
-
|
|
65514
|
-
|
|
65768
|
+
and78(
|
|
65769
|
+
eq89(aiUserFetchEventsHourly.sourceId, sourceRow2.id),
|
|
65515
65770
|
gte24(aiUserFetchEventsHourly.tsHour, windowStartIso),
|
|
65516
65771
|
lte25(aiUserFetchEventsHourly.tsHour, windowEndIso)
|
|
65517
65772
|
)
|
|
65518
65773
|
).run();
|
|
65519
65774
|
clearLikelyFakeAgentHits(tx, { sourceId: sourceRow2.id, windowStart: windowStartIso, windowEnd: windowEndIso });
|
|
65520
65775
|
tx.delete(aiReferralEventsHourly).where(
|
|
65521
|
-
|
|
65522
|
-
|
|
65776
|
+
and78(
|
|
65777
|
+
eq89(aiReferralEventsHourly.sourceId, sourceRow2.id),
|
|
65523
65778
|
gte24(aiReferralEventsHourly.tsHour, windowStartIso),
|
|
65524
65779
|
lte25(aiReferralEventsHourly.tsHour, windowEndIso)
|
|
65525
65780
|
)
|
|
65526
65781
|
).run();
|
|
65527
65782
|
tx.delete(rawEventSamples).where(
|
|
65528
|
-
|
|
65529
|
-
|
|
65783
|
+
and78(
|
|
65784
|
+
eq89(rawEventSamples.sourceId, sourceRow2.id),
|
|
65530
65785
|
gte24(rawEventSamples.ts, windowStartIso),
|
|
65531
65786
|
lte25(rawEventSamples.ts, windowEndIso)
|
|
65532
65787
|
)
|
|
65533
65788
|
).run();
|
|
65534
65789
|
tx.delete(unrecognizedAgentEventsHourly).where(
|
|
65535
|
-
|
|
65536
|
-
|
|
65790
|
+
and78(
|
|
65791
|
+
eq89(unrecognizedAgentEventsHourly.sourceId, sourceRow2.id),
|
|
65537
65792
|
gte24(unrecognizedAgentEventsHourly.tsHour, windowStartIso),
|
|
65538
65793
|
lte25(unrecognizedAgentEventsHourly.tsHour, windowEndIso)
|
|
65539
65794
|
)
|
|
@@ -65564,7 +65819,7 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65564
65819
|
crawlerEventsHourly.status
|
|
65565
65820
|
],
|
|
65566
65821
|
set: {
|
|
65567
|
-
hits:
|
|
65822
|
+
hits: sql38`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
65568
65823
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
65569
65824
|
updatedAt: finishedAt
|
|
65570
65825
|
}
|
|
@@ -65595,7 +65850,7 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65595
65850
|
crawlerVerificationManifestsHourly.manifestId
|
|
65596
65851
|
],
|
|
65597
65852
|
set: {
|
|
65598
|
-
hits:
|
|
65853
|
+
hits: sql38`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
65599
65854
|
manifestJson: bucket.verificationManifest,
|
|
65600
65855
|
updatedAt: finishedAt
|
|
65601
65856
|
}
|
|
@@ -65628,7 +65883,7 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65628
65883
|
aiUserFetchEventsHourly.status
|
|
65629
65884
|
],
|
|
65630
65885
|
set: {
|
|
65631
|
-
hits:
|
|
65886
|
+
hits: sql38`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
65632
65887
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
65633
65888
|
updatedAt: finishedAt
|
|
65634
65889
|
}
|
|
@@ -65659,7 +65914,7 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65659
65914
|
aiUserFetchVerificationManifestsHourly.manifestId
|
|
65660
65915
|
],
|
|
65661
65916
|
set: {
|
|
65662
|
-
hits:
|
|
65917
|
+
hits: sql38`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
65663
65918
|
manifestJson: bucket.verificationManifest,
|
|
65664
65919
|
updatedAt: finishedAt
|
|
65665
65920
|
}
|
|
@@ -65741,8 +65996,8 @@ async function runBackfillTaskHoldingLease(options, signal) {
|
|
|
65741
65996
|
lastEventIds: newRingBuffer,
|
|
65742
65997
|
...skipRecovered ? { skippedThroughAt: null } : {},
|
|
65743
65998
|
updatedAt: finishedAt
|
|
65744
|
-
}).where(
|
|
65745
|
-
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
65999
|
+
}).where(eq89(trafficSources.id, sourceRow2.id)).run();
|
|
66000
|
+
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq89(runs.id, runId)).run();
|
|
65746
66001
|
return "committed";
|
|
65747
66002
|
});
|
|
65748
66003
|
reportBackfill(commitOutcome === "committed" ? {
|
|
@@ -65915,7 +66170,7 @@ async function trafficRoutes(app, opts) {
|
|
|
65915
66170
|
};
|
|
65916
66171
|
const fallbackName = displayName ?? `Cloud Run \xB7 ${gcpProjectId}${serviceName ? ` / ${serviceName}` : ""}`;
|
|
65917
66172
|
const { sourceRow: sourceRow2, scheduleChanged } = app.db.transaction((tx) => {
|
|
65918
|
-
const activeSource = tx.select().from(trafficSources).where(
|
|
66173
|
+
const activeSource = tx.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes["cloud-run"] && row.status !== TrafficSourceStatuses.archived);
|
|
65919
66174
|
const sourceId = activeSource?.id ?? crypto55.randomUUID();
|
|
65920
66175
|
const sourceStatus = trafficConnectStatus(tx, project.id, activeSource);
|
|
65921
66176
|
if (activeSource) {
|
|
@@ -65925,7 +66180,7 @@ async function trafficRoutes(app, opts) {
|
|
|
65925
66180
|
lastError: null,
|
|
65926
66181
|
configJson: config,
|
|
65927
66182
|
updatedAt: now
|
|
65928
|
-
}).where(
|
|
66183
|
+
}).where(eq89(trafficSources.id, sourceId)).run();
|
|
65929
66184
|
} else {
|
|
65930
66185
|
tx.insert(trafficSources).values({
|
|
65931
66186
|
id: sourceId,
|
|
@@ -65951,7 +66206,7 @@ async function trafficRoutes(app, opts) {
|
|
|
65951
66206
|
entityId: sourceId
|
|
65952
66207
|
});
|
|
65953
66208
|
return {
|
|
65954
|
-
sourceRow: tx.select().from(trafficSources).where(
|
|
66209
|
+
sourceRow: tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceId)).get(),
|
|
65955
66210
|
scheduleChanged: scheduleBinding.changed
|
|
65956
66211
|
};
|
|
65957
66212
|
}, { behavior: "immediate" });
|
|
@@ -66006,7 +66261,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66006
66261
|
const config = { baseUrl, username };
|
|
66007
66262
|
const fallbackName = displayName ?? `WordPress \xB7 ${new URL(baseUrl).host}`;
|
|
66008
66263
|
const { sourceRow: sourceRow2, scheduleChanged } = app.db.transaction((tx) => {
|
|
66009
|
-
const activeSource = tx.select().from(trafficSources).where(
|
|
66264
|
+
const activeSource = tx.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.wordpress && row.status !== TrafficSourceStatuses.archived);
|
|
66010
66265
|
const endpointChanged = activeSource !== void 0 && wordpressBaseUrlIdentity(activeSource.configJson.baseUrl) !== wordpressBaseUrlIdentity(baseUrl);
|
|
66011
66266
|
const sourceId = endpointChanged ? crypto55.randomUUID() : activeSource?.id ?? crypto55.randomUUID();
|
|
66012
66267
|
const sourceStatus = trafficConnectStatus(tx, project.id, activeSource);
|
|
@@ -66017,14 +66272,14 @@ async function trafficRoutes(app, opts) {
|
|
|
66017
66272
|
lastError: null,
|
|
66018
66273
|
configJson: config,
|
|
66019
66274
|
updatedAt: now
|
|
66020
|
-
}).where(
|
|
66275
|
+
}).where(eq89(trafficSources.id, sourceId)).run();
|
|
66021
66276
|
} else {
|
|
66022
66277
|
if (activeSource) {
|
|
66023
66278
|
tx.update(trafficSources).set({
|
|
66024
66279
|
status: TrafficSourceStatuses.archived,
|
|
66025
66280
|
archivedAt: now,
|
|
66026
66281
|
updatedAt: now
|
|
66027
|
-
}).where(
|
|
66282
|
+
}).where(eq89(trafficSources.id, activeSource.id)).run();
|
|
66028
66283
|
}
|
|
66029
66284
|
tx.insert(trafficSources).values({
|
|
66030
66285
|
id: sourceId,
|
|
@@ -66050,7 +66305,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66050
66305
|
entityId: sourceId
|
|
66051
66306
|
});
|
|
66052
66307
|
return {
|
|
66053
|
-
sourceRow: tx.select().from(trafficSources).where(
|
|
66308
|
+
sourceRow: tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceId)).get(),
|
|
66054
66309
|
scheduleChanged: scheduleBinding.changed
|
|
66055
66310
|
};
|
|
66056
66311
|
}, { behavior: "immediate" });
|
|
@@ -66107,7 +66362,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66107
66362
|
const config = { projectId, teamId, environment };
|
|
66108
66363
|
const fallbackName = displayName ?? `Vercel \xB7 ${projectId}`;
|
|
66109
66364
|
const { sourceRow: sourceRow2, scheduleChanged } = app.db.transaction((tx) => {
|
|
66110
|
-
const activeSource = tx.select().from(trafficSources).where(
|
|
66365
|
+
const activeSource = tx.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all().find((row2) => row2.sourceType === TrafficSourceTypes.vercel && row2.status !== TrafficSourceStatuses.archived);
|
|
66111
66366
|
const sourceStatus = trafficConnectStatus(tx, project.id, activeSource);
|
|
66112
66367
|
let row;
|
|
66113
66368
|
if (activeSource) {
|
|
@@ -66117,8 +66372,8 @@ async function trafficRoutes(app, opts) {
|
|
|
66117
66372
|
lastError: null,
|
|
66118
66373
|
configJson: config,
|
|
66119
66374
|
updatedAt: now
|
|
66120
|
-
}).where(
|
|
66121
|
-
row = tx.select().from(trafficSources).where(
|
|
66375
|
+
}).where(eq89(trafficSources.id, activeSource.id)).run();
|
|
66376
|
+
row = tx.select().from(trafficSources).where(eq89(trafficSources.id, activeSource.id)).get();
|
|
66122
66377
|
} else {
|
|
66123
66378
|
const newId = crypto55.randomUUID();
|
|
66124
66379
|
tx.insert(trafficSources).values({
|
|
@@ -66143,7 +66398,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66143
66398
|
createdAt: now,
|
|
66144
66399
|
updatedAt: now
|
|
66145
66400
|
}).run();
|
|
66146
|
-
row = tx.select().from(trafficSources).where(
|
|
66401
|
+
row = tx.select().from(trafficSources).where(eq89(trafficSources.id, newId)).get();
|
|
66147
66402
|
}
|
|
66148
66403
|
const scheduleBinding = sourceStatus === TrafficSourceStatuses.connected ? bindTrafficSyncSchedule(tx, project.id, row.id, now) : { changed: false, created: false };
|
|
66149
66404
|
writeAuditLog(tx, {
|
|
@@ -66187,7 +66442,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66187
66442
|
const { displayName: displayName2, zoneId: zoneId2, accountId: accountId2, queueId, queueName, retentionSeconds, apiToken } = parsed.data;
|
|
66188
66443
|
const workerRouteHost2 = resolveCloudflareWorkerRouteHost(project.canonicalDomain);
|
|
66189
66444
|
const now2 = (/* @__PURE__ */ new Date()).toISOString();
|
|
66190
|
-
const cloudflareSources = app.db.select().from(trafficSources).where(
|
|
66445
|
+
const cloudflareSources = app.db.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all().filter((row) => row.sourceType === TrafficSourceTypes.cloudflare && row.status !== TrafficSourceStatuses.archived);
|
|
66191
66446
|
const sameModeSource = cloudflareSources.find((row) => parseQueuePullCloudflareSourceConfig(row.configJson));
|
|
66192
66447
|
if (sameModeSource) attempt.update({ action: "reauth" });
|
|
66193
66448
|
const sourceId2 = sameModeSource?.id ?? crypto55.randomUUID();
|
|
@@ -66241,7 +66496,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66241
66496
|
let scheduleChanged = false;
|
|
66242
66497
|
try {
|
|
66243
66498
|
const result = app.db.transaction((tx) => {
|
|
66244
|
-
const currentSameModeSource = tx.select().from(trafficSources).where(
|
|
66499
|
+
const currentSameModeSource = tx.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.cloudflare && row.status !== TrafficSourceStatuses.archived && parseQueuePullCloudflareSourceConfig(row.configJson) !== null);
|
|
66245
66500
|
if ((currentSameModeSource?.id ?? null) !== (sameModeSource?.id ?? null)) {
|
|
66246
66501
|
throw operationInProgress("Cloudflare Queue source changed during connect; retry");
|
|
66247
66502
|
}
|
|
@@ -66259,7 +66514,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66259
66514
|
queueBacklogObservedAt: null
|
|
66260
66515
|
} : {},
|
|
66261
66516
|
updatedAt: now2
|
|
66262
|
-
}).where(
|
|
66517
|
+
}).where(eq89(trafficSources.id, sourceId2)).run();
|
|
66263
66518
|
} else {
|
|
66264
66519
|
tx.insert(trafficSources).values({
|
|
66265
66520
|
id: sourceId2,
|
|
@@ -66279,7 +66534,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66279
66534
|
}
|
|
66280
66535
|
const changed = status === TrafficSourceStatuses.connected ? bindTrafficSyncSchedule(tx, project.id, sourceId2, now2).changed : false;
|
|
66281
66536
|
return {
|
|
66282
|
-
row: tx.select().from(trafficSources).where(
|
|
66537
|
+
row: tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceId2)).get(),
|
|
66283
66538
|
scheduleChanged: changed
|
|
66284
66539
|
};
|
|
66285
66540
|
}, { behavior: "immediate" });
|
|
@@ -66329,7 +66584,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66329
66584
|
const { deliveryMode, displayName, zoneId, accountId } = parsed.data;
|
|
66330
66585
|
const workerRouteHost = resolveCloudflareWorkerRouteHost(project.canonicalDomain);
|
|
66331
66586
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
66332
|
-
const activeSource = app.db.select().from(trafficSources).where(
|
|
66587
|
+
const activeSource = app.db.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.cloudflare && row.status !== TrafficSourceStatuses.archived && parseDirectPushCloudflareSourceConfig(row.configJson) !== null);
|
|
66333
66588
|
if (activeSource) attempt.update({ action: "reauth" });
|
|
66334
66589
|
const sourceId = activeSource?.id ?? crypto55.randomUUID();
|
|
66335
66590
|
const parsedActiveConfig = activeSource ? parseDirectPushCloudflareSourceConfig(activeSource.configJson) : null;
|
|
@@ -66397,7 +66652,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66397
66652
|
let scheduleRemoved = false;
|
|
66398
66653
|
try {
|
|
66399
66654
|
const result = app.db.transaction((tx) => {
|
|
66400
|
-
const currentActiveSource = tx.select().from(trafficSources).where(
|
|
66655
|
+
const currentActiveSource = tx.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all().find((row2) => row2.sourceType === TrafficSourceTypes.cloudflare && row2.status !== TrafficSourceStatuses.archived && parseDirectPushCloudflareSourceConfig(row2.configJson) !== null);
|
|
66401
66656
|
if ((currentActiveSource?.id ?? null) !== (activeSource?.id ?? null)) {
|
|
66402
66657
|
throw operationInProgress("Cloudflare direct-push source changed during connect; retry");
|
|
66403
66658
|
}
|
|
@@ -66411,8 +66666,8 @@ async function trafficRoutes(app, opts) {
|
|
|
66411
66666
|
configJson: config,
|
|
66412
66667
|
ingestTokenHash,
|
|
66413
66668
|
updatedAt: now
|
|
66414
|
-
}).where(
|
|
66415
|
-
row = tx.select().from(trafficSources).where(
|
|
66669
|
+
}).where(eq89(trafficSources.id, currentActiveSource.id)).run();
|
|
66670
|
+
row = tx.select().from(trafficSources).where(eq89(trafficSources.id, currentActiveSource.id)).get();
|
|
66416
66671
|
} else {
|
|
66417
66672
|
tx.insert(trafficSources).values({
|
|
66418
66673
|
id: sourceId,
|
|
@@ -66432,16 +66687,16 @@ async function trafficRoutes(app, opts) {
|
|
|
66432
66687
|
createdAt: now,
|
|
66433
66688
|
updatedAt: now
|
|
66434
66689
|
}).run();
|
|
66435
|
-
row = tx.select().from(trafficSources).where(
|
|
66690
|
+
row = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceId)).get();
|
|
66436
66691
|
}
|
|
66437
66692
|
let removed = false;
|
|
66438
66693
|
if (row.status === TrafficSourceStatuses.connected) {
|
|
66439
|
-
const schedule = tx.select().from(schedules).where(
|
|
66440
|
-
|
|
66441
|
-
|
|
66694
|
+
const schedule = tx.select().from(schedules).where(and78(
|
|
66695
|
+
eq89(schedules.projectId, project.id),
|
|
66696
|
+
eq89(schedules.kind, SchedulableRunKinds["traffic-sync"])
|
|
66442
66697
|
)).get();
|
|
66443
66698
|
if (schedule) {
|
|
66444
|
-
tx.delete(schedules).where(
|
|
66699
|
+
tx.delete(schedules).where(eq89(schedules.id, schedule.id)).run();
|
|
66445
66700
|
removed = true;
|
|
66446
66701
|
}
|
|
66447
66702
|
}
|
|
@@ -66497,47 +66752,47 @@ async function trafficRoutes(app, opts) {
|
|
|
66497
66752
|
};
|
|
66498
66753
|
}));
|
|
66499
66754
|
app.post("/projects/:name/traffic/sources/:id/activate", connectionRoute(app, (request) => {
|
|
66500
|
-
const source = app.db.select({ sourceType: trafficSources.sourceType }).from(trafficSources).where(
|
|
66755
|
+
const source = app.db.select({ sourceType: trafficSources.sourceType }).from(trafficSources).where(eq89(trafficSources.id, request.params.id)).get();
|
|
66501
66756
|
const integration = source ? TRAFFIC_SOURCE_INTEGRATIONS[source.sourceType] : void 0;
|
|
66502
66757
|
return integration ? { integration, action: "select" } : void 0;
|
|
66503
66758
|
}, async (request) => {
|
|
66504
66759
|
const project = resolveProject(app.db, request.params.name);
|
|
66505
66760
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
66506
66761
|
const result = app.db.transaction((tx) => {
|
|
66507
|
-
const target = tx.select().from(trafficSources).where(
|
|
66762
|
+
const target = tx.select().from(trafficSources).where(eq89(trafficSources.id, request.params.id)).get();
|
|
66508
66763
|
if (!target || target.projectId !== project.id || target.status === TrafficSourceStatuses.archived) {
|
|
66509
66764
|
throw notFound("Traffic source", request.params.id);
|
|
66510
66765
|
}
|
|
66511
66766
|
const { usesPullSchedule } = validateTrafficSourceCredential(target, project.name);
|
|
66512
|
-
const siblings = tx.select().from(trafficSources).where(
|
|
66767
|
+
const siblings = tx.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all();
|
|
66513
66768
|
for (const sibling of siblings) {
|
|
66514
66769
|
if (sibling.id === target.id || sibling.status === TrafficSourceStatuses.archived) continue;
|
|
66515
66770
|
tx.update(trafficSources).set({
|
|
66516
66771
|
status: TrafficSourceStatuses.paused,
|
|
66517
66772
|
updatedAt: now
|
|
66518
|
-
}).where(
|
|
66773
|
+
}).where(eq89(trafficSources.id, sibling.id)).run();
|
|
66519
66774
|
}
|
|
66520
66775
|
tx.update(trafficSources).set({
|
|
66521
66776
|
status: TrafficSourceStatuses.connected,
|
|
66522
66777
|
lastError: null,
|
|
66523
66778
|
archivedAt: null,
|
|
66524
66779
|
updatedAt: now
|
|
66525
|
-
}).where(
|
|
66780
|
+
}).where(eq89(trafficSources.id, target.id)).run();
|
|
66526
66781
|
let scheduleAction = null;
|
|
66527
66782
|
if (usesPullSchedule) {
|
|
66528
66783
|
bindTrafficSyncSchedule(tx, project.id, target.id, now);
|
|
66529
66784
|
scheduleAction = "upsert";
|
|
66530
66785
|
} else {
|
|
66531
|
-
const schedule = tx.select().from(schedules).where(
|
|
66532
|
-
|
|
66533
|
-
|
|
66786
|
+
const schedule = tx.select().from(schedules).where(and78(
|
|
66787
|
+
eq89(schedules.projectId, project.id),
|
|
66788
|
+
eq89(schedules.kind, SchedulableRunKinds["traffic-sync"])
|
|
66534
66789
|
)).get();
|
|
66535
66790
|
if (schedule) {
|
|
66536
|
-
tx.delete(schedules).where(
|
|
66791
|
+
tx.delete(schedules).where(eq89(schedules.id, schedule.id)).run();
|
|
66537
66792
|
scheduleAction = "delete";
|
|
66538
66793
|
}
|
|
66539
66794
|
}
|
|
66540
|
-
const source = tx.select().from(trafficSources).where(
|
|
66795
|
+
const source = tx.select().from(trafficSources).where(eq89(trafficSources.id, target.id)).get();
|
|
66541
66796
|
writeAuditLog(tx, {
|
|
66542
66797
|
projectId: project.id,
|
|
66543
66798
|
actor: "api",
|
|
@@ -66605,7 +66860,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66605
66860
|
}
|
|
66606
66861
|
const { bearerHash, credential } = authenticated;
|
|
66607
66862
|
const sourceId = credential.sourceId;
|
|
66608
|
-
const sourceRow2 = app.db.select().from(trafficSources).where(
|
|
66863
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(eq89(trafficSources.id, sourceId)).get();
|
|
66609
66864
|
if (!sourceRow2 || sourceRow2.sourceType !== TrafficSourceTypes.cloudflare || sourceRow2.status !== TrafficSourceStatuses.connected || !parseDirectPushCloudflareSourceConfig(sourceRow2.configJson) || !timingSafeEqualHex(bearerHash, sourceRow2.ingestTokenHash)) {
|
|
66610
66865
|
throw authRequired();
|
|
66611
66866
|
}
|
|
@@ -66683,7 +66938,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66683
66938
|
});
|
|
66684
66939
|
app.post("/projects/:name/traffic/sources/:id/sync", async (request) => {
|
|
66685
66940
|
const project = resolveProject(app.db, request.params.name);
|
|
66686
|
-
const initialSourceRow = app.db.select().from(trafficSources).where(
|
|
66941
|
+
const initialSourceRow = app.db.select().from(trafficSources).where(eq89(trafficSources.id, request.params.id)).get();
|
|
66687
66942
|
if (!initialSourceRow || initialSourceRow.projectId !== project.id) {
|
|
66688
66943
|
throw notFound("Traffic source", request.params.id);
|
|
66689
66944
|
}
|
|
@@ -66756,7 +67011,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66756
67011
|
});
|
|
66757
67012
|
const finishSync = (status, error, stopReason) => {
|
|
66758
67013
|
const queueDrain = drainSummary(stopReason);
|
|
66759
|
-
app.db.update(runs).set({ status, finishedAt: committedAt, syncSummary: queueDrain, ...error ? { error } : {} }).where(
|
|
67014
|
+
app.db.update(runs).set({ status, finishedAt: committedAt, syncSummary: queueDrain, ...error ? { error } : {} }).where(eq89(runs.id, runId)).run();
|
|
66760
67015
|
writeAuditLog(app.db, {
|
|
66761
67016
|
projectId: project.id,
|
|
66762
67017
|
actor: "api",
|
|
@@ -66931,7 +67186,7 @@ async function trafficRoutes(app, opts) {
|
|
|
66931
67186
|
now: (/* @__PURE__ */ new Date()).toISOString(),
|
|
66932
67187
|
ttlMs: CLOUDFLARE_QUEUE_SYNC_LEASE_TTL_MS
|
|
66933
67188
|
})) throw new Error("Queue sync lease was lost before acknowledgement");
|
|
66934
|
-
const latestSource = app.db.select().from(trafficSources).where(
|
|
67189
|
+
const latestSource = app.db.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
66935
67190
|
if (!latestSource || latestSource.projectId !== project.id || latestSource.status !== TrafficSourceStatuses.connected || latestSource.syncLeaseOwner !== leaseOwner || !isDeepStrictEqual(latestSource.configJson, sourceRow2.configJson)) {
|
|
66936
67191
|
throw validationError("Cloudflare Queue source was reconfigured before acknowledgement");
|
|
66937
67192
|
}
|
|
@@ -67020,13 +67275,13 @@ async function trafficRoutes(app, opts) {
|
|
|
67020
67275
|
error: safeError,
|
|
67021
67276
|
finishedAt: failedAt,
|
|
67022
67277
|
syncSummary: drainSummary(CloudflareQueueDrainStopReasons.error)
|
|
67023
|
-
}).where(
|
|
67024
|
-
const latestSource = tx.select().from(trafficSources).where(
|
|
67278
|
+
}).where(eq89(runs.id, runId)).run();
|
|
67279
|
+
const latestSource = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
67025
67280
|
if (latestSource && latestSource.status === TrafficSourceStatuses.connected && latestSource.syncLeaseOwner === leaseOwner && isDeepStrictEqual(latestSource.configJson, sourceRow2.configJson)) {
|
|
67026
|
-
tx.update(trafficSources).set({ lastError: safeError, updatedAt: failedAt }).where(
|
|
67027
|
-
|
|
67028
|
-
|
|
67029
|
-
|
|
67281
|
+
tx.update(trafficSources).set({ lastError: safeError, updatedAt: failedAt }).where(and78(
|
|
67282
|
+
eq89(trafficSources.id, sourceRow2.id),
|
|
67283
|
+
eq89(trafficSources.status, TrafficSourceStatuses.connected),
|
|
67284
|
+
eq89(trafficSources.syncLeaseOwner, leaseOwner)
|
|
67030
67285
|
)).run();
|
|
67031
67286
|
}
|
|
67032
67287
|
});
|
|
@@ -67045,7 +67300,7 @@ async function trafficRoutes(app, opts) {
|
|
|
67045
67300
|
`Sync for source type "${sourceRow2.sourceType}" is not implemented yet \u2014 only cloud-run, wordpress, and vercel are supported in v1.`
|
|
67046
67301
|
);
|
|
67047
67302
|
}
|
|
67048
|
-
const hasConnectedSibling = sourceRow2.status === TrafficSourceStatuses.error && app.db.select().from(trafficSources).where(
|
|
67303
|
+
const hasConnectedSibling = sourceRow2.status === TrafficSourceStatuses.error && app.db.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all().some((row) => row.id !== sourceRow2.id && row.status === TrafficSourceStatuses.connected);
|
|
67049
67304
|
if (sourceRow2.status === TrafficSourceStatuses.paused || sourceRow2.status === TrafficSourceStatuses.archived || hasConnectedSibling) {
|
|
67050
67305
|
throw validationError("Traffic source must be active before it can sync");
|
|
67051
67306
|
}
|
|
@@ -67082,10 +67337,10 @@ async function trafficRoutes(app, opts) {
|
|
|
67082
67337
|
const markFailed = (msg, errorCode2) => {
|
|
67083
67338
|
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
67084
67339
|
app.db.transaction((tx) => {
|
|
67085
|
-
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(
|
|
67086
|
-
const latestSource = tx.select().from(trafficSources).where(
|
|
67340
|
+
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(eq89(runs.id, runId)).run();
|
|
67341
|
+
const latestSource = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
67087
67342
|
if (latestSource?.status === TrafficSourceStatuses.connected && isSameTrafficSourceGeneration(latestSource, sourceRow2) && latestSource.syncLeaseOwner === syncLeaseOwner) {
|
|
67088
|
-
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(
|
|
67343
|
+
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(eq89(trafficSources.id, sourceRow2.id)).run();
|
|
67089
67344
|
}
|
|
67090
67345
|
});
|
|
67091
67346
|
try {
|
|
@@ -67179,7 +67434,7 @@ async function trafficRoutes(app, opts) {
|
|
|
67179
67434
|
}
|
|
67180
67435
|
const credential = credentialStore.getConnection(project.name);
|
|
67181
67436
|
if (!credential) {
|
|
67182
|
-
app.db.delete(runs).where(
|
|
67437
|
+
app.db.delete(runs).where(eq89(runs.id, runId)).run();
|
|
67183
67438
|
throw validationError(
|
|
67184
67439
|
`No WordPress credential found for project "${project.name}". Run "canonry traffic connect wordpress" first.`
|
|
67185
67440
|
);
|
|
@@ -67213,14 +67468,14 @@ async function trafficRoutes(app, opts) {
|
|
|
67213
67468
|
const windowEndIso2 = windowEnd.toISOString();
|
|
67214
67469
|
const reservedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
67215
67470
|
const reservedSource = app.db.transaction((tx) => {
|
|
67216
|
-
const latestSource = tx.select().from(trafficSources).where(
|
|
67471
|
+
const latestSource = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
67217
67472
|
if (!latestSource || !isAuthoritativeTrafficSource(tx, latestSource) || !isSameTrafficSourceGeneration(latestSource, sourceRow2) || latestSource.syncLeaseOwner !== syncLeaseOwner || latestSource.lastCursor || latestSource.wordpressPendingUntil) return void 0;
|
|
67218
67473
|
tx.update(trafficSources).set({
|
|
67219
67474
|
lastSyncedAt: windowStartIso2,
|
|
67220
67475
|
wordpressPendingUntil: windowEndIso2,
|
|
67221
67476
|
updatedAt: reservedAt
|
|
67222
|
-
}).where(
|
|
67223
|
-
return tx.select().from(trafficSources).where(
|
|
67477
|
+
}).where(eq89(trafficSources.id, sourceRow2.id)).run();
|
|
67478
|
+
return tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
67224
67479
|
}, { behavior: "immediate" });
|
|
67225
67480
|
if (!reservedSource) {
|
|
67226
67481
|
const message = "Traffic source changed while reserving the WordPress sync window; retry the sync.";
|
|
@@ -67280,12 +67535,12 @@ async function trafficRoutes(app, opts) {
|
|
|
67280
67535
|
auditAction = "traffic.vercel.synced";
|
|
67281
67536
|
const credentialStore = opts.vercelTrafficCredentialStore;
|
|
67282
67537
|
if (!credentialStore) {
|
|
67283
|
-
app.db.delete(runs).where(
|
|
67538
|
+
app.db.delete(runs).where(eq89(runs.id, runId)).run();
|
|
67284
67539
|
throw validationError("Vercel traffic credential storage is not configured for this deployment");
|
|
67285
67540
|
}
|
|
67286
67541
|
const credential = credentialStore.getConnection(project.name);
|
|
67287
67542
|
if (!credential) {
|
|
67288
|
-
app.db.delete(runs).where(
|
|
67543
|
+
app.db.delete(runs).where(eq89(runs.id, runId)).run();
|
|
67289
67544
|
throw validationError(
|
|
67290
67545
|
`No Vercel credential found for project "${project.name}". Run "canonry traffic connect vercel" first.`
|
|
67291
67546
|
);
|
|
@@ -67303,7 +67558,7 @@ async function trafficRoutes(app, opts) {
|
|
|
67303
67558
|
if (cappedStartMs > clampedStartMs) {
|
|
67304
67559
|
const previousSkip = sourceRow2.skippedThroughAt ? Date.parse(sourceRow2.skippedThroughAt) : Number.NaN;
|
|
67305
67560
|
const skippedThrough = Number.isFinite(previousSkip) ? new Date(Math.max(previousSkip, cappedStartMs)) : new Date(cappedStartMs);
|
|
67306
|
-
app.db.update(trafficSources).set({ skippedThroughAt: skippedThrough.toISOString() }).where(
|
|
67561
|
+
app.db.update(trafficSources).set({ skippedThroughAt: skippedThrough.toISOString() }).where(eq89(trafficSources.id, sourceRow2.id)).run();
|
|
67307
67562
|
request.log.warn(
|
|
67308
67563
|
{
|
|
67309
67564
|
sourceId: sourceRow2.id,
|
|
@@ -67397,15 +67652,15 @@ async function trafficRoutes(app, opts) {
|
|
|
67397
67652
|
throw providerError(`WordPress sync failed: ${msg}`);
|
|
67398
67653
|
}
|
|
67399
67654
|
const commitOutcome = app.db.transaction((tx) => {
|
|
67400
|
-
const latestRow = tx.select().from(trafficSources).where(
|
|
67401
|
-
const latestHasConnectedSibling = tx.select().from(trafficSources).where(
|
|
67655
|
+
const latestRow = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
67656
|
+
const latestHasConnectedSibling = tx.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all().some((row) => row.id !== sourceRow2.id && row.status === TrafficSourceStatuses.connected);
|
|
67402
67657
|
if (!latestRow || latestRow.status !== TrafficSourceStatuses.connected && latestRow.status !== TrafficSourceStatuses.error || latestHasConnectedSibling || latestRow.syncLeaseOwner !== syncLeaseOwner || !isSameTrafficSourceGeneration(latestRow, sourceRow2)) {
|
|
67403
67658
|
const abortedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
67404
67659
|
tx.update(runs).set({
|
|
67405
67660
|
status: RunStatuses.failed,
|
|
67406
67661
|
error: "Traffic source was deactivated or reconfigured during sync",
|
|
67407
67662
|
finishedAt: abortedAt
|
|
67408
|
-
}).where(
|
|
67663
|
+
}).where(eq89(runs.id, runId)).run();
|
|
67409
67664
|
return "source-inactive";
|
|
67410
67665
|
}
|
|
67411
67666
|
const previousIds = latestRow.lastEventIds ?? [];
|
|
@@ -67469,7 +67724,7 @@ async function trafficRoutes(app, opts) {
|
|
|
67469
67724
|
crawlerEventsHourly.status
|
|
67470
67725
|
],
|
|
67471
67726
|
set: {
|
|
67472
|
-
hits:
|
|
67727
|
+
hits: sql38`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
67473
67728
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
67474
67729
|
updatedAt: finishedAt
|
|
67475
67730
|
}
|
|
@@ -67500,7 +67755,7 @@ async function trafficRoutes(app, opts) {
|
|
|
67500
67755
|
crawlerVerificationManifestsHourly.manifestId
|
|
67501
67756
|
],
|
|
67502
67757
|
set: {
|
|
67503
|
-
hits:
|
|
67758
|
+
hits: sql38`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
67504
67759
|
manifestJson: bucket.verificationManifest,
|
|
67505
67760
|
updatedAt: finishedAt
|
|
67506
67761
|
}
|
|
@@ -67534,7 +67789,7 @@ async function trafficRoutes(app, opts) {
|
|
|
67534
67789
|
aiUserFetchEventsHourly.status
|
|
67535
67790
|
],
|
|
67536
67791
|
set: {
|
|
67537
|
-
hits:
|
|
67792
|
+
hits: sql38`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
67538
67793
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
67539
67794
|
updatedAt: finishedAt
|
|
67540
67795
|
}
|
|
@@ -67565,7 +67820,7 @@ async function trafficRoutes(app, opts) {
|
|
|
67565
67820
|
aiUserFetchVerificationManifestsHourly.manifestId
|
|
67566
67821
|
],
|
|
67567
67822
|
set: {
|
|
67568
|
-
hits:
|
|
67823
|
+
hits: sql38`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
67569
67824
|
manifestJson: bucket.verificationManifest,
|
|
67570
67825
|
updatedAt: finishedAt
|
|
67571
67826
|
}
|
|
@@ -67607,9 +67862,9 @@ async function trafficRoutes(app, opts) {
|
|
|
67607
67862
|
// colliding bucket adds its paid and organic arrivals to the ones
|
|
67608
67863
|
// already there rather than overwriting a single label.
|
|
67609
67864
|
set: {
|
|
67610
|
-
sessionsOrHits:
|
|
67611
|
-
paidSessionsOrHits:
|
|
67612
|
-
organicSessionsOrHits:
|
|
67865
|
+
sessionsOrHits: sql38`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
67866
|
+
paidSessionsOrHits: sql38`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
|
|
67867
|
+
organicSessionsOrHits: sql38`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
|
|
67613
67868
|
updatedAt: finishedAt
|
|
67614
67869
|
}
|
|
67615
67870
|
}).run();
|
|
@@ -67691,8 +67946,8 @@ async function trafficRoutes(app, opts) {
|
|
|
67691
67946
|
sourceUpdate.lastCursor = nextCursor ?? null;
|
|
67692
67947
|
sourceUpdate.wordpressPendingUntil = nextCursor ? windowEnd.toISOString() : null;
|
|
67693
67948
|
}
|
|
67694
|
-
tx.update(trafficSources).set(sourceUpdate).where(
|
|
67695
|
-
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
67949
|
+
tx.update(trafficSources).set(sourceUpdate).where(eq89(trafficSources.id, sourceRow2.id)).run();
|
|
67950
|
+
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq89(runs.id, runId)).run();
|
|
67696
67951
|
return "committed";
|
|
67697
67952
|
}, { behavior: "immediate" });
|
|
67698
67953
|
if (commitOutcome === "source-inactive") {
|
|
@@ -67761,7 +68016,7 @@ async function trafficRoutes(app, opts) {
|
|
|
67761
68016
|
});
|
|
67762
68017
|
app.post("/projects/:name/traffic/sources/:id/backfill", async (request) => withFeatureOutcome(app, { feature: "server_traffic", operation: "backfill" }, async (recordOutcome) => {
|
|
67763
68018
|
const project = resolveProject(app.db, request.params.name);
|
|
67764
|
-
const sourceRow2 = app.db.select().from(trafficSources).where(
|
|
68019
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(eq89(trafficSources.id, request.params.id)).get();
|
|
67765
68020
|
if (!sourceRow2 || sourceRow2.projectId !== project.id) {
|
|
67766
68021
|
throw notFound("Traffic source", request.params.id);
|
|
67767
68022
|
}
|
|
@@ -67886,7 +68141,7 @@ async function trafficRoutes(app, opts) {
|
|
|
67886
68141
|
}
|
|
67887
68142
|
try {
|
|
67888
68143
|
app.db.transaction((tx) => {
|
|
67889
|
-
const latestSource = tx.select().from(trafficSources).where(
|
|
68144
|
+
const latestSource = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
67890
68145
|
if (!latestSource || !isAuthoritativeTrafficSource(tx, latestSource) || !isSameTrafficSourceGeneration(latestSource, sourceRow2)) {
|
|
67891
68146
|
throw validationError("Traffic source must remain active and unchanged before it can backfill");
|
|
67892
68147
|
}
|
|
@@ -67934,10 +68189,10 @@ async function trafficRoutes(app, opts) {
|
|
|
67934
68189
|
function buildSourceDetail(projectId, row, since) {
|
|
67935
68190
|
const crawlerPathRows = app.db.select({
|
|
67936
68191
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
67937
|
-
hits:
|
|
68192
|
+
hits: sql38`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
67938
68193
|
}).from(crawlerEventsHourly).where(
|
|
67939
|
-
|
|
67940
|
-
|
|
68194
|
+
and78(
|
|
68195
|
+
eq89(crawlerEventsHourly.sourceId, row.id),
|
|
67941
68196
|
gte24(crawlerEventsHourly.tsHour, since)
|
|
67942
68197
|
)
|
|
67943
68198
|
).groupBy(crawlerEventsHourly.pathNormalized).all();
|
|
@@ -67945,41 +68200,41 @@ async function trafficRoutes(app, opts) {
|
|
|
67945
68200
|
crawlerPathRows.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
|
|
67946
68201
|
);
|
|
67947
68202
|
const crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
67948
|
-
const aiUserFetchTotals = app.db.select({ total:
|
|
67949
|
-
|
|
67950
|
-
|
|
68203
|
+
const aiUserFetchTotals = app.db.select({ total: sql38`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
68204
|
+
and78(
|
|
68205
|
+
eq89(aiUserFetchEventsHourly.sourceId, row.id),
|
|
67951
68206
|
gte24(aiUserFetchEventsHourly.tsHour, since)
|
|
67952
68207
|
)
|
|
67953
68208
|
).get();
|
|
67954
|
-
const likelyFakeTotals = app.db.select({ total:
|
|
67955
|
-
|
|
67956
|
-
|
|
68209
|
+
const likelyFakeTotals = app.db.select({ total: sql38`COALESCE(SUM(${aiUserFetchLikelyFakeHourly.hits}), 0)` }).from(aiUserFetchLikelyFakeHourly).where(
|
|
68210
|
+
and78(
|
|
68211
|
+
eq89(aiUserFetchLikelyFakeHourly.sourceId, row.id),
|
|
67957
68212
|
gte24(aiUserFetchLikelyFakeHourly.tsHour, since)
|
|
67958
68213
|
)
|
|
67959
68214
|
).get();
|
|
67960
68215
|
const aiTotals = app.db.select({
|
|
67961
|
-
total:
|
|
67962
|
-
nonhop:
|
|
67963
|
-
landed:
|
|
68216
|
+
total: sql38`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
68217
|
+
nonhop: sql38`COALESCE(SUM(CASE WHEN ${referralLandedCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`,
|
|
68218
|
+
landed: sql38`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`
|
|
67964
68219
|
}).from(aiReferralEventsHourly).where(
|
|
67965
|
-
|
|
67966
|
-
|
|
68220
|
+
and78(
|
|
68221
|
+
eq89(aiReferralEventsHourly.sourceId, row.id),
|
|
67967
68222
|
gte24(aiReferralEventsHourly.tsHour, since)
|
|
67968
68223
|
)
|
|
67969
68224
|
).get();
|
|
67970
|
-
const sampleTotals = app.db.select({ total:
|
|
67971
|
-
|
|
67972
|
-
|
|
68225
|
+
const sampleTotals = app.db.select({ total: sql38`COUNT(*)` }).from(rawEventSamples).where(
|
|
68226
|
+
and78(
|
|
68227
|
+
eq89(rawEventSamples.sourceId, row.id),
|
|
67973
68228
|
gte24(rawEventSamples.ts, since)
|
|
67974
68229
|
)
|
|
67975
68230
|
).get();
|
|
67976
68231
|
const latestRun = app.db.select().from(runs).where(
|
|
67977
|
-
|
|
67978
|
-
|
|
67979
|
-
|
|
67980
|
-
|
|
68232
|
+
and78(
|
|
68233
|
+
eq89(runs.projectId, projectId),
|
|
68234
|
+
eq89(runs.kind, RunKinds["traffic-sync"]),
|
|
68235
|
+
eq89(runs.sourceId, row.id)
|
|
67981
68236
|
)
|
|
67982
|
-
).orderBy(
|
|
68237
|
+
).orderBy(desc41(runs.startedAt)).limit(1).get();
|
|
67983
68238
|
return {
|
|
67984
68239
|
...rowToDto(row),
|
|
67985
68240
|
totals24h: {
|
|
@@ -68015,7 +68270,7 @@ async function trafficRoutes(app, opts) {
|
|
|
68015
68270
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
68016
68271
|
let updatedRow;
|
|
68017
68272
|
app.db.transaction((tx) => {
|
|
68018
|
-
const sourceRow2 = tx.select().from(trafficSources).where(
|
|
68273
|
+
const sourceRow2 = tx.select().from(trafficSources).where(and78(eq89(trafficSources.projectId, project.id), eq89(trafficSources.id, request.params.id))).get();
|
|
68019
68274
|
if (!sourceRow2) {
|
|
68020
68275
|
throw notFound("traffic source", request.params.id);
|
|
68021
68276
|
}
|
|
@@ -68029,7 +68284,7 @@ async function trafficRoutes(app, opts) {
|
|
|
68029
68284
|
`Traffic source "${sourceRow2.id}" is staged. Activate it explicitly instead of resetting it.`
|
|
68030
68285
|
);
|
|
68031
68286
|
}
|
|
68032
|
-
const hasConnectedSibling = tx.select().from(trafficSources).where(
|
|
68287
|
+
const hasConnectedSibling = tx.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).all().some((row) => row.id !== sourceRow2.id && row.status === TrafficSourceStatuses.connected);
|
|
68033
68288
|
if (hasConnectedSibling) {
|
|
68034
68289
|
throw validationError(
|
|
68035
68290
|
`Traffic source "${sourceRow2.id}" is not authoritative. Activate it explicitly instead of resetting it.`
|
|
@@ -68046,7 +68301,7 @@ async function trafficRoutes(app, opts) {
|
|
|
68046
68301
|
status: TrafficSourceStatuses.connected,
|
|
68047
68302
|
lastError: null,
|
|
68048
68303
|
updatedAt: now
|
|
68049
|
-
}).where(
|
|
68304
|
+
}).where(eq89(trafficSources.id, sourceRow2.id)).run();
|
|
68050
68305
|
writeAuditLog(tx, auditFromRequest(request, {
|
|
68051
68306
|
projectId: project.id,
|
|
68052
68307
|
actor: "api",
|
|
@@ -68054,7 +68309,7 @@ async function trafficRoutes(app, opts) {
|
|
|
68054
68309
|
entityType: "traffic_source",
|
|
68055
68310
|
entityId: sourceRow2.id
|
|
68056
68311
|
}));
|
|
68057
|
-
updatedRow = tx.select().from(trafficSources).where(
|
|
68312
|
+
updatedRow = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
68058
68313
|
});
|
|
68059
68314
|
return buildSourceDetail(project.id, updatedRow, new Date(Date.now() - 24 * 60 * 6e4).toISOString());
|
|
68060
68315
|
}));
|
|
@@ -68066,7 +68321,7 @@ async function trafficRoutes(app, opts) {
|
|
|
68066
68321
|
}
|
|
68067
68322
|
let updatedRow;
|
|
68068
68323
|
app.db.transaction((tx) => {
|
|
68069
|
-
const sourceRow2 = tx.select().from(trafficSources).where(
|
|
68324
|
+
const sourceRow2 = tx.select().from(trafficSources).where(and78(eq89(trafficSources.projectId, project.id), eq89(trafficSources.id, request.params.id))).get();
|
|
68070
68325
|
if (!sourceRow2) {
|
|
68071
68326
|
throw notFound("traffic source", request.params.id);
|
|
68072
68327
|
}
|
|
@@ -68079,7 +68334,7 @@ async function trafficRoutes(app, opts) {
|
|
|
68079
68334
|
);
|
|
68080
68335
|
}
|
|
68081
68336
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
68082
|
-
tx.update(trafficSources).set({ skippedThroughAt: null, updatedAt: now }).where(
|
|
68337
|
+
tx.update(trafficSources).set({ skippedThroughAt: null, updatedAt: now }).where(eq89(trafficSources.id, sourceRow2.id)).run();
|
|
68083
68338
|
writeAuditLog(tx, auditFromRequest(request, {
|
|
68084
68339
|
projectId: project.id,
|
|
68085
68340
|
actor: "api",
|
|
@@ -68088,20 +68343,20 @@ async function trafficRoutes(app, opts) {
|
|
|
68088
68343
|
entityId: sourceRow2.id,
|
|
68089
68344
|
diff: { skippedThroughAt: sourceRow2.skippedThroughAt, sourceType: sourceRow2.sourceType }
|
|
68090
68345
|
}));
|
|
68091
|
-
updatedRow = tx.select().from(trafficSources).where(
|
|
68346
|
+
updatedRow = tx.select().from(trafficSources).where(eq89(trafficSources.id, sourceRow2.id)).get();
|
|
68092
68347
|
});
|
|
68093
68348
|
return buildSourceDetail(project.id, updatedRow, new Date(Date.now() - 24 * 60 * 6e4).toISOString());
|
|
68094
68349
|
});
|
|
68095
68350
|
app.get("/projects/:name/traffic/sources", async (request) => {
|
|
68096
68351
|
const project = resolveProject(app.db, request.params.name);
|
|
68097
|
-
const rows = app.db.select().from(trafficSources).where(
|
|
68352
|
+
const rows = app.db.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).orderBy(desc41(trafficSources.createdAt)).all();
|
|
68098
68353
|
const sources = rows.filter((row) => row.status !== TrafficSourceStatuses.archived).map(rowToDto);
|
|
68099
68354
|
const response = { sources };
|
|
68100
68355
|
return response;
|
|
68101
68356
|
});
|
|
68102
68357
|
app.get("/projects/:name/traffic/status", async (request) => {
|
|
68103
68358
|
const project = resolveProject(app.db, request.params.name);
|
|
68104
|
-
const rows = app.db.select().from(trafficSources).where(
|
|
68359
|
+
const rows = app.db.select().from(trafficSources).where(eq89(trafficSources.projectId, project.id)).orderBy(desc41(trafficSources.createdAt)).all();
|
|
68105
68360
|
const since = new Date(Date.now() - 24 * 60 * 6e4).toISOString();
|
|
68106
68361
|
const sources = rows.filter((row) => row.status !== TrafficSourceStatuses.archived).map((row) => buildSourceDetail(project.id, row, since));
|
|
68107
68362
|
const response = { sources };
|
|
@@ -68111,7 +68366,7 @@ async function trafficRoutes(app, opts) {
|
|
|
68111
68366
|
"/projects/:name/traffic/sources/:id",
|
|
68112
68367
|
async (request) => {
|
|
68113
68368
|
const project = resolveProject(app.db, request.params.name);
|
|
68114
|
-
const row = app.db.select().from(trafficSources).where(
|
|
68369
|
+
const row = app.db.select().from(trafficSources).where(eq89(trafficSources.id, request.params.id)).get();
|
|
68115
68370
|
if (!row || row.projectId !== project.id) {
|
|
68116
68371
|
throw notFound("Traffic source", request.params.id);
|
|
68117
68372
|
}
|
|
@@ -68184,23 +68439,23 @@ async function trafficRoutes(app, opts) {
|
|
|
68184
68439
|
const seriesByBucket = /* @__PURE__ */ new Map();
|
|
68185
68440
|
if (kind === "all" || kind === TrafficEventKinds.crawler) {
|
|
68186
68441
|
const crawlerFilters = [
|
|
68187
|
-
|
|
68442
|
+
eq89(crawlerEventsHourly.projectId, project.id),
|
|
68188
68443
|
gte24(crawlerEventsHourly.tsHour, sinceIso),
|
|
68189
68444
|
lte25(crawlerEventsHourly.tsHour, untilIso)
|
|
68190
68445
|
];
|
|
68191
|
-
if (sourceIdParam) crawlerFilters.push(
|
|
68192
|
-
const crawlerWhere =
|
|
68446
|
+
if (sourceIdParam) crawlerFilters.push(eq89(crawlerEventsHourly.sourceId, sourceIdParam));
|
|
68447
|
+
const crawlerWhere = and78(...crawlerFilters);
|
|
68193
68448
|
const pathTotals = app.db.select({
|
|
68194
68449
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
68195
|
-
hits:
|
|
68196
|
-
rows:
|
|
68450
|
+
hits: sql38`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
68451
|
+
rows: sql38`COUNT(*)`
|
|
68197
68452
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerEventsHourly.pathNormalized).all();
|
|
68198
68453
|
crawlerSegments = segmentCrawlerHits(
|
|
68199
68454
|
pathTotals.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
|
|
68200
68455
|
);
|
|
68201
68456
|
crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
68202
68457
|
totalEventRows += pathTotals.reduce((sum, row) => sum + Number(row.rows), 0);
|
|
68203
|
-
const crawlerSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
68458
|
+
const crawlerSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql38`substr(${crawlerEventsHourly.tsHour}, 1, 10)` : crawlerEventsHourly.tsHour;
|
|
68204
68459
|
const distinctPaths = pathTotals.map((r) => ({ pathNormalized: r.pathNormalized }));
|
|
68205
68460
|
const contentPaths = [];
|
|
68206
68461
|
const infraPaths = [];
|
|
@@ -68212,12 +68467,12 @@ async function trafficRoutes(app, opts) {
|
|
|
68212
68467
|
const listed = useContentList ? contentPaths : infraPaths;
|
|
68213
68468
|
const IN_LIST_CAP = 2e4;
|
|
68214
68469
|
const listFitsInSql = listed.length <= IN_LIST_CAP;
|
|
68215
|
-
const contentExpr = listed.length === 0 ? useContentList ?
|
|
68470
|
+
const contentExpr = listed.length === 0 ? useContentList ? sql38`0` : sql38`${crawlerEventsHourly.hits}` : useContentList ? sql38`CASE WHEN ${inArray40(crawlerEventsHourly.pathNormalized, listed)} THEN ${crawlerEventsHourly.hits} ELSE 0 END` : sql38`CASE WHEN ${inArray40(crawlerEventsHourly.pathNormalized, listed)} THEN 0 ELSE ${crawlerEventsHourly.hits} END`;
|
|
68216
68471
|
if (listFitsInSql) {
|
|
68217
68472
|
const crawlerSeries = app.db.select({
|
|
68218
68473
|
bucket: crawlerSeriesBucket,
|
|
68219
|
-
hits:
|
|
68220
|
-
content:
|
|
68474
|
+
hits: sql38`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
68475
|
+
content: sql38`COALESCE(SUM(${contentExpr}), 0)`
|
|
68221
68476
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerSeriesBucket).all();
|
|
68222
68477
|
for (const row of crawlerSeries) {
|
|
68223
68478
|
const point = trafficSeriesPoint(seriesByBucket, String(row.bucket));
|
|
@@ -68229,7 +68484,7 @@ async function trafficRoutes(app, opts) {
|
|
|
68229
68484
|
const grouped = app.db.select({
|
|
68230
68485
|
bucket: crawlerSeriesBucket,
|
|
68231
68486
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
68232
|
-
hits:
|
|
68487
|
+
hits: sql38`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
68233
68488
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerSeriesBucket, crawlerEventsHourly.pathNormalized).all();
|
|
68234
68489
|
for (const row of grouped) {
|
|
68235
68490
|
const point = trafficSeriesPoint(seriesByBucket, String(row.bucket));
|
|
@@ -68239,7 +68494,7 @@ async function trafficRoutes(app, opts) {
|
|
|
68239
68494
|
if (useContentList ? inList : !inList) point.crawlerContentHits += hits;
|
|
68240
68495
|
}
|
|
68241
68496
|
}
|
|
68242
|
-
const rows = app.db.select().from(crawlerEventsHourly).where(crawlerWhere).orderBy(
|
|
68497
|
+
const rows = app.db.select().from(crawlerEventsHourly).where(crawlerWhere).orderBy(desc41(crawlerEventsHourly.tsHour)).limit(limit).all();
|
|
68243
68498
|
for (const r of rows) {
|
|
68244
68499
|
events.push({
|
|
68245
68500
|
kind: TrafficEventKinds.crawler,
|
|
@@ -68259,27 +68514,27 @@ async function trafficRoutes(app, opts) {
|
|
|
68259
68514
|
}
|
|
68260
68515
|
if (kind === "all" || kind === TrafficEventKinds["ai-user-fetch"]) {
|
|
68261
68516
|
const userFetchFilters = [
|
|
68262
|
-
|
|
68517
|
+
eq89(aiUserFetchEventsHourly.projectId, project.id),
|
|
68263
68518
|
gte24(aiUserFetchEventsHourly.tsHour, sinceIso),
|
|
68264
68519
|
lte25(aiUserFetchEventsHourly.tsHour, untilIso)
|
|
68265
68520
|
];
|
|
68266
|
-
if (sourceIdParam) userFetchFilters.push(
|
|
68267
|
-
const userFetchWhere =
|
|
68521
|
+
if (sourceIdParam) userFetchFilters.push(eq89(aiUserFetchEventsHourly.sourceId, sourceIdParam));
|
|
68522
|
+
const userFetchWhere = and78(...userFetchFilters);
|
|
68268
68523
|
const total = app.db.select({
|
|
68269
|
-
total:
|
|
68270
|
-
rows:
|
|
68524
|
+
total: sql38`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`,
|
|
68525
|
+
rows: sql38`COUNT(*)`
|
|
68271
68526
|
}).from(aiUserFetchEventsHourly).where(userFetchWhere).get();
|
|
68272
68527
|
aiUserFetchTotal = Number(total?.total ?? 0);
|
|
68273
68528
|
totalEventRows += Number(total?.rows ?? 0);
|
|
68274
|
-
const userFetchSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
68529
|
+
const userFetchSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql38`substr(${aiUserFetchEventsHourly.tsHour}, 1, 10)` : aiUserFetchEventsHourly.tsHour;
|
|
68275
68530
|
const userFetchSeries = app.db.select({
|
|
68276
68531
|
bucket: userFetchSeriesBucket,
|
|
68277
|
-
hits:
|
|
68532
|
+
hits: sql38`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`
|
|
68278
68533
|
}).from(aiUserFetchEventsHourly).where(userFetchWhere).groupBy(userFetchSeriesBucket).all();
|
|
68279
68534
|
for (const row of userFetchSeries) {
|
|
68280
68535
|
trafficSeriesPoint(seriesByBucket, row.bucket).aiUserFetchHits = Number(row.hits);
|
|
68281
68536
|
}
|
|
68282
|
-
const rows = app.db.select().from(aiUserFetchEventsHourly).where(userFetchWhere).orderBy(
|
|
68537
|
+
const rows = app.db.select().from(aiUserFetchEventsHourly).where(userFetchWhere).orderBy(desc41(aiUserFetchEventsHourly.tsHour)).limit(limit).all();
|
|
68283
68538
|
for (const r of rows) {
|
|
68284
68539
|
events.push({
|
|
68285
68540
|
kind: TrafficEventKinds["ai-user-fetch"],
|
|
@@ -68298,19 +68553,19 @@ async function trafficRoutes(app, opts) {
|
|
|
68298
68553
|
}
|
|
68299
68554
|
if (kind === "all" || kind === TrafficEventKinds["ai-referral"]) {
|
|
68300
68555
|
const aiFilters = [
|
|
68301
|
-
|
|
68556
|
+
eq89(aiReferralEventsHourly.projectId, project.id),
|
|
68302
68557
|
gte24(aiReferralEventsHourly.tsHour, sinceIso),
|
|
68303
68558
|
lte25(aiReferralEventsHourly.tsHour, untilIso)
|
|
68304
68559
|
];
|
|
68305
|
-
if (sourceIdParam) aiFilters.push(
|
|
68306
|
-
const aiWhere =
|
|
68560
|
+
if (sourceIdParam) aiFilters.push(eq89(aiReferralEventsHourly.sourceId, sourceIdParam));
|
|
68561
|
+
const aiWhere = and78(...aiFilters);
|
|
68307
68562
|
const total = app.db.select({
|
|
68308
|
-
total:
|
|
68309
|
-
nonhop:
|
|
68310
|
-
landed:
|
|
68311
|
-
paid:
|
|
68312
|
-
organic:
|
|
68313
|
-
rows:
|
|
68563
|
+
total: sql38`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
68564
|
+
nonhop: sql38`COALESCE(SUM(CASE WHEN ${referralLandedCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`,
|
|
68565
|
+
landed: sql38`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`,
|
|
68566
|
+
paid: sql38`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.paidSessionsOrHits} ELSE 0 END), 0)`,
|
|
68567
|
+
organic: sql38`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.organicSessionsOrHits} ELSE 0 END), 0)`,
|
|
68568
|
+
rows: sql38`COUNT(*)`
|
|
68314
68569
|
}).from(aiReferralEventsHourly).where(aiWhere).get();
|
|
68315
68570
|
aiReferralTotalHits = Number(total?.total ?? 0);
|
|
68316
68571
|
aiReferralNonHopHits = Number(total?.nonhop ?? 0);
|
|
@@ -68321,18 +68576,18 @@ async function trafficRoutes(app, opts) {
|
|
|
68321
68576
|
Number(total?.organic ?? 0)
|
|
68322
68577
|
);
|
|
68323
68578
|
totalEventRows += Number(total?.rows ?? 0);
|
|
68324
|
-
const referralSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
68579
|
+
const referralSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql38`substr(${aiReferralEventsHourly.tsHour}, 1, 10)` : aiReferralEventsHourly.tsHour;
|
|
68325
68580
|
const referralSeries = app.db.select({
|
|
68326
68581
|
bucket: referralSeriesBucket,
|
|
68327
|
-
hits:
|
|
68328
|
-
landed:
|
|
68582
|
+
hits: sql38`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
68583
|
+
landed: sql38`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`
|
|
68329
68584
|
}).from(aiReferralEventsHourly).where(aiWhere).groupBy(referralSeriesBucket).all();
|
|
68330
68585
|
for (const row of referralSeries) {
|
|
68331
68586
|
const point = trafficSeriesPoint(seriesByBucket, row.bucket);
|
|
68332
68587
|
point.aiReferralHits = Number(row.hits);
|
|
68333
68588
|
point.aiReferralLandedHits = Number(row.landed);
|
|
68334
68589
|
}
|
|
68335
|
-
const rows = app.db.select().from(aiReferralEventsHourly).where(aiWhere).orderBy(
|
|
68590
|
+
const rows = app.db.select().from(aiReferralEventsHourly).where(aiWhere).orderBy(desc41(aiReferralEventsHourly.tsHour)).limit(limit).all();
|
|
68336
68591
|
for (const r of rows) {
|
|
68337
68592
|
const counts2 = aiReferralClassCounts(r.sessionsOrHits, r.paidSessionsOrHits, r.organicSessionsOrHits);
|
|
68338
68593
|
events.push({
|
|
@@ -68378,11 +68633,11 @@ async function trafficRoutes(app, opts) {
|
|
|
68378
68633
|
windowEnd: untilIso,
|
|
68379
68634
|
series: (() => {
|
|
68380
68635
|
const points = completeTrafficSeries(since, until, granularity, seriesByBucket);
|
|
68381
|
-
const coverageScope = (col, srcCol) => sourceIdParam ?
|
|
68636
|
+
const coverageScope = (col, srcCol) => sourceIdParam ? and78(eq89(col, project.id), eq89(srcCol, sourceIdParam)) : eq89(col, project.id);
|
|
68382
68637
|
const firsts = [
|
|
68383
|
-
app.db.select({ v:
|
|
68384
|
-
app.db.select({ v:
|
|
68385
|
-
app.db.select({ v:
|
|
68638
|
+
app.db.select({ v: sql38`MIN(${crawlerEventsHourly.tsHour})` }).from(crawlerEventsHourly).where(coverageScope(crawlerEventsHourly.projectId, crawlerEventsHourly.sourceId)).get()?.v,
|
|
68639
|
+
app.db.select({ v: sql38`MIN(${aiUserFetchEventsHourly.tsHour})` }).from(aiUserFetchEventsHourly).where(coverageScope(aiUserFetchEventsHourly.projectId, aiUserFetchEventsHourly.sourceId)).get()?.v,
|
|
68640
|
+
app.db.select({ v: sql38`MIN(${aiReferralEventsHourly.tsHour})` }).from(aiReferralEventsHourly).where(coverageScope(aiReferralEventsHourly.projectId, aiReferralEventsHourly.sourceId)).get()?.v
|
|
68386
68641
|
].filter((v) => typeof v === "string" && v.length > 0);
|
|
68387
68642
|
const coverageStart = firsts.length ? firsts.slice().sort()[0] : null;
|
|
68388
68643
|
const bucketStartMs = (bucket) => Date.parse(
|
|
@@ -68456,7 +68711,7 @@ async function trafficRoutes(app, opts) {
|
|
|
68456
68711
|
}
|
|
68457
68712
|
|
|
68458
68713
|
// ../api-routes/src/doctor/checks/report-sweeps.ts
|
|
68459
|
-
import { and as
|
|
68714
|
+
import { and as and79, asc as asc23, eq as eq90, gte as gte25, lte as lte26 } from "drizzle-orm";
|
|
68460
68715
|
|
|
68461
68716
|
// ../api-routes/src/visibility-attribution.ts
|
|
68462
68717
|
function buildQueryAttribution2(projectQueries) {
|
|
@@ -68490,11 +68745,11 @@ function inspectSweep(ctx, run) {
|
|
|
68490
68745
|
answerMentioned: querySnapshots.answerMentioned,
|
|
68491
68746
|
answerText: querySnapshots.answerText,
|
|
68492
68747
|
measurementExecutionId: querySnapshots.measurementExecutionId
|
|
68493
|
-
}).from(querySnapshots).where(
|
|
68748
|
+
}).from(querySnapshots).where(eq90(querySnapshots.runId, run.id)).all();
|
|
68494
68749
|
if (snapshots.length === 0) return { detail: { ...base, complete: false, reason: "no-answers" } };
|
|
68495
68750
|
try {
|
|
68496
68751
|
if (run.measurementPlanVersionId) {
|
|
68497
|
-
const version = ctx.db.select().from(measurementPlanVersions).where(
|
|
68752
|
+
const version = ctx.db.select().from(measurementPlanVersions).where(and79(eq90(measurementPlanVersions.projectId, ctx.project.id), eq90(measurementPlanVersions.id, run.measurementPlanVersionId))).get();
|
|
68498
68753
|
if (!version) return { detail: { ...base, complete: false, reason: "missing-plan" } };
|
|
68499
68754
|
const plan = parseStoredMeasurementPlanAnyVersion(version.canonicalJson);
|
|
68500
68755
|
const manifest = measurementRunExpectedSlots(run, plan);
|
|
@@ -68526,10 +68781,10 @@ function inspectSweep(ctx, run) {
|
|
|
68526
68781
|
};
|
|
68527
68782
|
return unbound ? { detail } : { detail, coverage: slotCoverage(expected2, answered3) };
|
|
68528
68783
|
}
|
|
68529
|
-
const stored = ctx.db.select().from(simpleMeasurementDefinitions).where(
|
|
68784
|
+
const stored = ctx.db.select().from(simpleMeasurementDefinitions).where(eq90(simpleMeasurementDefinitions.runId, run.id)).get();
|
|
68530
68785
|
const frozen = stored ? simpleMeasurementDefinitionSchema.parse(stored.definition) : null;
|
|
68531
|
-
const project = ctx.db.select().from(projects).where(
|
|
68532
|
-
const basket = frozen ? frozen.queries.map((query2) => ({ id: query2.queryId, query: query2.queryText })) : ctx.db.select({ id: queries.id, query: queries.query }).from(queries).where(
|
|
68786
|
+
const project = ctx.db.select().from(projects).where(eq90(projects.id, ctx.project.id)).get();
|
|
68787
|
+
const basket = frozen ? frozen.queries.map((query2) => ({ id: query2.queryId, query: query2.queryText })) : ctx.db.select({ id: queries.id, query: queries.query }).from(queries).where(eq90(queries.projectId, project.id)).all();
|
|
68533
68788
|
const providers = frozen ? frozen.engines.map((engine) => engine.provider) : project.providers.length > 0 ? project.providers : [...new Set(snapshots.map((snapshot) => snapshot.provider))];
|
|
68534
68789
|
const attribution = buildQueryAttribution2(basket);
|
|
68535
68790
|
const expected = new Set(basket.flatMap((query2) => providers.map((provider) => JSON.stringify([query2.id, provider.trim().toLowerCase()]))));
|
|
@@ -68578,7 +68833,7 @@ var reportSweepsCheck = {
|
|
|
68578
68833
|
const graded = gradedReportMonths(reportMonths, now);
|
|
68579
68834
|
const months = reportMonths.map((month) => {
|
|
68580
68835
|
const bounds = calendarMonthBounds(month);
|
|
68581
|
-
const monthRuns = ctx.db.select().from(runs).where(
|
|
68836
|
+
const monthRuns = ctx.db.select().from(runs).where(and79(eq90(runs.projectId, ctx.project.id), eq90(runs.kind, RunKinds["answer-visibility"]), notProbeRun(), gte25(runs.createdAt, bounds.since), lte26(runs.createdAt, bounds.until))).orderBy(asc23(runs.createdAt), asc23(runs.id)).all();
|
|
68582
68837
|
const evidence = monthRuns.map((run) => inspectSweep(ctx, run));
|
|
68583
68838
|
return {
|
|
68584
68839
|
month,
|
|
@@ -68601,7 +68856,7 @@ var reportSweepsCheck = {
|
|
|
68601
68856
|
};
|
|
68602
68857
|
|
|
68603
68858
|
// ../api-routes/src/doctor/checks/report-models.ts
|
|
68604
|
-
import { and as
|
|
68859
|
+
import { and as and80, eq as eq91, gte as gte26, inArray as inArray41, lte as lte27 } from "drizzle-orm";
|
|
68605
68860
|
var reportModelsCheck = {
|
|
68606
68861
|
id: "report.models",
|
|
68607
68862
|
category: CheckCategories.providers,
|
|
@@ -68616,7 +68871,7 @@ var reportModelsCheck = {
|
|
|
68616
68871
|
const months = reportMonths.flatMap((month) => {
|
|
68617
68872
|
const bounds = calendarMonthBounds(month);
|
|
68618
68873
|
const previous = new Date(Date.parse(bounds.since) - 1).toISOString().slice(0, 7);
|
|
68619
|
-
const observations = ctx.db.select({ provider: querySnapshots.provider, model: querySnapshots.model, at: runs.createdAt }).from(querySnapshots).innerJoin(runs,
|
|
68874
|
+
const observations = ctx.db.select({ provider: querySnapshots.provider, model: querySnapshots.model, at: runs.createdAt }).from(querySnapshots).innerJoin(runs, eq91(runs.id, querySnapshots.runId)).where(and80(eq91(runs.projectId, ctx.project.id), eq91(runs.kind, RunKinds["answer-visibility"]), inArray41(runs.status, [RunStatuses.completed, RunStatuses.partial]), notProbeRun(), gte26(runs.createdAt, bounds.since), lte27(runs.createdAt, bounds.until))).all();
|
|
68620
68875
|
return readVisibilityContinuity(ctx.db, ctx.project.name, { from: previous, to: month }).map(({ frame, continuity: continuity2 }) => ({
|
|
68621
68876
|
month,
|
|
68622
68877
|
previousMonth: previous,
|
|
@@ -68657,7 +68912,7 @@ var reportModelsCheck = {
|
|
|
68657
68912
|
};
|
|
68658
68913
|
|
|
68659
68914
|
// ../api-routes/src/doctor/checks/report-daily-data.ts
|
|
68660
|
-
import { eq as
|
|
68915
|
+
import { eq as eq92 } from "drizzle-orm";
|
|
68661
68916
|
var REPORTING_LAG_DAYS = 3;
|
|
68662
68917
|
function dateSpan(dates) {
|
|
68663
68918
|
const sorted = [...dates].sort();
|
|
@@ -68666,31 +68921,31 @@ function dateSpan(dates) {
|
|
|
68666
68921
|
function latestSyncRange(ctx, source) {
|
|
68667
68922
|
const projectId = ctx.project.id;
|
|
68668
68923
|
if (source === "ga") {
|
|
68669
|
-
const rows2 = ctx.db.select({ date: gaDailyTotals.date, syncedAt: gaDailyTotals.syncedAt }).from(gaDailyTotals).where(
|
|
68924
|
+
const rows2 = ctx.db.select({ date: gaDailyTotals.date, syncedAt: gaDailyTotals.syncedAt }).from(gaDailyTotals).where(eq92(gaDailyTotals.projectId, projectId)).all();
|
|
68670
68925
|
const latest2 = rows2.reduce((max2, row) => max2 === null || row.syncedAt > max2 ? row.syncedAt : max2, null);
|
|
68671
|
-
const summary3 = ctx.db.select({ start: gaTrafficSummaries.periodStart, end: gaTrafficSummaries.periodEnd, syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(
|
|
68926
|
+
const summary3 = ctx.db.select({ start: gaTrafficSummaries.periodStart, end: gaTrafficSummaries.periodEnd, syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(eq92(gaTrafficSummaries.projectId, projectId)).get();
|
|
68672
68927
|
if (summary3 && (latest2 === null || summary3.syncedAt >= latest2)) return { start: summary3.start, end: summary3.end, syncedAt: summary3.syncedAt };
|
|
68673
68928
|
const span2 = dateSpan(rows2.filter((row) => row.syncedAt === latest2).map((row) => row.date));
|
|
68674
68929
|
return span2 && latest2 !== null ? { ...span2, syncedAt: latest2 } : null;
|
|
68675
68930
|
}
|
|
68676
|
-
const rows = ctx.db.select({ date: gscDailyTotals.date, createdAt: gscDailyTotals.createdAt }).from(gscDailyTotals).where(
|
|
68931
|
+
const rows = ctx.db.select({ date: gscDailyTotals.date, createdAt: gscDailyTotals.createdAt }).from(gscDailyTotals).where(eq92(gscDailyTotals.projectId, projectId)).all();
|
|
68677
68932
|
const latest = rows.reduce((max2, row) => max2 === null || row.createdAt > max2 ? row.createdAt : max2, null);
|
|
68678
68933
|
const span = dateSpan(rows.filter((row) => row.createdAt === latest).map((row) => row.date));
|
|
68679
68934
|
if (!span || latest === null) return null;
|
|
68680
|
-
const watermark = ctx.db.select({ through: gscDataWatermarks.syncedThroughDate, updatedAt: gscDataWatermarks.updatedAt }).from(gscDataWatermarks).where(
|
|
68935
|
+
const watermark = ctx.db.select({ through: gscDataWatermarks.syncedThroughDate, updatedAt: gscDataWatermarks.updatedAt }).from(gscDataWatermarks).where(eq92(gscDataWatermarks.projectId, projectId)).get();
|
|
68681
68936
|
return watermark?.through && watermark.updatedAt === latest && watermark.through > span.end ? { start: span.start, end: watermark.through, syncedAt: latest } : { ...span, syncedAt: latest };
|
|
68682
68937
|
}
|
|
68683
68938
|
function dailyCoverage(ctx, month, source) {
|
|
68684
68939
|
const project = ctx.project;
|
|
68685
68940
|
const connection = source === "ga" ? ctx.ga4CredentialStore?.getConnection(project.name) ?? ctx.googleConnectionStore?.getConnection(project.canonicalDomain, "ga4") : ctx.googleConnectionStore?.getConnection(project.canonicalDomain, "gsc");
|
|
68686
|
-
const stored = source === "ga" ? ctx.db.select({ date: gaDailyTotals.date, count: gaDailyTotals.sessions }).from(gaDailyTotals).where(
|
|
68941
|
+
const stored = source === "ga" ? ctx.db.select({ date: gaDailyTotals.date, count: gaDailyTotals.sessions }).from(gaDailyTotals).where(eq92(gaDailyTotals.projectId, project.id)).all() : ctx.db.select({ date: gscDailyTotals.date, count: gscDailyTotals.impressions }).from(gscDailyTotals).where(eq92(gscDailyTotals.projectId, project.id)).all();
|
|
68687
68942
|
if (!connection && stored.length === 0) return null;
|
|
68688
68943
|
const bounds = calendarMonthBounds(month);
|
|
68689
68944
|
const start = bounds.since.slice(0, 10);
|
|
68690
68945
|
const end = bounds.until.slice(0, 10);
|
|
68691
68946
|
const timeZone = source === "gsc" ? "America/Los_Angeles" : "UTC";
|
|
68692
68947
|
const matureThrough = isoDateDaysBeforeInTimeZone((/* @__PURE__ */ new Date()).toISOString(), REPORTING_LAG_DAYS, timeZone);
|
|
68693
|
-
const created = ctx.db.select({ date: projects.createdAt }).from(projects).where(
|
|
68948
|
+
const created = ctx.db.select({ date: projects.createdAt }).from(projects).where(eq92(projects.id, project.id)).get().date.slice(0, 10);
|
|
68694
68949
|
const connectedAt = connection?.createdAt?.slice(0, 10) ?? created;
|
|
68695
68950
|
const onboardingDate = [connectedAt, ...stored.map((row) => row.date)].sort()[0];
|
|
68696
68951
|
const byDate = new Map(stored.map((row) => [row.date, row.count]));
|
|
@@ -68772,7 +69027,7 @@ var reportDailyDataCheck = {
|
|
|
68772
69027
|
};
|
|
68773
69028
|
|
|
68774
69029
|
// ../api-routes/src/doctor/checks/referral-assessment.ts
|
|
68775
|
-
import { and as
|
|
69030
|
+
import { and as and81, eq as eq93, ne as ne16 } from "drizzle-orm";
|
|
68776
69031
|
var REFERRAL_ASSESSMENT_CHECKS = [{
|
|
68777
69032
|
id: "report.ai-referral-bursts",
|
|
68778
69033
|
category: CheckCategories.integrations,
|
|
@@ -68781,7 +69036,7 @@ var REFERRAL_ASSESSMENT_CHECKS = [{
|
|
|
68781
69036
|
notificationPolicy: CheckNotificationPolicies.silent,
|
|
68782
69037
|
run: (ctx) => {
|
|
68783
69038
|
if (!ctx.project) return { status: CheckStatuses.skipped, code: "report.ai-referral-bursts.no-project", summary: "Select a project to inspect referral evidence." };
|
|
68784
|
-
const configured = ctx.db.select({ id: trafficSources.id }).from(trafficSources).where(
|
|
69039
|
+
const configured = ctx.db.select({ id: trafficSources.id }).from(trafficSources).where(and81(eq93(trafficSources.projectId, ctx.project.id), ne16(trafficSources.status, TrafficSourceStatuses.archived))).get();
|
|
68785
69040
|
if (!configured) {
|
|
68786
69041
|
return { status: CheckStatuses.skipped, code: "report.ai-referral-bursts.not-configured", summary: "No non-archived server traffic source is configured for this project." };
|
|
68787
69042
|
}
|
|
@@ -69193,19 +69448,19 @@ var BACKLINKS_CHECKS = [
|
|
|
69193
69448
|
];
|
|
69194
69449
|
|
|
69195
69450
|
// ../api-routes/src/doctor/checks/bing-auth.ts
|
|
69196
|
-
import { and as
|
|
69451
|
+
import { and as and82, desc as desc42, eq as eq94, inArray as inArray42 } from "drizzle-orm";
|
|
69197
69452
|
var BING_CONNECTION_ENTITY = "bing_connection";
|
|
69198
69453
|
var BING_CONNECTED_ACTION = "bing.connected";
|
|
69199
69454
|
var BING_DISCONNECTED_ACTION = "bing.disconnected";
|
|
69200
69455
|
function lostBingConnection(ctx, project) {
|
|
69201
69456
|
const hasRow = (row) => row !== void 0;
|
|
69202
|
-
const storedData = hasRow(ctx.db.select({ id: bingCoverageSnapshots.id }).from(bingCoverageSnapshots).where(
|
|
69457
|
+
const storedData = hasRow(ctx.db.select({ id: bingCoverageSnapshots.id }).from(bingCoverageSnapshots).where(eq94(bingCoverageSnapshots.projectId, project.id)).limit(1).get()) || hasRow(ctx.db.select({ id: bingUrlInspections.id }).from(bingUrlInspections).where(eq94(bingUrlInspections.projectId, project.id)).limit(1).get()) || hasRow(ctx.db.select({ id: bingKeywordStats.id }).from(bingKeywordStats).where(eq94(bingKeywordStats.projectId, project.id)).limit(1).get()) || hasRow(ctx.db.select({ id: bingTrafficDaily.id }).from(bingTrafficDaily).where(eq94(bingTrafficDaily.projectId, project.id)).limit(1).get());
|
|
69203
69458
|
if (!storedData) return false;
|
|
69204
|
-
const lastChange = ctx.db.select({ action: auditLog.action }).from(auditLog).where(
|
|
69205
|
-
|
|
69206
|
-
|
|
69207
|
-
|
|
69208
|
-
)).orderBy(
|
|
69459
|
+
const lastChange = ctx.db.select({ action: auditLog.action }).from(auditLog).where(and82(
|
|
69460
|
+
eq94(auditLog.entityType, BING_CONNECTION_ENTITY),
|
|
69461
|
+
eq94(auditLog.entityId, project.canonicalDomain),
|
|
69462
|
+
inArray42(auditLog.action, [BING_CONNECTED_ACTION, BING_DISCONNECTED_ACTION])
|
|
69463
|
+
)).orderBy(desc42(auditLog.createdAt)).limit(1).get();
|
|
69209
69464
|
return lastChange?.action !== BING_DISCONNECTED_ACTION;
|
|
69210
69465
|
}
|
|
69211
69466
|
var BING_AUTH_CHECKS = [
|
|
@@ -69362,7 +69617,7 @@ var BING_AUTH_CHECKS = [
|
|
|
69362
69617
|
];
|
|
69363
69618
|
|
|
69364
69619
|
// ../api-routes/src/doctor/checks/content.ts
|
|
69365
|
-
import { eq as
|
|
69620
|
+
import { eq as eq95 } from "drizzle-orm";
|
|
69366
69621
|
var WINNABILITY_COVERAGE_WARN_THRESHOLD = 0.8;
|
|
69367
69622
|
var UNCLASSIFIED_DOMAIN_SAMPLE_LIMIT = 10;
|
|
69368
69623
|
function skippedNoProject2() {
|
|
@@ -69375,7 +69630,7 @@ function skippedNoProject2() {
|
|
|
69375
69630
|
}
|
|
69376
69631
|
function loadProject(ctx) {
|
|
69377
69632
|
if (!ctx.project) return null;
|
|
69378
|
-
return ctx.db.select().from(projects).where(
|
|
69633
|
+
return ctx.db.select().from(projects).where(eq95(projects.id, ctx.project.id)).get() ?? null;
|
|
69379
69634
|
}
|
|
69380
69635
|
var winnabilityCoverageCheck = {
|
|
69381
69636
|
id: "content.winnability.coverage",
|
|
@@ -69468,7 +69723,7 @@ var CONTENT_CHECK_BY_ID = Object.fromEntries(
|
|
|
69468
69723
|
);
|
|
69469
69724
|
|
|
69470
69725
|
// ../api-routes/src/doctor/checks/ads.ts
|
|
69471
|
-
import { eq as
|
|
69726
|
+
import { eq as eq96 } from "drizzle-orm";
|
|
69472
69727
|
var ADS_RECENT_SYNC_WARN_DAYS = 7;
|
|
69473
69728
|
var ADS_RECENT_SYNC_FAIL_DAYS = 30;
|
|
69474
69729
|
var adsConnectionCheck = {
|
|
@@ -69485,7 +69740,7 @@ var adsConnectionCheck = {
|
|
|
69485
69740
|
remediation: null
|
|
69486
69741
|
};
|
|
69487
69742
|
}
|
|
69488
|
-
const row = ctx.db.select().from(adsConnections).where(
|
|
69743
|
+
const row = ctx.db.select().from(adsConnections).where(eq96(adsConnections.projectId, ctx.project.id)).get();
|
|
69489
69744
|
if (!row) {
|
|
69490
69745
|
return {
|
|
69491
69746
|
status: CheckStatuses.skipped,
|
|
@@ -69535,7 +69790,7 @@ var adsRecentSyncCheck = {
|
|
|
69535
69790
|
remediation: null
|
|
69536
69791
|
};
|
|
69537
69792
|
}
|
|
69538
|
-
const row = ctx.db.select().from(adsConnections).where(
|
|
69793
|
+
const row = ctx.db.select().from(adsConnections).where(eq96(adsConnections.projectId, ctx.project.id)).get();
|
|
69539
69794
|
if (!row) {
|
|
69540
69795
|
return {
|
|
69541
69796
|
status: CheckStatuses.skipped,
|
|
@@ -69750,7 +70005,7 @@ var ga4ConnectionCheck = {
|
|
|
69750
70005
|
var GA_AUTH_CHECKS = [ga4ConnectionCheck];
|
|
69751
70006
|
|
|
69752
70007
|
// ../api-routes/src/doctor/checks/data-freshness.ts
|
|
69753
|
-
import { and as
|
|
70008
|
+
import { and as and83, desc as desc43, eq as eq97, inArray as inArray43 } from "drizzle-orm";
|
|
69754
70009
|
var GA_DATA_AGING_DAYS = 3;
|
|
69755
70010
|
var GA_DATA_STALE_DAYS = 5;
|
|
69756
70011
|
var GSC_DATA_AGING_DAYS = 5;
|
|
@@ -69780,7 +70035,7 @@ function skipped(code, summary3) {
|
|
|
69780
70035
|
return { status: CheckStatuses.skipped, code, summary: summary3, remediation: null };
|
|
69781
70036
|
}
|
|
69782
70037
|
function lastSyncAgeDays(ctx, kinds) {
|
|
69783
|
-
const row = ctx.db.select({ createdAt: runs.createdAt }).from(runs).where(
|
|
70038
|
+
const row = ctx.db.select({ createdAt: runs.createdAt }).from(runs).where(and83(eq97(runs.projectId, ctx.project.id), inArray43(runs.kind, kinds), eq97(runs.status, "completed"))).orderBy(desc43(runs.createdAt)).limit(1).get();
|
|
69784
70039
|
if (!row?.createdAt) return null;
|
|
69785
70040
|
const age = Math.floor((Date.now() - Date.parse(row.createdAt)) / DAY_MS5);
|
|
69786
70041
|
return Number.isFinite(age) ? age : null;
|
|
@@ -69882,7 +70137,7 @@ var gaRecentDataCheck = {
|
|
|
69882
70137
|
const connection = gaConnected(ctx);
|
|
69883
70138
|
if (connection === "store-unavailable") return skipped("ga.data.store-unavailable", "No GA4 credential store configured for this deployment.");
|
|
69884
70139
|
if (connection === "not-connected") return skipped("ga.data.not-connected", "GA4 is not connected for this project.");
|
|
69885
|
-
const newest = ctx.db.select({ date: gaDailyTotals.date }).from(gaDailyTotals).where(
|
|
70140
|
+
const newest = ctx.db.select({ date: gaDailyTotals.date }).from(gaDailyTotals).where(eq97(gaDailyTotals.projectId, ctx.project.id)).orderBy(desc43(gaDailyTotals.date)).limit(1).get();
|
|
69886
70141
|
return grade(ctx, GA_SPEC, newest?.date);
|
|
69887
70142
|
}
|
|
69888
70143
|
};
|
|
@@ -69897,16 +70152,16 @@ var gscRecentDataCheck = {
|
|
|
69897
70152
|
if (!ctx.googleConnectionStore.getConnection(ctx.project.canonicalDomain, "gsc")) {
|
|
69898
70153
|
return skipped("gsc.data.not-connected", "Search Console is not connected for this project.");
|
|
69899
70154
|
}
|
|
69900
|
-
const watermark = ctx.db.select({ dataThroughDate: gscDataWatermarks.dataThroughDate }).from(gscDataWatermarks).where(
|
|
70155
|
+
const watermark = ctx.db.select({ dataThroughDate: gscDataWatermarks.dataThroughDate }).from(gscDataWatermarks).where(eq97(gscDataWatermarks.projectId, ctx.project.id)).get();
|
|
69901
70156
|
if (watermark?.dataThroughDate) return grade(ctx, GSC_SPEC, watermark.dataThroughDate);
|
|
69902
|
-
const newest = ctx.db.select({ date: gscDailyTotals.date }).from(gscDailyTotals).where(
|
|
70157
|
+
const newest = ctx.db.select({ date: gscDailyTotals.date }).from(gscDailyTotals).where(eq97(gscDailyTotals.projectId, ctx.project.id)).orderBy(desc43(gscDailyTotals.date)).limit(1).get();
|
|
69903
70158
|
return grade(ctx, GSC_SPEC, newest?.date);
|
|
69904
70159
|
}
|
|
69905
70160
|
};
|
|
69906
70161
|
var DATA_FRESHNESS_CHECKS = [gaRecentDataCheck, gscRecentDataCheck];
|
|
69907
70162
|
|
|
69908
70163
|
// ../api-routes/src/doctor/checks/seo-refresh.ts
|
|
69909
|
-
import { and as
|
|
70164
|
+
import { and as and84, asc as asc24, desc as desc44, eq as eq98, gt as gt8, inArray as inArray44, sql as sql39 } from "drizzle-orm";
|
|
69910
70165
|
var GRACE_DAYS = 2;
|
|
69911
70166
|
var DAY_MS6 = 24 * 60 * 60 * 1e3;
|
|
69912
70167
|
var FIRING_SPAN_MS = 6e4;
|
|
@@ -69915,7 +70170,7 @@ function skipped2(code, summary3) {
|
|
|
69915
70170
|
return { status: CheckStatuses.skipped, code, summary: summary3, remediation: null };
|
|
69916
70171
|
}
|
|
69917
70172
|
function seoSyncRuns(projectId) {
|
|
69918
|
-
return
|
|
70173
|
+
return and84(eq98(runs.projectId, projectId), eq98(runs.kind, RunKinds["seo-sync"]));
|
|
69919
70174
|
}
|
|
69920
70175
|
function scheduleTiming(schedule) {
|
|
69921
70176
|
if (!schedule.recurrence) return { cronExpr: schedule.cronExpr, timezone: schedule.timezone, recurrence: null };
|
|
@@ -69923,22 +70178,22 @@ function scheduleTiming(schedule) {
|
|
|
69923
70178
|
return recurrence ? { cronExpr: schedule.cronExpr, timezone: schedule.timezone, recurrence } : null;
|
|
69924
70179
|
}
|
|
69925
70180
|
function latestFiring(ctx, projectId, enabledSince) {
|
|
69926
|
-
const scheduled =
|
|
69927
|
-
const newest = ctx.db.select({ createdAt: runs.createdAt }).from(runs).where(scheduled).orderBy(
|
|
70181
|
+
const scheduled = and84(seoSyncRuns(projectId), eq98(runs.trigger, RunTriggers.scheduled), gt8(runs.createdAt, enabledSince));
|
|
70182
|
+
const newest = ctx.db.select({ createdAt: runs.createdAt }).from(runs).where(scheduled).orderBy(desc44(runs.createdAt)).limit(1).get();
|
|
69928
70183
|
if (!newest) return [];
|
|
69929
70184
|
const newestMs = Date.parse(newest.createdAt);
|
|
69930
70185
|
const after = Number.isFinite(newestMs) ? new Date(newestMs - FIRING_SPAN_MS).toISOString() : newest.createdAt;
|
|
69931
|
-
return ctx.db.select({ id: runs.id, status: runs.status, error: runs.error, createdAt: runs.createdAt }).from(runs).where(
|
|
70186
|
+
return ctx.db.select({ id: runs.id, status: runs.status, error: runs.error, createdAt: runs.createdAt }).from(runs).where(and84(scheduled, gt8(runs.createdAt, after))).orderBy(asc24(runs.createdAt), asc24(runs.id)).all();
|
|
69932
70187
|
}
|
|
69933
70188
|
function lastCompleted(ctx, projectId) {
|
|
69934
|
-
const at =
|
|
69935
|
-
return ctx.db.select({ id: runs.id, at }).from(runs).where(
|
|
70189
|
+
const at = sql39`coalesce(${runs.finishedAt}, ${runs.createdAt})`;
|
|
70190
|
+
return ctx.db.select({ id: runs.id, at }).from(runs).where(and84(seoSyncRuns(projectId), eq98(runs.status, RunStatuses.completed))).orderBy(desc44(at)).limit(1).get() ?? null;
|
|
69936
70191
|
}
|
|
69937
70192
|
function isIncomplete(run) {
|
|
69938
70193
|
return run.status === RunStatuses.failed || run.status === RunStatuses.partial;
|
|
69939
70194
|
}
|
|
69940
70195
|
function runProfiles(ctx, firing) {
|
|
69941
|
-
const jobs = firing.length === 0 ? [] : ctx.db.select({ runId: seoJobs.runId, profileId: seoJobs.profileId }).from(seoJobs).where(
|
|
70196
|
+
const jobs = firing.length === 0 ? [] : ctx.db.select({ runId: seoJobs.runId, profileId: seoJobs.profileId }).from(seoJobs).where(inArray44(seoJobs.runId, firing.map((run) => run.id))).all();
|
|
69942
70197
|
const byRun = new Map(jobs.flatMap((job) => job.runId ? [[job.runId, job.profileId]] : []));
|
|
69943
70198
|
return new Map(firing.map((run) => [run.id, byRun.get(run.id) ?? skippedRefreshProfile(run.error)]));
|
|
69944
70199
|
}
|
|
@@ -69946,7 +70201,7 @@ function unsettledRuns(ctx, projectId, firing, profiles) {
|
|
|
69946
70201
|
const incomplete = firing.filter(isIncomplete);
|
|
69947
70202
|
const firedAt = firing[firing.length - 1]?.createdAt;
|
|
69948
70203
|
if (incomplete.length === 0 || !firedAt) return incomplete;
|
|
69949
|
-
const refreshed = new Set(ctx.db.select({ profileId: seoJobs.profileId }).from(runs).innerJoin(seoJobs,
|
|
70204
|
+
const refreshed = new Set(ctx.db.select({ profileId: seoJobs.profileId }).from(runs).innerJoin(seoJobs, eq98(seoJobs.runId, runs.id)).where(and84(seoSyncRuns(projectId), eq98(runs.status, RunStatuses.completed), gt8(runs.createdAt, firedAt))).all().map((row) => row.profileId));
|
|
69950
70205
|
return incomplete.filter((run) => {
|
|
69951
70206
|
const profile = profiles.get(run.id) ?? null;
|
|
69952
70207
|
return profile === null ? refreshed.size === 0 : !refreshed.has(profile);
|
|
@@ -69956,7 +70211,7 @@ var COUNT_LIST = new Intl.ListFormat("en", { style: "long", type: "conjunction"
|
|
|
69956
70211
|
function refreshReason(ctx, run) {
|
|
69957
70212
|
const stored = parseRunError(run.error);
|
|
69958
70213
|
if (stored) return formatRunErrorOneLine(stored);
|
|
69959
|
-
const tasks = currentSeoTasks(ctx.db.select({ id: seoTasks.id, retryOf: seoTasks.retryOf, status: seoTasks.status, error: seoTasks.error, request: seoTasks.request }).from(seoTasks).innerJoin(seoJobs,
|
|
70214
|
+
const tasks = currentSeoTasks(ctx.db.select({ id: seoTasks.id, retryOf: seoTasks.retryOf, status: seoTasks.status, error: seoTasks.error, request: seoTasks.request }).from(seoTasks).innerJoin(seoJobs, eq98(seoTasks.jobId, seoJobs.id)).where(eq98(seoJobs.runId, run.id)).all());
|
|
69960
70215
|
const failed = tasks.filter((task) => task.status === SeoRunStatuses.failed || task.status === SeoRunStatuses["reconciliation-required"]);
|
|
69961
70216
|
const partial = tasks.filter((task) => task.status === SeoRunStatuses.partial).length;
|
|
69962
70217
|
const cancelled = tasks.filter((task) => task.status === SeoRunStatuses.cancelled).length;
|
|
@@ -69976,7 +70231,7 @@ function refreshReason(ctx, run) {
|
|
|
69976
70231
|
return `Of ${tasks.length} ${unit}, ${COUNT_LIST.format(counts2)}.${errors ? ` ${errors}` : ""}`;
|
|
69977
70232
|
}
|
|
69978
70233
|
function gradeSeoRefresh(ctx, project) {
|
|
69979
|
-
const schedule = ctx.db.select().from(schedules).where(
|
|
70234
|
+
const schedule = ctx.db.select().from(schedules).where(and84(eq98(schedules.projectId, project.id), eq98(schedules.kind, SchedulableRunKinds["seo-sync"]))).get();
|
|
69980
70235
|
if (!schedule) return skipped2("seo.data.not-scheduled", "No SEO refresh schedule is set for this project.");
|
|
69981
70236
|
if (!schedule.enabled) return skipped2("seo.data.not-scheduled", "The SEO refresh schedule is disabled.");
|
|
69982
70237
|
const name = project.name;
|
|
@@ -70048,7 +70303,7 @@ function gradeSeoRefresh(ctx, project) {
|
|
|
70048
70303
|
};
|
|
70049
70304
|
}
|
|
70050
70305
|
if (now >= Date.parse(deadlineAt)) {
|
|
70051
|
-
const active = ctx.db.select({ id: runs.id, createdAt: runs.createdAt }).from(runs).where(
|
|
70306
|
+
const active = ctx.db.select({ id: runs.id, createdAt: runs.createdAt }).from(runs).where(and84(seoSyncRuns(project.id), inArray44(runs.status, [RunStatuses.queued, RunStatuses.running]))).orderBy(asc24(runs.createdAt)).limit(1).get();
|
|
70052
70307
|
const since = last ? `No SEO refresh has completed since ${day(last.at)}.` : "No SEO refresh has completed since the schedule was enabled.";
|
|
70053
70308
|
const missed = last ? `The scheduled refresh due ${day(dueAt)} did not complete within ${GRACE_DAYS} days.` : `Its first refresh, due ${day(dueAt)}, did not complete within ${GRACE_DAYS} days.`;
|
|
70054
70309
|
return {
|
|
@@ -70358,7 +70613,7 @@ var siteReachabilityCheck = {
|
|
|
70358
70613
|
var SITE_REACHABILITY_CHECKS = [siteReachabilityCheck];
|
|
70359
70614
|
|
|
70360
70615
|
// ../api-routes/src/doctor/checks/api-hosts.ts
|
|
70361
|
-
import { eq as
|
|
70616
|
+
import { eq as eq100 } from "drizzle-orm";
|
|
70362
70617
|
|
|
70363
70618
|
// ../api-routes/src/api-host-probe.ts
|
|
70364
70619
|
import dns4 from "dns/promises";
|
|
@@ -70464,7 +70719,7 @@ async function probeApiHost(host, options = {}) {
|
|
|
70464
70719
|
}
|
|
70465
70720
|
|
|
70466
70721
|
// ../api-routes/src/doctor/checks/integration-sync.ts
|
|
70467
|
-
import { and as
|
|
70722
|
+
import { and as and85, count as count7, desc as desc45, eq as eq99, gte as gte27, gt as gt9, inArray as inArray45, isNull as isNull14, notInArray as notInArray3, or as or19 } from "drizzle-orm";
|
|
70468
70723
|
var SYNC_FAILURE_WINDOW = 3;
|
|
70469
70724
|
var SUMMARY_ERROR_LIMIT = 300;
|
|
70470
70725
|
var FINISHED_STATUSES = [RunStatuses.completed, RunStatuses.partial, RunStatuses.failed];
|
|
@@ -70491,7 +70746,7 @@ function gscConnection(ctx, project) {
|
|
|
70491
70746
|
function gbpConnection(ctx, project) {
|
|
70492
70747
|
if (!ctx.googleConnectionStore) return { state: "store-unavailable" };
|
|
70493
70748
|
if (!ctx.googleConnectionStore.getConnection(project.canonicalDomain, "gbp")) return { state: "not-connected" };
|
|
70494
|
-
const selected = ctx.db.select({ locationName: gbpLocations.locationName }).from(gbpLocations).where(
|
|
70749
|
+
const selected = ctx.db.select({ locationName: gbpLocations.locationName }).from(gbpLocations).where(and85(eq99(gbpLocations.projectId, project.id), eq99(gbpLocations.selected, true))).limit(1).get();
|
|
70495
70750
|
return selected ? CONNECTED : { state: "no-selected-locations" };
|
|
70496
70751
|
}
|
|
70497
70752
|
function gaConnection(ctx, project) {
|
|
@@ -70564,16 +70819,16 @@ function latestErrorText(raw) {
|
|
|
70564
70819
|
return parsed ? formatRunErrorOneLine(parsed) : "no error was recorded";
|
|
70565
70820
|
}
|
|
70566
70821
|
function readSyncRunState(db, projectId, kind, options = {}) {
|
|
70567
|
-
const scope =
|
|
70568
|
-
|
|
70569
|
-
|
|
70570
|
-
...options.sourceId ? [
|
|
70822
|
+
const scope = and85(
|
|
70823
|
+
eq99(runs.projectId, projectId),
|
|
70824
|
+
eq99(runs.kind, kind),
|
|
70825
|
+
...options.sourceId ? [eq99(runs.sourceId, options.sourceId)] : [],
|
|
70571
70826
|
...options.nothingToSyncErrors?.length ? [or19(isNull14(runs.error), notInArray3(runs.error, [...options.nothingToSyncErrors]))] : []
|
|
70572
70827
|
);
|
|
70573
|
-
const graded = options.since ?
|
|
70574
|
-
const recent = db.select({ id: runs.id, status: runs.status, trigger: runs.trigger, error: runs.error, createdAt: runs.createdAt }).from(runs).where(
|
|
70575
|
-
const lastSuccess = db.select({ createdAt: runs.createdAt, finishedAt: runs.finishedAt }).from(runs).where(
|
|
70576
|
-
const failureCount = db.select({ count: count7() }).from(runs).where(
|
|
70828
|
+
const graded = options.since ? and85(scope, gte27(runs.createdAt, options.since)) : scope;
|
|
70829
|
+
const recent = db.select({ id: runs.id, status: runs.status, trigger: runs.trigger, error: runs.error, createdAt: runs.createdAt }).from(runs).where(and85(graded, inArray45(runs.status, FINISHED_STATUSES))).orderBy(desc45(runs.createdAt)).limit(SYNC_FAILURE_WINDOW).all();
|
|
70830
|
+
const lastSuccess = db.select({ createdAt: runs.createdAt, finishedAt: runs.finishedAt }).from(runs).where(and85(graded, inArray45(runs.status, SUCCESS_STATUSES))).orderBy(desc45(runs.createdAt)).limit(1).get();
|
|
70831
|
+
const failureCount = db.select({ count: count7() }).from(runs).where(and85(lastSuccess ? and85(graded, gt9(runs.createdAt, lastSuccess.createdAt)) : graded, eq99(runs.status, RunStatuses.failed))).get();
|
|
70577
70832
|
const failuresSinceSuccess = Number(failureCount?.count ?? 0);
|
|
70578
70833
|
const latest = recent[0];
|
|
70579
70834
|
return {
|
|
@@ -70670,7 +70925,7 @@ function connectionFor(prefix, ctx, project) {
|
|
|
70670
70925
|
if (prefix === "google-ads") return googleMarketingConnection("googleAds")(ctx);
|
|
70671
70926
|
if (prefix === "gtm") return googleMarketingConnection("gtm")(ctx);
|
|
70672
70927
|
if (prefix === "bing") return ctx.bingConnectionStore?.getConnection(project.canonicalDomain) ? { state: "connected", since: null } : { state: "not-connected" };
|
|
70673
|
-
return ctx.db.select({ id: adsConnections.id }).from(adsConnections).where(
|
|
70928
|
+
return ctx.db.select({ id: adsConnections.id }).from(adsConnections).where(eq100(adsConnections.projectId, project.id)).limit(1).get() ? { state: "connected", since: null } : { state: "not-connected" };
|
|
70674
70929
|
}
|
|
70675
70930
|
function syncCommand(prefix) {
|
|
70676
70931
|
if (prefix === "gsc") return "google sync";
|
|
@@ -70715,7 +70970,7 @@ function definition(prefix) {
|
|
|
70715
70970
|
var API_HOST_CHECKS = Object.keys(API_HOST_TABLE).map(definition);
|
|
70716
70971
|
|
|
70717
70972
|
// ../api-routes/src/doctor/checks/gbp-auth.ts
|
|
70718
|
-
import { and as
|
|
70973
|
+
import { and as and86, eq as eq101 } from "drizzle-orm";
|
|
70719
70974
|
var RECENT_SYNC_WARN_DAYS = 4;
|
|
70720
70975
|
var RECENT_SYNC_FAIL_DAYS = 30;
|
|
70721
70976
|
function skippedNoProject3() {
|
|
@@ -70735,7 +70990,7 @@ function storeUnavailable() {
|
|
|
70735
70990
|
};
|
|
70736
70991
|
}
|
|
70737
70992
|
function hasSelectedLocation(ctx, projectId) {
|
|
70738
|
-
return !!ctx.db.select({ id: gbpLocations.id }).from(gbpLocations).where(
|
|
70993
|
+
return !!ctx.db.select({ id: gbpLocations.id }).from(gbpLocations).where(and86(eq101(gbpLocations.projectId, projectId), eq101(gbpLocations.selected, true))).limit(1).get();
|
|
70739
70994
|
}
|
|
70740
70995
|
async function resolveGbpToken(ctx) {
|
|
70741
70996
|
if (!ctx.project) return { ok: false, output: skippedNoProject3() };
|
|
@@ -70995,7 +71250,7 @@ var recentSyncCheck = {
|
|
|
70995
71250
|
title: "GBP recent sync",
|
|
70996
71251
|
run: (ctx) => {
|
|
70997
71252
|
if (!ctx.project) return skippedNoProject3();
|
|
70998
|
-
const selected = ctx.db.select({ locationName: gbpLocations.locationName, syncedAt: gbpLocations.syncedAt }).from(gbpLocations).where(
|
|
71253
|
+
const selected = ctx.db.select({ locationName: gbpLocations.locationName, syncedAt: gbpLocations.syncedAt }).from(gbpLocations).where(and86(eq101(gbpLocations.projectId, ctx.project.id), eq101(gbpLocations.selected, true))).all();
|
|
70999
71254
|
if (selected.length === 0) {
|
|
71000
71255
|
return {
|
|
71001
71256
|
status: CheckStatuses.skipped,
|
|
@@ -71055,7 +71310,7 @@ var GBP_AUTH_CHECK_BY_ID = Object.fromEntries(
|
|
|
71055
71310
|
);
|
|
71056
71311
|
|
|
71057
71312
|
// ../api-routes/src/doctor/checks/places.ts
|
|
71058
|
-
import { eq as
|
|
71313
|
+
import { eq as eq102 } from "drizzle-orm";
|
|
71059
71314
|
var apiKeyCheck = {
|
|
71060
71315
|
id: "gbp.places.api-key",
|
|
71061
71316
|
category: CheckCategories.auth,
|
|
@@ -71100,7 +71355,7 @@ var apiKeyCheck = {
|
|
|
71100
71355
|
details: { tier: cfg.tier }
|
|
71101
71356
|
};
|
|
71102
71357
|
}
|
|
71103
|
-
const rows = ctx.db.select({ placeId: gbpLocations.placeId, selected: gbpLocations.selected }).from(gbpLocations).where(
|
|
71358
|
+
const rows = ctx.db.select({ placeId: gbpLocations.placeId, selected: gbpLocations.selected }).from(gbpLocations).where(eq102(gbpLocations.projectId, ctx.project.id)).all();
|
|
71104
71359
|
const selected = rows.filter((r) => r.selected);
|
|
71105
71360
|
const locationsWithPlaceId = selected.filter((r) => Boolean(r.placeId)).length;
|
|
71106
71361
|
const details = {
|
|
@@ -71133,7 +71388,7 @@ var PLACES_CHECK_BY_ID = Object.fromEntries(
|
|
|
71133
71388
|
);
|
|
71134
71389
|
|
|
71135
71390
|
// ../api-routes/src/doctor/checks/gbp-reviews.ts
|
|
71136
|
-
import { and as
|
|
71391
|
+
import { and as and87, desc as desc46, eq as eq103, gte as gte28, inArray as inArray46, sql as sql40 } from "drizzle-orm";
|
|
71137
71392
|
var REVIEW_EVENTS = /* @__PURE__ */ new Set(["review.negative", "review.rating-dropped"]);
|
|
71138
71393
|
var V4_ACCESS_REMEDIATION = "Business Profile reviews use the legacy My Business API (mybusiness.googleapis.com), which Google enables per Cloud project only on request; the API Library and `gcloud services enable` cannot turn it on. Open the enable link in the API access approval email as the approved account, or reply to the access-request thread asking Google to enable mybusiness.googleapis.com for your project number, then run `canonry gbp sync`.";
|
|
71139
71394
|
var reviewsAccessCheck = {
|
|
@@ -71161,7 +71416,7 @@ var reviewsAccessCheck = {
|
|
|
71161
71416
|
reviewsAccess: gbpLocations.reviewsAccess,
|
|
71162
71417
|
reviewsAccessReason: gbpLocations.reviewsAccessReason,
|
|
71163
71418
|
reviewsCheckedAt: gbpLocations.reviewsCheckedAt
|
|
71164
|
-
}).from(gbpLocations).where(
|
|
71419
|
+
}).from(gbpLocations).where(and87(eq103(gbpLocations.projectId, project.id), eq103(gbpLocations.selected, true))).all();
|
|
71165
71420
|
if (locations.length === 0) {
|
|
71166
71421
|
return {
|
|
71167
71422
|
status: CheckStatuses.skipped,
|
|
@@ -71181,7 +71436,7 @@ var reviewsAccessCheck = {
|
|
|
71181
71436
|
const places = ctx.getPlacesConfig?.();
|
|
71182
71437
|
const placesFallback = places !== void 0 && places.tier === "atmosphere" && Boolean(places.apiKey);
|
|
71183
71438
|
const v4Worked = new Set(
|
|
71184
|
-
ctx.db.select({ locationName: gbpReviewRatings.locationName }).from(gbpReviewRatings).where(
|
|
71439
|
+
ctx.db.select({ locationName: gbpReviewRatings.locationName }).from(gbpReviewRatings).where(and87(eq103(gbpReviewRatings.projectId, project.id), eq103(gbpReviewRatings.origin, "gbp"))).all().map((row) => row.locationName)
|
|
71185
71440
|
);
|
|
71186
71441
|
const coverageOf = (loc) => {
|
|
71187
71442
|
if (businessProfileReviewsCover(loc.reviewsAccess ?? null, v4Worked.has(loc.locationName))) return "full";
|
|
@@ -71215,7 +71470,7 @@ var reviewsAccessCheck = {
|
|
|
71215
71470
|
none > 0 ? `${none} have no review source` : null
|
|
71216
71471
|
].filter((part) => part !== null);
|
|
71217
71472
|
const summary3 = `Business Profile reviews are not available${why}. ${parts.join("; ")}.`;
|
|
71218
|
-
const subscribed = ctx.db.select({ config: notifications.config, enabled: notifications.enabled }).from(notifications).where(
|
|
71473
|
+
const subscribed = ctx.db.select({ config: notifications.config, enabled: notifications.enabled }).from(notifications).where(eq103(notifications.projectId, project.id)).all().some((n) => n.enabled && (n.config.events ?? []).some((event) => REVIEW_EVENTS.has(event)));
|
|
71219
71474
|
const placesHint = placesFallback ? "" : " For partial coverage meanwhile, set a Places API key (`GOOGLE_PLACES_API_KEY` or `places.apiKey`) with `places.tier: atmosphere`.";
|
|
71220
71475
|
if (none > 0 && subscribed) {
|
|
71221
71476
|
return {
|
|
@@ -71246,7 +71501,7 @@ var reviewDeliveryCheck = {
|
|
|
71246
71501
|
return { status: CheckStatuses.skipped, code: "gbp.reviews.delivery.no-project", summary: "Project context required.", remediation: null };
|
|
71247
71502
|
}
|
|
71248
71503
|
const since = new Date(Date.now() - UNDELIVERED_ATTENTION_DAYS * 864e5).toISOString();
|
|
71249
|
-
const lastTouched =
|
|
71504
|
+
const lastTouched = sql40`coalesce(${reviewWebhookDeliveries.attemptedAt}, ${reviewWebhookDeliveries.createdAt})`;
|
|
71250
71505
|
const rows = ctx.db.select({
|
|
71251
71506
|
id: reviewWebhookDeliveries.id,
|
|
71252
71507
|
notificationId: reviewWebhookDeliveries.notificationId,
|
|
@@ -71254,11 +71509,11 @@ var reviewDeliveryCheck = {
|
|
|
71254
71509
|
attempts: reviewWebhookDeliveries.attempts,
|
|
71255
71510
|
lastTouched,
|
|
71256
71511
|
payload: reviewWebhookDeliveries.payload
|
|
71257
|
-
}).from(reviewWebhookDeliveries).where(
|
|
71258
|
-
|
|
71259
|
-
|
|
71512
|
+
}).from(reviewWebhookDeliveries).where(and87(
|
|
71513
|
+
eq103(reviewWebhookDeliveries.projectId, ctx.project.id),
|
|
71514
|
+
inArray46(reviewWebhookDeliveries.state, [ReviewDeliveryStates.Uncertain, ReviewDeliveryStates.Failed]),
|
|
71260
71515
|
gte28(lastTouched, since)
|
|
71261
|
-
)).orderBy(
|
|
71516
|
+
)).orderBy(desc46(lastTouched)).all();
|
|
71262
71517
|
if (rows.length === 0) {
|
|
71263
71518
|
return {
|
|
71264
71519
|
status: CheckStatuses.ok,
|
|
@@ -71267,7 +71522,7 @@ var reviewDeliveryCheck = {
|
|
|
71267
71522
|
remediation: null
|
|
71268
71523
|
};
|
|
71269
71524
|
}
|
|
71270
|
-
const destinations = new Map(ctx.db.select({ id: notifications.id, config: notifications.config }).from(notifications).where(
|
|
71525
|
+
const destinations = new Map(ctx.db.select({ id: notifications.id, config: notifications.config }).from(notifications).where(eq103(notifications.projectId, ctx.project.id)).all().map((row) => [row.id, redactNotificationUrl(row.config.url).urlDisplay]));
|
|
71271
71526
|
const deliveries = rows.map((row) => {
|
|
71272
71527
|
const review = row.payload.reviews[0];
|
|
71273
71528
|
return {
|
|
@@ -71841,7 +72096,7 @@ var RUNTIME_STATE_CHECKS = [
|
|
|
71841
72096
|
];
|
|
71842
72097
|
|
|
71843
72098
|
// ../api-routes/src/doctor/checks/traffic-source.ts
|
|
71844
|
-
import { and as
|
|
72099
|
+
import { and as and88, desc as desc47, eq as eq104, gte as gte29, inArray as inArray47, isNotNull as isNotNull9, ne as ne17, sql as sql41 } from "drizzle-orm";
|
|
71845
72100
|
var RECENT_DATA_WARN_DAYS = 7;
|
|
71846
72101
|
var RECENT_DATA_FAIL_DAYS = 30;
|
|
71847
72102
|
function isCloudflareDirectPush(source) {
|
|
@@ -71880,8 +72135,8 @@ function skippedNoProject5() {
|
|
|
71880
72135
|
function loadProbes(ctx) {
|
|
71881
72136
|
if (!ctx.project) return [];
|
|
71882
72137
|
const rows = ctx.db.select().from(trafficSources).where(
|
|
71883
|
-
|
|
71884
|
-
|
|
72138
|
+
and88(
|
|
72139
|
+
eq104(trafficSources.projectId, ctx.project.id),
|
|
71885
72140
|
ne17(trafficSources.status, TrafficSourceStatuses.archived)
|
|
71886
72141
|
)
|
|
71887
72142
|
).all();
|
|
@@ -72018,28 +72273,28 @@ var recentDataCheck = {
|
|
|
72018
72273
|
const warnCutoff = new Date(now.getTime() - RECENT_DATA_WARN_DAYS * 24 * 60 * 6e4).toISOString();
|
|
72019
72274
|
const failCutoff = new Date(now.getTime() - RECENT_DATA_FAIL_DAYS * 24 * 60 * 6e4).toISOString();
|
|
72020
72275
|
const recentCrawlers = Number(
|
|
72021
|
-
ctx.db.select({ total:
|
|
72022
|
-
|
|
72023
|
-
|
|
72024
|
-
|
|
72276
|
+
ctx.db.select({ total: sql41`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
72277
|
+
and88(
|
|
72278
|
+
eq104(crawlerEventsHourly.projectId, ctx.project.id),
|
|
72279
|
+
inArray47(crawlerEventsHourly.sourceId, activeSourceIds),
|
|
72025
72280
|
gte29(crawlerEventsHourly.tsHour, warnCutoff)
|
|
72026
72281
|
)
|
|
72027
72282
|
).get()?.total ?? 0
|
|
72028
72283
|
);
|
|
72029
72284
|
const recentReferrals = Number(
|
|
72030
|
-
ctx.db.select({ total:
|
|
72031
|
-
|
|
72032
|
-
|
|
72033
|
-
|
|
72285
|
+
ctx.db.select({ total: sql41`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
72286
|
+
and88(
|
|
72287
|
+
eq104(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
72288
|
+
inArray47(aiReferralEventsHourly.sourceId, activeSourceIds),
|
|
72034
72289
|
gte29(aiReferralEventsHourly.tsHour, warnCutoff)
|
|
72035
72290
|
)
|
|
72036
72291
|
).get()?.total ?? 0
|
|
72037
72292
|
);
|
|
72038
72293
|
const recentUserFetches = Number(
|
|
72039
|
-
ctx.db.select({ total:
|
|
72040
|
-
|
|
72041
|
-
|
|
72042
|
-
|
|
72294
|
+
ctx.db.select({ total: sql41`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
72295
|
+
and88(
|
|
72296
|
+
eq104(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
72297
|
+
inArray47(aiUserFetchEventsHourly.sourceId, activeSourceIds),
|
|
72043
72298
|
gte29(aiUserFetchEventsHourly.tsHour, warnCutoff)
|
|
72044
72299
|
)
|
|
72045
72300
|
).get()?.total ?? 0
|
|
@@ -72062,28 +72317,28 @@ var recentDataCheck = {
|
|
|
72062
72317
|
};
|
|
72063
72318
|
}
|
|
72064
72319
|
const olderCrawlers = Number(
|
|
72065
|
-
ctx.db.select({ total:
|
|
72066
|
-
|
|
72067
|
-
|
|
72068
|
-
|
|
72320
|
+
ctx.db.select({ total: sql41`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
72321
|
+
and88(
|
|
72322
|
+
eq104(crawlerEventsHourly.projectId, ctx.project.id),
|
|
72323
|
+
inArray47(crawlerEventsHourly.sourceId, activeSourceIds),
|
|
72069
72324
|
gte29(crawlerEventsHourly.tsHour, failCutoff)
|
|
72070
72325
|
)
|
|
72071
72326
|
).get()?.total ?? 0
|
|
72072
72327
|
);
|
|
72073
72328
|
const olderReferrals = Number(
|
|
72074
|
-
ctx.db.select({ total:
|
|
72075
|
-
|
|
72076
|
-
|
|
72077
|
-
|
|
72329
|
+
ctx.db.select({ total: sql41`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
72330
|
+
and88(
|
|
72331
|
+
eq104(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
72332
|
+
inArray47(aiReferralEventsHourly.sourceId, activeSourceIds),
|
|
72078
72333
|
gte29(aiReferralEventsHourly.tsHour, failCutoff)
|
|
72079
72334
|
)
|
|
72080
72335
|
).get()?.total ?? 0
|
|
72081
72336
|
);
|
|
72082
72337
|
const olderUserFetches = Number(
|
|
72083
|
-
ctx.db.select({ total:
|
|
72084
|
-
|
|
72085
|
-
|
|
72086
|
-
|
|
72338
|
+
ctx.db.select({ total: sql41`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
72339
|
+
and88(
|
|
72340
|
+
eq104(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
72341
|
+
inArray47(aiUserFetchEventsHourly.sourceId, activeSourceIds),
|
|
72087
72342
|
gte29(aiUserFetchEventsHourly.tsHour, failCutoff)
|
|
72088
72343
|
)
|
|
72089
72344
|
).get()?.total ?? 0
|
|
@@ -72409,12 +72664,12 @@ var syncLagCheck = {
|
|
|
72409
72664
|
}
|
|
72410
72665
|
};
|
|
72411
72666
|
function lastQueueStopReason(ctx, sourceId) {
|
|
72412
|
-
const row = ctx.db.select({ syncSummary: runs.syncSummary }).from(runs).where(
|
|
72413
|
-
|
|
72414
|
-
|
|
72415
|
-
|
|
72667
|
+
const row = ctx.db.select({ syncSummary: runs.syncSummary }).from(runs).where(and88(
|
|
72668
|
+
eq104(runs.projectId, ctx.project.id),
|
|
72669
|
+
eq104(runs.kind, RunKinds["traffic-sync"]),
|
|
72670
|
+
eq104(runs.sourceId, sourceId),
|
|
72416
72671
|
isNotNull9(runs.syncSummary)
|
|
72417
|
-
)).orderBy(
|
|
72672
|
+
)).orderBy(desc47(runs.createdAt)).limit(1).get();
|
|
72418
72673
|
return row?.syncSummary?.type === "cloudflare-queue" ? row.syncSummary.stopReason : null;
|
|
72419
72674
|
}
|
|
72420
72675
|
var QUEUE_LIMIT_STOPS = [
|
|
@@ -73212,7 +73467,7 @@ async function doctorRoutes(app, opts) {
|
|
|
73212
73467
|
}
|
|
73213
73468
|
|
|
73214
73469
|
// ../api-routes/src/status.ts
|
|
73215
|
-
import { and as
|
|
73470
|
+
import { and as and89, desc as desc48, eq as eq105, isNull as isNull15 } from "drizzle-orm";
|
|
73216
73471
|
function ageDays(iso) {
|
|
73217
73472
|
if (!iso) return null;
|
|
73218
73473
|
const ms = Date.now() - Date.parse(iso);
|
|
@@ -73262,7 +73517,7 @@ function makeRow(integration, label, runKind, state, staleAfterDays, dataThrough
|
|
|
73262
73517
|
};
|
|
73263
73518
|
}
|
|
73264
73519
|
function latestDate2(db, table, projectId) {
|
|
73265
|
-
const row = db.select({ date: table.date }).from(table).where(
|
|
73520
|
+
const row = db.select({ date: table.date }).from(table).where(eq105(table.projectId, projectId)).orderBy(desc48(table.date)).limit(1).get();
|
|
73266
73521
|
return row?.date ?? null;
|
|
73267
73522
|
}
|
|
73268
73523
|
function projectRows(opts, project) {
|
|
@@ -73277,7 +73532,7 @@ function projectRows(opts, project) {
|
|
|
73277
73532
|
const rows = [];
|
|
73278
73533
|
const gsc = gscConnection(ctx, info);
|
|
73279
73534
|
if (connected(gsc)) {
|
|
73280
|
-
const watermarkRow = opts.db.select({ dataThroughDate: gscDataWatermarks.dataThroughDate }).from(gscDataWatermarks).where(
|
|
73535
|
+
const watermarkRow = opts.db.select({ dataThroughDate: gscDataWatermarks.dataThroughDate }).from(gscDataWatermarks).where(eq105(gscDataWatermarks.projectId, project.id)).limit(1).get();
|
|
73281
73536
|
const state = readSyncRunState(opts.db, project.id, RunKinds["gsc-sync"], { since: gsc.since });
|
|
73282
73537
|
const dataThrough = watermarkRow?.dataThroughDate ?? latestDate2(opts.db, gscDailyTotals, project.id);
|
|
73283
73538
|
rows.push(makeRow("gsc", "Search Console", RunKinds["gsc-sync"], state, SYNC_IDLE_DAYS, dataThrough, null, ["gsc.sync.recent-failures"], GSC_DATA_AGING_DAYS));
|
|
@@ -73293,12 +73548,12 @@ function projectRows(opts, project) {
|
|
|
73293
73548
|
rows.push(makeRow("bing", "Bing Webmaster Tools", RunKinds["bing-inspect-sitemap"], readSyncRunState(opts.db, project.id, RunKinds["bing-inspect-sitemap"]), null));
|
|
73294
73549
|
rows.push(makeRow("bing-search", "Bing search data", RunKinds["bing-sync"], readSyncRunState(opts.db, project.id, RunKinds["bing-sync"]), SYNC_IDLE_DAYS, latestDate2(opts.db, bingTrafficDaily, project.id), null, ["bing.sync.recent-failures"], BING_DATA_AGING_DAYS));
|
|
73295
73550
|
}
|
|
73296
|
-
const ads = opts.db.select({ id: adsConnections.id, lastSyncedAt: adsConnections.lastSyncedAt }).from(adsConnections).where(
|
|
73551
|
+
const ads = opts.db.select({ id: adsConnections.id, lastSyncedAt: adsConnections.lastSyncedAt }).from(adsConnections).where(eq105(adsConnections.projectId, project.id)).limit(1).get();
|
|
73297
73552
|
if (ads) rows.push(makeRow("ads", "OpenAI Ads", RunKinds["ads-sync"], readSyncRunState(opts.db, project.id, RunKinds["ads-sync"]), ADS_RECENT_SYNC_WARN_DAYS, null, null, ["ads.data.recent-sync"], ADS_RECENT_SYNC_WARN_DAYS, ads.lastSyncedAt));
|
|
73298
73553
|
const marketing = opts.getGoogleMarketingDoctorInput?.(ctx);
|
|
73299
73554
|
if (marketing?.googleAds) rows.push(makeRow("google-ads", "Google Ads", RunKinds["google-ads-sync"], readSyncRunState(opts.db, project.id, RunKinds["google-ads-sync"]), FRESH_SNAPSHOT_WARN_DAYS, null, null, ["google-ads.sync.recent-failures"], FRESH_SNAPSHOT_WARN_DAYS, marketing.googleAds.latestSnapshotAt));
|
|
73300
73555
|
if (marketing?.gtm) rows.push(makeRow("gtm", "Google Tag Manager", RunKinds["gtm-sync"], readSyncRunState(opts.db, project.id, RunKinds["gtm-sync"]), FRESH_SNAPSHOT_WARN_DAYS, null, null, ["gtm.sync.recent-failures"], FRESH_SNAPSHOT_WARN_DAYS, marketing.gtm.latestSnapshotAt));
|
|
73301
|
-
const sources = opts.db.select().from(trafficSources).where(
|
|
73556
|
+
const sources = opts.db.select().from(trafficSources).where(and89(eq105(trafficSources.projectId, project.id), isNull15(trafficSources.archivedAt))).all();
|
|
73302
73557
|
for (const source of sources) {
|
|
73303
73558
|
const row = makeRow("traffic", source.displayName, RunKinds["traffic-sync"], readSyncRunState(opts.db, project.id, RunKinds["traffic-sync"], { sourceId: source.id }), null, null, source.id, ["traffic.source.sync-lag", "traffic.source.recent-data"]);
|
|
73304
73559
|
const lag = source.lastSyncedAt ? Date.now() - Date.parse(source.lastSyncedAt) : Number.POSITIVE_INFINITY;
|
|
@@ -73310,7 +73565,7 @@ function projectRows(opts, project) {
|
|
|
73310
73565
|
}
|
|
73311
73566
|
function projectStatus(opts, project) {
|
|
73312
73567
|
const integrations = projectRows(opts, project);
|
|
73313
|
-
const healthRow = opts.db.select().from(doctorHealthState).where(
|
|
73568
|
+
const healthRow = opts.db.select().from(doctorHealthState).where(eq105(doctorHealthState.projectId, project.id)).limit(1).get();
|
|
73314
73569
|
const health = healthRow ? { status: healthRow.status, code: healthRow.code, summary: healthRow.summary, checkedAt: healthRow.checkedAt, notifiedAt: healthRow.notifiedAt } : null;
|
|
73315
73570
|
const rank = { ok: 0, "never-synced": 1, warning: 2, stale: 3, failing: 4 };
|
|
73316
73571
|
const healthState = health?.status === "fail" ? "failing" : health?.status === "warn" ? "warning" : "ok";
|
|
@@ -73330,7 +73585,7 @@ async function statusRoutes(app, opts) {
|
|
|
73330
73585
|
if (request.query.problems === "true") result.projects = result.projects.map((item2) => ({ ...item2, integrations: item2.integrations.filter((row) => row.state !== "ok") }));
|
|
73331
73586
|
return instanceStatusDtoSchema.parse(result);
|
|
73332
73587
|
}
|
|
73333
|
-
const projectsRows = (scopedId ? opts.db.select().from(projects).where(
|
|
73588
|
+
const projectsRows = (scopedId ? opts.db.select().from(projects).where(eq105(projects.id, scopedId)) : opts.db.select().from(projects)).all();
|
|
73334
73589
|
const output = { generatedAt: (/* @__PURE__ */ new Date()).toISOString(), ...opts.getMcpHealth ? { mcp: opts.getMcpHealth() } : {}, summary: { projects: projectsRows.length, ok: 0, failing: 0, warning: 0, stale: 0, neverSynced: 0 }, projects: projectsRows.map((project) => projectStatus(opts, project)) };
|
|
73335
73590
|
for (const project of output.projects) {
|
|
73336
73591
|
const key = project.state === "never-synced" ? "neverSynced" : project.state;
|
|
@@ -73343,7 +73598,7 @@ async function statusRoutes(app, opts) {
|
|
|
73343
73598
|
|
|
73344
73599
|
// ../api-routes/src/discovery/routes.ts
|
|
73345
73600
|
import crypto57 from "crypto";
|
|
73346
|
-
import { and as
|
|
73601
|
+
import { and as and90, desc as desc49, eq as eq106, gte as gte30, inArray as inArray48, isNull as isNull16, or as or20 } from "drizzle-orm";
|
|
73347
73602
|
var MAX_INFLIGHT_DISCOVERY_AGE_MS = 2 * 60 * 60 * 1e3;
|
|
73348
73603
|
async function discoveryRoutes(app, opts) {
|
|
73349
73604
|
app.post("/projects/:name/discover/run", async (request, reply) => {
|
|
@@ -73377,13 +73632,13 @@ async function discoveryRoutes(app, opts) {
|
|
|
73377
73632
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
73378
73633
|
const ageFloorIso = new Date(Date.now() - MAX_INFLIGHT_DISCOVERY_AGE_MS).toISOString();
|
|
73379
73634
|
const decision = app.db.transaction((tx) => {
|
|
73380
|
-
const existing = tx.select({ id: discoverySessions.id, runId: discoverySessions.runId }).from(discoverySessions).where(
|
|
73381
|
-
|
|
73382
|
-
|
|
73635
|
+
const existing = tx.select({ id: discoverySessions.id, runId: discoverySessions.runId }).from(discoverySessions).where(and90(
|
|
73636
|
+
eq106(discoverySessions.projectId, project.id),
|
|
73637
|
+
eq106(discoverySessions.icpDescription, icpDescription),
|
|
73383
73638
|
// Buyer is part of session identity: it changes the seed prompt's
|
|
73384
73639
|
// semantics, so a request with a different (or no) buyer must start
|
|
73385
73640
|
// its own session, never adopt another buyer's probes.
|
|
73386
|
-
parsed.data.buyerDescription == null ? isNull16(discoverySessions.buyerDescription) :
|
|
73641
|
+
parsed.data.buyerDescription == null ? isNull16(discoverySessions.buyerDescription) : eq106(discoverySessions.buyerDescription, parsed.data.buyerDescription),
|
|
73387
73642
|
// Locations are identity too: a different service-area subset seeds
|
|
73388
73643
|
// and probes a different geo, so it must never reuse another geo's
|
|
73389
73644
|
// session. resolveLocations is deterministic (project-config order),
|
|
@@ -73393,18 +73648,18 @@ async function discoveryRoutes(app, opts) {
|
|
|
73393
73648
|
// locations a NULL row's subset is unknowable, so it conservatively
|
|
73394
73649
|
// never reuses — a one-time, bounded (2h window) non-reuse after
|
|
73395
73650
|
// upgrade, never a wrong reuse.
|
|
73396
|
-
locations.length === 0 ? or20(isNull16(discoverySessions.locations),
|
|
73651
|
+
locations.length === 0 ? or20(isNull16(discoverySessions.locations), eq106(discoverySessions.locations, locations)) : eq106(discoverySessions.locations, locations),
|
|
73397
73652
|
// Seed provider set is identity: a different phrasing distribution
|
|
73398
73653
|
// must never reuse another set's session. Null (= Gemini-only
|
|
73399
73654
|
// default, including explicit ['gemini']) matches legacy rows.
|
|
73400
|
-
seedProviders == null ? isNull16(discoverySessions.seedProviders) :
|
|
73401
|
-
|
|
73655
|
+
seedProviders == null ? isNull16(discoverySessions.seedProviders) : eq106(discoverySessions.seedProviders, seedProviders),
|
|
73656
|
+
inArray48(discoverySessions.status, [
|
|
73402
73657
|
DiscoverySessionStatuses.queued,
|
|
73403
73658
|
DiscoverySessionStatuses.seeding,
|
|
73404
73659
|
DiscoverySessionStatuses.probing
|
|
73405
73660
|
]),
|
|
73406
73661
|
gte30(discoverySessions.createdAt, ageFloorIso)
|
|
73407
|
-
)).orderBy(
|
|
73662
|
+
)).orderBy(desc49(discoverySessions.createdAt)).get();
|
|
73408
73663
|
if (existing && existing.runId) {
|
|
73409
73664
|
return { reused: true, sessionId: existing.id, runId: existing.runId };
|
|
73410
73665
|
}
|
|
@@ -73473,7 +73728,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
73473
73728
|
const project = resolveProject(app.db, request.params.name);
|
|
73474
73729
|
const parsedLimit = parseInt(request.query.limit ?? "", 10);
|
|
73475
73730
|
const limit = Number.isNaN(parsedLimit) || parsedLimit <= 0 ? 50 : parsedLimit;
|
|
73476
|
-
const rows = app.db.select().from(discoverySessions).where(
|
|
73731
|
+
const rows = app.db.select().from(discoverySessions).where(eq106(discoverySessions.projectId, project.id)).orderBy(desc49(discoverySessions.createdAt)).limit(limit).all();
|
|
73477
73732
|
return reply.send(rows.map(serializeSession));
|
|
73478
73733
|
}
|
|
73479
73734
|
);
|
|
@@ -73481,11 +73736,11 @@ async function discoveryRoutes(app, opts) {
|
|
|
73481
73736
|
"/projects/:name/discover/sessions/:id",
|
|
73482
73737
|
async (request, reply) => {
|
|
73483
73738
|
const project = resolveProject(app.db, request.params.name);
|
|
73484
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
73739
|
+
const session = app.db.select().from(discoverySessions).where(eq106(discoverySessions.id, request.params.id)).get();
|
|
73485
73740
|
if (!session || session.projectId !== project.id) {
|
|
73486
73741
|
throw notFound("Discovery session", request.params.id);
|
|
73487
73742
|
}
|
|
73488
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
73743
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq106(discoveryProbes.sessionId, session.id)).all();
|
|
73489
73744
|
const detail = {
|
|
73490
73745
|
...serializeSession(session),
|
|
73491
73746
|
probes: probeRows.map(serializeProbe)
|
|
@@ -73497,7 +73752,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
73497
73752
|
"/projects/:name/discover/sessions/:id/harvest",
|
|
73498
73753
|
async (request, reply) => {
|
|
73499
73754
|
const project = resolveProject(app.db, request.params.name);
|
|
73500
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
73755
|
+
const session = app.db.select().from(discoverySessions).where(eq106(discoverySessions.id, request.params.id)).get();
|
|
73501
73756
|
if (!session || session.projectId !== project.id) {
|
|
73502
73757
|
throw notFound("Discovery session", request.params.id);
|
|
73503
73758
|
}
|
|
@@ -73505,7 +73760,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
73505
73760
|
const minProbeHits = Number.isNaN(parsedFloor) || parsedFloor < 1 ? 1 : parsedFloor;
|
|
73506
73761
|
const applyAnchor = request.query.anchor !== "false";
|
|
73507
73762
|
const provider = session.seedProvider ?? "gemini";
|
|
73508
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
73763
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq106(discoveryProbes.sessionId, session.id)).all();
|
|
73509
73764
|
const extract = opts.harvestSearchQueries;
|
|
73510
73765
|
const probesWithQueries = probeRows.map((row) => {
|
|
73511
73766
|
if (!extract || !row.rawResponse) return { searchQueries: [] };
|
|
@@ -73516,7 +73771,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
73516
73771
|
return { searchQueries: [] };
|
|
73517
73772
|
}
|
|
73518
73773
|
});
|
|
73519
|
-
const trackedQueries = app.db.select({ query: queries.query }).from(queries).where(
|
|
73774
|
+
const trackedQueries = app.db.select({ query: queries.query }).from(queries).where(eq106(queries.projectId, project.id)).all().map((r) => r.query);
|
|
73520
73775
|
const anchorTerms = buildHarvestAnchorTerms(
|
|
73521
73776
|
[session.icpDescription ?? "", ...trackedQueries],
|
|
73522
73777
|
effectiveDomains(project)
|
|
@@ -73563,11 +73818,11 @@ async function discoveryRoutes(app, opts) {
|
|
|
73563
73818
|
"/projects/:name/discover/sessions/:id/promote",
|
|
73564
73819
|
async (request, reply) => {
|
|
73565
73820
|
const project = resolveProject(app.db, request.params.name);
|
|
73566
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
73821
|
+
const session = app.db.select().from(discoverySessions).where(eq106(discoverySessions.id, request.params.id)).get();
|
|
73567
73822
|
if (!session || session.projectId !== project.id) {
|
|
73568
73823
|
throw notFound("Discovery session", request.params.id);
|
|
73569
73824
|
}
|
|
73570
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
73825
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq106(discoveryProbes.sessionId, session.id)).all();
|
|
73571
73826
|
const cited = /* @__PURE__ */ new Set();
|
|
73572
73827
|
const aspirational = /* @__PURE__ */ new Set();
|
|
73573
73828
|
const wasted = /* @__PURE__ */ new Set();
|
|
@@ -73602,7 +73857,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
73602
73857
|
);
|
|
73603
73858
|
app.post("/projects/:name/discover/sessions/:id/promote", async (request, reply) => {
|
|
73604
73859
|
const project = resolveProject(app.db, request.params.name);
|
|
73605
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
73860
|
+
const session = app.db.select().from(discoverySessions).where(eq106(discoverySessions.id, request.params.id)).get();
|
|
73606
73861
|
if (!session || session.projectId !== project.id) {
|
|
73607
73862
|
throw notFound("Discovery session", request.params.id);
|
|
73608
73863
|
}
|
|
@@ -73625,7 +73880,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
73625
73880
|
const bucketSet = new Set(buckets);
|
|
73626
73881
|
const includeCompetitors = parsed.data.includeCompetitors ?? true;
|
|
73627
73882
|
const competitorTypes = parsed.data.competitorTypes ?? DEFAULT_DISCOVERY_PROMOTE_COMPETITOR_TYPES;
|
|
73628
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
73883
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq106(discoveryProbes.sessionId, session.id)).all();
|
|
73629
73884
|
const candidateQueries = /* @__PURE__ */ new Set();
|
|
73630
73885
|
for (const probe of probeRows) {
|
|
73631
73886
|
if (!probe.bucket) continue;
|
|
@@ -73633,7 +73888,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
73633
73888
|
if (bucket.success && bucketSet.has(bucket.data)) candidateQueries.add(probe.query);
|
|
73634
73889
|
}
|
|
73635
73890
|
const existingQueries = new Set(
|
|
73636
|
-
app.db.select({ query: queries.query }).from(queries).where(
|
|
73891
|
+
app.db.select({ query: queries.query }).from(queries).where(eq106(queries.projectId, project.id)).all().map((r) => r.query.toLowerCase())
|
|
73637
73892
|
);
|
|
73638
73893
|
const promotedQueries = [];
|
|
73639
73894
|
const skippedQueries = [];
|
|
@@ -73847,7 +74102,7 @@ function planCompetitorPromotion(input) {
|
|
|
73847
74102
|
|
|
73848
74103
|
// ../api-routes/src/discovery/orchestrate.ts
|
|
73849
74104
|
import crypto58 from "crypto";
|
|
73850
|
-
import { eq as
|
|
74105
|
+
import { eq as eq107 } from "drizzle-orm";
|
|
73851
74106
|
var DEFAULT_MAX_PROBES = 100;
|
|
73852
74107
|
var ABSOLUTE_MAX_PROBES = 500;
|
|
73853
74108
|
function classifyProbeBucket(input) {
|
|
@@ -73933,7 +74188,7 @@ async function executeDiscovery(opts) {
|
|
|
73933
74188
|
status: DiscoverySessionStatuses.seeding,
|
|
73934
74189
|
dedupThreshold,
|
|
73935
74190
|
startedAt
|
|
73936
|
-
}).where(
|
|
74191
|
+
}).where(eq107(discoverySessions.id, opts.sessionId)).run();
|
|
73937
74192
|
const seedResult = await opts.deps.seed({
|
|
73938
74193
|
project: opts.project,
|
|
73939
74194
|
icpDescription: opts.icpDescription,
|
|
@@ -73994,7 +74249,7 @@ async function executeDiscovery(opts) {
|
|
|
73994
74249
|
dedupBandPairFraction: dedupStats.bandPairFraction,
|
|
73995
74250
|
dedupPairsTotal: dedupStats.pairsTotal,
|
|
73996
74251
|
warning
|
|
73997
|
-
}).where(
|
|
74252
|
+
}).where(eq107(discoverySessions.id, opts.sessionId)).run();
|
|
73998
74253
|
const probeLocation = opts.locations?.[0];
|
|
73999
74254
|
const probeResults = await mapWithConcurrency(
|
|
74000
74255
|
probedCanonicals,
|
|
@@ -74048,7 +74303,7 @@ async function executeDiscovery(opts) {
|
|
|
74048
74303
|
wastedCount: buckets["wasted-surface"],
|
|
74049
74304
|
competitorMap,
|
|
74050
74305
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
74051
|
-
}).where(
|
|
74306
|
+
}).where(eq107(discoverySessions.id, opts.sessionId)).run();
|
|
74052
74307
|
upsertDomainClassifications(opts.db, opts.project.id, opts.sessionId, competitorMap);
|
|
74053
74308
|
return {
|
|
74054
74309
|
buckets,
|
|
@@ -74088,7 +74343,7 @@ function markSessionFailed(db, sessionId, error) {
|
|
|
74088
74343
|
status: DiscoverySessionStatuses.failed,
|
|
74089
74344
|
error,
|
|
74090
74345
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
74091
|
-
}).where(
|
|
74346
|
+
}).where(eq107(discoverySessions.id, sessionId)).run();
|
|
74092
74347
|
}
|
|
74093
74348
|
function dedupeStrings(input) {
|
|
74094
74349
|
const seen2 = /* @__PURE__ */ new Set();
|
|
@@ -74106,12 +74361,12 @@ function dedupeStrings(input) {
|
|
|
74106
74361
|
|
|
74107
74362
|
// ../api-routes/src/technical-aeo.ts
|
|
74108
74363
|
import crypto59 from "crypto";
|
|
74109
|
-
import { and as
|
|
74364
|
+
import { and as and92, asc as asc26, count as count8, desc as desc50, eq as eq109, gte as gte31, inArray as inArray49, isNotNull as isNotNull10, isNull as isNull17, lt as lt17, or as or21, sql as sql43 } from "drizzle-orm";
|
|
74110
74365
|
import { alias } from "drizzle-orm/sqlite-core";
|
|
74111
74366
|
|
|
74112
74367
|
// ../api-routes/src/technical-aeo-issues.ts
|
|
74113
74368
|
import { createHash as createHash10 } from "crypto";
|
|
74114
|
-
import { and as
|
|
74369
|
+
import { and as and91, asc as asc25, eq as eq108, gt as gt10, sql as sql42 } from "drizzle-orm";
|
|
74115
74370
|
var ADVISORY_CODES = /* @__PURE__ */ new Set([
|
|
74116
74371
|
"entity-consistency.title.short",
|
|
74117
74372
|
"site.duplicate-title",
|
|
@@ -74256,12 +74511,12 @@ function readSiteAuditIssues(input) {
|
|
|
74256
74511
|
}
|
|
74257
74512
|
}
|
|
74258
74513
|
if (snapshot.detailsAvailable && snapshot.attemptId) {
|
|
74259
|
-
const scope =
|
|
74260
|
-
|
|
74261
|
-
|
|
74262
|
-
|
|
74514
|
+
const scope = and91(
|
|
74515
|
+
eq108(siteCrawlPages.projectId, project.id),
|
|
74516
|
+
eq108(siteCrawlPages.runId, snapshot.runId),
|
|
74517
|
+
eq108(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
74263
74518
|
);
|
|
74264
|
-
const size = db.select({ bytes:
|
|
74519
|
+
const size = db.select({ bytes: sql42`coalesce(sum(length(cast(${siteCrawlPages.auditFields} as blob)) + length(cast(${siteCrawlPages.url} as blob))), 0)` }).from(siteCrawlPages).where(scope).get()?.bytes ?? 0;
|
|
74265
74520
|
if (size + crossBytes > MAX_EVIDENCE_BYTES) {
|
|
74266
74521
|
throw validationError("Stored audit exceeds the 128 MiB issue-evidence read budget");
|
|
74267
74522
|
}
|
|
@@ -74272,7 +74527,7 @@ function readSiteAuditIssues(input) {
|
|
|
74272
74527
|
url: siteCrawlPages.url,
|
|
74273
74528
|
auditScore: siteCrawlPages.auditScore,
|
|
74274
74529
|
auditFields: siteCrawlPages.auditFields
|
|
74275
|
-
}).from(siteCrawlPages).where(
|
|
74530
|
+
}).from(siteCrawlPages).where(and91(scope, after === void 0 ? void 0 : gt10(siteCrawlPages.nodeKey, after))).orderBy(asc25(siteCrawlPages.nodeKey)).limit(200).all();
|
|
74276
74531
|
for (const page2 of pages) {
|
|
74277
74532
|
evidenceHash.update(JSON.stringify(page2));
|
|
74278
74533
|
const fields = record(page2.auditFields);
|
|
@@ -74409,10 +74664,10 @@ var FETCHED_SITE_CRAWL_STATES = /* @__PURE__ */ new Set([
|
|
|
74409
74664
|
var SURFACEABLE_STATUSES = [RunStatuses.completed, RunStatuses.partial];
|
|
74410
74665
|
function surfaceableSiteAuditFilters(projectId) {
|
|
74411
74666
|
return [
|
|
74412
|
-
|
|
74413
|
-
|
|
74414
|
-
|
|
74415
|
-
|
|
74667
|
+
eq109(siteAuditSnapshots.projectId, projectId),
|
|
74668
|
+
eq109(runs.projectId, projectId),
|
|
74669
|
+
eq109(runs.kind, RunKinds["site-audit"]),
|
|
74670
|
+
inArray49(runs.status, SURFACEABLE_STATUSES),
|
|
74416
74671
|
notProbeRun()
|
|
74417
74672
|
];
|
|
74418
74673
|
}
|
|
@@ -74652,13 +74907,13 @@ function parseLinkKind(value) {
|
|
|
74652
74907
|
return allowed.data;
|
|
74653
74908
|
}
|
|
74654
74909
|
function linkKindFilter(linkKind) {
|
|
74655
|
-
if (linkKind === SiteHealthLinkKinds.content) return
|
|
74656
|
-
if (linkKind === SiteHealthLinkKinds.template) return
|
|
74910
|
+
if (linkKind === SiteHealthLinkKinds.content) return eq109(siteCrawlEdges.isTemplate, false);
|
|
74911
|
+
if (linkKind === SiteHealthLinkKinds.template) return eq109(siteCrawlEdges.isTemplate, true);
|
|
74657
74912
|
return void 0;
|
|
74658
74913
|
}
|
|
74659
74914
|
var graphSourceNode = alias(siteCrawlGraphNodes, "site_crawl_graph_source_node");
|
|
74660
74915
|
var graphTargetNode = alias(siteCrawlGraphNodes, "site_crawl_graph_target_node");
|
|
74661
|
-
var PAGE_CHANGE_PREDICATE =
|
|
74916
|
+
var PAGE_CHANGE_PREDICATE = sql43`(
|
|
74662
74917
|
current.url IS NOT previous.url
|
|
74663
74918
|
OR current.final_url IS NOT previous.final_url
|
|
74664
74919
|
OR current.path IS NOT previous.path
|
|
@@ -74680,7 +74935,7 @@ var PAGE_CHANGE_PREDICATE = sql42`(
|
|
|
74680
74935
|
OR current.link_score_raw IS NOT previous.link_score_raw
|
|
74681
74936
|
OR current.link_score_normalized IS NOT previous.link_score_normalized
|
|
74682
74937
|
)`;
|
|
74683
|
-
var LINK_CHANGE_PREDICATE =
|
|
74938
|
+
var LINK_CHANGE_PREDICATE = sql43`(
|
|
74684
74939
|
current.source_node_key IS NOT previous.source_node_key
|
|
74685
74940
|
OR current.source_url IS NOT previous.source_url
|
|
74686
74941
|
OR current.target_node_key IS NOT previous.target_node_key
|
|
@@ -74757,20 +75012,20 @@ function changedFields(before, after, fields) {
|
|
|
74757
75012
|
}
|
|
74758
75013
|
async function technicalAeoRoutes(app, opts) {
|
|
74759
75014
|
const visibleCrawlFilters = (projectId) => [
|
|
74760
|
-
|
|
74761
|
-
|
|
74762
|
-
|
|
74763
|
-
|
|
75015
|
+
eq109(siteCrawlSnapshots.projectId, projectId),
|
|
75016
|
+
eq109(runs.projectId, projectId),
|
|
75017
|
+
eq109(runs.kind, RunKinds["site-audit"]),
|
|
75018
|
+
inArray49(runs.status, SURFACEABLE_STATUSES),
|
|
74764
75019
|
notProbeRun()
|
|
74765
75020
|
];
|
|
74766
75021
|
const availableScanDatesFor = (projectId, requestedDate) => {
|
|
74767
75022
|
if (!requestedDate) return void 0;
|
|
74768
|
-
const date =
|
|
75023
|
+
const date = sql43`substr(${siteCrawlSnapshots.createdAt}, 1, 10)`;
|
|
74769
75024
|
const filters = visibleCrawlFilters(projectId);
|
|
74770
|
-
const allDates =
|
|
74771
|
-
const matchingDates =
|
|
74772
|
-
const readDates = (where) => app.db.select({ date }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
74773
|
-
const countDates = (where) => app.db.select({ value:
|
|
75025
|
+
const allDates = and92(...filters);
|
|
75026
|
+
const matchingDates = and92(...filters, eq109(sql43`substr(${siteCrawlSnapshots.createdAt}, 6, 5)`, requestedDate.slice(5)));
|
|
75027
|
+
const readDates = (where) => app.db.select({ date }).from(siteCrawlSnapshots).innerJoin(runs, eq109(siteCrawlSnapshots.runId, runs.id)).where(where).groupBy(date).orderBy(desc50(date)).limit(10).all().map((row) => row.date);
|
|
75028
|
+
const countDates = (where) => app.db.select({ value: sql43`count(distinct ${date})` }).from(siteCrawlSnapshots).innerJoin(runs, eq109(siteCrawlSnapshots.runId, runs.id)).where(where).get()?.value ?? 0;
|
|
74774
75029
|
return {
|
|
74775
75030
|
recentDates: readDates(allDates),
|
|
74776
75031
|
totalDates: countDates(allDates),
|
|
@@ -74786,34 +75041,34 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
74786
75041
|
nextDate2.setUTCDate(nextDate2.getUTCDate() + 1);
|
|
74787
75042
|
filters.push(gte31(siteCrawlSnapshots.createdAt, requestedDate), lt17(siteCrawlSnapshots.createdAt, nextDate2.toISOString()));
|
|
74788
75043
|
}
|
|
74789
|
-
const selectionFilters = scope.scoredOnly ? [...filters,
|
|
74790
|
-
const selected = app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
74791
|
-
|
|
74792
|
-
|
|
74793
|
-
|
|
74794
|
-
|
|
74795
|
-
|
|
75044
|
+
const selectionFilters = scope.scoredOnly ? [...filters, sql43`exists (select 1 from ${siteAuditSnapshots} where ${siteAuditSnapshots.projectId} = ${siteCrawlSnapshots.projectId} and ${siteAuditSnapshots.runId} = ${siteCrawlSnapshots.runId})`] : filters;
|
|
75045
|
+
const selected = app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq109(siteCrawlSnapshots.runId, runs.id)).where(and92(...selectionFilters, runId ? eq109(siteCrawlSnapshots.runId, runId) : void 0)).orderBy(
|
|
75046
|
+
desc50(sql43`substr(${siteCrawlSnapshots.createdAt}, 1, 10)`),
|
|
75047
|
+
desc50(siteCrawlSnapshots.complete),
|
|
75048
|
+
desc50(siteCrawlSnapshots.pagesFetched),
|
|
75049
|
+
desc50(siteCrawlSnapshots.createdAt),
|
|
75050
|
+
desc50(siteCrawlSnapshots.runId)
|
|
74796
75051
|
).limit(1).get();
|
|
74797
75052
|
if (!selected) return void 0;
|
|
74798
75053
|
const date = selected.snapshot.createdAt.slice(0, 10);
|
|
74799
75054
|
const nextDate = /* @__PURE__ */ new Date(`${date}T00:00:00.000Z`);
|
|
74800
75055
|
nextDate.setUTCDate(nextDate.getUTCDate() + 1);
|
|
74801
|
-
const sameDate =
|
|
75056
|
+
const sameDate = and92(
|
|
74802
75057
|
...filters,
|
|
74803
75058
|
gte31(siteCrawlSnapshots.createdAt, date),
|
|
74804
75059
|
lt17(siteCrawlSnapshots.createdAt, nextDate.toISOString())
|
|
74805
75060
|
);
|
|
74806
|
-
const sameDateRunCount = app.db.select({ value: count8() }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
75061
|
+
const sameDateRunCount = app.db.select({ value: count8() }).from(siteCrawlSnapshots).innerJoin(runs, eq109(siteCrawlSnapshots.runId, runs.id)).where(sameDate).get()?.value ?? 0;
|
|
74807
75062
|
const candidates = app.db.select({
|
|
74808
75063
|
runId: siteCrawlSnapshots.runId,
|
|
74809
75064
|
createdAt: siteCrawlSnapshots.createdAt,
|
|
74810
75065
|
complete: siteCrawlSnapshots.complete,
|
|
74811
75066
|
pages: siteCrawlSnapshots.pagesFetched
|
|
74812
|
-
}).from(siteCrawlSnapshots).innerJoin(runs,
|
|
74813
|
-
|
|
74814
|
-
|
|
74815
|
-
|
|
74816
|
-
|
|
75067
|
+
}).from(siteCrawlSnapshots).innerJoin(runs, eq109(siteCrawlSnapshots.runId, runs.id)).where(sameDate).orderBy(
|
|
75068
|
+
desc50(siteCrawlSnapshots.complete),
|
|
75069
|
+
desc50(siteCrawlSnapshots.pagesFetched),
|
|
75070
|
+
desc50(siteCrawlSnapshots.createdAt),
|
|
75071
|
+
desc50(siteCrawlSnapshots.runId)
|
|
74817
75072
|
).limit(10).all();
|
|
74818
75073
|
const runSelection = {
|
|
74819
75074
|
reason: runId ? "explicit-run" : scope.scoredOnly ? "latest-scored-scan" : requestedDate ? selected.snapshot.complete ? "requested-date-complete" : "requested-date-most-pages" : selected.snapshot.complete ? "latest-date-complete" : "latest-date-most-pages",
|
|
@@ -74830,13 +75085,13 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
74830
75085
|
...surfaceableSiteAuditFilters(projectId),
|
|
74831
75086
|
...recordedBefore ? [lt17(siteAuditSnapshots.createdAt, recordedBefore)] : []
|
|
74832
75087
|
];
|
|
74833
|
-
const complete =
|
|
74834
|
-
const selectAudit = (pinnedRunId) => app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs,
|
|
74835
|
-
|
|
74836
|
-
|
|
74837
|
-
|
|
74838
|
-
|
|
74839
|
-
|
|
75088
|
+
const complete = sql43`case when ${runs.status} = ${RunStatuses.completed} then 1 else 0 end`;
|
|
75089
|
+
const selectAudit = (pinnedRunId) => app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs, eq109(siteAuditSnapshots.runId, runs.id)).where(and92(...filters, pinnedRunId ? eq109(siteAuditSnapshots.runId, pinnedRunId) : void 0)).orderBy(
|
|
75090
|
+
desc50(sql43`substr(${siteAuditSnapshots.createdAt}, 1, 10)`),
|
|
75091
|
+
desc50(complete),
|
|
75092
|
+
desc50(siteAuditSnapshots.pagesAudited),
|
|
75093
|
+
desc50(siteAuditSnapshots.createdAt),
|
|
75094
|
+
desc50(siteAuditSnapshots.runId)
|
|
74840
75095
|
).limit(1).get();
|
|
74841
75096
|
const auditOfCrawl = (target) => {
|
|
74842
75097
|
const selected2 = target ? selectAudit(target.snapshot.runId) : void 0;
|
|
@@ -74852,22 +75107,22 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
74852
75107
|
const date = selected.snap.createdAt.slice(0, 10);
|
|
74853
75108
|
const nextDate = /* @__PURE__ */ new Date(`${date}T00:00:00.000Z`);
|
|
74854
75109
|
nextDate.setUTCDate(nextDate.getUTCDate() + 1);
|
|
74855
|
-
const sameDate =
|
|
75110
|
+
const sameDate = and92(
|
|
74856
75111
|
...filters,
|
|
74857
75112
|
gte31(siteAuditSnapshots.createdAt, date),
|
|
74858
75113
|
lt17(siteAuditSnapshots.createdAt, nextDate.toISOString())
|
|
74859
75114
|
);
|
|
74860
|
-
const sameDateRunCount = app.db.select({ value: count8() }).from(siteAuditSnapshots).innerJoin(runs,
|
|
75115
|
+
const sameDateRunCount = app.db.select({ value: count8() }).from(siteAuditSnapshots).innerJoin(runs, eq109(siteAuditSnapshots.runId, runs.id)).where(sameDate).get()?.value ?? 0;
|
|
74861
75116
|
const candidates = app.db.select({
|
|
74862
75117
|
runId: siteAuditSnapshots.runId,
|
|
74863
75118
|
createdAt: siteAuditSnapshots.createdAt,
|
|
74864
75119
|
complete,
|
|
74865
75120
|
pages: siteAuditSnapshots.pagesAudited
|
|
74866
|
-
}).from(siteAuditSnapshots).innerJoin(runs,
|
|
74867
|
-
|
|
74868
|
-
|
|
74869
|
-
|
|
74870
|
-
|
|
75121
|
+
}).from(siteAuditSnapshots).innerJoin(runs, eq109(siteAuditSnapshots.runId, runs.id)).where(sameDate).orderBy(
|
|
75122
|
+
desc50(complete),
|
|
75123
|
+
desc50(siteAuditSnapshots.pagesAudited),
|
|
75124
|
+
desc50(siteAuditSnapshots.createdAt),
|
|
75125
|
+
desc50(siteAuditSnapshots.runId)
|
|
74871
75126
|
).limit(10).all().map((candidate) => ({ ...candidate, complete: Boolean(candidate.complete) }));
|
|
74872
75127
|
const runSelection = {
|
|
74873
75128
|
// A crawl exists here only when none stored a scorecard: this score is a fallback.
|
|
@@ -74882,9 +75137,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
74882
75137
|
};
|
|
74883
75138
|
const detailScopeFor = (projectId, snapshot) => snapshot.detailsAvailable && snapshot.attemptId ? { projectId, runId: snapshot.runId, attemptId: snapshot.attemptId } : null;
|
|
74884
75139
|
const crawlReasonGroups = (where) => {
|
|
74885
|
-
const canonicalOther =
|
|
75140
|
+
const canonicalOther = sql43`case when ${siteCrawlPages.canonicalNodeKey} is not null
|
|
74886
75141
|
and ${siteCrawlPages.canonicalNodeKey} <> ${siteCrawlPages.nodeKey} then 1 else 0 end`;
|
|
74887
|
-
const canonicalReason =
|
|
75142
|
+
const canonicalReason = sql43`exists (select 1 from json_each(${siteCrawlPages.indexabilityReasons})
|
|
74888
75143
|
where value = ${SiteCrawlIndexabilityReasons.canonicalToOther})`;
|
|
74889
75144
|
const groups = app.db.select({
|
|
74890
75145
|
fetchState: siteCrawlPages.fetchState,
|
|
@@ -74893,7 +75148,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
74893
75148
|
canonicalOther,
|
|
74894
75149
|
canonicalReason,
|
|
74895
75150
|
pages: count8(),
|
|
74896
|
-
exampleUrl:
|
|
75151
|
+
exampleUrl: sql43`min(${siteCrawlPages.url})`
|
|
74897
75152
|
}).from(siteCrawlPages).where(where).groupBy(
|
|
74898
75153
|
siteCrawlPages.fetchState,
|
|
74899
75154
|
siteCrawlPages.indexabilityState,
|
|
@@ -74929,19 +75184,19 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
74929
75184
|
};
|
|
74930
75185
|
const inventorySummaryFor = (scope) => {
|
|
74931
75186
|
if (!scope) return null;
|
|
74932
|
-
const inScope =
|
|
74933
|
-
|
|
74934
|
-
|
|
74935
|
-
|
|
75187
|
+
const inScope = and92(
|
|
75188
|
+
eq109(siteCrawlPages.projectId, scope.projectId),
|
|
75189
|
+
eq109(siteCrawlPages.runId, scope.runId),
|
|
75190
|
+
eq109(siteCrawlPages.attemptId, scope.attemptId)
|
|
74936
75191
|
);
|
|
74937
75192
|
const groups = crawlReasonGroups(inScope);
|
|
74938
75193
|
const total = groups.reduce((sum, group) => sum + group.pages, 0);
|
|
74939
75194
|
const eligible = groups.reduce((sum, group) => sum + (group.inventoryEligible ? group.pages : 0), 0);
|
|
74940
|
-
const urlVersions = app.db.select({ value: count8() }).from(siteCrawlPages).where(
|
|
74941
|
-
const urlVersionsNotFetched = app.db.select({ value: siteCrawlAttempts.urlVersionsSkipped }).from(siteCrawlAttempts).where(
|
|
74942
|
-
|
|
74943
|
-
|
|
74944
|
-
|
|
75195
|
+
const urlVersions = app.db.select({ value: count8() }).from(siteCrawlPages).where(and92(inScope, isNotNull10(siteCrawlPages.urlVersionOf))).get()?.value ?? 0;
|
|
75196
|
+
const urlVersionsNotFetched = app.db.select({ value: siteCrawlAttempts.urlVersionsSkipped }).from(siteCrawlAttempts).where(and92(
|
|
75197
|
+
eq109(siteCrawlAttempts.projectId, scope.projectId),
|
|
75198
|
+
eq109(siteCrawlAttempts.runId, scope.runId),
|
|
75199
|
+
eq109(siteCrawlAttempts.id, scope.attemptId)
|
|
74945
75200
|
)).get()?.value ?? 0;
|
|
74946
75201
|
return {
|
|
74947
75202
|
scope: "selected-snapshot",
|
|
@@ -74961,34 +75216,34 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
74961
75216
|
const NO_CRAWL_COMPLETENESS = { complete: false, termination: null };
|
|
74962
75217
|
const pageInScope = (scope, selector) => {
|
|
74963
75218
|
if (!selector.nodeKey && !selector.url) return null;
|
|
74964
|
-
return app.db.select().from(siteCrawlPages).where(
|
|
74965
|
-
|
|
74966
|
-
|
|
74967
|
-
|
|
74968
|
-
selector.nodeKey ?
|
|
74969
|
-
)).orderBy(
|
|
74970
|
-
};
|
|
74971
|
-
const rootPageInScope = (scope, rootUrl) => pageInScope(scope, { url: rootUrl }) ?? app.db.select().from(siteCrawlPages).where(
|
|
74972
|
-
|
|
74973
|
-
|
|
74974
|
-
|
|
74975
|
-
|
|
74976
|
-
)).orderBy(
|
|
74977
|
-
const isSurfaceableAuditRun = (projectId, runId) => Boolean(app.db.select({ id: runs.id }).from(runs).where(
|
|
74978
|
-
|
|
74979
|
-
|
|
74980
|
-
|
|
74981
|
-
|
|
75219
|
+
return app.db.select().from(siteCrawlPages).where(and92(
|
|
75220
|
+
eq109(siteCrawlPages.projectId, scope.projectId),
|
|
75221
|
+
eq109(siteCrawlPages.runId, scope.runId),
|
|
75222
|
+
eq109(siteCrawlPages.attemptId, scope.attemptId),
|
|
75223
|
+
selector.nodeKey ? eq109(siteCrawlPages.nodeKey, selector.nodeKey) : eq109(siteCrawlPages.url, selector.url)
|
|
75224
|
+
)).orderBy(asc26(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
75225
|
+
};
|
|
75226
|
+
const rootPageInScope = (scope, rootUrl) => pageInScope(scope, { url: rootUrl }) ?? app.db.select().from(siteCrawlPages).where(and92(
|
|
75227
|
+
eq109(siteCrawlPages.projectId, scope.projectId),
|
|
75228
|
+
eq109(siteCrawlPages.runId, scope.runId),
|
|
75229
|
+
eq109(siteCrawlPages.attemptId, scope.attemptId),
|
|
75230
|
+
eq109(siteCrawlPages.depth, 0)
|
|
75231
|
+
)).orderBy(asc26(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
75232
|
+
const isSurfaceableAuditRun = (projectId, runId) => Boolean(app.db.select({ id: runs.id }).from(runs).where(and92(
|
|
75233
|
+
eq109(runs.id, runId),
|
|
75234
|
+
eq109(runs.projectId, projectId),
|
|
75235
|
+
eq109(runs.kind, RunKinds["site-audit"]),
|
|
75236
|
+
inArray49(runs.status, SURFACEABLE_STATUSES),
|
|
74982
75237
|
notProbeRun()
|
|
74983
75238
|
)).limit(1).get());
|
|
74984
75239
|
const assertKnownAuditRun = (projectId, runId) => {
|
|
74985
75240
|
if (runId && !isSurfaceableAuditRun(projectId, runId)) throw notFound("Site crawl run", runId);
|
|
74986
75241
|
};
|
|
74987
|
-
const hasLegacyAudit = (projectId) => Boolean(app.db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).innerJoin(runs,
|
|
74988
|
-
|
|
74989
|
-
|
|
74990
|
-
|
|
74991
|
-
|
|
75242
|
+
const hasLegacyAudit = (projectId) => Boolean(app.db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).innerJoin(runs, eq109(siteAuditSnapshots.runId, runs.id)).where(and92(
|
|
75243
|
+
eq109(siteAuditSnapshots.projectId, projectId),
|
|
75244
|
+
eq109(runs.projectId, projectId),
|
|
75245
|
+
eq109(runs.kind, RunKinds["site-audit"]),
|
|
75246
|
+
inArray49(runs.status, SURFACEABLE_STATUSES),
|
|
74992
75247
|
notProbeRun()
|
|
74993
75248
|
)).limit(1).get());
|
|
74994
75249
|
const emptyCrawlSummary = (projectName, legacyAuditAvailable) => ({
|
|
@@ -75089,14 +75344,14 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75089
75344
|
};
|
|
75090
75345
|
}
|
|
75091
75346
|
const statusFilter = request.query.status === "success" || request.query.status === "error" ? request.query.status : null;
|
|
75092
|
-
const conds = [
|
|
75093
|
-
if (statusFilter) conds.push(
|
|
75094
|
-
const where =
|
|
75347
|
+
const conds = [eq109(siteAuditPages.projectId, project.id), eq109(siteAuditPages.runId, latest.snap.runId)];
|
|
75348
|
+
if (statusFilter) conds.push(eq109(siteAuditPages.status, statusFilter));
|
|
75349
|
+
const where = and92(...conds);
|
|
75095
75350
|
const totalRow = app.db.select({ value: count8() }).from(siteAuditPages).where(where).get();
|
|
75096
75351
|
const total = totalRow?.value ?? 0;
|
|
75097
75352
|
const limit = parsePositiveInt(request.query.limit, 100, 500);
|
|
75098
75353
|
const offset = parsePositiveInt(request.query.offset, 0, Number.MAX_SAFE_INTEGER);
|
|
75099
|
-
const orderBy = request.query.sort === "score-desc" ?
|
|
75354
|
+
const orderBy = request.query.sort === "score-desc" ? desc50(siteAuditPages.overallScore) : request.query.sort === "url" ? asc26(siteAuditPages.url) : asc26(siteAuditPages.overallScore);
|
|
75100
75355
|
const rows = app.db.select().from(siteAuditPages).where(where).orderBy(orderBy).limit(limit).offset(offset).all();
|
|
75101
75356
|
const pages = rows.map((row) => ({
|
|
75102
75357
|
url: row.url,
|
|
@@ -75115,7 +75370,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75115
75370
|
auditedAt: siteAuditSnapshots.auditedAt,
|
|
75116
75371
|
aggregateScore: siteAuditSnapshots.aggregateScore,
|
|
75117
75372
|
pagesAudited: siteAuditSnapshots.pagesAudited
|
|
75118
|
-
}).from(siteAuditSnapshots).innerJoin(runs,
|
|
75373
|
+
}).from(siteAuditSnapshots).innerJoin(runs, eq109(siteAuditSnapshots.runId, runs.id)).where(and92(...surfaceableSiteAuditFilters(project.id))).orderBy(desc50(siteAuditSnapshots.createdAt)).limit(limit).all();
|
|
75119
75374
|
return { project: project.name, points: rows.reverse() };
|
|
75120
75375
|
});
|
|
75121
75376
|
app.get("/projects/:name/technical-aeo/crawl", async (request) => {
|
|
@@ -75215,10 +75470,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75215
75470
|
{ projectId: project.id, runId: snapshot.runId, attemptId: snapshot.attemptId },
|
|
75216
75471
|
snapshot.rootUrl
|
|
75217
75472
|
)?.nodeKey ?? null;
|
|
75218
|
-
const persistedLayout = app.db.select().from(siteCrawlGraphLayouts).where(
|
|
75219
|
-
|
|
75220
|
-
|
|
75221
|
-
|
|
75473
|
+
const persistedLayout = app.db.select().from(siteCrawlGraphLayouts).where(and92(
|
|
75474
|
+
eq109(siteCrawlGraphLayouts.projectId, project.id),
|
|
75475
|
+
eq109(siteCrawlGraphLayouts.runId, snapshot.runId),
|
|
75476
|
+
eq109(siteCrawlGraphLayouts.attemptId, snapshot.attemptId)
|
|
75222
75477
|
)).limit(1).get();
|
|
75223
75478
|
if (!persistedLayout) {
|
|
75224
75479
|
return {
|
|
@@ -75271,9 +75526,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75271
75526
|
const maxNodes = parseBoundedLimit(request.query.maxNodes, SITE_CRAWL_GRAPH_DEFAULT_MAX_NODES, SITE_CRAWL_GRAPH_MAX_NODES);
|
|
75272
75527
|
const maxEdges = parseBoundedLimit(request.query.maxEdges, SITE_CRAWL_GRAPH_DEFAULT_MAX_EDGES, SITE_CRAWL_GRAPH_MAX_EDGES);
|
|
75273
75528
|
const graphScope = [
|
|
75274
|
-
|
|
75275
|
-
|
|
75276
|
-
|
|
75529
|
+
eq109(siteCrawlGraphNodes.projectId, project.id),
|
|
75530
|
+
eq109(siteCrawlGraphNodes.runId, snapshot.runId),
|
|
75531
|
+
eq109(siteCrawlGraphNodes.attemptId, snapshot.attemptId)
|
|
75277
75532
|
];
|
|
75278
75533
|
const nodeRows = app.db.select({
|
|
75279
75534
|
nodeKey: siteCrawlPages.nodeKey,
|
|
@@ -75292,18 +75547,18 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75292
75547
|
linkScoreNormalized: siteCrawlPages.linkScoreNormalized,
|
|
75293
75548
|
x: siteCrawlGraphNodes.x,
|
|
75294
75549
|
y: siteCrawlGraphNodes.y
|
|
75295
|
-
}).from(siteCrawlGraphNodes).innerJoin(siteCrawlPages,
|
|
75296
|
-
|
|
75297
|
-
|
|
75298
|
-
|
|
75299
|
-
|
|
75300
|
-
)).where(
|
|
75550
|
+
}).from(siteCrawlGraphNodes).innerJoin(siteCrawlPages, and92(
|
|
75551
|
+
eq109(siteCrawlPages.projectId, siteCrawlGraphNodes.projectId),
|
|
75552
|
+
eq109(siteCrawlPages.runId, siteCrawlGraphNodes.runId),
|
|
75553
|
+
eq109(siteCrawlPages.attemptId, siteCrawlGraphNodes.attemptId),
|
|
75554
|
+
eq109(siteCrawlPages.nodeKey, siteCrawlGraphNodes.nodeKey)
|
|
75555
|
+
)).where(and92(...graphScope, lt17(siteCrawlGraphNodes.sampleRank, maxNodes))).orderBy(asc26(siteCrawlGraphNodes.sampleRank)).all();
|
|
75301
75556
|
const nodes = nodeRows.map(({ indexabilityReasons, canonicalNodeKey, ...row }) => ({
|
|
75302
75557
|
...row,
|
|
75303
75558
|
healthState: deriveSiteHealthState({ ...row, indexabilityReasons, canonicalNodeKey }),
|
|
75304
75559
|
healthReason: deriveSiteHealthReason({ ...row, indexabilityReasons, canonicalNodeKey })
|
|
75305
75560
|
}));
|
|
75306
|
-
const graphLinkKindFilter = linkKind === SiteHealthLinkKinds.all ? void 0 :
|
|
75561
|
+
const graphLinkKindFilter = linkKind === SiteHealthLinkKinds.all ? void 0 : eq109(siteCrawlGraphEdges.isTemplate, linkKind === SiteHealthLinkKinds.template);
|
|
75307
75562
|
const edges = app.db.select({
|
|
75308
75563
|
edgeKey: siteCrawlGraphEdges.edgeKey,
|
|
75309
75564
|
sourceNodeKey: siteCrawlGraphEdges.sourceNodeKey,
|
|
@@ -75311,25 +75566,25 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75311
75566
|
followable: siteCrawlGraphEdges.followable,
|
|
75312
75567
|
occurrences: siteCrawlGraphEdges.occurrences,
|
|
75313
75568
|
isTemplate: siteCrawlGraphEdges.isTemplate
|
|
75314
|
-
}).from(siteCrawlGraphEdges).innerJoin(graphSourceNode,
|
|
75315
|
-
|
|
75316
|
-
|
|
75317
|
-
|
|
75318
|
-
|
|
75569
|
+
}).from(siteCrawlGraphEdges).innerJoin(graphSourceNode, and92(
|
|
75570
|
+
eq109(graphSourceNode.projectId, siteCrawlGraphEdges.projectId),
|
|
75571
|
+
eq109(graphSourceNode.runId, siteCrawlGraphEdges.runId),
|
|
75572
|
+
eq109(graphSourceNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
75573
|
+
eq109(graphSourceNode.nodeKey, siteCrawlGraphEdges.sourceNodeKey),
|
|
75319
75574
|
lt17(graphSourceNode.sampleRank, maxNodes)
|
|
75320
|
-
)).innerJoin(graphTargetNode,
|
|
75321
|
-
|
|
75322
|
-
|
|
75323
|
-
|
|
75324
|
-
|
|
75575
|
+
)).innerJoin(graphTargetNode, and92(
|
|
75576
|
+
eq109(graphTargetNode.projectId, siteCrawlGraphEdges.projectId),
|
|
75577
|
+
eq109(graphTargetNode.runId, siteCrawlGraphEdges.runId),
|
|
75578
|
+
eq109(graphTargetNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
75579
|
+
eq109(graphTargetNode.nodeKey, siteCrawlGraphEdges.targetNodeKey),
|
|
75325
75580
|
lt17(graphTargetNode.sampleRank, maxNodes)
|
|
75326
|
-
)).where(
|
|
75327
|
-
|
|
75328
|
-
|
|
75329
|
-
|
|
75581
|
+
)).where(and92(
|
|
75582
|
+
eq109(siteCrawlGraphEdges.projectId, project.id),
|
|
75583
|
+
eq109(siteCrawlGraphEdges.runId, snapshot.runId),
|
|
75584
|
+
eq109(siteCrawlGraphEdges.attemptId, snapshot.attemptId),
|
|
75330
75585
|
lt17(siteCrawlGraphEdges.sampleRank, maxEdges),
|
|
75331
75586
|
graphLinkKindFilter
|
|
75332
|
-
)).orderBy(
|
|
75587
|
+
)).orderBy(asc26(siteCrawlGraphEdges.sampleRank)).all();
|
|
75333
75588
|
const totalNodes = persistedLayout.totalNodes;
|
|
75334
75589
|
const totalEdges = persistedLayout.totalEdges;
|
|
75335
75590
|
const totalTemplateEdges = persistedLayout.totalTemplateEdges;
|
|
@@ -75421,17 +75676,17 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75421
75676
|
let frontier = [focus.nodeKey];
|
|
75422
75677
|
let queryTruncated = false;
|
|
75423
75678
|
for (let distance = 0; distance < hops && frontier.length > 0; distance += 1) {
|
|
75424
|
-
const candidates = app.db.select().from(siteCrawlEdges).where(
|
|
75425
|
-
|
|
75426
|
-
|
|
75427
|
-
|
|
75428
|
-
|
|
75679
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and92(
|
|
75680
|
+
eq109(siteCrawlEdges.projectId, scope.projectId),
|
|
75681
|
+
eq109(siteCrawlEdges.runId, scope.runId),
|
|
75682
|
+
eq109(siteCrawlEdges.attemptId, scope.attemptId),
|
|
75683
|
+
eq109(siteCrawlEdges.internal, true),
|
|
75429
75684
|
isNotNull10(siteCrawlEdges.targetNodeKey),
|
|
75430
75685
|
or21(
|
|
75431
|
-
|
|
75432
|
-
|
|
75686
|
+
inArray49(siteCrawlEdges.sourceNodeKey, frontier),
|
|
75687
|
+
inArray49(siteCrawlEdges.targetNodeKey, frontier)
|
|
75433
75688
|
)
|
|
75434
|
-
)).orderBy(
|
|
75689
|
+
)).orderBy(asc26(siteCrawlEdges.edgeKey)).limit(maxEdges + maxNodes + 1).all();
|
|
75435
75690
|
if (candidates.length > maxEdges + maxNodes) queryTruncated = true;
|
|
75436
75691
|
const next = /* @__PURE__ */ new Set();
|
|
75437
75692
|
for (const edge of candidates.slice(0, maxEdges + maxNodes)) {
|
|
@@ -75462,22 +75717,22 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75462
75717
|
frontier = [...next].sort();
|
|
75463
75718
|
}
|
|
75464
75719
|
const nodeKeys = [...distances.keys()];
|
|
75465
|
-
const pageRows = nodeKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(
|
|
75466
|
-
|
|
75467
|
-
|
|
75468
|
-
|
|
75469
|
-
|
|
75720
|
+
const pageRows = nodeKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and92(
|
|
75721
|
+
eq109(siteCrawlPages.projectId, scope.projectId),
|
|
75722
|
+
eq109(siteCrawlPages.runId, scope.runId),
|
|
75723
|
+
eq109(siteCrawlPages.attemptId, scope.attemptId),
|
|
75724
|
+
inArray49(siteCrawlPages.nodeKey, nodeKeys)
|
|
75470
75725
|
)).all();
|
|
75471
75726
|
const persistedNodeKeys = new Set(pageRows.map((row) => row.nodeKey));
|
|
75472
75727
|
const outermostNodeKeys = pageRows.filter((row) => distances.get(row.nodeKey) === hops).map((row) => row.nodeKey);
|
|
75473
|
-
const omittedOutermostEdge = outermostNodeKeys.length === 0 ? void 0 : app.db.select({ edgeKey: siteCrawlEdges.edgeKey }).from(siteCrawlEdges).where(
|
|
75474
|
-
|
|
75475
|
-
|
|
75476
|
-
|
|
75477
|
-
|
|
75728
|
+
const omittedOutermostEdge = outermostNodeKeys.length === 0 ? void 0 : app.db.select({ edgeKey: siteCrawlEdges.edgeKey }).from(siteCrawlEdges).where(and92(
|
|
75729
|
+
eq109(siteCrawlEdges.projectId, scope.projectId),
|
|
75730
|
+
eq109(siteCrawlEdges.runId, scope.runId),
|
|
75731
|
+
eq109(siteCrawlEdges.attemptId, scope.attemptId),
|
|
75732
|
+
eq109(siteCrawlEdges.internal, true),
|
|
75478
75733
|
isNotNull10(siteCrawlEdges.targetNodeKey),
|
|
75479
|
-
|
|
75480
|
-
|
|
75734
|
+
inArray49(siteCrawlEdges.sourceNodeKey, outermostNodeKeys),
|
|
75735
|
+
inArray49(siteCrawlEdges.targetNodeKey, outermostNodeKeys)
|
|
75481
75736
|
)).limit(1).get();
|
|
75482
75737
|
if (omittedOutermostEdge && !seenEdgeKeys.has(omittedOutermostEdge.edgeKey)) {
|
|
75483
75738
|
omittedEdgeKeys.add(omittedOutermostEdge.edgeKey);
|
|
@@ -75570,23 +75825,23 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75570
75825
|
let found = fromPage.nodeKey === toPage.nodeKey;
|
|
75571
75826
|
let truncated = false;
|
|
75572
75827
|
for (let depth = 0; depth < maxDepth && frontier.length > 0 && !found; depth += 1) {
|
|
75573
|
-
const candidates = app.db.select().from(siteCrawlEdges).where(
|
|
75574
|
-
|
|
75575
|
-
|
|
75576
|
-
|
|
75577
|
-
|
|
75578
|
-
|
|
75579
|
-
|
|
75828
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and92(
|
|
75829
|
+
eq109(siteCrawlEdges.projectId, scope.projectId),
|
|
75830
|
+
eq109(siteCrawlEdges.runId, scope.runId),
|
|
75831
|
+
eq109(siteCrawlEdges.attemptId, scope.attemptId),
|
|
75832
|
+
eq109(siteCrawlEdges.internal, true),
|
|
75833
|
+
eq109(siteCrawlEdges.followable, true),
|
|
75834
|
+
eq109(siteCrawlEdges.relation, "anchor"),
|
|
75580
75835
|
isNotNull10(siteCrawlEdges.targetNodeKey),
|
|
75581
|
-
|
|
75582
|
-
)).orderBy(
|
|
75836
|
+
inArray49(siteCrawlEdges.sourceNodeKey, frontier)
|
|
75837
|
+
)).orderBy(asc26(siteCrawlEdges.edgeKey)).limit(SITE_HEALTH_PATH_MAX_VISITED_NODES + 1).all();
|
|
75583
75838
|
if (candidates.length > SITE_HEALTH_PATH_MAX_VISITED_NODES) truncated = true;
|
|
75584
75839
|
const candidateTargetKeys = [...new Set(candidates.map((edge) => edge.targetNodeKey).filter((nodeKey) => nodeKey != null))];
|
|
75585
|
-
const persistedTargetKeys = new Set(candidateTargetKeys.length === 0 ? [] : app.db.select({ nodeKey: siteCrawlPages.nodeKey }).from(siteCrawlPages).where(
|
|
75586
|
-
|
|
75587
|
-
|
|
75588
|
-
|
|
75589
|
-
|
|
75840
|
+
const persistedTargetKeys = new Set(candidateTargetKeys.length === 0 ? [] : app.db.select({ nodeKey: siteCrawlPages.nodeKey }).from(siteCrawlPages).where(and92(
|
|
75841
|
+
eq109(siteCrawlPages.projectId, scope.projectId),
|
|
75842
|
+
eq109(siteCrawlPages.runId, scope.runId),
|
|
75843
|
+
eq109(siteCrawlPages.attemptId, scope.attemptId),
|
|
75844
|
+
inArray49(siteCrawlPages.nodeKey, candidateTargetKeys)
|
|
75590
75845
|
)).all().map((row) => row.nodeKey));
|
|
75591
75846
|
const next = /* @__PURE__ */ new Set();
|
|
75592
75847
|
for (const edge of candidates.slice(0, SITE_HEALTH_PATH_MAX_VISITED_NODES)) {
|
|
@@ -75627,11 +75882,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75627
75882
|
pathEdges.unshift(step.edge);
|
|
75628
75883
|
pathKeys.unshift(step.nodeKey);
|
|
75629
75884
|
}
|
|
75630
|
-
const pathRows = app.db.select().from(siteCrawlPages).where(
|
|
75631
|
-
|
|
75632
|
-
|
|
75633
|
-
|
|
75634
|
-
|
|
75885
|
+
const pathRows = app.db.select().from(siteCrawlPages).where(and92(
|
|
75886
|
+
eq109(siteCrawlPages.projectId, scope.projectId),
|
|
75887
|
+
eq109(siteCrawlPages.runId, scope.runId),
|
|
75888
|
+
eq109(siteCrawlPages.attemptId, scope.attemptId),
|
|
75889
|
+
inArray49(siteCrawlPages.nodeKey, pathKeys)
|
|
75635
75890
|
)).all();
|
|
75636
75891
|
const pageByKey = new Map(pathRows.map((row) => [row.nodeKey, row]));
|
|
75637
75892
|
const pathDetection = templateDetectionOf(snapshot.templateDetection);
|
|
@@ -75659,25 +75914,25 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75659
75914
|
throw validationError("change must be all, added, removed, or changed");
|
|
75660
75915
|
}
|
|
75661
75916
|
const completeSnapshotFilters = [
|
|
75662
|
-
|
|
75663
|
-
|
|
75664
|
-
|
|
75665
|
-
|
|
75666
|
-
|
|
75917
|
+
eq109(siteCrawlSnapshots.projectId, project.id),
|
|
75918
|
+
eq109(runs.projectId, project.id),
|
|
75919
|
+
eq109(runs.kind, RunKinds["site-audit"]),
|
|
75920
|
+
eq109(runs.status, RunStatuses.completed),
|
|
75921
|
+
eq109(siteCrawlSnapshots.complete, true),
|
|
75667
75922
|
notProbeRun()
|
|
75668
75923
|
];
|
|
75669
|
-
const afterTarget = request.query.toRunId ? resolveCrawl(project.id, request.query.toRunId) : app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
75924
|
+
const afterTarget = request.query.toRunId ? resolveCrawl(project.id, request.query.toRunId) : app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq109(siteCrawlSnapshots.runId, runs.id)).where(and92(...completeSnapshotFilters)).orderBy(desc50(siteCrawlSnapshots.createdAt), desc50(siteCrawlSnapshots.runId)).limit(1).get();
|
|
75670
75925
|
if (request.query.toRunId && !afterTarget) throw notFound("Site crawl run", request.query.toRunId);
|
|
75671
|
-
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
75926
|
+
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq109(siteCrawlSnapshots.runId, runs.id)).where(and92(
|
|
75672
75927
|
...completeSnapshotFilters,
|
|
75673
75928
|
or21(
|
|
75674
75929
|
lt17(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
75675
|
-
|
|
75676
|
-
|
|
75930
|
+
and92(
|
|
75931
|
+
eq109(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
75677
75932
|
lt17(siteCrawlSnapshots.runId, afterTarget.snapshot.runId)
|
|
75678
75933
|
)
|
|
75679
75934
|
)
|
|
75680
|
-
)).orderBy(
|
|
75935
|
+
)).orderBy(desc50(siteCrawlSnapshots.createdAt), desc50(siteCrawlSnapshots.runId)).limit(1).get() : void 0;
|
|
75681
75936
|
if (request.query.fromRunId && !beforeTarget) throw notFound("Site crawl run", request.query.fromRunId);
|
|
75682
75937
|
if (!afterTarget) {
|
|
75683
75938
|
const latestCrawl = resolveCrawl(project.id);
|
|
@@ -75761,9 +76016,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75761
76016
|
const summary4 = { pages: zero2(), links: zero2() };
|
|
75762
76017
|
if (scopeFilter !== "links") {
|
|
75763
76018
|
if (includeAdded || includeChanged) {
|
|
75764
|
-
const addedExpression = includeAdded ?
|
|
75765
|
-
const changedExpression = includeChanged ?
|
|
75766
|
-
const [counts2 = { added: 0, changed: 0 }] = app.db.all(
|
|
76019
|
+
const addedExpression = includeAdded ? sql43`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql43`0`;
|
|
76020
|
+
const changedExpression = includeChanged ? sql43`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${PAGE_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql43`0`;
|
|
76021
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql43`
|
|
75767
76022
|
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
75768
76023
|
FROM site_crawl_pages AS current
|
|
75769
76024
|
LEFT JOIN site_crawl_pages AS previous
|
|
@@ -75779,7 +76034,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75779
76034
|
summary4.pages.changed = Number(counts2.changed);
|
|
75780
76035
|
}
|
|
75781
76036
|
if (includeRemoved) {
|
|
75782
|
-
const [counts2 = { removed: 0 }] = app.db.all(
|
|
76037
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql43`
|
|
75783
76038
|
SELECT COUNT(*) AS removed
|
|
75784
76039
|
FROM site_crawl_pages AS previous
|
|
75785
76040
|
LEFT JOIN site_crawl_pages AS current
|
|
@@ -75797,9 +76052,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75797
76052
|
}
|
|
75798
76053
|
if (scopeFilter !== "pages") {
|
|
75799
76054
|
if (includeAdded || includeChanged) {
|
|
75800
|
-
const addedExpression = includeAdded ?
|
|
75801
|
-
const changedExpression = includeChanged ?
|
|
75802
|
-
const [counts2 = { added: 0, changed: 0 }] = app.db.all(
|
|
76055
|
+
const addedExpression = includeAdded ? sql43`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql43`0`;
|
|
76056
|
+
const changedExpression = includeChanged ? sql43`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${LINK_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql43`0`;
|
|
76057
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql43`
|
|
75803
76058
|
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
75804
76059
|
FROM site_crawl_edges AS current
|
|
75805
76060
|
LEFT JOIN site_crawl_edges AS previous
|
|
@@ -75817,7 +76072,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75817
76072
|
summary4.links.changed = Number(counts2.changed);
|
|
75818
76073
|
}
|
|
75819
76074
|
if (includeRemoved) {
|
|
75820
|
-
const [counts2 = { removed: 0 }] = app.db.all(
|
|
76075
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql43`
|
|
75821
76076
|
SELECT COUNT(*) AS removed
|
|
75822
76077
|
FROM site_crawl_edges AS previous
|
|
75823
76078
|
LEFT JOIN site_crawl_edges AS current
|
|
@@ -75847,48 +76102,48 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75847
76102
|
const includeRemoved = changeFilter === "all" || changeFilter === "removed";
|
|
75848
76103
|
const table = entity === "page" ? "site_crawl_pages" : "site_crawl_edges";
|
|
75849
76104
|
const keyColumn = entity === "page" ? "node_key" : "edge_key";
|
|
75850
|
-
const internalCurrent = entity === "link" ?
|
|
75851
|
-
const internalPreviousJoin = entity === "link" ?
|
|
75852
|
-
const internalPrevious = entity === "link" ?
|
|
75853
|
-
const internalCurrentJoin = entity === "link" ?
|
|
76105
|
+
const internalCurrent = entity === "link" ? sql43`AND current.internal = 1` : sql43``;
|
|
76106
|
+
const internalPreviousJoin = entity === "link" ? sql43`AND previous.internal = 1` : sql43``;
|
|
76107
|
+
const internalPrevious = entity === "link" ? sql43`AND previous.internal = 1` : sql43``;
|
|
76108
|
+
const internalCurrentJoin = entity === "link" ? sql43`AND current.internal = 1` : sql43``;
|
|
75854
76109
|
const difference3 = entity === "page" ? PAGE_CHANGE_PREDICATE : LINK_CHANGE_PREDICATE;
|
|
75855
|
-
const afterKind = changeFilter === "added" ?
|
|
75856
|
-
const afterRows = includeAfter ? app.db.all(
|
|
76110
|
+
const afterKind = changeFilter === "added" ? sql43`previous.id IS NULL` : changeFilter === "changed" ? sql43`previous.id IS NOT NULL AND ${difference3}` : sql43`previous.id IS NULL OR (previous.id IS NOT NULL AND ${difference3})`;
|
|
76111
|
+
const afterRows = includeAfter ? app.db.all(sql43`
|
|
75857
76112
|
SELECT ${entity} AS entity,
|
|
75858
76113
|
CASE WHEN previous.id IS NULL THEN 'added' ELSE 'changed' END AS change,
|
|
75859
|
-
current.${
|
|
75860
|
-
FROM ${
|
|
75861
|
-
LEFT JOIN ${
|
|
76114
|
+
current.${sql43.raw(keyColumn)} AS key
|
|
76115
|
+
FROM ${sql43.raw(table)} AS current
|
|
76116
|
+
LEFT JOIN ${sql43.raw(table)} AS previous
|
|
75862
76117
|
ON previous.project_id = ${beforeScope.projectId}
|
|
75863
76118
|
AND previous.run_id = ${beforeScope.runId}
|
|
75864
76119
|
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
75865
76120
|
${internalPreviousJoin}
|
|
75866
|
-
AND previous.${
|
|
76121
|
+
AND previous.${sql43.raw(keyColumn)} = current.${sql43.raw(keyColumn)}
|
|
75867
76122
|
WHERE current.project_id = ${afterScope.projectId}
|
|
75868
76123
|
AND current.run_id = ${afterScope.runId}
|
|
75869
76124
|
AND current.attempt_id = ${afterScope.attemptId}
|
|
75870
76125
|
${internalCurrent}
|
|
75871
|
-
AND current.${
|
|
76126
|
+
AND current.${sql43.raw(keyColumn)} > ${afterKey}
|
|
75872
76127
|
AND (${afterKind})
|
|
75873
|
-
ORDER BY current.${
|
|
76128
|
+
ORDER BY current.${sql43.raw(keyColumn)} ASC
|
|
75874
76129
|
LIMIT ${wanted2}
|
|
75875
76130
|
`) : [];
|
|
75876
|
-
const removedRows = includeRemoved ? app.db.all(
|
|
75877
|
-
SELECT ${entity} AS entity, 'removed' AS change, previous.${
|
|
75878
|
-
FROM ${
|
|
75879
|
-
LEFT JOIN ${
|
|
76131
|
+
const removedRows = includeRemoved ? app.db.all(sql43`
|
|
76132
|
+
SELECT ${entity} AS entity, 'removed' AS change, previous.${sql43.raw(keyColumn)} AS key
|
|
76133
|
+
FROM ${sql43.raw(table)} AS previous
|
|
76134
|
+
LEFT JOIN ${sql43.raw(table)} AS current
|
|
75880
76135
|
ON current.project_id = ${afterScope.projectId}
|
|
75881
76136
|
AND current.run_id = ${afterScope.runId}
|
|
75882
76137
|
AND current.attempt_id = ${afterScope.attemptId}
|
|
75883
76138
|
${internalCurrentJoin}
|
|
75884
|
-
AND current.${
|
|
76139
|
+
AND current.${sql43.raw(keyColumn)} = previous.${sql43.raw(keyColumn)}
|
|
75885
76140
|
WHERE previous.project_id = ${beforeScope.projectId}
|
|
75886
76141
|
AND previous.run_id = ${beforeScope.runId}
|
|
75887
76142
|
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
75888
76143
|
${internalPrevious}
|
|
75889
|
-
AND previous.${
|
|
76144
|
+
AND previous.${sql43.raw(keyColumn)} > ${afterKey}
|
|
75890
76145
|
AND current.id IS NULL
|
|
75891
|
-
ORDER BY previous.${
|
|
76146
|
+
ORDER BY previous.${sql43.raw(keyColumn)} ASC
|
|
75892
76147
|
LIMIT ${wanted2}
|
|
75893
76148
|
`) : [];
|
|
75894
76149
|
const merged = [];
|
|
@@ -75918,18 +76173,18 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
75918
76173
|
const visibleKeys = keyRows.slice(0, limit);
|
|
75919
76174
|
const pageKeys = visibleKeys.filter((row) => row.entity === "page").map((row) => row.key);
|
|
75920
76175
|
const linkKeys = visibleKeys.filter((row) => row.entity === "link").map((row) => row.key);
|
|
75921
|
-
const pageRowsFor = (scope) => pageKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(
|
|
75922
|
-
|
|
75923
|
-
|
|
75924
|
-
|
|
75925
|
-
|
|
76176
|
+
const pageRowsFor = (scope) => pageKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and92(
|
|
76177
|
+
eq109(siteCrawlPages.projectId, scope.projectId),
|
|
76178
|
+
eq109(siteCrawlPages.runId, scope.runId),
|
|
76179
|
+
eq109(siteCrawlPages.attemptId, scope.attemptId),
|
|
76180
|
+
inArray49(siteCrawlPages.nodeKey, pageKeys)
|
|
75926
76181
|
)).all();
|
|
75927
|
-
const edgeRowsFor = (scope) => linkKeys.length === 0 ? [] : app.db.select().from(siteCrawlEdges).where(
|
|
75928
|
-
|
|
75929
|
-
|
|
75930
|
-
|
|
75931
|
-
|
|
75932
|
-
|
|
76182
|
+
const edgeRowsFor = (scope) => linkKeys.length === 0 ? [] : app.db.select().from(siteCrawlEdges).where(and92(
|
|
76183
|
+
eq109(siteCrawlEdges.projectId, scope.projectId),
|
|
76184
|
+
eq109(siteCrawlEdges.runId, scope.runId),
|
|
76185
|
+
eq109(siteCrawlEdges.attemptId, scope.attemptId),
|
|
76186
|
+
eq109(siteCrawlEdges.internal, true),
|
|
76187
|
+
inArray49(siteCrawlEdges.edgeKey, linkKeys)
|
|
75933
76188
|
)).all();
|
|
75934
76189
|
const beforePages = new Map(pageRowsFor(beforeScope).map((row) => [row.nodeKey, mapCrawlPage(row)]));
|
|
75935
76190
|
const afterPages = new Map(pageRowsFor(afterScope).map((row) => [row.nodeKey, mapCrawlPage(row)]));
|
|
@@ -76102,16 +76357,16 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76102
76357
|
}
|
|
76103
76358
|
const inventorySummary = inventorySummaryFor(detailScopeFor(project.id, snapshot));
|
|
76104
76359
|
const filters = [
|
|
76105
|
-
|
|
76106
|
-
|
|
76107
|
-
|
|
76360
|
+
eq109(siteCrawlPages.projectId, project.id),
|
|
76361
|
+
eq109(siteCrawlPages.runId, snapshot.runId),
|
|
76362
|
+
eq109(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
76108
76363
|
];
|
|
76109
|
-
if (request.query.nodeKey) filters.push(
|
|
76364
|
+
if (request.query.nodeKey) filters.push(eq109(siteCrawlPages.nodeKey, request.query.nodeKey));
|
|
76110
76365
|
const inventoryEligible = parseBoolean2(request.query.inventoryEligible);
|
|
76111
|
-
if (inventoryEligible != null) filters.push(
|
|
76112
|
-
if (request.query.fetchState) filters.push(
|
|
76113
|
-
if (request.query.indexabilityState) filters.push(
|
|
76114
|
-
if (request.query.auditState) filters.push(
|
|
76366
|
+
if (inventoryEligible != null) filters.push(eq109(siteCrawlPages.inventoryEligible, inventoryEligible));
|
|
76367
|
+
if (request.query.fetchState) filters.push(eq109(siteCrawlPages.fetchState, request.query.fetchState));
|
|
76368
|
+
if (request.query.indexabilityState) filters.push(eq109(siteCrawlPages.indexabilityState, request.query.indexabilityState));
|
|
76369
|
+
if (request.query.auditState) filters.push(eq109(siteCrawlPages.auditState, request.query.auditState));
|
|
76115
76370
|
const urlVersions = parseUrlVersionFilter(request.query.urlVersions);
|
|
76116
76371
|
if (urlVersions === SiteCrawlUrlVersionFilters.exclude) filters.push(isNull17(siteCrawlPages.urlVersionOf));
|
|
76117
76372
|
if (urlVersions === SiteCrawlUrlVersionFilters.only) filters.push(isNotNull10(siteCrawlPages.urlVersionOf));
|
|
@@ -76134,17 +76389,17 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76134
76389
|
sort
|
|
76135
76390
|
})).digest("base64url");
|
|
76136
76391
|
const offset = decodeCrawlPageCursor(request.query.cursor, selectionKey);
|
|
76137
|
-
const orderBy = sort === "score-desc" ? [
|
|
76392
|
+
const orderBy = sort === "score-desc" ? [desc50(siteCrawlPages.auditScore), asc26(siteCrawlPages.nodeKey)] : sort === "score-asc" ? [asc26(siteCrawlPages.auditScore), asc26(siteCrawlPages.nodeKey)] : sort === "path" ? [asc26(siteCrawlPages.path), asc26(siteCrawlPages.nodeKey)] : [asc26(siteCrawlPages.url), asc26(siteCrawlPages.nodeKey)];
|
|
76138
76393
|
let healthStateFilter = null;
|
|
76139
76394
|
if (healthState) {
|
|
76140
|
-
const legacyRow = app.db.select({ id: siteCrawlPages.id }).from(siteCrawlPages).where(
|
|
76141
|
-
|
|
76142
|
-
|
|
76143
|
-
|
|
76395
|
+
const legacyRow = app.db.select({ id: siteCrawlPages.id }).from(siteCrawlPages).where(and92(
|
|
76396
|
+
eq109(siteCrawlPages.projectId, project.id),
|
|
76397
|
+
eq109(siteCrawlPages.runId, snapshot.runId),
|
|
76398
|
+
eq109(siteCrawlPages.attemptId, snapshot.attemptId),
|
|
76144
76399
|
isNull17(siteCrawlPages.healthState)
|
|
76145
76400
|
)).limit(1).get();
|
|
76146
76401
|
healthStateFilter = legacyRow ? "unavailable-legacy-scan" : "applied";
|
|
76147
|
-
if (healthStateFilter === "applied") filters.push(
|
|
76402
|
+
if (healthStateFilter === "applied") filters.push(eq109(siteCrawlPages.healthState, healthState));
|
|
76148
76403
|
}
|
|
76149
76404
|
if (healthStateFilter === "unavailable-legacy-scan") {
|
|
76150
76405
|
return {
|
|
@@ -76161,7 +76416,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76161
76416
|
pages: []
|
|
76162
76417
|
};
|
|
76163
76418
|
}
|
|
76164
|
-
const where =
|
|
76419
|
+
const where = and92(...filters);
|
|
76165
76420
|
const total = app.db.select({ value: count8() }).from(siteCrawlPages).where(where).get()?.value ?? 0;
|
|
76166
76421
|
const rows = app.db.select().from(siteCrawlPages).where(where).orderBy(...orderBy).limit(limit).offset(offset).all();
|
|
76167
76422
|
const nextOffset = offset + rows.length;
|
|
@@ -76196,10 +76451,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76196
76451
|
url: siteCrawlPages.url,
|
|
76197
76452
|
inventoryEligible: siteCrawlPages.inventoryEligible,
|
|
76198
76453
|
fetchState: siteCrawlPages.fetchState
|
|
76199
|
-
}).from(siteCrawlPages).where(
|
|
76200
|
-
|
|
76201
|
-
|
|
76202
|
-
|
|
76454
|
+
}).from(siteCrawlPages).where(and92(
|
|
76455
|
+
eq109(siteCrawlPages.projectId, project.id),
|
|
76456
|
+
eq109(siteCrawlPages.runId, snapshot.runId),
|
|
76457
|
+
eq109(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
76203
76458
|
)).limit(MAX_STRUCTURE_SOURCE_ROWS + 1).all();
|
|
76204
76459
|
if (sourceRows.length > MAX_STRUCTURE_SOURCE_ROWS) {
|
|
76205
76460
|
throw validationError(`Persisted crawl exceeds the ${MAX_STRUCTURE_SOURCE_ROWS}-page structure limit`);
|
|
@@ -76281,21 +76536,21 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76281
76536
|
};
|
|
76282
76537
|
}
|
|
76283
76538
|
const filters = [
|
|
76284
|
-
|
|
76285
|
-
|
|
76286
|
-
|
|
76287
|
-
|
|
76539
|
+
eq109(siteCrawlEdges.projectId, project.id),
|
|
76540
|
+
eq109(siteCrawlEdges.runId, snapshot.runId),
|
|
76541
|
+
eq109(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
76542
|
+
eq109(siteCrawlEdges.internal, true),
|
|
76288
76543
|
linkKindFilter(linkKind)
|
|
76289
76544
|
];
|
|
76290
|
-
if (request.query.sourceUrl) filters.push(
|
|
76291
|
-
if (request.query.targetUrl) filters.push(
|
|
76545
|
+
if (request.query.sourceUrl) filters.push(eq109(siteCrawlEdges.sourceUrl, request.query.sourceUrl));
|
|
76546
|
+
if (request.query.targetUrl) filters.push(eq109(siteCrawlEdges.targetUrl, request.query.targetUrl));
|
|
76292
76547
|
const followable = parseBoolean2(request.query.followable);
|
|
76293
|
-
if (followable != null) filters.push(
|
|
76294
|
-
const where =
|
|
76548
|
+
if (followable != null) filters.push(eq109(siteCrawlEdges.followable, followable));
|
|
76549
|
+
const where = and92(...filters);
|
|
76295
76550
|
const total = app.db.select({ value: count8() }).from(siteCrawlEdges).where(where).get()?.value ?? 0;
|
|
76296
76551
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
76297
76552
|
const offset = decodeCursor(request.query.cursor);
|
|
76298
|
-
const rows = app.db.select().from(siteCrawlEdges).where(where).orderBy(
|
|
76553
|
+
const rows = app.db.select().from(siteCrawlEdges).where(where).orderBy(asc26(siteCrawlEdges.edgeKey)).limit(limit).offset(offset).all();
|
|
76299
76554
|
const nextOffset = offset + rows.length;
|
|
76300
76555
|
return {
|
|
76301
76556
|
project: project.name,
|
|
@@ -76352,17 +76607,17 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76352
76607
|
};
|
|
76353
76608
|
}
|
|
76354
76609
|
const scope = [
|
|
76355
|
-
|
|
76356
|
-
|
|
76357
|
-
|
|
76358
|
-
|
|
76610
|
+
eq109(siteCrawlEdges.projectId, project.id),
|
|
76611
|
+
eq109(siteCrawlEdges.runId, snapshot.runId),
|
|
76612
|
+
eq109(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
76613
|
+
eq109(siteCrawlEdges.internal, true),
|
|
76359
76614
|
linkKindFilter(linkKind)
|
|
76360
76615
|
];
|
|
76361
|
-
const inboundMatch = request.query.nodeKey && request.query.url ? or21(
|
|
76362
|
-
const outboundMatch = request.query.nodeKey && request.query.url ? or21(
|
|
76616
|
+
const inboundMatch = request.query.nodeKey && request.query.url ? or21(eq109(siteCrawlEdges.targetNodeKey, request.query.nodeKey), eq109(siteCrawlEdges.targetUrl, request.query.url)) : request.query.nodeKey ? eq109(siteCrawlEdges.targetNodeKey, request.query.nodeKey) : eq109(siteCrawlEdges.targetUrl, request.query.url);
|
|
76617
|
+
const outboundMatch = request.query.nodeKey && request.query.url ? or21(eq109(siteCrawlEdges.sourceNodeKey, request.query.nodeKey), eq109(siteCrawlEdges.sourceUrl, request.query.url)) : request.query.nodeKey ? eq109(siteCrawlEdges.sourceNodeKey, request.query.nodeKey) : eq109(siteCrawlEdges.sourceUrl, request.query.url);
|
|
76363
76618
|
const limit = parseBoundedLimit(request.query.limit, 50, 100);
|
|
76364
|
-
const inboundRows = app.db.select().from(siteCrawlEdges).where(
|
|
76365
|
-
const outboundRows = app.db.select().from(siteCrawlEdges).where(
|
|
76619
|
+
const inboundRows = app.db.select().from(siteCrawlEdges).where(and92(...scope, inboundMatch)).orderBy(asc26(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
76620
|
+
const outboundRows = app.db.select().from(siteCrawlEdges).where(and92(...scope, outboundMatch)).orderBy(asc26(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
76366
76621
|
return {
|
|
76367
76622
|
project: project.name,
|
|
76368
76623
|
hasCrawlData: true,
|
|
@@ -76396,11 +76651,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76396
76651
|
if (!snapshot.attemptId) {
|
|
76397
76652
|
return { project: project.name, runId: snapshot.runId, state: "partial", checkDeadLinks: true, checked: snapshot.deadLinksChecked, found: snapshot.deadLinksFound, unverified: snapshot.deadLinksUnverified, total: 0, nextCursor: null, deadLinks: [], runSelection: target.runSelection };
|
|
76398
76653
|
}
|
|
76399
|
-
const where =
|
|
76400
|
-
|
|
76401
|
-
|
|
76402
|
-
|
|
76403
|
-
|
|
76654
|
+
const where = and92(
|
|
76655
|
+
eq109(siteCrawlFindings.projectId, project.id),
|
|
76656
|
+
eq109(siteCrawlFindings.runId, snapshot.runId),
|
|
76657
|
+
eq109(siteCrawlFindings.attemptId, snapshot.attemptId),
|
|
76658
|
+
eq109(siteCrawlFindings.findingType, "dead-link"),
|
|
76404
76659
|
// Belt and braces on the client-facing surface: a row without a status
|
|
76405
76660
|
// code is not evidence of a broken link, and this response is where such
|
|
76406
76661
|
// a row would be read as one. Migration 140 removed every stored row that
|
|
@@ -76410,12 +76665,12 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76410
76665
|
// a fabricated broken link shown to a client is not worth the saved
|
|
76411
76666
|
// predicate. Applied to `total` as well as the rows, or the count would
|
|
76412
76667
|
// promise a page the list cannot fill.
|
|
76413
|
-
isNotNull10(
|
|
76668
|
+
isNotNull10(sql43`json_extract(${siteCrawlFindings.evidence}, '$.statusCode')`)
|
|
76414
76669
|
);
|
|
76415
76670
|
const total = app.db.select({ value: count8() }).from(siteCrawlFindings).where(where).get()?.value ?? 0;
|
|
76416
76671
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
76417
76672
|
const offset = decodeCursor(request.query.cursor);
|
|
76418
|
-
const rows = app.db.select().from(siteCrawlFindings).where(where).orderBy(
|
|
76673
|
+
const rows = app.db.select().from(siteCrawlFindings).where(where).orderBy(asc26(siteCrawlFindings.findingKey)).limit(limit).offset(offset).all();
|
|
76419
76674
|
const nextOffset = offset + rows.length;
|
|
76420
76675
|
const deadLinks = rows.map((row) => ({
|
|
76421
76676
|
findingKey: row.findingKey,
|
|
@@ -76450,19 +76705,19 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76450
76705
|
createdAt: runs.createdAt,
|
|
76451
76706
|
startedAt: runs.startedAt,
|
|
76452
76707
|
finishedAt: runs.finishedAt
|
|
76453
|
-
}).from(runs).where(
|
|
76454
|
-
|
|
76455
|
-
|
|
76708
|
+
}).from(runs).where(and92(
|
|
76709
|
+
eq109(runs.projectId, project.id),
|
|
76710
|
+
eq109(runs.kind, RunKinds["site-audit"]),
|
|
76456
76711
|
notProbeRun()
|
|
76457
|
-
)).orderBy(
|
|
76712
|
+
)).orderBy(desc50(runs.createdAt), desc50(runs.id)).limit(limit).all();
|
|
76458
76713
|
const runIds = rows.map((row) => row.runId);
|
|
76459
|
-
const crawlRunIds = new Set(runIds.length === 0 ? [] : app.db.select({ runId: siteCrawlSnapshots.runId }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
76460
|
-
|
|
76461
|
-
|
|
76462
|
-
|
|
76463
|
-
|
|
76714
|
+
const crawlRunIds = new Set(runIds.length === 0 ? [] : app.db.select({ runId: siteCrawlSnapshots.runId }).from(siteCrawlSnapshots).innerJoin(runs, eq109(siteCrawlSnapshots.runId, runs.id)).where(and92(
|
|
76715
|
+
eq109(siteCrawlSnapshots.projectId, project.id),
|
|
76716
|
+
eq109(runs.projectId, project.id),
|
|
76717
|
+
eq109(runs.kind, RunKinds["site-audit"]),
|
|
76718
|
+
inArray49(runs.status, SURFACEABLE_STATUSES),
|
|
76464
76719
|
notProbeRun(),
|
|
76465
|
-
|
|
76720
|
+
inArray49(siteCrawlSnapshots.runId, runIds)
|
|
76466
76721
|
)).all().map((row) => row.runId));
|
|
76467
76722
|
return {
|
|
76468
76723
|
project: project.name,
|
|
@@ -76476,21 +76731,21 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76476
76731
|
});
|
|
76477
76732
|
app.get("/projects/:name/technical-aeo/runs/:runId/progress", async (request) => {
|
|
76478
76733
|
const project = resolveProject(app.db, request.params.name);
|
|
76479
|
-
const run = app.db.select().from(runs).where(
|
|
76480
|
-
|
|
76481
|
-
|
|
76482
|
-
|
|
76734
|
+
const run = app.db.select().from(runs).where(and92(
|
|
76735
|
+
eq109(runs.id, request.params.runId),
|
|
76736
|
+
eq109(runs.projectId, project.id),
|
|
76737
|
+
eq109(runs.kind, RunKinds["site-audit"]),
|
|
76483
76738
|
notProbeRun()
|
|
76484
76739
|
)).get();
|
|
76485
76740
|
if (!run) throw notFound("Site audit run", request.params.runId);
|
|
76486
|
-
const attempt = app.db.select().from(siteCrawlAttempts).where(
|
|
76487
|
-
|
|
76488
|
-
|
|
76489
|
-
)).orderBy(
|
|
76490
|
-
const persistedLayout = attempt ? app.db.select().from(siteCrawlGraphLayouts).where(
|
|
76491
|
-
|
|
76492
|
-
|
|
76493
|
-
|
|
76741
|
+
const attempt = app.db.select().from(siteCrawlAttempts).where(and92(
|
|
76742
|
+
eq109(siteCrawlAttempts.projectId, project.id),
|
|
76743
|
+
eq109(siteCrawlAttempts.runId, run.id)
|
|
76744
|
+
)).orderBy(desc50(siteCrawlAttempts.attemptNumber), desc50(siteCrawlAttempts.updatedAt)).limit(1).get();
|
|
76745
|
+
const persistedLayout = attempt ? app.db.select().from(siteCrawlGraphLayouts).where(and92(
|
|
76746
|
+
eq109(siteCrawlGraphLayouts.projectId, project.id),
|
|
76747
|
+
eq109(siteCrawlGraphLayouts.runId, run.id),
|
|
76748
|
+
eq109(siteCrawlGraphLayouts.attemptId, attempt.id)
|
|
76494
76749
|
)).get() : void 0;
|
|
76495
76750
|
const layoutState = persistedLayout?.state === "ready" || persistedLayout?.state === "unavailable" ? persistedLayout.state : run.status === RunStatuses.completed || run.status === RunStatuses.partial ? "unavailable" : "pending";
|
|
76496
76751
|
return {
|
|
@@ -76521,19 +76776,19 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76521
76776
|
};
|
|
76522
76777
|
});
|
|
76523
76778
|
app.get("/projects/:name/technical-aeo/runs/:runId/page-health-preview", async (request) => app.db.transaction((tx) => {
|
|
76524
|
-
const project = tx.select().from(projects).where(
|
|
76779
|
+
const project = tx.select().from(projects).where(eq109(projects.name, request.params.name)).get();
|
|
76525
76780
|
if (!project) throw notFound("Project", request.params.name);
|
|
76526
|
-
const run = tx.select().from(runs).where(
|
|
76527
|
-
|
|
76528
|
-
|
|
76529
|
-
|
|
76781
|
+
const run = tx.select().from(runs).where(and92(
|
|
76782
|
+
eq109(runs.id, request.params.runId),
|
|
76783
|
+
eq109(runs.projectId, project.id),
|
|
76784
|
+
eq109(runs.kind, RunKinds["site-audit"]),
|
|
76530
76785
|
notProbeRun()
|
|
76531
76786
|
)).get();
|
|
76532
76787
|
if (!run) throw notFound("Site audit run", request.params.runId);
|
|
76533
|
-
const attempt = tx.select().from(siteCrawlAttempts).where(
|
|
76534
|
-
|
|
76535
|
-
|
|
76536
|
-
)).orderBy(
|
|
76788
|
+
const attempt = tx.select().from(siteCrawlAttempts).where(and92(
|
|
76789
|
+
eq109(siteCrawlAttempts.projectId, project.id),
|
|
76790
|
+
eq109(siteCrawlAttempts.runId, run.id)
|
|
76791
|
+
)).orderBy(desc50(siteCrawlAttempts.attemptNumber), desc50(siteCrawlAttempts.updatedAt)).limit(1).get();
|
|
76537
76792
|
const state = run.status === RunStatuses.queued ? "waiting" : run.status === RunStatuses.running ? "collecting" : "terminal";
|
|
76538
76793
|
const base = {
|
|
76539
76794
|
project: project.name,
|
|
@@ -76546,11 +76801,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76546
76801
|
if (!attempt) {
|
|
76547
76802
|
return { ...base, pagesAudited: 0, examples: [] };
|
|
76548
76803
|
}
|
|
76549
|
-
const auditedWhere =
|
|
76550
|
-
|
|
76551
|
-
|
|
76552
|
-
|
|
76553
|
-
|
|
76804
|
+
const auditedWhere = and92(
|
|
76805
|
+
eq109(siteCrawlPages.projectId, project.id),
|
|
76806
|
+
eq109(siteCrawlPages.runId, run.id),
|
|
76807
|
+
eq109(siteCrawlPages.attemptId, attempt.id),
|
|
76808
|
+
eq109(siteCrawlPages.auditState, "success")
|
|
76554
76809
|
);
|
|
76555
76810
|
const pagesAudited = tx.select({ value: count8() }).from(siteCrawlPages).where(auditedWhere).get()?.value ?? 0;
|
|
76556
76811
|
if (state !== "collecting") {
|
|
@@ -76561,11 +76816,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76561
76816
|
url: siteCrawlPages.url,
|
|
76562
76817
|
auditScore: siteCrawlPages.auditScore,
|
|
76563
76818
|
auditFields: siteCrawlPages.auditFields
|
|
76564
|
-
}).from(siteCrawlPages).where(
|
|
76819
|
+
}).from(siteCrawlPages).where(and92(
|
|
76565
76820
|
auditedWhere,
|
|
76566
76821
|
isNotNull10(siteCrawlPages.auditScore),
|
|
76567
76822
|
lt17(siteCrawlPages.auditScore, 70)
|
|
76568
|
-
)).orderBy(
|
|
76823
|
+
)).orderBy(asc26(siteCrawlPages.auditScore), asc26(siteCrawlPages.nodeKey)).limit(LIVE_PAGE_HEALTH_CANDIDATE_LIMIT).all();
|
|
76569
76824
|
const examples = candidates.flatMap((candidate) => {
|
|
76570
76825
|
if (candidate.auditScore == null) return [];
|
|
76571
76826
|
const checksNeedingAttention = countLivePageHealthChecks(candidate.auditFields);
|
|
@@ -76597,13 +76852,13 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76597
76852
|
status: runs.status,
|
|
76598
76853
|
identityKey: siteCrawlRunRequests.identityKey,
|
|
76599
76854
|
effectiveOptions: siteCrawlRunRequests.effectiveOptions
|
|
76600
|
-
}).from(runs).leftJoin(siteCrawlRunRequests,
|
|
76601
|
-
|
|
76602
|
-
|
|
76603
|
-
)).where(
|
|
76604
|
-
|
|
76605
|
-
|
|
76606
|
-
|
|
76855
|
+
}).from(runs).leftJoin(siteCrawlRunRequests, and92(
|
|
76856
|
+
eq109(siteCrawlRunRequests.projectId, runs.projectId),
|
|
76857
|
+
eq109(siteCrawlRunRequests.runId, runs.id)
|
|
76858
|
+
)).where(and92(
|
|
76859
|
+
eq109(runs.projectId, project.id),
|
|
76860
|
+
eq109(runs.kind, RunKinds["site-audit"]),
|
|
76861
|
+
inArray49(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
76607
76862
|
)).get();
|
|
76608
76863
|
if (existing) {
|
|
76609
76864
|
if (existing.identityKey === identityKey) {
|
|
@@ -76657,7 +76912,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
76657
76912
|
}
|
|
76658
76913
|
|
|
76659
76914
|
// ../api-routes/src/trends.ts
|
|
76660
|
-
import { and as
|
|
76915
|
+
import { and as and93, asc as asc27, eq as eq110 } from "drizzle-orm";
|
|
76661
76916
|
function parseSurface(value) {
|
|
76662
76917
|
const parsed = trendSurfaceSchema.safeParse(value);
|
|
76663
76918
|
if (!parsed.success) {
|
|
@@ -76713,8 +76968,8 @@ async function trendsRoutes(app) {
|
|
|
76713
76968
|
function readSurface(app, projectId, surface) {
|
|
76714
76969
|
switch (surface) {
|
|
76715
76970
|
case TrendSurfaces["index-coverage"]: {
|
|
76716
|
-
const google = app.db.select({ date: gscCoverageSnapshots.date, indexed: gscCoverageSnapshots.indexed, notIndexed: gscCoverageSnapshots.notIndexed }).from(gscCoverageSnapshots).where(
|
|
76717
|
-
const bing = app.db.select({ date: bingCoverageSnapshots.date, indexed: bingCoverageSnapshots.indexed, notIndexed: bingCoverageSnapshots.notIndexed }).from(bingCoverageSnapshots).where(
|
|
76971
|
+
const google = app.db.select({ date: gscCoverageSnapshots.date, indexed: gscCoverageSnapshots.indexed, notIndexed: gscCoverageSnapshots.notIndexed }).from(gscCoverageSnapshots).where(eq110(gscCoverageSnapshots.projectId, projectId)).orderBy(asc27(gscCoverageSnapshots.date)).all();
|
|
76972
|
+
const bing = app.db.select({ date: bingCoverageSnapshots.date, indexed: bingCoverageSnapshots.indexed, notIndexed: bingCoverageSnapshots.notIndexed }).from(bingCoverageSnapshots).where(eq110(bingCoverageSnapshots.projectId, projectId)).orderBy(asc27(bingCoverageSnapshots.date)).all();
|
|
76718
76973
|
const coverage = (key, label, rows) => ({
|
|
76719
76974
|
key,
|
|
76720
76975
|
label,
|
|
@@ -76726,7 +76981,7 @@ function readSurface(app, projectId, surface) {
|
|
|
76726
76981
|
return [coverage("google", "Google", google), coverage("bing", "Bing", bing)];
|
|
76727
76982
|
}
|
|
76728
76983
|
case TrendSurfaces["site-health"]: {
|
|
76729
|
-
const rows = app.db.select({ auditedAt: siteAuditSnapshots.auditedAt, aggregateScore: siteAuditSnapshots.aggregateScore, pagesAudited: siteAuditSnapshots.pagesAudited }).from(siteAuditSnapshots).innerJoin(runs,
|
|
76984
|
+
const rows = app.db.select({ auditedAt: siteAuditSnapshots.auditedAt, aggregateScore: siteAuditSnapshots.aggregateScore, pagesAudited: siteAuditSnapshots.pagesAudited }).from(siteAuditSnapshots).innerJoin(runs, eq110(siteAuditSnapshots.runId, runs.id)).where(and93(...surfaceableSiteAuditFilters(projectId))).orderBy(asc27(siteAuditSnapshots.auditedAt)).all();
|
|
76730
76985
|
return [{
|
|
76731
76986
|
key: "site-health",
|
|
76732
76987
|
label: "Site Health",
|
|
@@ -76738,7 +76993,7 @@ function readSurface(app, projectId, surface) {
|
|
|
76738
76993
|
}
|
|
76739
76994
|
case TrendSurfaces.backlinks: {
|
|
76740
76995
|
const totals = computeFilteredReleaseTotals(app.db, projectId, BacklinkSources.commoncrawl);
|
|
76741
|
-
const rows = app.db.select({ release: backlinkSummaries.release, totalLinkingDomains: backlinkSummaries.totalLinkingDomains, queriedAt: backlinkSummaries.queriedAt }).from(backlinkSummaries).where(
|
|
76996
|
+
const rows = app.db.select({ release: backlinkSummaries.release, totalLinkingDomains: backlinkSummaries.totalLinkingDomains, queriedAt: backlinkSummaries.queriedAt }).from(backlinkSummaries).where(and93(eq110(backlinkSummaries.projectId, projectId), eq110(backlinkSummaries.source, BacklinkSources.commoncrawl))).all().sort((a, b) => compareReleaseIds(a.release, b.release)).map((row) => {
|
|
76742
76997
|
const filtered = totals.get(row.release);
|
|
76743
76998
|
const known = (filtered?.domainRows ?? 0) > 0 || row.totalLinkingDomains === 0;
|
|
76744
76999
|
const windowEnd = releaseWindowEndMonth(row.release);
|
|
@@ -76768,7 +77023,7 @@ function readSurface(app, projectId, surface) {
|
|
|
76768
77023
|
// ../api-routes/src/research.ts
|
|
76769
77024
|
import crypto60 from "crypto";
|
|
76770
77025
|
import { z as z7 } from "zod";
|
|
76771
|
-
import { and as
|
|
77026
|
+
import { and as and94, count as count9, desc as desc51, eq as eq111, gte as gte32, lt as lt18, or as or22, sql as sql44 } from "drizzle-orm";
|
|
76772
77027
|
var sameLocation = (a, b) => a.label === b.label && a.city === b.city && a.region === b.region && a.country === b.country && a.timezone === b.timezone;
|
|
76773
77028
|
var BATCH_RECEIPT_PREFIX = "__canonry_research_batch__:";
|
|
76774
77029
|
async function researchRoutes(app, opts) {
|
|
@@ -76787,7 +77042,7 @@ async function researchRoutes(app, opts) {
|
|
|
76787
77042
|
const input = parsed.data;
|
|
76788
77043
|
if (input.idempotencyKey?.startsWith(BATCH_RECEIPT_PREFIX)) throw validationError("This idempotency key prefix is reserved for internal research batch receipts.");
|
|
76789
77044
|
const requestHash2 = directResearchRequestHash(input);
|
|
76790
|
-
const existingReceipt = input.idempotencyKey ? app.db.select().from(researchRuns).where(
|
|
77045
|
+
const existingReceipt = input.idempotencyKey ? app.db.select().from(researchRuns).where(and94(eq111(researchRuns.projectId, project.id), eq111(researchRuns.idempotencyKey, input.idempotencyKey))).get() : void 0;
|
|
76791
77046
|
if (existingReceipt) {
|
|
76792
77047
|
assertSameDirectResearchRequest(existingReceipt, input, requestHash2);
|
|
76793
77048
|
const result2 = getDetail(app, project.id, existingReceipt.id);
|
|
@@ -76822,7 +77077,7 @@ async function researchRoutes(app, opts) {
|
|
|
76822
77077
|
const initiatedBy = researchPrincipal(request);
|
|
76823
77078
|
const decision = app.db.transaction((tx) => {
|
|
76824
77079
|
if (input.idempotencyKey) {
|
|
76825
|
-
const existing = tx.select().from(researchRuns).where(
|
|
77080
|
+
const existing = tx.select().from(researchRuns).where(and94(eq111(researchRuns.projectId, project.id), eq111(researchRuns.idempotencyKey, input.idempotencyKey))).get();
|
|
76826
77081
|
if (existing) {
|
|
76827
77082
|
assertSameDirectResearchRequest(existing, input, requestHash2);
|
|
76828
77083
|
return { reused: true, id: existing.id, shouldDispatch: existing.status === ResearchRunStatuses.queued };
|
|
@@ -76830,11 +77085,11 @@ async function researchRoutes(app, opts) {
|
|
|
76830
77085
|
}
|
|
76831
77086
|
if (initiatedBy?.limited) {
|
|
76832
77087
|
const { start, end, date } = utcDayBounds(now);
|
|
76833
|
-
const used = tx.select({ value: count9() }).from(researchRuns).where(
|
|
76834
|
-
|
|
77088
|
+
const used = tx.select({ value: count9() }).from(researchRuns).where(and94(
|
|
77089
|
+
eq111(researchRuns.projectId, project.id),
|
|
76835
77090
|
gte32(researchRuns.createdAt, start),
|
|
76836
77091
|
lt18(researchRuns.createdAt, end),
|
|
76837
|
-
|
|
77092
|
+
sql44`(json_extract(${researchRuns.initiatedBy}, '$.role') = ${UserRoles.viewer} OR json_extract(${researchRuns.initiatedBy}, '$.limited') = 1)`
|
|
76838
77093
|
)).get()?.value ?? 0;
|
|
76839
77094
|
if (used >= viewerDailyRunLimit) {
|
|
76840
77095
|
throw researchDailyLimitExceeded(project.name, viewerDailyRunLimit, date);
|
|
@@ -76862,15 +77117,15 @@ async function researchRoutes(app, opts) {
|
|
|
76862
77117
|
if (input.idempotencyKey.startsWith(BATCH_RECEIPT_PREFIX)) throw validationError("This idempotency key prefix is reserved for internal research batch receipts.");
|
|
76863
77118
|
const requestHash2 = researchBatchRequestHash(input);
|
|
76864
77119
|
const receiptKeys = input.runs.map((_run, index) => batchChildReceiptKey(input.idempotencyKey, index));
|
|
76865
|
-
const firstReceipt = app.db.select().from(researchRuns).where(
|
|
76866
|
-
|
|
76867
|
-
|
|
77120
|
+
const firstReceipt = app.db.select().from(researchRuns).where(and94(
|
|
77121
|
+
eq111(researchRuns.projectId, project.id),
|
|
77122
|
+
eq111(researchRuns.idempotencyKey, receiptKeys[0])
|
|
76868
77123
|
)).get();
|
|
76869
77124
|
if (firstReceipt) {
|
|
76870
77125
|
if (firstReceipt.requestHash !== requestHash2) throw alreadyExists("Research batch idempotency key", input.idempotencyKey);
|
|
76871
|
-
const saved = receiptKeys.map((key) => app.db.select().from(researchRuns).where(
|
|
76872
|
-
|
|
76873
|
-
|
|
77126
|
+
const saved = receiptKeys.map((key) => app.db.select().from(researchRuns).where(and94(
|
|
77127
|
+
eq111(researchRuns.projectId, project.id),
|
|
77128
|
+
eq111(researchRuns.idempotencyKey, key)
|
|
76874
77129
|
)).get());
|
|
76875
77130
|
if (saved.some((row) => !row || row.requestHash !== requestHash2)) throw alreadyExists("Research batch idempotency key", input.idempotencyKey);
|
|
76876
77131
|
const runs3 = saved.map((row) => getDetail(app, project.id, row.id));
|
|
@@ -76886,11 +77141,11 @@ async function researchRoutes(app, opts) {
|
|
|
76886
77141
|
const decision = app.db.transaction((tx) => {
|
|
76887
77142
|
if (initiatedBy?.limited) {
|
|
76888
77143
|
const { start, end, date } = utcDayBounds(now);
|
|
76889
|
-
const used = tx.select({ value: count9() }).from(researchRuns).where(
|
|
76890
|
-
|
|
77144
|
+
const used = tx.select({ value: count9() }).from(researchRuns).where(and94(
|
|
77145
|
+
eq111(researchRuns.projectId, project.id),
|
|
76891
77146
|
gte32(researchRuns.createdAt, start),
|
|
76892
77147
|
lt18(researchRuns.createdAt, end),
|
|
76893
|
-
|
|
77148
|
+
sql44`(json_extract(${researchRuns.initiatedBy}, '$.role') = ${UserRoles.viewer} OR json_extract(${researchRuns.initiatedBy}, '$.limited') = 1)`
|
|
76894
77149
|
)).get()?.value ?? 0;
|
|
76895
77150
|
if (used + prepared.length > viewerDailyRunLimit) throw researchDailyLimitExceeded(project.name, viewerDailyRunLimit, date);
|
|
76896
77151
|
}
|
|
@@ -76941,10 +77196,10 @@ async function researchRoutes(app, opts) {
|
|
|
76941
77196
|
const requested = Number.parseInt(request.query.limit ?? "", 10);
|
|
76942
77197
|
const limit = Number.isInteger(requested) && requested > 0 ? Math.min(requested, 100) : 20;
|
|
76943
77198
|
const cursor = request.query.cursor ? parseResearchCursor(request.query.cursor, project.id) : null;
|
|
76944
|
-
const rows = app.db.select().from(researchRuns).where(
|
|
76945
|
-
|
|
76946
|
-
cursor ? or22(lt18(researchRuns.createdAt, cursor.createdAt),
|
|
76947
|
-
)).orderBy(
|
|
77199
|
+
const rows = app.db.select().from(researchRuns).where(and94(
|
|
77200
|
+
eq111(researchRuns.projectId, project.id),
|
|
77201
|
+
cursor ? or22(lt18(researchRuns.createdAt, cursor.createdAt), and94(eq111(researchRuns.createdAt, cursor.createdAt), lt18(researchRuns.id, cursor.id))) : void 0
|
|
77202
|
+
)).orderBy(desc51(researchRuns.createdAt), desc51(researchRuns.id)).limit(limit + 1).all();
|
|
76948
77203
|
const runs2 = rows.slice(0, limit).map(serializeRun);
|
|
76949
77204
|
const last = runs2.at(-1);
|
|
76950
77205
|
const nextCursor = rows.length > limit && last ? Buffer.from(JSON.stringify({ projectId: project.id, createdAt: last.createdAt, id: last.id })).toString("base64url") : null;
|
|
@@ -77093,9 +77348,9 @@ function resolveStoredResearchTemplate(stored, selection, idempotencyKey) {
|
|
|
77093
77348
|
return stored;
|
|
77094
77349
|
}
|
|
77095
77350
|
function resolveResearchTemplate(app, projectId, selection, scope, location) {
|
|
77096
|
-
const savedTemplate = app.db.select().from(measurementQueryTemplates).where(
|
|
77097
|
-
|
|
77098
|
-
|
|
77351
|
+
const savedTemplate = app.db.select().from(measurementQueryTemplates).where(and94(
|
|
77352
|
+
eq111(measurementQueryTemplates.projectId, projectId),
|
|
77353
|
+
eq111(measurementQueryTemplates.id, selection.templateId)
|
|
77099
77354
|
)).get();
|
|
77100
77355
|
if (!savedTemplate) throw validationError("Unknown or stale research template. Reload the selected template and try again.");
|
|
77101
77356
|
const templateVersion = savedTemplate.updatedAt;
|
|
@@ -77104,7 +77359,7 @@ function resolveResearchTemplate(app, projectId, selection, scope, location) {
|
|
|
77104
77359
|
}
|
|
77105
77360
|
const bindingLocation = selection.bindingLocation === void 0 ? location : selection.bindingLocation;
|
|
77106
77361
|
if (bindingLocation) {
|
|
77107
|
-
const project = app.db.select({ locations: projects.locations }).from(projects).where(
|
|
77362
|
+
const project = app.db.select({ locations: projects.locations }).from(projects).where(eq111(projects.id, projectId)).get();
|
|
77108
77363
|
if (!project.locations.some((item2) => sameLocation(item2, bindingLocation))) throw validationError("Research template binding location must match a configured project location.");
|
|
77109
77364
|
}
|
|
77110
77365
|
const { bindings, output } = expandResearchTemplate(savedTemplate, scope, bindingLocation);
|
|
@@ -77117,9 +77372,9 @@ function classifyResearchQueries(project, plan, queries2) {
|
|
|
77117
77372
|
return queries2.map((query2) => classifier?.classify(query2) ?? null);
|
|
77118
77373
|
}
|
|
77119
77374
|
function getDetail(app, projectId, id) {
|
|
77120
|
-
const row = app.db.select().from(researchRuns).where(
|
|
77375
|
+
const row = app.db.select().from(researchRuns).where(and94(eq111(researchRuns.id, id), eq111(researchRuns.projectId, projectId))).get();
|
|
77121
77376
|
if (!row) throw notFound("Research run", id);
|
|
77122
|
-
const queries2 = app.db.select().from(researchRunQueries).where(
|
|
77377
|
+
const queries2 = app.db.select().from(researchRunQueries).where(eq111(researchRunQueries.researchRunId, id)).orderBy(researchRunQueries.position).all().map(serializeQuery);
|
|
77123
77378
|
return { ...serializeRun(row), queries: queries2 };
|
|
77124
77379
|
}
|
|
77125
77380
|
function serializeRun(row) {
|
|
@@ -77174,7 +77429,7 @@ function utcDayBounds(now) {
|
|
|
77174
77429
|
|
|
77175
77430
|
// ../api-routes/src/simple-measurement-definitions.ts
|
|
77176
77431
|
import crypto61 from "crypto";
|
|
77177
|
-
import { and as
|
|
77432
|
+
import { and as and95, eq as eq112 } from "drizzle-orm";
|
|
77178
77433
|
function captureSimpleMeasurementDefinition(db, input) {
|
|
77179
77434
|
return db.transaction((tx) => {
|
|
77180
77435
|
const run = tx.select({
|
|
@@ -77182,7 +77437,7 @@ function captureSimpleMeasurementDefinition(db, input) {
|
|
|
77182
77437
|
trigger: runs.trigger,
|
|
77183
77438
|
status: runs.status,
|
|
77184
77439
|
measurementPlanVersionId: runs.measurementPlanVersionId
|
|
77185
|
-
}).from(runs).where(
|
|
77440
|
+
}).from(runs).where(and95(eq112(runs.projectId, input.projectId), eq112(runs.id, input.runId))).get();
|
|
77186
77441
|
if (!run) throw notFound("Run", input.runId);
|
|
77187
77442
|
if (run.kind !== RunKinds["answer-visibility"] || run.trigger === RunTriggers.probe || run.measurementPlanVersionId !== null) {
|
|
77188
77443
|
return null;
|
|
@@ -77203,7 +77458,7 @@ function captureSimpleMeasurementDefinition(db, input) {
|
|
|
77203
77458
|
throw validationError("Qualified aliases must be a sorted subset of the captured aliases.");
|
|
77204
77459
|
}
|
|
77205
77460
|
}
|
|
77206
|
-
const existing = tx.select().from(simpleMeasurementDefinitions).where(
|
|
77461
|
+
const existing = tx.select().from(simpleMeasurementDefinitions).where(and95(eq112(simpleMeasurementDefinitions.projectId, input.projectId), eq112(simpleMeasurementDefinitions.runId, input.runId))).get();
|
|
77207
77462
|
if (existing) {
|
|
77208
77463
|
const frozen = simpleMeasurementDefinitionSchema.parse(existing.definition);
|
|
77209
77464
|
const { qualifiedAliases: _liveQualifiedAliases, ...liveIdentity } = definition2.identity;
|
|
@@ -77229,15 +77484,15 @@ function captureSimpleMeasurementDefinition(db, input) {
|
|
|
77229
77484
|
if (run.status !== RunStatuses.running) {
|
|
77230
77485
|
throw validationError("Only a running run can capture a new measurement definition.");
|
|
77231
77486
|
}
|
|
77232
|
-
if (tx.select({ id: querySnapshots.id }).from(querySnapshots).where(
|
|
77487
|
+
if (tx.select({ id: querySnapshots.id }).from(querySnapshots).where(eq112(querySnapshots.runId, input.runId)).limit(1).get()) {
|
|
77233
77488
|
throw validationError("This run already has stored answers. Its measurement definition cannot be inferred afterward.");
|
|
77234
77489
|
}
|
|
77235
|
-
const projectQueryIds = new Set(tx.select({ id: queries.id }).from(queries).where(
|
|
77490
|
+
const projectQueryIds = new Set(tx.select({ id: queries.id }).from(queries).where(eq112(queries.projectId, input.projectId)).all().map((query2) => query2.id));
|
|
77236
77491
|
if (definition2.queries.some((query2) => !projectQueryIds.has(query2.queryId))) {
|
|
77237
77492
|
throw validationError("Every captured query must belong to the run project.");
|
|
77238
77493
|
}
|
|
77239
77494
|
if (definition2.competitors !== void 0) {
|
|
77240
|
-
const liveCompetitorDomains = new Set(tx.select({ domain: competitors.domain }).from(competitors).where(
|
|
77495
|
+
const liveCompetitorDomains = new Set(tx.select({ domain: competitors.domain }).from(competitors).where(eq112(competitors.projectId, input.projectId)).orderBy(competitors.domain).all().map((competitor) => competitor.domain));
|
|
77241
77496
|
const capturedDomains = new Set(definition2.competitors.map((competitor) => competitor.domain));
|
|
77242
77497
|
if (liveCompetitorDomains.size !== capturedDomains.size || [...liveCompetitorDomains].some((domain) => !capturedDomains.has(domain))) {
|
|
77243
77498
|
throw validationError("Captured competitors must exactly match the project competitors dispatched for this run.");
|
|
@@ -77514,6 +77769,10 @@ async function apiRoutes(app, opts) {
|
|
|
77514
77769
|
onSyncRequested: opts.onBingSyncRequested,
|
|
77515
77770
|
claimScheduledSync: opts.claimScheduledSync
|
|
77516
77771
|
});
|
|
77772
|
+
await api.register(sitemapRoutes, {
|
|
77773
|
+
googleConnectionStore: opts.googleConnectionStore,
|
|
77774
|
+
bingConnectionStore: opts.bingConnectionStore
|
|
77775
|
+
});
|
|
77517
77776
|
await api.register(googleRoutes, {
|
|
77518
77777
|
getGoogleAuthConfig: opts.getGoogleAuthConfig,
|
|
77519
77778
|
googleConnectionStore: opts.googleConnectionStore,
|
|
@@ -77977,6 +78236,7 @@ export {
|
|
|
77977
78236
|
getAuthUrl,
|
|
77978
78237
|
exchangeCode,
|
|
77979
78238
|
refreshAccessToken,
|
|
78239
|
+
listSitemaps,
|
|
77980
78240
|
fetchSearchAnalytics,
|
|
77981
78241
|
inspectUrl,
|
|
77982
78242
|
EgressRefusedError,
|