@canonry/canonry 7.61.1 → 7.62.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agent-workspace/skills/aero/SKILL.md +2 -2
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +8 -4
- package/assets/agent-workspace/skills/canonry/references/server-side-traffic.md +14 -3
- package/assets/assets/{AgentSetupPage-CCMNQwRE.js → AgentSetupPage-DtPnVaeF.js} +1 -1
- package/assets/assets/{AuditHistoryPanel-B8mIRI3I.js → AuditHistoryPanel-Ce_oy4OV.js} +1 -1
- package/assets/assets/{BacklinksPage-BNLyIR97.js → BacklinksPage-CrqPWRd8.js} +1 -1
- package/assets/assets/{HistoryPage-DSr9KE16.js → HistoryPage-5v0JM7DN.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-BTg-SFW7.js → MeasurementPropertyPage-Bm5Mrd5a.js} +1 -1
- package/assets/assets/{ProjectPage-SqE0fkEq.js → ProjectPage-DaWUh6iw.js} +1 -1
- package/assets/assets/{RunsPage-gJR17ho1.js → RunsPage-CfdAR2v2.js} +1 -1
- package/assets/assets/{SettingsPage-DY5f_oN4.js → SettingsPage-BFooA7JG.js} +1 -1
- package/assets/assets/{SiteHealthSection-BYx91hP-.js → SiteHealthSection-CerUQgAp.js} +2 -2
- package/assets/assets/{ToneBadge-C3S7aVf3.js → ToneBadge-DKvZcONW.js} +1 -1
- package/assets/assets/{TrafficPage-CCXH5nhh.js → TrafficPage-C5fhZGJ2.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-DTjCsu4-.js → TrafficSourceDetailPage-BoqARHYz.js} +1 -1
- package/assets/assets/{account-context-DozTQM44.js → account-context-D_CDYrs8.js} +1 -1
- package/assets/assets/{card-D8MU7RYV.js → card-C6z0XyOo.js} +1 -1
- package/assets/assets/{extract-error-message-CsVYPTyt.js → extract-error-message-By_wcU8g.js} +1 -1
- package/assets/assets/{format-helpers-CWBooMJJ.js → format-helpers-D3Ubm8vk.js} +1 -1
- package/assets/assets/{index-D5o-caTl.js → index-DSVFZqv6.js} +4 -4
- package/assets/assets/{query-client-DbxlHfUm.js → query-client-BETh6SO9.js} +1 -1
- package/assets/assets/{react-query-D4ZSMg7h.js → react-query-DzDr4w41.js} +1 -1
- package/assets/assets/{react-sigma_core.esm.min-CSRp4hew.js → react-sigma_core.esm.min-3kgtOWbQ.js} +1 -1
- package/assets/assets/{service-CYzJmCXU.js → service-BUaUaize.js} +1 -1
- package/assets/index.html +7 -7
- package/dist/{ads-Z2SNR2F5.js → ads-7WUY5CVC.js} +3 -3
- package/dist/{agent-UUDYXXRD.js → agent-NOA33LVM.js} +7 -7
- package/dist/{agent-ask-KCZKPBY7.js → agent-ask-PKQDFJ7K.js} +3 -3
- package/dist/{agent-conversations-CDQA7HOU.js → agent-conversations-7HVWPSFB.js} +3 -3
- package/dist/{agent-memory-MER75L4L.js → agent-memory-Y4M2EWKA.js} +3 -3
- package/dist/{agent-providers-RH7W5HYA.js → agent-providers-P67FWELV.js} +3 -3
- package/dist/{agent-transcript-VU2VHSZ6.js → agent-transcript-C2BYWPRT.js} +3 -3
- package/dist/{analytics-BDVOFX6O.js → analytics-XHYWNNW7.js} +3 -3
- package/dist/{apply-XJULCLGT.js → apply-6OIWDXQP.js} +3 -3
- package/dist/{backfill-SCB3KOTF.js → backfill-KQPXCUUJ.js} +9 -9
- package/dist/{backlinks-HCSL6KKT.js → backlinks-ASYIZFLQ.js} +3 -3
- package/dist/{bing-AEUPBLV6.js → bing-ZLKJ3MIL.js} +3 -3
- package/dist/{bootstrap-J73OTQWI.js → bootstrap-KMVQY5JD.js} +10 -10
- package/dist/{cdp-2UQER2VR.js → cdp-V2HX4ZLE.js} +7 -7
- package/dist/{chunk-QP36LXT5.js → chunk-2HNTNKFC.js} +165 -63
- package/dist/{chunk-CZSHCZHW.js → chunk-2LY4FY4N.js} +1 -1
- package/dist/{chunk-TGZKOIZ3.js → chunk-3B2GBWIA.js} +5 -5
- package/dist/{chunk-3E7NFLYB.js → chunk-3NICRFTU.js} +333 -333
- package/dist/{chunk-YHTU4XBR.js → chunk-5NYS3ABL.js} +2 -2
- package/dist/{chunk-4UTPB5KB.js → chunk-7EWDQUNH.js} +8 -8
- package/dist/{chunk-G3DZSUYD.js → chunk-AO4P75DI.js} +5 -5
- package/dist/{chunk-GAA7JGB7.js → chunk-APQ5EAL2.js} +2 -2
- package/dist/{chunk-VHHFAAMI.js → chunk-B5CSCGVD.js} +1 -1
- package/dist/{chunk-SQ6AOE57.js → chunk-CJC4AQ2M.js} +2 -2
- package/dist/{chunk-VCSJHXW6.js → chunk-DAKJBAUN.js} +1 -1
- package/dist/{chunk-V7Q2FUHQ.js → chunk-GMP7OZDW.js} +4 -4
- package/dist/{chunk-B5KK54RE.js → chunk-GSHHYR24.js} +3 -3
- package/dist/{chunk-P5DZ56CR.js → chunk-IALK4DQJ.js} +1 -1
- package/dist/{chunk-FWRHDY4R.js → chunk-IVZMXHXE.js} +1 -1
- package/dist/{chunk-EKY6GHJ4.js → chunk-KLGHY67V.js} +4 -4
- package/dist/{chunk-XCA546YC.js → chunk-KTQVRQDM.js} +1 -1
- package/dist/{chunk-KX5IRN55.js → chunk-LAILXR2F.js} +1 -1
- package/dist/{chunk-MRNMDQRB.js → chunk-MMRJUOE2.js} +1 -1
- package/dist/{chunk-5PYPNDIE.js → chunk-OJVXKPIN.js} +3 -3
- package/dist/{chunk-7VEXNHRX.js → chunk-Q7PTBHNG.js} +1 -1
- package/dist/{chunk-HD7AZIOP.js → chunk-RGD5SDT3.js} +17 -1
- package/dist/{chunk-SZXYGJMI.js → chunk-SDZKHJ6W.js} +2 -2
- package/dist/{chunk-34HGKT6I.js → chunk-SYQLUOR7.js} +1 -1
- package/dist/{chunk-G3ZNUINX.js → chunk-U2WPZZPQ.js} +1 -1
- package/dist/{chunk-IG6ISQYD.js → chunk-U6CNJC2Y.js} +6310 -6261
- package/dist/{chunk-BKNIXYMI.js → chunk-UUGT7UMF.js} +4 -4
- package/dist/{chunk-SA4CG775.js → chunk-UVDBUMYV.js} +2 -2
- package/dist/{chunk-W5FR6AJP.js → chunk-WKZZILBI.js} +2 -2
- package/dist/{chunk-3A7LI5KI.js → chunk-X3GWBK7N.js} +3 -3
- package/dist/{chunk-5AV2OUCJ.js → chunk-YBDUZBGU.js} +1 -1
- package/dist/{chunk-TYWTIYYM.js → chunk-YJAN2ZDV.js} +3 -3
- package/dist/{chunk-RL7LPPFU.js → chunk-YQGVA64Q.js} +3 -3
- package/dist/{chunk-QGXPQHDN.js → chunk-YQMQWDTI.js} +20 -20
- package/dist/{citations-3X5EJNJ3.js → citations-NOCVQPPU.js} +3 -3
- package/dist/{cli-catalog-F3IOEPJK.js → cli-catalog-WOWYZ7Q2.js} +3 -3
- package/dist/{cli-commands-J7CI2RNW.js → cli-commands-4YSZAPKZ.js} +7 -7
- package/dist/cli.js +13 -13
- package/dist/{competitor-CUPGN4NZ.js → competitor-KQK66W3M.js} +3 -3
- package/dist/{content-FYQT7V5A.js → content-OHOQO5LP.js} +3 -3
- package/dist/{dashboard-WHWMMX7W.js → dashboard-W6MII62Z.js} +6 -6
- package/dist/{demo-CNTR3ZXS.js → demo-23H25IOO.js} +1 -1
- package/dist/{demo-server-ZOV5HCDE.js → demo-server-WYYWTH4H.js} +11 -11
- package/dist/{discover-4IGXLRRE.js → discover-557WRGEK.js} +3 -3
- package/dist/{discover-eval-DGLZO5MD.js → discover-eval-GGRZZHHI.js} +3 -3
- package/dist/{doctor-3MFTUIEU.js → doctor-D5JNJKVE.js} +3 -3
- package/dist/{evidence-ANE4SFP5.js → evidence-DHYYVPWW.js} +3 -3
- package/dist/{export-cmd-X6S5F463.js → export-cmd-M24S3YBY.js} +3 -3
- package/dist/{ga-2UKFAVDJ.js → ga-XM6WXZNP.js} +3 -3
- package/dist/{gbp-Q3V3SQMV.js → gbp-M3S5XYPG.js} +3 -3
- package/dist/{google-D4O4JCM4.js → google-PULCSUOU.js} +4 -4
- package/dist/{google-marketing-6L5327YK.js → google-marketing-AA625HA4.js} +1 -1
- package/dist/{health-cmd-7TPQHGKK.js → health-cmd-LVEW6OBD.js} +3 -3
- package/dist/{history-NNUOPNA7.js → history-32ZYGZGQ.js} +3 -3
- package/dist/index.d.ts +115 -68
- package/dist/index.js +24 -24
- package/dist/{init-NCJ6KXTS.js → init-4YT2HMKY.js} +11 -11
- package/dist/{insights-DDAP3MRC.js → insights-RK3MJ4UM.js} +3 -3
- package/dist/intelligence-service-UNKFCG5N.js +15 -0
- package/dist/{journeys-I7H25PPF.js → journeys-S4CPNZYU.js} +3 -3
- package/dist/{keys-YUJNNNV3.js → keys-WCXE7WXV.js} +3 -3
- package/dist/{keyword-25DDYT45.js → keyword-HHVLC4HI.js} +3 -3
- package/dist/{logs-FKPJ4L5R.js → logs-VYESWGAN.js} +3 -3
- package/dist/{mcp-ZPLVNLL3.js → mcp-4LRFIOLC.js} +2 -2
- package/dist/mcp.js +9 -9
- package/dist/{measurement-plan-CDY63CLU.js → measurement-plan-AL3EHWRK.js} +5 -5
- package/dist/{notify-67IIWOSF.js → notify-LUDG5PWU.js} +3 -3
- package/dist/{organic-evidence-P2BQK2BV.js → organic-evidence-CIOHMKLD.js} +3 -3
- package/dist/{overview-WHSTSNS2.js → overview-NAZJ6PJG.js} +4 -4
- package/dist/{project-7AMXFZMD.js → project-N3MU5V6T.js} +3 -3
- package/dist/{query-RLKE6JHY.js → query-XMI2FV6G.js} +3 -3
- package/dist/{research-PDOE77HZ.js → research-MQ6QA5TV.js} +3 -3
- package/dist/{results-export-5VKTQLOR.js → results-export-OFEZXCJK.js} +3 -3
- package/dist/{run-QXCB4AWO.js → run-YQS3LCFT.js} +4 -4
- package/dist/{schedule-NDM2GBBW.js → schedule-DF6HCMXN.js} +3 -3
- package/dist/{search-DDRSJUWZ.js → search-MRDJKVOL.js} +3 -3
- package/dist/{serve-YWBWBUBR.js → serve-5A4K242B.js} +25 -25
- package/dist/{session-HCGHUWAV.js → session-5OXLS7NP.js} +12 -12
- package/dist/{settings-Q2ZOEA7H.js → settings-NIF34QPH.js} +3 -3
- package/dist/{skills-PKSEX44J.js → skills-JOSNWG7B.js} +2 -2
- package/dist/{snapshot-URNBVDTA.js → snapshot-32DLVRIJ.js} +3 -3
- package/dist/{src-MAMJVGSJ.js → src-F63SHN5T.js} +2 -2
- package/dist/{status-DDXVX4KE.js → status-IERHHP4C.js} +4 -4
- package/dist/{technical-aeo-OS6KQXAP.js → technical-aeo-D7CA3CK6.js} +3 -3
- package/dist/{telemetry-G7GX2LIY.js → telemetry-DRURFCCJ.js} +4 -4
- package/dist/{tools-C3I2X3IS.js → tools-3IZMSHSY.js} +11 -11
- package/dist/{traffic-A4QW7455.js → traffic-AME5NDMN.js} +27 -5
- package/dist/{users-4WPDDCCR.js → users-4CQ6ATAG.js} +3 -3
- package/dist/{visibility-stats-5BY7GVUV.js → visibility-stats-R5JBISBX.js} +3 -3
- package/dist/{wordpress-T5EYN2WX.js → wordpress-GFK4GR3O.js} +3 -3
- package/package.json +1 -1
- package/dist/intelligence-service-O4BALXZK.js +0 -15
|
@@ -9,14 +9,14 @@ import {
|
|
|
9
9
|
normalizeCloudflareWorkerEvent,
|
|
10
10
|
sessionizeJourneyRequests,
|
|
11
11
|
verifyRequestSignature
|
|
12
|
-
} from "./chunk-
|
|
12
|
+
} from "./chunk-SYQLUOR7.js";
|
|
13
13
|
import {
|
|
14
14
|
currentOutcomeAttribution,
|
|
15
15
|
getRequestContext,
|
|
16
16
|
registerRequestContext,
|
|
17
17
|
requestAuditIdentity,
|
|
18
18
|
sanitizeRequestContext
|
|
19
|
-
} from "./chunk-
|
|
19
|
+
} from "./chunk-LAILXR2F.js";
|
|
20
20
|
import {
|
|
21
21
|
SentimentIdempotencyConflict,
|
|
22
22
|
SentimentRepository,
|
|
@@ -154,7 +154,7 @@ import {
|
|
|
154
154
|
userSessions,
|
|
155
155
|
users,
|
|
156
156
|
writeNegativeReviewMaxStars
|
|
157
|
-
} from "./chunk-
|
|
157
|
+
} from "./chunk-RGD5SDT3.js";
|
|
158
158
|
import {
|
|
159
159
|
ADS_ACTIVATE_SCOPE,
|
|
160
160
|
ADS_ACTIVATION_ABSOLUTE_MAX_ENTITIES,
|
|
@@ -208,6 +208,7 @@ import {
|
|
|
208
208
|
CheckScopes,
|
|
209
209
|
CheckStatuses,
|
|
210
210
|
CitationStates,
|
|
211
|
+
CloudflareQueueDrainStopReasons,
|
|
211
212
|
CloudflareTrafficDeliveryModes,
|
|
212
213
|
CompetitorChangeKinds,
|
|
213
214
|
CompetitorShareImpactCaveats,
|
|
@@ -1157,7 +1158,7 @@ import {
|
|
|
1157
1158
|
wordpressSchemaDeployResultDtoSchema,
|
|
1158
1159
|
wordpressSchemaStatusResultDtoSchema,
|
|
1159
1160
|
wordpressStatusDtoSchema
|
|
1160
|
-
} from "./chunk-
|
|
1161
|
+
} from "./chunk-U6CNJC2Y.js";
|
|
1161
1162
|
|
|
1162
1163
|
// ../api-routes/src/user-session.ts
|
|
1163
1164
|
import crypto2 from "crypto";
|
|
@@ -6193,9 +6194,9 @@ function queryValue(request, key) {
|
|
|
6193
6194
|
function requestEmbedProjectTabs(request, configuredTabs) {
|
|
6194
6195
|
const headerTabs = normalizeIdTokens(splitList(request.headers["x-canonry-embed-tabs"]));
|
|
6195
6196
|
const configured = configuredTabs && configuredTabs.length > 0 ? configuredTabs : void 0;
|
|
6196
|
-
if (!configured) return headerTabs;
|
|
6197
|
-
if (!headerTabs) return [...configured];
|
|
6198
|
-
return headerTabs.filter((tab) => configured.includes(tab));
|
|
6197
|
+
if (!configured) return headerTabs ? { tabs: headerTabs, fromEmbedSurface: true } : void 0;
|
|
6198
|
+
if (!headerTabs) return { tabs: [...configured], fromEmbedSurface: false };
|
|
6199
|
+
return { tabs: headerTabs.filter((tab) => configured.includes(tab)), fromEmbedSurface: true };
|
|
6199
6200
|
}
|
|
6200
6201
|
function projectRouteRest(url) {
|
|
6201
6202
|
const match = url.match(/\/projects\/[^/]+(?:\/([^?#]*))?$/);
|
|
@@ -6255,12 +6256,14 @@ function isTechnicalAeoRead(request, url) {
|
|
|
6255
6256
|
return rest === "technical-aeo" || rest === "technical-aeo/pages" || rest === "technical-aeo/trend" || rest === "technical-aeo/crawl" || rest === "technical-aeo/graph" || rest === "technical-aeo/subgraph" || rest === "technical-aeo/path" || rest === "technical-aeo/changes" || rest === "technical-aeo/crawl/pages/audit" || rest === "technical-aeo/crawl/pages" || rest === "technical-aeo/runs" || rest === "technical-aeo/structure" || rest === "technical-aeo/internal-links" || rest === "technical-aeo/internal-links/neighbors" || rest === "technical-aeo/dead-links";
|
|
6256
6257
|
}
|
|
6257
6258
|
function enforceEmbedProjectTabs(request, configuredTabs) {
|
|
6258
|
-
const
|
|
6259
|
-
if (
|
|
6259
|
+
const policy = requestEmbedProjectTabs(request, configuredTabs);
|
|
6260
|
+
if (policy === void 0) return;
|
|
6260
6261
|
if (request.method === "OPTIONS") return;
|
|
6261
|
-
|
|
6262
|
+
const { tabs } = policy;
|
|
6263
|
+
if (policy.fromEmbedSurface && WRITE_METHODS2.has(request.method)) {
|
|
6262
6264
|
throw forbidden("This endpoint is not available in embed mode.");
|
|
6263
6265
|
}
|
|
6266
|
+
if (WRITE_METHODS2.has(request.method)) return;
|
|
6264
6267
|
const url = request.url.split("?")[0];
|
|
6265
6268
|
if (isProjectShellRead(request, url)) return;
|
|
6266
6269
|
if (tabs.includes("overview") && isOverviewRead(url)) return;
|
|
@@ -15941,6 +15944,8 @@ function formatRun(row) {
|
|
|
15941
15944
|
} : {},
|
|
15942
15945
|
// Only on a run that skipped a provider, so other runs read as before.
|
|
15943
15946
|
...row.skippedProviders && Object.keys(row.skippedProviders).length > 0 ? { skippedProviders: row.skippedProviders } : {},
|
|
15947
|
+
// Only on a sync that recorded one, so other runs read as before.
|
|
15948
|
+
...row.syncSummary ? { syncSummary: row.syncSummary } : {},
|
|
15944
15949
|
createdAt: row.createdAt
|
|
15945
15950
|
};
|
|
15946
15951
|
}
|
|
@@ -63705,8 +63710,33 @@ async function trafficRoutes(app, opts) {
|
|
|
63705
63710
|
let committedAt = startedAt;
|
|
63706
63711
|
let remainingBacklogCount = 0;
|
|
63707
63712
|
let completedBatches = 0;
|
|
63708
|
-
|
|
63709
|
-
|
|
63713
|
+
let pulls = 0;
|
|
63714
|
+
let emptyPulls = 0;
|
|
63715
|
+
let messagesPulled = 0;
|
|
63716
|
+
let messagesAcknowledged = 0;
|
|
63717
|
+
let poisonMessages = 0;
|
|
63718
|
+
let skippedUnleasedMessages = 0;
|
|
63719
|
+
let duplicateEvents = 0;
|
|
63720
|
+
let backlogAtFirstPull = null;
|
|
63721
|
+
let backlogAtLastPull = null;
|
|
63722
|
+
const drainSummary = (stopReason) => ({
|
|
63723
|
+
type: "cloudflare-queue",
|
|
63724
|
+
stopReason,
|
|
63725
|
+
pulls,
|
|
63726
|
+
emptyPulls,
|
|
63727
|
+
messagesPulled,
|
|
63728
|
+
messagesAcknowledged,
|
|
63729
|
+
poisonMessages,
|
|
63730
|
+
skippedUnleasedMessages,
|
|
63731
|
+
acceptedEvents,
|
|
63732
|
+
duplicateEvents,
|
|
63733
|
+
backlogAtFirstPull,
|
|
63734
|
+
backlogAtLastPull,
|
|
63735
|
+
elapsedMs: Date.now() - startedMs
|
|
63736
|
+
});
|
|
63737
|
+
const finishSync = (status, error, stopReason) => {
|
|
63738
|
+
const queueDrain = drainSummary(stopReason);
|
|
63739
|
+
app.db.update(runs).set({ status, finishedAt: committedAt, syncSummary: queueDrain, ...error ? { error } : {} }).where(eq80(runs.id, runId)).run();
|
|
63710
63740
|
writeAuditLog(app.db, {
|
|
63711
63741
|
projectId: project.id,
|
|
63712
63742
|
actor: "api",
|
|
@@ -63743,6 +63773,7 @@ async function trafficRoutes(app, opts) {
|
|
|
63743
63773
|
aiReferralBucketRows,
|
|
63744
63774
|
sampleRows,
|
|
63745
63775
|
remainingBacklogCount,
|
|
63776
|
+
queueDrain,
|
|
63746
63777
|
windowStart: startedAt,
|
|
63747
63778
|
windowEnd: committedAt
|
|
63748
63779
|
};
|
|
@@ -63761,6 +63792,7 @@ async function trafficRoutes(app, opts) {
|
|
|
63761
63792
|
const canonicalHost = resolveCloudflareWorkerRouteHost(project.canonicalDomain);
|
|
63762
63793
|
const drainDeadlineMs = startedMs + cloudflareQueueSyncDeadlineMs;
|
|
63763
63794
|
let consecutiveEmptyPulls = 0;
|
|
63795
|
+
let stopReason = null;
|
|
63764
63796
|
let receiptPolicy = null;
|
|
63765
63797
|
const resolveReceiptPolicy = () => receiptPolicy ??= loadQueueReceiptPolicy(
|
|
63766
63798
|
app.db,
|
|
@@ -63781,6 +63813,12 @@ async function trafficRoutes(app, opts) {
|
|
|
63781
63813
|
visibilityTimeoutMs: CLOUDFLARE_QUEUE_VISIBILITY_TIMEOUT_MS
|
|
63782
63814
|
});
|
|
63783
63815
|
remainingBacklogCount = pulled.messageBacklogCount;
|
|
63816
|
+
pulls += 1;
|
|
63817
|
+
if (pulled.messages.length === 0) emptyPulls += 1;
|
|
63818
|
+
messagesPulled += pulled.messages.length;
|
|
63819
|
+
skippedUnleasedMessages += pulled.skippedUnleasedMessageCount;
|
|
63820
|
+
backlogAtFirstPull ??= pulled.messageBacklogCount;
|
|
63821
|
+
backlogAtLastPull = pulled.messageBacklogCount;
|
|
63784
63822
|
if (pulled.skippedUnleasedMessageCount > 0) {
|
|
63785
63823
|
request.log.warn({
|
|
63786
63824
|
sourceId: sourceRow2.id,
|
|
@@ -63834,6 +63872,7 @@ async function trafficRoutes(app, opts) {
|
|
|
63834
63872
|
}
|
|
63835
63873
|
validLeases.push(message.leaseId);
|
|
63836
63874
|
}
|
|
63875
|
+
poisonMessages += poisonLeases.length;
|
|
63837
63876
|
const policy = normalized.length > 0 ? await resolveReceiptPolicy() : null;
|
|
63838
63877
|
committedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
63839
63878
|
const writeResult = writeTrafficEventBatch({
|
|
@@ -63876,6 +63915,7 @@ async function trafficRoutes(app, opts) {
|
|
|
63876
63915
|
throw validationError("Cloudflare Queue source was reconfigured before acknowledgement");
|
|
63877
63916
|
}
|
|
63878
63917
|
const ackResult = await ackQueueMessages(queueClient, { acks: [...validLeases, ...poisonLeases] });
|
|
63918
|
+
messagesAcknowledged += validLeases.length + poisonLeases.length;
|
|
63879
63919
|
if (ackResult.warningCount > 0) {
|
|
63880
63920
|
request.log.warn({
|
|
63881
63921
|
sourceId: sourceRow2.id,
|
|
@@ -63884,6 +63924,7 @@ async function trafficRoutes(app, opts) {
|
|
|
63884
63924
|
}
|
|
63885
63925
|
}
|
|
63886
63926
|
acceptedEvents += writeResult.acceptedEvents;
|
|
63927
|
+
duplicateEvents += writeResult.duplicateEvents;
|
|
63887
63928
|
selfTrafficExcluded += writeResult.selfTrafficExcluded;
|
|
63888
63929
|
crawlerHits += writeResult.crawlerHits;
|
|
63889
63930
|
aiUserFetchHits += writeResult.aiUserFetchHits;
|
|
@@ -63894,16 +63935,26 @@ async function trafficRoutes(app, opts) {
|
|
|
63894
63935
|
aiReferralBucketRows += writeResult.aiReferralBucketRows;
|
|
63895
63936
|
sampleRows += writeResult.sampleRows;
|
|
63896
63937
|
completedBatches += 1;
|
|
63897
|
-
if (remainingBacklogCount === 0)
|
|
63938
|
+
if (remainingBacklogCount === 0) {
|
|
63939
|
+
stopReason = CloudflareQueueDrainStopReasons["backlog-empty"];
|
|
63940
|
+
break;
|
|
63941
|
+
}
|
|
63898
63942
|
if (pulled.messages.length === 0) {
|
|
63899
|
-
if (pulled.skippedUnleasedMessageCount > 0 && pulled.skippedUnleasedMessageCount < CLOUDFLARE_QUEUE_BATCH_SIZE)
|
|
63943
|
+
if (pulled.skippedUnleasedMessageCount > 0 && pulled.skippedUnleasedMessageCount < CLOUDFLARE_QUEUE_BATCH_SIZE) {
|
|
63944
|
+
stopReason = CloudflareQueueDrainStopReasons["unleased-only"];
|
|
63945
|
+
break;
|
|
63946
|
+
}
|
|
63900
63947
|
consecutiveEmptyPulls = pulled.skippedUnleasedMessageCount === 0 ? consecutiveEmptyPulls + 1 : 0;
|
|
63901
|
-
if (consecutiveEmptyPulls >= 2)
|
|
63948
|
+
if (consecutiveEmptyPulls >= 2) {
|
|
63949
|
+
stopReason = CloudflareQueueDrainStopReasons["empty-pulls"];
|
|
63950
|
+
break;
|
|
63951
|
+
}
|
|
63902
63952
|
} else {
|
|
63903
63953
|
consecutiveEmptyPulls = 0;
|
|
63904
63954
|
}
|
|
63905
63955
|
}
|
|
63906
|
-
|
|
63956
|
+
stopReason ??= completedBatches >= cloudflareQueueMaxBatches ? CloudflareQueueDrainStopReasons["batch-cap"] : CloudflareQueueDrainStopReasons.deadline;
|
|
63957
|
+
return finishSync(RunStatuses.completed, null, stopReason);
|
|
63907
63958
|
} catch (error) {
|
|
63908
63959
|
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
63909
63960
|
const safeError = error instanceof CloudflareQueueApiError ? error.message : "Cloudflare Queue sync failed; retry the active source.";
|
|
@@ -63912,7 +63963,7 @@ async function trafficRoutes(app, opts) {
|
|
|
63912
63963
|
{ sourceId: sourceRow2.id, completedBatches, error: safeError },
|
|
63913
63964
|
"Cloudflare Queue sync stopped after saving some batches"
|
|
63914
63965
|
);
|
|
63915
|
-
return finishSync(RunStatuses.partial, safeError);
|
|
63966
|
+
return finishSync(RunStatuses.partial, safeError, CloudflareQueueDrainStopReasons.error);
|
|
63916
63967
|
}
|
|
63917
63968
|
try {
|
|
63918
63969
|
opts.onTrafficSynced?.({
|
|
@@ -63931,7 +63982,12 @@ async function trafficRoutes(app, opts) {
|
|
|
63931
63982
|
} catch {
|
|
63932
63983
|
}
|
|
63933
63984
|
app.db.transaction((tx) => {
|
|
63934
|
-
tx.update(runs).set({
|
|
63985
|
+
tx.update(runs).set({
|
|
63986
|
+
status: RunStatuses.failed,
|
|
63987
|
+
error: safeError,
|
|
63988
|
+
finishedAt: failedAt,
|
|
63989
|
+
syncSummary: drainSummary(CloudflareQueueDrainStopReasons.error)
|
|
63990
|
+
}).where(eq80(runs.id, runId)).run();
|
|
63935
63991
|
const latestSource = tx.select().from(trafficSources).where(eq80(trafficSources.id, sourceRow2.id)).get();
|
|
63936
63992
|
if (latestSource && latestSource.status === TrafficSourceStatuses.connected && latestSource.syncLeaseOwner === leaseOwner && isDeepStrictEqual(latestSource.configJson, sourceRow2.configJson)) {
|
|
63937
63993
|
tx.update(trafficSources).set({ lastError: safeError, updatedAt: failedAt }).where(and69(
|
|
@@ -64864,7 +64920,8 @@ async function trafficRoutes(app, opts) {
|
|
|
64864
64920
|
status: latestRun.status,
|
|
64865
64921
|
startedAt: latestRun.startedAt,
|
|
64866
64922
|
finishedAt: latestRun.finishedAt ?? null,
|
|
64867
|
-
error: latestRun.error ?? null
|
|
64923
|
+
error: latestRun.error ?? null,
|
|
64924
|
+
...latestRun.syncSummary ? { syncSummary: latestRun.syncSummary } : {}
|
|
64868
64925
|
} : null
|
|
64869
64926
|
};
|
|
64870
64927
|
}
|
|
@@ -68447,7 +68504,7 @@ var RUNTIME_STATE_CHECKS = [
|
|
|
68447
68504
|
];
|
|
68448
68505
|
|
|
68449
68506
|
// ../api-routes/src/doctor/checks/traffic-source.ts
|
|
68450
|
-
import { and as and78, eq as eq94, gte as gte24, inArray as inArray40, ne as ne16, sql as sql32 } from "drizzle-orm";
|
|
68507
|
+
import { and as and78, desc as desc44, eq as eq94, gte as gte24, inArray as inArray40, isNotNull as isNotNull9, ne as ne16, sql as sql32 } from "drizzle-orm";
|
|
68451
68508
|
var RECENT_DATA_WARN_DAYS = 7;
|
|
68452
68509
|
var RECENT_DATA_FAIL_DAYS = 30;
|
|
68453
68510
|
function isCloudflareDirectPush(source) {
|
|
@@ -68992,6 +69049,48 @@ var syncLagCheck = {
|
|
|
68992
69049
|
};
|
|
68993
69050
|
}
|
|
68994
69051
|
};
|
|
69052
|
+
function lastQueueStopReason(ctx, sourceId) {
|
|
69053
|
+
const row = ctx.db.select({ syncSummary: runs.syncSummary }).from(runs).where(and78(
|
|
69054
|
+
eq94(runs.projectId, ctx.project.id),
|
|
69055
|
+
eq94(runs.kind, RunKinds["traffic-sync"]),
|
|
69056
|
+
eq94(runs.sourceId, sourceId),
|
|
69057
|
+
isNotNull9(runs.syncSummary)
|
|
69058
|
+
)).orderBy(desc44(runs.createdAt)).limit(1).get();
|
|
69059
|
+
return row?.syncSummary?.type === "cloudflare-queue" ? row.syncSummary.stopReason : null;
|
|
69060
|
+
}
|
|
69061
|
+
var QUEUE_LIMIT_STOPS = [
|
|
69062
|
+
CloudflareQueueDrainStopReasons.deadline,
|
|
69063
|
+
CloudflareQueueDrainStopReasons["batch-cap"]
|
|
69064
|
+
];
|
|
69065
|
+
var QUEUE_NOT_VISIBLE_STOPS = [
|
|
69066
|
+
CloudflareQueueDrainStopReasons["empty-pulls"],
|
|
69067
|
+
CloudflareQueueDrainStopReasons["unleased-only"]
|
|
69068
|
+
];
|
|
69069
|
+
function queueResidualCause(stopReasons) {
|
|
69070
|
+
const readRuns = "Read why each sync stopped with `canonry runs <project> --kind traffic-sync --format json` (`syncSummary.stopReason`).";
|
|
69071
|
+
if (stopReasons.every((reason) => reason !== null && QUEUE_LIMIT_STOPS.includes(reason))) {
|
|
69072
|
+
return {
|
|
69073
|
+
cause: "The drain stopped at its time limit or pull cap before the Queue was empty.",
|
|
69074
|
+
remediation: "If the backlog keeps growing, messages will pass the Queue's retention and be dropped. Shorten the traffic-sync schedule interval, or raise `CANONRY_CLOUDFLARE_QUEUE_SYNC_DEADLINE_MS` (default 4 minutes per sync)."
|
|
69075
|
+
};
|
|
69076
|
+
}
|
|
69077
|
+
if (stopReasons.every((reason) => reason !== null && QUEUE_NOT_VISIBLE_STOPS.includes(reason))) {
|
|
69078
|
+
return {
|
|
69079
|
+
cause: "Cloudflare returned no more messages to pull while still counting them, so they were leased or not yet visible. The drain stopped before its time limit, so raising the limit does not help.",
|
|
69080
|
+
remediation: "Later syncs pull them as they become visible. If the count does not fall across syncs, check the Queue in the Cloudflare dashboard for in-flight or delayed messages before they pass the Queue's retention."
|
|
69081
|
+
};
|
|
69082
|
+
}
|
|
69083
|
+
if (stopReasons.every((reason) => reason === CloudflareQueueDrainStopReasons.error)) {
|
|
69084
|
+
return {
|
|
69085
|
+
cause: "The last sync failed before the Queue was empty.",
|
|
69086
|
+
remediation: `Fix the sync error, then run \`canonry traffic sync <project> --source <id>\`. ${readRuns}`
|
|
69087
|
+
};
|
|
69088
|
+
}
|
|
69089
|
+
return {
|
|
69090
|
+
cause: "The last sync stopped before the Queue was empty.",
|
|
69091
|
+
remediation: `${readRuns} If it hit its time limit, shorten the traffic-sync schedule interval or raise \`CANONRY_CLOUDFLARE_QUEUE_SYNC_DEADLINE_MS\` (default 4 minutes per sync).`
|
|
69092
|
+
};
|
|
69093
|
+
}
|
|
68995
69094
|
var queueBacklogCheck = {
|
|
68996
69095
|
id: "traffic.source.queue-backlog",
|
|
68997
69096
|
category: CheckCategories.integrations,
|
|
@@ -69010,12 +69109,14 @@ var queueBacklogCheck = {
|
|
|
69010
69109
|
};
|
|
69011
69110
|
}
|
|
69012
69111
|
const measured = sources.filter((source) => source.queueBacklogCount !== null && source.queueBacklogCount !== void 0);
|
|
69112
|
+
const stopReasons = new Map(sources.map((source) => [source.id, lastQueueStopReason(ctx, source.id)]));
|
|
69013
69113
|
const details = {
|
|
69014
69114
|
sources: sources.map((source) => ({
|
|
69015
69115
|
id: source.id,
|
|
69016
69116
|
displayName: source.displayName,
|
|
69017
69117
|
queueBacklogCount: source.queueBacklogCount ?? null,
|
|
69018
|
-
queueBacklogObservedAt: source.queueBacklogObservedAt ?? null
|
|
69118
|
+
queueBacklogObservedAt: source.queueBacklogObservedAt ?? null,
|
|
69119
|
+
lastStopReason: stopReasons.get(source.id) ?? null
|
|
69019
69120
|
}))
|
|
69020
69121
|
};
|
|
69021
69122
|
if (measured.length === 0) {
|
|
@@ -69030,11 +69131,12 @@ var queueBacklogCheck = {
|
|
|
69030
69131
|
const remaining = measured.filter((source) => (source.queueBacklogCount ?? 0) > 0);
|
|
69031
69132
|
const total = remaining.reduce((sum, source) => sum + source.queueBacklogCount, 0);
|
|
69032
69133
|
if (total > CLOUDFLARE_QUEUE_RESIDUAL_WARN_COUNT) {
|
|
69134
|
+
const { cause, remediation } = queueResidualCause(remaining.map((source) => stopReasons.get(source.id) ?? null));
|
|
69033
69135
|
return {
|
|
69034
69136
|
status: CheckStatuses.warn,
|
|
69035
69137
|
code: "traffic.queue-backlog.remaining",
|
|
69036
|
-
summary: `${total.toLocaleString("en-US")} message(s) remained across ${remaining.length} Cloudflare Queue source(s) after the last sync, more than the ${CLOUDFLARE_QUEUE_RESIDUAL_WARN_COUNT.toLocaleString("en-US")} that can arrive while a drain runs.
|
|
69037
|
-
remediation
|
|
69138
|
+
summary: `${total.toLocaleString("en-US")} message(s) remained across ${remaining.length} Cloudflare Queue source(s) after the last sync, more than the ${CLOUDFLARE_QUEUE_RESIDUAL_WARN_COUNT.toLocaleString("en-US")} that can arrive while a drain runs. ${cause}`,
|
|
69139
|
+
remediation,
|
|
69038
69140
|
details
|
|
69039
69141
|
};
|
|
69040
69142
|
}
|
|
@@ -69710,7 +69812,7 @@ async function doctorRoutes(app, opts) {
|
|
|
69710
69812
|
}
|
|
69711
69813
|
|
|
69712
69814
|
// ../api-routes/src/status.ts
|
|
69713
|
-
import { and as and79, desc as
|
|
69815
|
+
import { and as and79, desc as desc45, eq as eq95, isNull as isNull15 } from "drizzle-orm";
|
|
69714
69816
|
function ageDays(iso) {
|
|
69715
69817
|
if (!iso) return null;
|
|
69716
69818
|
const ms = Date.now() - Date.parse(iso);
|
|
@@ -69760,7 +69862,7 @@ function makeRow(integration, label, runKind, state, staleAfterDays, dataThrough
|
|
|
69760
69862
|
};
|
|
69761
69863
|
}
|
|
69762
69864
|
function latestDate2(db, table, projectId) {
|
|
69763
|
-
const row = db.select({ date: table.date }).from(table).where(eq95(table.projectId, projectId)).orderBy(
|
|
69865
|
+
const row = db.select({ date: table.date }).from(table).where(eq95(table.projectId, projectId)).orderBy(desc45(table.date)).limit(1).get();
|
|
69764
69866
|
return row?.date ?? null;
|
|
69765
69867
|
}
|
|
69766
69868
|
function projectRows(opts, project) {
|
|
@@ -69838,7 +69940,7 @@ async function statusRoutes(app, opts) {
|
|
|
69838
69940
|
|
|
69839
69941
|
// ../api-routes/src/discovery/routes.ts
|
|
69840
69942
|
import crypto54 from "crypto";
|
|
69841
|
-
import { and as and80, desc as
|
|
69943
|
+
import { and as and80, desc as desc46, eq as eq96, gte as gte25, inArray as inArray41, isNull as isNull16, or as or20 } from "drizzle-orm";
|
|
69842
69944
|
var MAX_INFLIGHT_DISCOVERY_AGE_MS = 2 * 60 * 60 * 1e3;
|
|
69843
69945
|
async function discoveryRoutes(app, opts) {
|
|
69844
69946
|
app.post("/projects/:name/discover/run", async (request, reply) => {
|
|
@@ -69899,7 +70001,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
69899
70001
|
DiscoverySessionStatuses.probing
|
|
69900
70002
|
]),
|
|
69901
70003
|
gte25(discoverySessions.createdAt, ageFloorIso)
|
|
69902
|
-
)).orderBy(
|
|
70004
|
+
)).orderBy(desc46(discoverySessions.createdAt)).get();
|
|
69903
70005
|
if (existing && existing.runId) {
|
|
69904
70006
|
return { reused: true, sessionId: existing.id, runId: existing.runId };
|
|
69905
70007
|
}
|
|
@@ -69968,7 +70070,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
69968
70070
|
const project = resolveProject(app.db, request.params.name);
|
|
69969
70071
|
const parsedLimit = parseInt(request.query.limit ?? "", 10);
|
|
69970
70072
|
const limit = Number.isNaN(parsedLimit) || parsedLimit <= 0 ? 50 : parsedLimit;
|
|
69971
|
-
const rows = app.db.select().from(discoverySessions).where(eq96(discoverySessions.projectId, project.id)).orderBy(
|
|
70073
|
+
const rows = app.db.select().from(discoverySessions).where(eq96(discoverySessions.projectId, project.id)).orderBy(desc46(discoverySessions.createdAt)).limit(limit).all();
|
|
69972
70074
|
return reply.send(rows.map(serializeSession));
|
|
69973
70075
|
}
|
|
69974
70076
|
);
|
|
@@ -70601,7 +70703,7 @@ function dedupeStrings(input) {
|
|
|
70601
70703
|
|
|
70602
70704
|
// ../api-routes/src/technical-aeo.ts
|
|
70603
70705
|
import crypto56 from "crypto";
|
|
70604
|
-
import { and as and82, asc as asc21, count as count8, desc as
|
|
70706
|
+
import { and as and82, asc as asc21, count as count8, desc as desc47, eq as eq99, gte as gte26, inArray as inArray42, isNotNull as isNotNull10, isNull as isNull17, lt as lt14, or as or21, sql as sql34 } from "drizzle-orm";
|
|
70605
70707
|
import { alias } from "drizzle-orm/sqlite-core";
|
|
70606
70708
|
|
|
70607
70709
|
// ../api-routes/src/technical-aeo-issues.ts
|
|
@@ -71246,7 +71348,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
71246
71348
|
const filters = visibleCrawlFilters(projectId);
|
|
71247
71349
|
const allDates = and82(...filters);
|
|
71248
71350
|
const matchingDates = and82(...filters, eq99(sql34`substr(${siteCrawlSnapshots.createdAt}, 6, 5)`, requestedDate.slice(5)));
|
|
71249
|
-
const readDates = (where) => app.db.select({ date }).from(siteCrawlSnapshots).innerJoin(runs, eq99(siteCrawlSnapshots.runId, runs.id)).where(where).groupBy(date).orderBy(
|
|
71351
|
+
const readDates = (where) => app.db.select({ date }).from(siteCrawlSnapshots).innerJoin(runs, eq99(siteCrawlSnapshots.runId, runs.id)).where(where).groupBy(date).orderBy(desc47(date)).limit(10).all().map((row) => row.date);
|
|
71250
71352
|
const countDates = (where) => app.db.select({ value: sql34`count(distinct ${date})` }).from(siteCrawlSnapshots).innerJoin(runs, eq99(siteCrawlSnapshots.runId, runs.id)).where(where).get()?.value ?? 0;
|
|
71251
71353
|
return {
|
|
71252
71354
|
recentDates: readDates(allDates),
|
|
@@ -71265,11 +71367,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
71265
71367
|
}
|
|
71266
71368
|
const selectionFilters = scope.scoredOnly ? [...filters, sql34`exists (select 1 from ${siteAuditSnapshots} where ${siteAuditSnapshots.projectId} = ${siteCrawlSnapshots.projectId} and ${siteAuditSnapshots.runId} = ${siteCrawlSnapshots.runId})`] : filters;
|
|
71267
71369
|
const selected = app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq99(siteCrawlSnapshots.runId, runs.id)).where(and82(...selectionFilters, runId ? eq99(siteCrawlSnapshots.runId, runId) : void 0)).orderBy(
|
|
71268
|
-
|
|
71269
|
-
|
|
71270
|
-
|
|
71271
|
-
|
|
71272
|
-
|
|
71370
|
+
desc47(sql34`substr(${siteCrawlSnapshots.createdAt}, 1, 10)`),
|
|
71371
|
+
desc47(siteCrawlSnapshots.complete),
|
|
71372
|
+
desc47(siteCrawlSnapshots.pagesFetched),
|
|
71373
|
+
desc47(siteCrawlSnapshots.createdAt),
|
|
71374
|
+
desc47(siteCrawlSnapshots.runId)
|
|
71273
71375
|
).limit(1).get();
|
|
71274
71376
|
if (!selected) return void 0;
|
|
71275
71377
|
const date = selected.snapshot.createdAt.slice(0, 10);
|
|
@@ -71287,10 +71389,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
71287
71389
|
complete: siteCrawlSnapshots.complete,
|
|
71288
71390
|
pages: siteCrawlSnapshots.pagesFetched
|
|
71289
71391
|
}).from(siteCrawlSnapshots).innerJoin(runs, eq99(siteCrawlSnapshots.runId, runs.id)).where(sameDate).orderBy(
|
|
71290
|
-
|
|
71291
|
-
|
|
71292
|
-
|
|
71293
|
-
|
|
71392
|
+
desc47(siteCrawlSnapshots.complete),
|
|
71393
|
+
desc47(siteCrawlSnapshots.pagesFetched),
|
|
71394
|
+
desc47(siteCrawlSnapshots.createdAt),
|
|
71395
|
+
desc47(siteCrawlSnapshots.runId)
|
|
71294
71396
|
).limit(10).all();
|
|
71295
71397
|
const runSelection = {
|
|
71296
71398
|
reason: runId ? "explicit-run" : scope.scoredOnly ? "latest-scored-scan" : requestedDate ? selected.snapshot.complete ? "requested-date-complete" : "requested-date-most-pages" : selected.snapshot.complete ? "latest-date-complete" : "latest-date-most-pages",
|
|
@@ -71313,11 +71415,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
71313
71415
|
];
|
|
71314
71416
|
const complete = sql34`case when ${runs.status} = ${RunStatuses.completed} then 1 else 0 end`;
|
|
71315
71417
|
const selectAudit = (pinnedRunId) => app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs, eq99(siteAuditSnapshots.runId, runs.id)).where(and82(...filters, pinnedRunId ? eq99(siteAuditSnapshots.runId, pinnedRunId) : void 0)).orderBy(
|
|
71316
|
-
|
|
71317
|
-
|
|
71318
|
-
|
|
71319
|
-
|
|
71320
|
-
|
|
71418
|
+
desc47(sql34`substr(${siteAuditSnapshots.createdAt}, 1, 10)`),
|
|
71419
|
+
desc47(complete),
|
|
71420
|
+
desc47(siteAuditSnapshots.pagesAudited),
|
|
71421
|
+
desc47(siteAuditSnapshots.createdAt),
|
|
71422
|
+
desc47(siteAuditSnapshots.runId)
|
|
71321
71423
|
).limit(1).get();
|
|
71322
71424
|
const auditOfCrawl = (target) => {
|
|
71323
71425
|
const selected2 = target ? selectAudit(target.snapshot.runId) : void 0;
|
|
@@ -71345,10 +71447,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
71345
71447
|
complete,
|
|
71346
71448
|
pages: siteAuditSnapshots.pagesAudited
|
|
71347
71449
|
}).from(siteAuditSnapshots).innerJoin(runs, eq99(siteAuditSnapshots.runId, runs.id)).where(sameDate).orderBy(
|
|
71348
|
-
|
|
71349
|
-
|
|
71350
|
-
|
|
71351
|
-
|
|
71450
|
+
desc47(complete),
|
|
71451
|
+
desc47(siteAuditSnapshots.pagesAudited),
|
|
71452
|
+
desc47(siteAuditSnapshots.createdAt),
|
|
71453
|
+
desc47(siteAuditSnapshots.runId)
|
|
71352
71454
|
).limit(10).all().map((candidate) => ({ ...candidate, complete: Boolean(candidate.complete) }));
|
|
71353
71455
|
const runSelection = {
|
|
71354
71456
|
// A crawl exists here only when none stored a scorecard: this score is a fallback.
|
|
@@ -71568,7 +71670,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
71568
71670
|
const total = totalRow?.value ?? 0;
|
|
71569
71671
|
const limit = parsePositiveInt(request.query.limit, 100, 500);
|
|
71570
71672
|
const offset = parsePositiveInt(request.query.offset, 0, Number.MAX_SAFE_INTEGER);
|
|
71571
|
-
const orderBy = request.query.sort === "score-desc" ?
|
|
71673
|
+
const orderBy = request.query.sort === "score-desc" ? desc47(siteAuditPages.overallScore) : request.query.sort === "url" ? asc21(siteAuditPages.url) : asc21(siteAuditPages.overallScore);
|
|
71572
71674
|
const rows = app.db.select().from(siteAuditPages).where(where).orderBy(orderBy).limit(limit).offset(offset).all();
|
|
71573
71675
|
const pages = rows.map((row) => ({
|
|
71574
71676
|
url: row.url,
|
|
@@ -71593,7 +71695,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
71593
71695
|
eq99(runs.kind, RunKinds["site-audit"]),
|
|
71594
71696
|
inArray42(runs.status, SURFACEABLE_STATUSES),
|
|
71595
71697
|
notProbeRun()
|
|
71596
|
-
)).orderBy(
|
|
71698
|
+
)).orderBy(desc47(siteAuditSnapshots.createdAt)).limit(limit).all();
|
|
71597
71699
|
return { project: project.name, points: rows.reverse() };
|
|
71598
71700
|
});
|
|
71599
71701
|
app.get("/projects/:name/technical-aeo/crawl", async (request) => {
|
|
@@ -71904,7 +72006,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
71904
72006
|
eq99(siteCrawlEdges.runId, scope.runId),
|
|
71905
72007
|
eq99(siteCrawlEdges.attemptId, scope.attemptId),
|
|
71906
72008
|
eq99(siteCrawlEdges.internal, true),
|
|
71907
|
-
|
|
72009
|
+
isNotNull10(siteCrawlEdges.targetNodeKey),
|
|
71908
72010
|
or21(
|
|
71909
72011
|
inArray42(siteCrawlEdges.sourceNodeKey, frontier),
|
|
71910
72012
|
inArray42(siteCrawlEdges.targetNodeKey, frontier)
|
|
@@ -71953,7 +72055,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
71953
72055
|
eq99(siteCrawlEdges.runId, scope.runId),
|
|
71954
72056
|
eq99(siteCrawlEdges.attemptId, scope.attemptId),
|
|
71955
72057
|
eq99(siteCrawlEdges.internal, true),
|
|
71956
|
-
|
|
72058
|
+
isNotNull10(siteCrawlEdges.targetNodeKey),
|
|
71957
72059
|
inArray42(siteCrawlEdges.sourceNodeKey, outermostNodeKeys),
|
|
71958
72060
|
inArray42(siteCrawlEdges.targetNodeKey, outermostNodeKeys)
|
|
71959
72061
|
)).limit(1).get();
|
|
@@ -72055,7 +72157,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
72055
72157
|
eq99(siteCrawlEdges.internal, true),
|
|
72056
72158
|
eq99(siteCrawlEdges.followable, true),
|
|
72057
72159
|
eq99(siteCrawlEdges.relation, "anchor"),
|
|
72058
|
-
|
|
72160
|
+
isNotNull10(siteCrawlEdges.targetNodeKey),
|
|
72059
72161
|
inArray42(siteCrawlEdges.sourceNodeKey, frontier)
|
|
72060
72162
|
)).orderBy(asc21(siteCrawlEdges.edgeKey)).limit(SITE_HEALTH_PATH_MAX_VISITED_NODES + 1).all();
|
|
72061
72163
|
if (candidates.length > SITE_HEALTH_PATH_MAX_VISITED_NODES) truncated = true;
|
|
@@ -72144,7 +72246,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
72144
72246
|
eq99(siteCrawlSnapshots.complete, true),
|
|
72145
72247
|
notProbeRun()
|
|
72146
72248
|
];
|
|
72147
|
-
const afterTarget = request.query.toRunId ? resolveCrawl(project.id, request.query.toRunId) : app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq99(siteCrawlSnapshots.runId, runs.id)).where(and82(...completeSnapshotFilters)).orderBy(
|
|
72249
|
+
const afterTarget = request.query.toRunId ? resolveCrawl(project.id, request.query.toRunId) : app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq99(siteCrawlSnapshots.runId, runs.id)).where(and82(...completeSnapshotFilters)).orderBy(desc47(siteCrawlSnapshots.createdAt), desc47(siteCrawlSnapshots.runId)).limit(1).get();
|
|
72148
72250
|
if (request.query.toRunId && !afterTarget) throw notFound("Site crawl run", request.query.toRunId);
|
|
72149
72251
|
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq99(siteCrawlSnapshots.runId, runs.id)).where(and82(
|
|
72150
72252
|
...completeSnapshotFilters,
|
|
@@ -72155,7 +72257,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
72155
72257
|
lt14(siteCrawlSnapshots.runId, afterTarget.snapshot.runId)
|
|
72156
72258
|
)
|
|
72157
72259
|
)
|
|
72158
|
-
)).orderBy(
|
|
72260
|
+
)).orderBy(desc47(siteCrawlSnapshots.createdAt), desc47(siteCrawlSnapshots.runId)).limit(1).get() : void 0;
|
|
72159
72261
|
if (request.query.fromRunId && !beforeTarget) throw notFound("Site crawl run", request.query.fromRunId);
|
|
72160
72262
|
if (!afterTarget) {
|
|
72161
72263
|
const latestCrawl = resolveCrawl(project.id);
|
|
@@ -72607,7 +72709,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
72607
72709
|
sort
|
|
72608
72710
|
})).digest("base64url");
|
|
72609
72711
|
const offset = decodeCrawlPageCursor(request.query.cursor, selectionKey);
|
|
72610
|
-
const orderBy = sort === "score-desc" ? [
|
|
72712
|
+
const orderBy = sort === "score-desc" ? [desc47(siteCrawlPages.auditScore), asc21(siteCrawlPages.nodeKey)] : sort === "score-asc" ? [asc21(siteCrawlPages.auditScore), asc21(siteCrawlPages.nodeKey)] : sort === "path" ? [asc21(siteCrawlPages.path), asc21(siteCrawlPages.nodeKey)] : [asc21(siteCrawlPages.url), asc21(siteCrawlPages.nodeKey)];
|
|
72611
72713
|
let healthStateFilter = null;
|
|
72612
72714
|
if (healthState) {
|
|
72613
72715
|
const legacyRow = app.db.select({ id: siteCrawlPages.id }).from(siteCrawlPages).where(and82(
|
|
@@ -72883,7 +72985,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
72883
72985
|
// a fabricated broken link shown to a client is not worth the saved
|
|
72884
72986
|
// predicate. Applied to `total` as well as the rows, or the count would
|
|
72885
72987
|
// promise a page the list cannot fill.
|
|
72886
|
-
|
|
72988
|
+
isNotNull10(sql34`json_extract(${siteCrawlFindings.evidence}, '$.statusCode')`)
|
|
72887
72989
|
);
|
|
72888
72990
|
const total = app.db.select({ value: count8() }).from(siteCrawlFindings).where(where).get()?.value ?? 0;
|
|
72889
72991
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
@@ -72927,7 +73029,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
72927
73029
|
eq99(runs.projectId, project.id),
|
|
72928
73030
|
eq99(runs.kind, RunKinds["site-audit"]),
|
|
72929
73031
|
notProbeRun()
|
|
72930
|
-
)).orderBy(
|
|
73032
|
+
)).orderBy(desc47(runs.createdAt), desc47(runs.id)).limit(limit).all();
|
|
72931
73033
|
const runIds = rows.map((row) => row.runId);
|
|
72932
73034
|
const crawlRunIds = new Set(runIds.length === 0 ? [] : app.db.select({ runId: siteCrawlSnapshots.runId }).from(siteCrawlSnapshots).innerJoin(runs, eq99(siteCrawlSnapshots.runId, runs.id)).where(and82(
|
|
72933
73035
|
eq99(siteCrawlSnapshots.projectId, project.id),
|
|
@@ -72959,7 +73061,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
72959
73061
|
const attempt = app.db.select().from(siteCrawlAttempts).where(and82(
|
|
72960
73062
|
eq99(siteCrawlAttempts.projectId, project.id),
|
|
72961
73063
|
eq99(siteCrawlAttempts.runId, run.id)
|
|
72962
|
-
)).orderBy(
|
|
73064
|
+
)).orderBy(desc47(siteCrawlAttempts.attemptNumber), desc47(siteCrawlAttempts.updatedAt)).limit(1).get();
|
|
72963
73065
|
const persistedLayout = attempt ? app.db.select().from(siteCrawlGraphLayouts).where(and82(
|
|
72964
73066
|
eq99(siteCrawlGraphLayouts.projectId, project.id),
|
|
72965
73067
|
eq99(siteCrawlGraphLayouts.runId, run.id),
|
|
@@ -73006,7 +73108,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
73006
73108
|
const attempt = tx.select().from(siteCrawlAttempts).where(and82(
|
|
73007
73109
|
eq99(siteCrawlAttempts.projectId, project.id),
|
|
73008
73110
|
eq99(siteCrawlAttempts.runId, run.id)
|
|
73009
|
-
)).orderBy(
|
|
73111
|
+
)).orderBy(desc47(siteCrawlAttempts.attemptNumber), desc47(siteCrawlAttempts.updatedAt)).limit(1).get();
|
|
73010
73112
|
const state = run.status === RunStatuses.queued ? "waiting" : run.status === RunStatuses.running ? "collecting" : "terminal";
|
|
73011
73113
|
const base = {
|
|
73012
73114
|
project: project.name,
|
|
@@ -73036,7 +73138,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
73036
73138
|
auditFields: siteCrawlPages.auditFields
|
|
73037
73139
|
}).from(siteCrawlPages).where(and82(
|
|
73038
73140
|
auditedWhere,
|
|
73039
|
-
|
|
73141
|
+
isNotNull10(siteCrawlPages.auditScore),
|
|
73040
73142
|
lt14(siteCrawlPages.auditScore, 70)
|
|
73041
73143
|
)).orderBy(asc21(siteCrawlPages.auditScore), asc21(siteCrawlPages.nodeKey)).limit(LIVE_PAGE_HEALTH_CANDIDATE_LIMIT).all();
|
|
73042
73144
|
const examples = candidates.flatMap((candidate) => {
|
|
@@ -73132,7 +73234,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
73132
73234
|
// ../api-routes/src/research.ts
|
|
73133
73235
|
import crypto57 from "crypto";
|
|
73134
73236
|
import { z as z7 } from "zod";
|
|
73135
|
-
import { and as and83, count as count9, desc as
|
|
73237
|
+
import { and as and83, count as count9, desc as desc48, eq as eq100, gte as gte27, lt as lt15, or as or22, sql as sql35 } from "drizzle-orm";
|
|
73136
73238
|
var sameLocation = (a, b) => a.label === b.label && a.city === b.city && a.region === b.region && a.country === b.country && a.timezone === b.timezone;
|
|
73137
73239
|
var BATCH_RECEIPT_PREFIX = "__canonry_research_batch__:";
|
|
73138
73240
|
async function researchRoutes(app, opts) {
|
|
@@ -73308,7 +73410,7 @@ async function researchRoutes(app, opts) {
|
|
|
73308
73410
|
const rows = app.db.select().from(researchRuns).where(and83(
|
|
73309
73411
|
eq100(researchRuns.projectId, project.id),
|
|
73310
73412
|
cursor ? or22(lt15(researchRuns.createdAt, cursor.createdAt), and83(eq100(researchRuns.createdAt, cursor.createdAt), lt15(researchRuns.id, cursor.id))) : void 0
|
|
73311
|
-
)).orderBy(
|
|
73413
|
+
)).orderBy(desc48(researchRuns.createdAt), desc48(researchRuns.id)).limit(limit + 1).all();
|
|
73312
73414
|
const runs2 = rows.slice(0, limit).map(serializeRun);
|
|
73313
73415
|
const last = runs2.at(-1);
|
|
73314
73416
|
const nextCursor = rows.length > limit && last ? Buffer.from(JSON.stringify({ projectId: project.id, createdAt: last.createdAt, id: last.id })).toString("base64url") : null;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
pickProjectCitedDomain
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-U2WPZZPQ.js";
|
|
4
4
|
import {
|
|
5
5
|
GBP_INSIGHT_PROVIDER,
|
|
6
6
|
PERSISTENT_GAP_THRESHOLD,
|
|
@@ -14,10 +14,10 @@ import {
|
|
|
14
14
|
mergeGscQueryTotalsWithFallback,
|
|
15
15
|
readGscQueryDailyRows,
|
|
16
16
|
readLatestGscDataDate
|
|
17
|
-
} from "./chunk-
|
|
17
|
+
} from "./chunk-2HNTNKFC.js";
|
|
18
18
|
import {
|
|
19
19
|
getRequestContext
|
|
20
|
-
} from "./chunk-
|
|
20
|
+
} from "./chunk-LAILXR2F.js";
|
|
21
21
|
import {
|
|
22
22
|
competitors,
|
|
23
23
|
gbpDailyMetrics,
|
|
@@ -35,7 +35,7 @@ import {
|
|
|
35
35
|
queries,
|
|
36
36
|
querySnapshots,
|
|
37
37
|
runs
|
|
38
|
-
} from "./chunk-
|
|
38
|
+
} from "./chunk-RGD5SDT3.js";
|
|
39
39
|
import {
|
|
40
40
|
CitationStates,
|
|
41
41
|
RunKinds,
|
|
@@ -45,7 +45,7 @@ import {
|
|
|
45
45
|
effectiveDomains,
|
|
46
46
|
redactLogString,
|
|
47
47
|
redactLogValue
|
|
48
|
-
} from "./chunk-
|
|
48
|
+
} from "./chunk-U6CNJC2Y.js";
|
|
49
49
|
|
|
50
50
|
// src/intelligence-service.ts
|
|
51
51
|
import { eq, desc, asc, and, ne, or, inArray, gte, lte, isNull, sql, exists } from "drizzle-orm";
|