@canonry/canonry 7.6.0 → 7.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agent-workspace/skills/aero/SKILL.md +70 -6
- package/assets/agent-workspace/skills/aero/references/agent-operations.md +4 -1
- package/assets/agent-workspace/skills/aero/references/portfolio-analysis.md +15 -7
- package/assets/agent-workspace/skills/canonry/SKILL.md +4 -1
- package/assets/agent-workspace/skills/canonry/references/aeo-analysis.md +2 -2
- package/assets/assets/{AuditHistoryPanel-DZbPVuDn.js → AuditHistoryPanel-AJlkzSmq.js} +1 -1
- package/assets/assets/{BacklinksPage-6iPxcQdF.js → BacklinksPage-IhPtSh39.js} +1 -1
- package/assets/assets/{HistoryPage-DQ4gqs5s.js → HistoryPage-B-vNinvd.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-X4EGa3a5.js → MeasurementPropertyPage-s1kKo2TD.js} +1 -1
- package/assets/assets/{ProjectPage-qvbLROZz.js → ProjectPage-iH_9g_lU.js} +1 -1
- package/assets/assets/{RunsPage-DBfy79uf.js → RunsPage-BSbEUlNZ.js} +1 -1
- package/assets/assets/{SettingsPage-BlNafKGd.js → SettingsPage-zHyljW2M.js} +1 -1
- package/assets/assets/SiteHealthSection-AYGPjoka.js +4 -0
- package/assets/assets/{TrafficPage-CPZWDxjp.js → TrafficPage-CY_qggoW.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-BIhpWiRg.js → TrafficSourceDetailPage-aJ2Vk6Pd.js} +1 -1
- package/assets/assets/{extract-error-message-C9XX_kMH.js → extract-error-message-CHEe8sOP.js} +1 -1
- package/assets/assets/index-DfamSNSD.js +86 -0
- package/assets/assets/{react-sigma_core.esm.min-FTRP6jVQ.js → react-sigma_core.esm.min-2sZRqoN4.js} +1 -1
- package/assets/assets/{v2-overview-adapter-DEuH0zoM.js → v2-overview-adapter-ZNgNU_Yp.js} +1 -1
- package/assets/index.html +1 -1
- package/dist/{chunk-HXAGDOYV.js → chunk-AAMKTL6B.js} +3 -3
- package/dist/{chunk-WO7XJJHH.js → chunk-DO7XPDG5.js} +5977 -5742
- package/dist/{chunk-FRSGZYQZ.js → chunk-KRFTA4I7.js} +1129 -477
- package/dist/{chunk-BLHPFAR6.js → chunk-L35TGPAV.js} +458 -161
- package/dist/{chunk-7FJZCG7A.js → chunk-LO56QXPO.js} +5 -4
- package/dist/{chunk-P72CWNTU.js → chunk-T2V2PESH.js} +4 -3
- package/dist/{chunk-B7IT47FE.js → chunk-ZLB7WX5D.js} +26 -33
- package/dist/cli.js +67 -28
- package/dist/{demo-server-Z7NM6DJT.js → demo-server-KDMG3APY.js} +4 -4
- package/dist/index.js +6 -6
- package/dist/{intelligence-service-P7AOOXLZ.js → intelligence-service-LAXJFE3U.js} +2 -2
- package/dist/mcp.js +4 -4
- package/package.json +8 -8
- package/assets/assets/SiteHealthSection-79l2PnfR.js +0 -4
- package/assets/assets/index-CUQtRdIf.js +0 -86
|
@@ -88,9 +88,11 @@ import {
|
|
|
88
88
|
MEASUREMENT_PLAN_V2_SCHEMA_VERSION,
|
|
89
89
|
MEASUREMENT_PORTFOLIO_ANSWER_SOURCES_LIMIT,
|
|
90
90
|
MEASUREMENT_PORTFOLIO_DEFAULT_LIMIT,
|
|
91
|
+
MEASUREMENT_PORTFOLIO_DEFAULT_ROW_EVIDENCE_LIMIT,
|
|
91
92
|
MEASUREMENT_PORTFOLIO_ROW_EVIDENCE_LIMIT,
|
|
92
93
|
MEASUREMENT_PORTFOLIO_TIE_NAMED_INSTEAD_LIMIT,
|
|
93
94
|
MEASUREMENT_PORTFOLIO_TIE_NOTE,
|
|
95
|
+
MEASUREMENT_PORTFOLIO_WEAKEST_MARKET_LIMIT,
|
|
94
96
|
MEASUREMENT_PROPERTY_CITED_DOMAINS_LIMIT,
|
|
95
97
|
MIN_BRAND_ALIAS_KEY_LENGTH,
|
|
96
98
|
MIN_DOMAIN_BRAND_KEY_LENGTH,
|
|
@@ -98,6 +100,7 @@ import {
|
|
|
98
100
|
MODEL_ATTRIBUTION_EVENT_LIMIT,
|
|
99
101
|
MeasurementEvidenceShapes,
|
|
100
102
|
MeasurementPlanValidationError,
|
|
103
|
+
MeasurementPortfolioLists,
|
|
101
104
|
MeasurementRunScopeError,
|
|
102
105
|
OPERATIONAL_LOG_FIELDS_HEADER,
|
|
103
106
|
OPERATIONAL_LOG_OPT_IN_CONTEXT_FIELDS,
|
|
@@ -143,6 +146,7 @@ import {
|
|
|
143
146
|
SentimentOutcomes,
|
|
144
147
|
SiteAuditTrendDirections,
|
|
145
148
|
SiteCrawlFetchedStates,
|
|
149
|
+
SiteCrawlIndexabilityReasons,
|
|
146
150
|
SiteHealthLinkKinds,
|
|
147
151
|
SiteHealthTemplateDetections,
|
|
148
152
|
TrafficEventConfidences,
|
|
@@ -250,6 +254,7 @@ import {
|
|
|
250
254
|
buildMeasurementRunManifestV1,
|
|
251
255
|
businessProfileReviewsCover,
|
|
252
256
|
calendarDateRange,
|
|
257
|
+
calendarDateSchema,
|
|
253
258
|
calendarMonthBounds,
|
|
254
259
|
calendarRecurrenceSchema,
|
|
255
260
|
canonicalMeasurementExecutionIdentityJson,
|
|
@@ -326,6 +331,7 @@ import {
|
|
|
326
331
|
deriveCitedUrlCandidates,
|
|
327
332
|
deriveConversionTrackingIntegrityStatus,
|
|
328
333
|
deriveGoogleAdsEffectiveGoalGraph,
|
|
334
|
+
deriveSiteHealthReason,
|
|
329
335
|
deriveSiteHealthState,
|
|
330
336
|
deriveWinnabilityClass,
|
|
331
337
|
describeCompetitorAliasRejection,
|
|
@@ -355,6 +361,7 @@ import {
|
|
|
355
361
|
extractAnswerMentions,
|
|
356
362
|
extractDomainsFromText,
|
|
357
363
|
extractRecommendedCompetitors,
|
|
364
|
+
factorStatusFromDistribution,
|
|
358
365
|
factorStatusFromScore,
|
|
359
366
|
feedbackAcceptedDtoSchema,
|
|
360
367
|
feedbackSubmissionSchema,
|
|
@@ -800,10 +807,10 @@ import {
|
|
|
800
807
|
wordpressSchemaDeployResultDtoSchema,
|
|
801
808
|
wordpressSchemaStatusResultDtoSchema,
|
|
802
809
|
wordpressStatusDtoSchema
|
|
803
|
-
} from "./chunk-
|
|
810
|
+
} from "./chunk-DO7XPDG5.js";
|
|
804
811
|
|
|
805
812
|
// src/intelligence-service.ts
|
|
806
|
-
import { eq as eq84, desc as desc38, asc as asc15, and as and71, ne as ne15, or as or20, inArray as inArray34, gte as
|
|
813
|
+
import { eq as eq84, desc as desc38, asc as asc15, and as and71, ne as ne15, or as or20, inArray as inArray34, gte as gte23, lte as lte20, isNull as isNull16, sql as sql32, exists as exists2 } from "drizzle-orm";
|
|
807
814
|
|
|
808
815
|
// ../db/src/client.ts
|
|
809
816
|
import { mkdirSync } from "fs";
|
|
@@ -10290,7 +10297,7 @@ function buildShareOfVoiceFrame(input) {
|
|
|
10290
10297
|
const competitorMentions = selected.reduce((sum, row) => sum + row.mentions, 0);
|
|
10291
10298
|
const denominator = input.projectMentions + competitorMentions;
|
|
10292
10299
|
const basis = input.tracked ? "tracked" : input.competitors.length > 0 ? "observed" : null;
|
|
10293
|
-
const reason = !input.classSelected ? "select-query-class" : basis === null ? "no-competitors" : !input.tracked && domains.length < MIN_OBSERVED_COMPETITORS ? "insufficient-observed" : input.answeredResults === 0 ? "no-answers" : denominator === 0 ? "no-mentions" : null;
|
|
10300
|
+
const reason = input.unavailableReason ?? (!input.classSelected ? "select-query-class" : basis === null ? "no-competitors" : !input.tracked && domains.length < MIN_OBSERVED_COMPETITORS ? "insufficient-observed" : input.answeredResults === 0 ? "no-answers" : denominator === 0 ? "no-mentions" : null);
|
|
10294
10301
|
return {
|
|
10295
10302
|
basis,
|
|
10296
10303
|
reason,
|
|
@@ -10458,6 +10465,7 @@ function buildCompetitorLandscapeHistory(options) {
|
|
|
10458
10465
|
const frame = buildShareOfVoiceFrame({
|
|
10459
10466
|
tracked: pinned.size > 0,
|
|
10460
10467
|
classSelected: options.shareOfVoiceEligible,
|
|
10468
|
+
unavailableReason: options.shareOfVoiceUnavailableReason,
|
|
10461
10469
|
projectMentions: projectRow.mentionCount,
|
|
10462
10470
|
answeredResults,
|
|
10463
10471
|
competitors: competitiveRows.map((row) => ({ domain: row.identity.domain, mentions: row.mentionCount }))
|
|
@@ -13525,18 +13533,12 @@ function sentimentSentenceSpans(text2) {
|
|
|
13525
13533
|
}
|
|
13526
13534
|
return spans.map((span, index2) => ({ id: `s${index2 + 1}`, ...span }));
|
|
13527
13535
|
}
|
|
13528
|
-
function
|
|
13536
|
+
function sentimentClassifierIdentity(source) {
|
|
13529
13537
|
const edge = source.edges[0];
|
|
13530
13538
|
const subject = { id: source.subject.key, displayName: source.subject.name, aliases: source.subject.aliases, qualifiedAliases: source.subject.identityAliases, urls: source.subject.urls, mentionNotApplicable: source.subject.mentionNotApplicable };
|
|
13531
13539
|
return {
|
|
13532
|
-
sourceSnapshotId: source.snapshotId,
|
|
13533
|
-
sourceText: source.sourceText,
|
|
13534
|
-
sourceTextHash: sentimentHash(source.sourceText),
|
|
13535
13540
|
subject,
|
|
13536
|
-
subjectHash: sentimentHash({ subject, context: edge.context }),
|
|
13537
13541
|
language: source.language,
|
|
13538
|
-
definition,
|
|
13539
|
-
sentences: sentimentSentenceSpans(source.sourceText),
|
|
13540
13542
|
context: {
|
|
13541
13543
|
queryId: edge.queryKey,
|
|
13542
13544
|
queryText: edge.queryText,
|
|
@@ -13551,12 +13553,27 @@ function sentimentClassifierInput(source, definition) {
|
|
|
13551
13553
|
}
|
|
13552
13554
|
};
|
|
13553
13555
|
}
|
|
13556
|
+
function sentimentSourceHashes(source, identity = sentimentClassifierIdentity(source)) {
|
|
13557
|
+
return { sourceTextHash: sentimentHash(source.sourceText), subjectHash: sentimentHash({ subject: identity.subject, context: source.edges[0].context }) };
|
|
13558
|
+
}
|
|
13559
|
+
function sentimentClassifierInput(source, definition) {
|
|
13560
|
+
const identity = sentimentClassifierIdentity(source);
|
|
13561
|
+
return {
|
|
13562
|
+
sourceSnapshotId: source.snapshotId,
|
|
13563
|
+
sourceText: source.sourceText,
|
|
13564
|
+
...identity,
|
|
13565
|
+
...sentimentSourceHashes(source, identity),
|
|
13566
|
+
definition,
|
|
13567
|
+
sentences: sentimentSentenceSpans(source.sourceText)
|
|
13568
|
+
};
|
|
13569
|
+
}
|
|
13554
13570
|
|
|
13555
13571
|
// ../api-routes/src/sentiment-service.ts
|
|
13556
13572
|
var DISCLOSURE = "Experimental English sentiment classification. Enabling sends each stored answer's text to TypeSafe with the frozen subject identity (name, aliases, qualified aliases and URLs), the tracked query text and query class, the answer engine with its requested and served models, the location, and internal query, subject, Property, group and market identifiers.";
|
|
13557
13573
|
var FULL_ROWS = { include: /* @__PURE__ */ new Set(["assessments", "locations"]), limit: null, after: null };
|
|
13558
13574
|
var COMPACT_ROWS = { include: /* @__PURE__ */ new Set(), limit: null, after: null };
|
|
13559
13575
|
var OPERATIONAL_OUTCOMES = /* @__PURE__ */ new Set([SentimentOutcomes.pending, SentimentOutcomes.running, SentimentOutcomes["waiting-to-retry"], SentimentOutcomes.failed, SentimentOutcomes.canceled]);
|
|
13576
|
+
var PREVIOUS_RATED_CANDIDATE_LIMIT = 50;
|
|
13560
13577
|
var RATED_OUTCOMES = /* @__PURE__ */ new Set([SentimentOutcomes.favorable, SentimentOutcomes.mixed, SentimentOutcomes.unfavorable]);
|
|
13561
13578
|
var SentimentService = class {
|
|
13562
13579
|
constructor(db, options) {
|
|
@@ -13761,11 +13778,11 @@ var SentimentService = class {
|
|
|
13761
13778
|
}
|
|
13762
13779
|
items(projectId, selection, source, withInput = false) {
|
|
13763
13780
|
if (!selection.runId || !selection.evaluationDefinitionId) return [];
|
|
13764
|
-
|
|
13781
|
+
this.definition(selection.evaluationDefinitionId);
|
|
13765
13782
|
const filter = sourceFilter(selection);
|
|
13766
13783
|
const allowed = new Map(source.assessments.map((item2) => {
|
|
13767
|
-
const
|
|
13768
|
-
return [`${item2.snapshotId}:${
|
|
13784
|
+
const hashes = sentimentSourceHashes(item2);
|
|
13785
|
+
return [`${item2.snapshotId}:${hashes.sourceTextHash}:${hashes.subjectHash}`, item2];
|
|
13769
13786
|
}));
|
|
13770
13787
|
const scope = and4(eq9(sentimentWorkItems.projectId, projectId), eq9(sentimentWorkItems.runId, selection.runId), eq9(sentimentWorkItems.evaluationDefinitionId, selection.evaluationDefinitionId));
|
|
13771
13788
|
const rows = this.db.select({
|
|
@@ -13792,7 +13809,7 @@ var SentimentService = class {
|
|
|
13792
13809
|
if (!inputs) return [item2];
|
|
13793
13810
|
const input = storedSentimentClassifierInputSchema.parse(inputs.get(work.id));
|
|
13794
13811
|
const primary = edges[0];
|
|
13795
|
-
const projected =
|
|
13812
|
+
const projected = sentimentClassifierIdentity({ ...member, edges }).context;
|
|
13796
13813
|
const context = { ...input.context, ...projected, queryText: primary.queryText };
|
|
13797
13814
|
const evidence = { assessmentId: work.id, runId: work.runId, sourceSnapshotId: work.snapshotId, sourceText: input.sourceText, sourceTextHash: work.sourceTextHash, subject: input.subject, subjectHash: work.subjectHash, context, evaluationDefinitionId: work.evaluationDefinitionId, outcome: item2.outcome, conclusion: classification?.kind === "classified" ? classification.conclusion : [], complaint: classification?.kind === "classified" ? classification.complaint : null, returnedModel: item2.returnedModel, reason: item2.reason };
|
|
13798
13815
|
return [{ ...item2, input, evidence }];
|
|
@@ -13928,7 +13945,8 @@ var SentimentService = class {
|
|
|
13928
13945
|
return { state: result.state, reason: result.reason, provisional: result.provisional, coverage: result.coverage, score: result.score, selection: result.selection, runIds: result.selection.runIds ?? (result.selection.runId ? [result.selection.runId] : []) };
|
|
13929
13946
|
};
|
|
13930
13947
|
const branded = headline("branded"), nonBrand = headline("non-brand");
|
|
13931
|
-
|
|
13948
|
+
const pooledOverall = { ...overallSentiment(reads, !configured), pooled: true };
|
|
13949
|
+
return { configured, branded, nonBrand, headlineQueryClass: "branded", pooledOverall, overall: pooledOverall };
|
|
13932
13950
|
}
|
|
13933
13951
|
evidence(projectId, query, limit, cursor) {
|
|
13934
13952
|
const { assessmentId, outcome: outcome2, ...sourceSelection } = query;
|
|
@@ -13949,13 +13967,52 @@ var SentimentService = class {
|
|
|
13949
13967
|
const page3 = items.slice(0, limit);
|
|
13950
13968
|
return { state, selection, items: page3.map((item2) => item2.evidence), nextCursor: items.length > limit ? this.sign({ projectId, selection, fingerprint: read.fingerprint, after: page3.at(-1).work.id }) : null };
|
|
13951
13969
|
}
|
|
13970
|
+
previousRatedRun(projectId, query, toRunId) {
|
|
13971
|
+
const target = this.db.select().from(runs).where(and4(eq9(runs.projectId, projectId), eq9(runs.id, toRunId))).get();
|
|
13972
|
+
if (!target) throw notFound("Run", toRunId);
|
|
13973
|
+
const selected = this.resolve(projectId, { ...query, runId: toRunId });
|
|
13974
|
+
if (!selected.source.assessments.length) {
|
|
13975
|
+
const incomplete = Boolean(selected.source.skipped["incomplete-run"]);
|
|
13976
|
+
throw validationError(incomplete ? "The target run is incomplete; no sentiment trend can be selected." : "The target run has no eligible sentiment population for this selection.", { reason: incomplete ? "target-incomplete" : "target-population-unavailable" });
|
|
13977
|
+
}
|
|
13978
|
+
const filter = sourceFilter(query);
|
|
13979
|
+
const population = (source) => new Set(source.assessments.map((member) => comparisonUnit(sentimentClassifierIdentity({ ...member, edges: member.edges.filter((edge) => matchesSentimentEdge(edge, filter)) }))));
|
|
13980
|
+
const targetUnits = population(selected.source);
|
|
13981
|
+
const requestedVersion = query.revision === void 0 ? void 0 : this.db.select({ id: measurementPlanVersions.id }).from(measurementPlanVersions).where(and4(eq9(measurementPlanVersions.projectId, projectId), eq9(measurementPlanVersions.revision, query.revision))).get()?.id ?? "__missing-revision__";
|
|
13982
|
+
const candidates = this.db.selectDistinct({ id: runs.id, createdAt: runs.createdAt }).from(runs).innerJoin(sentimentWorkItems, and4(eq9(sentimentWorkItems.runId, runs.id), eq9(sentimentWorkItems.projectId, projectId))).innerJoin(sentimentResults, eq9(sentimentResults.workItemId, sentimentWorkItems.id)).where(and4(
|
|
13983
|
+
eq9(runs.projectId, projectId),
|
|
13984
|
+
eq9(runs.kind, target.kind),
|
|
13985
|
+
notProbeRun(),
|
|
13986
|
+
inArray3(sentimentResults.outcome, [...RATED_OUTCOMES]),
|
|
13987
|
+
query.evaluationDefinitionId ? eq9(sentimentWorkItems.evaluationDefinitionId, query.evaluationDefinitionId) : void 0,
|
|
13988
|
+
query.queryClass ? sql6`exists (select 1 from json_each(${sentimentWorkItems.edges}) as edge where json_extract(edge.value, '$.queryClass') = ${query.queryClass})` : void 0,
|
|
13989
|
+
requestedVersion ? eq9(runs.measurementPlanVersionId, requestedVersion) : void 0,
|
|
13990
|
+
or2(lt2(runs.createdAt, target.createdAt), and4(eq9(runs.createdAt, target.createdAt), lt2(runs.id, target.id))),
|
|
13991
|
+
target.measurementPlanVersionId ? isNotNull2(runs.measurementPlanVersionId) : isNull2(runs.measurementPlanVersionId)
|
|
13992
|
+
)).orderBy(desc3(runs.createdAt), desc3(runs.id)).limit(PREVIOUS_RATED_CANDIDATE_LIMIT + 1).all();
|
|
13993
|
+
let newestRatedRunId;
|
|
13994
|
+
for (const candidate of candidates.slice(0, PREVIOUS_RATED_CANDIDATE_LIMIT)) {
|
|
13995
|
+
const read = this.resolve(projectId, { ...query, runId: candidate.id });
|
|
13996
|
+
if (!read.source.assessments.length) continue;
|
|
13997
|
+
const candidateUnits = population(read.source);
|
|
13998
|
+
if (candidateUnits.size !== targetUnits.size || [...targetUnits].some((key) => !candidateUnits.has(key))) {
|
|
13999
|
+
newestRatedRunId ??= candidate.id;
|
|
14000
|
+
continue;
|
|
14001
|
+
}
|
|
14002
|
+
if (this.items(projectId, read.selection, read.source).some((item2) => RATED_OUTCOMES.has(item2.outcome))) return candidate.id;
|
|
14003
|
+
}
|
|
14004
|
+
if (candidates.length > PREVIOUS_RATED_CANDIDATE_LIMIT) throw validationError("No compatible rated predecessor was found within the bounded search; select an older run explicitly.", { reason: "previous-rated-search-limit", candidateLimit: PREVIOUS_RATED_CANDIDATE_LIMIT, ...newestRatedRunId ? { newestRatedRunId } : {} });
|
|
14005
|
+
if (newestRatedRunId) throw validationError("Earlier rated runs exist, but their selected sentiment population differs from the target.", { reason: "previous-rated-population-changed", newestRatedRunId });
|
|
14006
|
+
throw validationError("No previous rated run exists for the selected sentiment scope.", { reason: "previous-rated-run-unavailable" });
|
|
14007
|
+
}
|
|
13952
14008
|
compare(projectId, query, fromRunId, toRunId) {
|
|
13953
14009
|
if (query.runIds) throw validationError("Comparison requires one explicit run per period.");
|
|
14010
|
+
if (fromRunId === toRunId) throw validationError("Sentiment comparison requires two distinct runs.", { reason: "same-run" });
|
|
14011
|
+
if (fromRunId === "previous-rated") fromRunId = this.previousRatedRun(projectId, query, toRunId);
|
|
13954
14012
|
const from = this.resolve(projectId, { ...query, runId: fromRunId }), to = this.resolve(projectId, { ...query, runId: toRunId });
|
|
13955
14013
|
const fromItems = this.items(projectId, from.selection, from.source, true), toItems = this.items(projectId, to.selection, to.source, true);
|
|
13956
|
-
const unit = (item2) => sentimentHash({ query: item2.evidence.context.queryId, queryText: item2.evidence.context.queryText, provider: item2.input.context.provider, subject: item2.input.subject, context: item2.evidence.context.locationContext, language: item2.input.language, usageEdges: item2.evidence.context.usageEdges });
|
|
13957
14014
|
const units = (items) => {
|
|
13958
|
-
const byItem = new Map(items.map((item2) => [item2,
|
|
14015
|
+
const byItem = new Map(items.map((item2) => [item2, comparisonUnit(item2.input, item2.evidence.context)])), byUnit = /* @__PURE__ */ new Map();
|
|
13959
14016
|
for (const [item2, key] of byItem) if (!byUnit.has(key)) byUnit.set(key, item2);
|
|
13960
14017
|
return { byItem, byUnit };
|
|
13961
14018
|
};
|
|
@@ -13988,6 +14045,9 @@ var SentimentService = class {
|
|
|
13988
14045
|
return { from: fromSummary, to: toSummary, verdict, favorableRateDelta: verdict && fromSummary.score.favorableRate !== null && toSummary.score.favorableRate !== null ? toSummary.score.favorableRate - fromSummary.score.favorableRate : null, refusalReasons: [...new Set(refusalReasons)], commonUnits: common.size, excludedFrom: expectedFrom - fromSummary.coverage.selected, excludedTo: expectedTo - toSummary.coverage.selected, changedScope, method: "wilson-independent-v1", limitation: SENTIMENT_INTERVAL_LIMITATION };
|
|
13989
14046
|
}
|
|
13990
14047
|
};
|
|
14048
|
+
function comparisonUnit(input, context = input.context) {
|
|
14049
|
+
return sentimentHash({ query: context.queryId, queryText: context.queryText, provider: input.context.provider, subject: input.subject, context: context.locationContext, language: input.language, usageEdges: context.usageEdges });
|
|
14050
|
+
}
|
|
13991
14051
|
var jobAttemptJoin = and4(eq9(sentimentJobItems.workItemId, sentimentAttempts.workItemId), eq9(sentimentJobItems.projectId, sentimentAttempts.projectId));
|
|
13992
14052
|
function emptySource() {
|
|
13993
14053
|
return { assessments: [], runIds: [], sourceCoverage: { expected: 0, completed: 0 }, sourceSlotIds: { expected: [], completed: [] }, skipped: {} };
|
|
@@ -14085,8 +14145,9 @@ function outcome(status, result) {
|
|
|
14085
14145
|
}
|
|
14086
14146
|
function parse(schema, value) {
|
|
14087
14147
|
const result = schema.safeParse(value);
|
|
14088
|
-
if (
|
|
14089
|
-
|
|
14148
|
+
if (result.success) return result.data;
|
|
14149
|
+
const first = Array.isArray(result.error.issues) ? result.error.issues[0]?.message : void 0;
|
|
14150
|
+
throw validationError(typeof first === "string" ? `Invalid sentiment request: ${first}` : "Invalid sentiment request.", { issues: result.error.issues });
|
|
14090
14151
|
}
|
|
14091
14152
|
|
|
14092
14153
|
// ../api-routes/src/sentiment.ts
|
|
@@ -15528,6 +15589,24 @@ function resolveMentionIdentity(text2, targets, aliases, citedTargetIds, ambigui
|
|
|
15528
15589
|
}
|
|
15529
15590
|
return state;
|
|
15530
15591
|
}
|
|
15592
|
+
function createTargetMentionReader(targets, ownedHosts = []) {
|
|
15593
|
+
const compiledAliases = compiledMentionAliases(targets);
|
|
15594
|
+
const mentionableIds = new Set(compiledAliases.map((alias2) => alias2.targetId));
|
|
15595
|
+
const aliases = indexMentionAliases(compiledAliases);
|
|
15596
|
+
const ambiguityPatterns = new Map(targets.map((target) => [target.id, identityAmbiguityPatterns(target.aliases)]));
|
|
15597
|
+
const routes = compiledTargetRoutes(targets);
|
|
15598
|
+
const normalizedOwnedHosts = ownedHosts.map(normalizedHost);
|
|
15599
|
+
return (answerText, targetIds, citedUrls = []) => {
|
|
15600
|
+
if (answerText === null || targetIds.length === 0) return null;
|
|
15601
|
+
const citedTargetIds = new Set(citedUrls.flatMap((url) => {
|
|
15602
|
+
const source = classifySourceAttribution(url, routes, normalizedOwnedHosts);
|
|
15603
|
+
return source.classification === "matched" ? source.matchedTargetIds : [];
|
|
15604
|
+
}));
|
|
15605
|
+
const state = resolveMentionIdentity(answerProseForMentions(answerText), targets, aliases, citedTargetIds, ambiguityPatterns);
|
|
15606
|
+
if (targetIds.some((id) => state.mentioned.has(id))) return true;
|
|
15607
|
+
return targetIds.some((id) => !mentionableIds.has(id) || state.unknown.has(id)) ? null : false;
|
|
15608
|
+
};
|
|
15609
|
+
}
|
|
15531
15610
|
function observationSource(observation) {
|
|
15532
15611
|
if (observation.citedUrls !== null) {
|
|
15533
15612
|
return {
|
|
@@ -16252,6 +16331,10 @@ function historicalEvidence(rawResponse) {
|
|
|
16252
16331
|
return { urls: [], complete: false };
|
|
16253
16332
|
}
|
|
16254
16333
|
}
|
|
16334
|
+
function measurementSnapshotSources(snapshot) {
|
|
16335
|
+
if (snapshot.citedUrls !== null) return { urls: snapshot.citedUrls, complete: snapshot.captureStatus === "complete", historical: false };
|
|
16336
|
+
return { ...historicalEvidence(snapshot.rawResponse), historical: true };
|
|
16337
|
+
}
|
|
16255
16338
|
function slotLocation(slot) {
|
|
16256
16339
|
return slot.context?.label ?? null;
|
|
16257
16340
|
}
|
|
@@ -16300,7 +16383,7 @@ function observationInputs(manifest, snapshots, legacy) {
|
|
|
16300
16383
|
if (!supportsRequestedContext(snapshot, slot)) return [];
|
|
16301
16384
|
validateSupportedLocation(snapshot, slot);
|
|
16302
16385
|
}
|
|
16303
|
-
const
|
|
16386
|
+
const sources = measurementSnapshotSources(snapshot);
|
|
16304
16387
|
return [{
|
|
16305
16388
|
id: snapshot.id,
|
|
16306
16389
|
executionId: snapshot.measurementExecutionId,
|
|
@@ -16308,12 +16391,9 @@ function observationInputs(manifest, snapshots, legacy) {
|
|
|
16308
16391
|
provider: snapshot.provider.trim().toLocaleLowerCase("en"),
|
|
16309
16392
|
location: snapshot.location,
|
|
16310
16393
|
answerText: snapshot.answerText,
|
|
16311
|
-
citedUrls:
|
|
16312
|
-
citedUrlsComplete:
|
|
16313
|
-
...
|
|
16314
|
-
const historical = historicalEvidence(snapshot.rawResponse);
|
|
16315
|
-
return { historicalCitedUrls: historical.urls, historicalCitedUrlsComplete: historical.complete };
|
|
16316
|
-
})() : {}
|
|
16394
|
+
citedUrls: sources.historical ? null : sources.urls,
|
|
16395
|
+
citedUrlsComplete: !sources.historical && sources.complete,
|
|
16396
|
+
...sources.historical ? { historicalCitedUrls: sources.urls, historicalCitedUrlsComplete: sources.complete } : {}
|
|
16317
16397
|
}];
|
|
16318
16398
|
});
|
|
16319
16399
|
}
|
|
@@ -16348,6 +16428,15 @@ function reportInput(revision, plan, manifest, snapshots, legacy) {
|
|
|
16348
16428
|
function v2UsageEdgeId(edge) {
|
|
16349
16429
|
return `target:${edge.targetKey}:${edge.queryId}:${edge.executionNodeKey}`;
|
|
16350
16430
|
}
|
|
16431
|
+
function measurementPlanV2ReportTargets(plan) {
|
|
16432
|
+
return plan.targets.map((target) => ({
|
|
16433
|
+
id: target.stableKey,
|
|
16434
|
+
label: target.label,
|
|
16435
|
+
aliases: target.mentionNotApplicable ? [] : target.aliases,
|
|
16436
|
+
...target.identityAliases === void 0 ? {} : { identityAliases: target.identityAliases },
|
|
16437
|
+
urls: target.urlMatchers.map((matcher, index2) => matcherInput(target.stableKey, matcher, index2))
|
|
16438
|
+
}));
|
|
16439
|
+
}
|
|
16351
16440
|
function buildMeasurementPlanV2ReportInput(revision, plan, manifest, snapshots) {
|
|
16352
16441
|
const classByAssignment = new Map(plan.assignments.map((assignment) => [
|
|
16353
16442
|
`${assignment.targetKey}:${assignment.queryId}`,
|
|
@@ -16365,15 +16454,7 @@ function buildMeasurementPlanV2ReportInput(revision, plan, manifest, snapshots)
|
|
|
16365
16454
|
ownedHosts: plan.identities.projectBrand.ownedHosts,
|
|
16366
16455
|
projectBrandNames: plan.identities.projectBrand.names,
|
|
16367
16456
|
projectDomain: plan.identities.projectBrand.canonicalHost,
|
|
16368
|
-
targets: plan
|
|
16369
|
-
id: target.stableKey,
|
|
16370
|
-
label: target.label,
|
|
16371
|
-
// The revision already decided this Property cannot be mentioned. Feeding
|
|
16372
|
-
// its aliases in anyway would turn "not applicable" into a 0% reading.
|
|
16373
|
-
aliases: target.mentionNotApplicable ? [] : target.aliases,
|
|
16374
|
-
...target.identityAliases === void 0 ? {} : { identityAliases: target.identityAliases },
|
|
16375
|
-
urls: target.urlMatchers.map((matcher, index2) => matcherInput(target.stableKey, matcher, index2))
|
|
16376
|
-
})),
|
|
16457
|
+
targets: measurementPlanV2ReportTargets(plan),
|
|
16377
16458
|
groups: plan.groups.map((group) => ({
|
|
16378
16459
|
id: group.stableKey,
|
|
16379
16460
|
label: group.label,
|
|
@@ -18357,7 +18438,7 @@ function parseBody(schema, value, message) {
|
|
|
18357
18438
|
}
|
|
18358
18439
|
|
|
18359
18440
|
// ../api-routes/src/competitor-landscape.ts
|
|
18360
|
-
import { and as and17, desc as desc8, eq as eq24, gte as gte3, inArray as inArray10, isNull as isNull6 } from "drizzle-orm";
|
|
18441
|
+
import { and as and17, desc as desc8, eq as eq24, gte as gte3, inArray as inArray10, isNull as isNull6, sql as sql9 } from "drizzle-orm";
|
|
18361
18442
|
|
|
18362
18443
|
// ../api-routes/src/mention-share-inputs.ts
|
|
18363
18444
|
import { eq as eq22 } from "drizzle-orm";
|
|
@@ -18493,7 +18574,11 @@ function createMeasurementOverviewCache(maxEntries = MEASUREMENT_OVERVIEW_CACHE_
|
|
|
18493
18574
|
};
|
|
18494
18575
|
}
|
|
18495
18576
|
function parseOverviewQuery(raw) {
|
|
18496
|
-
const candidate = {
|
|
18577
|
+
const candidate = {
|
|
18578
|
+
...raw,
|
|
18579
|
+
...raw.limit === void 0 ? {} : { limit: Number(raw.limit) },
|
|
18580
|
+
...raw.compact === "true" || raw.compact === "false" ? { compact: raw.compact === "true" } : {}
|
|
18581
|
+
};
|
|
18497
18582
|
const parsed = measurementOverviewQuerySchema.safeParse(candidate);
|
|
18498
18583
|
if (!parsed.success) throw validationError("Invalid measurement overview query", { issues: parsed.error.issues });
|
|
18499
18584
|
return parsed.data;
|
|
@@ -18563,7 +18648,8 @@ function overviewFilterFingerprint(query) {
|
|
|
18563
18648
|
location: query.location === void 0 ? null : normalizedText(query.location),
|
|
18564
18649
|
from: query.from ?? null,
|
|
18565
18650
|
to: query.to ?? null,
|
|
18566
|
-
search: query.search === void 0 ? null : normalizedText(query.search)
|
|
18651
|
+
search: query.search === void 0 ? null : normalizedText(query.search),
|
|
18652
|
+
compact: query.compact ?? false
|
|
18567
18653
|
};
|
|
18568
18654
|
return createHash4("sha256").update(JSON.stringify(filters)).digest("base64url");
|
|
18569
18655
|
}
|
|
@@ -18725,12 +18811,19 @@ function pageOf(rows, query, displayedRunId, activePlanVersionId, evidenceFinger
|
|
|
18725
18811
|
if (index2 < 0) throw validationError("The measurement overview cursor does not belong to this result set.");
|
|
18726
18812
|
offset = index2 + 1;
|
|
18727
18813
|
}
|
|
18728
|
-
const
|
|
18814
|
+
const selected = ordered.slice(offset, offset + limit);
|
|
18815
|
+
const items = query.compact ? selected.map((row) => ({ ...row, providers: [] })) : selected;
|
|
18816
|
+
if (query.compact) {
|
|
18817
|
+
while (JSON.stringify(items, null, 2).length > 12e3 && items.length > 1) items.pop();
|
|
18818
|
+
if (JSON.stringify(items, null, 2).length > 12e3) {
|
|
18819
|
+
throw validationError("One measurement Property row exceeds the compact page budget. Read a narrower scope without compact=true.");
|
|
18820
|
+
}
|
|
18821
|
+
}
|
|
18729
18822
|
const last = items.at(-1);
|
|
18730
18823
|
return {
|
|
18731
18824
|
page: {
|
|
18732
18825
|
items,
|
|
18733
|
-
nextCursor: last === void 0 || ordered.at(offset +
|
|
18826
|
+
nextCursor: last === void 0 || ordered.at(offset + items.length) === void 0 ? null : cursorOf(
|
|
18734
18827
|
last,
|
|
18735
18828
|
sort,
|
|
18736
18829
|
displayedRunId,
|
|
@@ -19097,6 +19190,16 @@ async function measurementOverviewRoutes(app, options = {}) {
|
|
|
19097
19190
|
if (!active) throw notFound("Active measurement plan", project.name);
|
|
19098
19191
|
const scope = resolveScope(active.plan, query);
|
|
19099
19192
|
const response = active.plan.schemaVersion === MEASUREMENT_PLAN_V2_SCHEMA_VERSION ? planV2Overview(app.db, project.id, active, active.plan, query, scope, cache) : planV1Overview(app.db, project.id, active, query, scope);
|
|
19193
|
+
if (query.compact) {
|
|
19194
|
+
response.detailsOmitted = ["properties.providers", ...response.detailsOmitted ?? []];
|
|
19195
|
+
if (response.namedShareOfVoice !== void 0) {
|
|
19196
|
+
delete response.namedShareOfVoice;
|
|
19197
|
+
response.detailsOmitted.push("namedShareOfVoice");
|
|
19198
|
+
}
|
|
19199
|
+
}
|
|
19200
|
+
if (query.compact && JSON.stringify(response, null, 2).length > 18e3) {
|
|
19201
|
+
throw validationError("Measurement overview metadata exceeds the compact page budget. Read a narrower scope without compact=true.");
|
|
19202
|
+
}
|
|
19100
19203
|
return measurementOverviewResponseSchema.parse(response);
|
|
19101
19204
|
}
|
|
19102
19205
|
);
|
|
@@ -19191,7 +19294,7 @@ function readCompetitorLandscape(app, projectName, query, selection) {
|
|
|
19191
19294
|
throw validationError("Invalid competitor landscape query", { issues: parsed.error.issues });
|
|
19192
19295
|
}
|
|
19193
19296
|
const filters = parsed.data;
|
|
19194
|
-
const classSplitByDefault = filters.scope === void 0 && filters.groupKey === void 0 && filters.queryClass !== void 0 && filters.queryClass !== "all";
|
|
19297
|
+
const classSplitByDefault = filters.scope === void 0 && filters.groupKey === void 0 && (filters.queryClass !== void 0 && filters.queryClass !== "all" || filters.answers === "not-mentioned");
|
|
19195
19298
|
if ((selection?.autoAdvanced || classSplitByDefault) && activeMeasurementPlan(app.db, project.id)?.plan.schemaVersion === 2) filters.scope = "all-markets";
|
|
19196
19299
|
const window = parseWindow(filters.window);
|
|
19197
19300
|
const cutoff = windowCutoff(window);
|
|
@@ -19225,6 +19328,7 @@ function readCompetitorLandscape(app, projectName, query, selection) {
|
|
|
19225
19328
|
citedDomains: querySnapshots.citedDomains,
|
|
19226
19329
|
citedUrls: querySnapshots.citedUrls,
|
|
19227
19330
|
captureStatus: querySnapshots.captureStatus,
|
|
19331
|
+
rawResponse: advanced && filters.answers === "not-mentioned" ? sql9`case when ${querySnapshots.citedUrls} is null then ${querySnapshots.rawResponse} else null end` : sql9`null`,
|
|
19228
19332
|
location: querySnapshots.location,
|
|
19229
19333
|
measurementExecutionId: querySnapshots.measurementExecutionId,
|
|
19230
19334
|
createdAt: querySnapshots.createdAt
|
|
@@ -19244,7 +19348,14 @@ function readCompetitorLandscape(app, projectName, query, selection) {
|
|
|
19244
19348
|
const run = runById.get(snapshot.runId);
|
|
19245
19349
|
return run !== void 0 && run.trigger !== "probe" && run.status !== RunStatuses.completed && run.status !== RunStatuses.partial;
|
|
19246
19350
|
}).length;
|
|
19247
|
-
const
|
|
19351
|
+
const eligibleSnapshots = inScope.filter((snapshot) => eligibleRuns.has(snapshot.runId));
|
|
19352
|
+
let unknownMentionAnswers = 0;
|
|
19353
|
+
const simpleMentionInputs = !advanced && filters.answers === "not-mentioned" ? buildMentionShareInputs({ project, competitors: [], snapshots: eligibleSnapshots, queryTextById }) : void 0;
|
|
19354
|
+
const snapshots = filters.answers !== "not-mentioned" ? eligibleSnapshots : eligibleSnapshots.filter((snapshot, index2) => {
|
|
19355
|
+
const mentioned = advanced ? advanced.runScopes.get(snapshot.runId)?.mentioned(snapshot.answerText, snapshot.measurementExecutionId, measurementSnapshotSources(snapshot).urls) ?? null : snapshot.answerText === null || !simpleMentionInputs.classified ? null : simpleMentionInputs.snapshots[index2].projectMentioned;
|
|
19356
|
+
if (mentioned === null) unknownMentionAnswers++;
|
|
19357
|
+
return mentioned === false;
|
|
19358
|
+
});
|
|
19248
19359
|
const countedRunIds = new Set(snapshots.map((snapshot) => snapshot.runId));
|
|
19249
19360
|
const countedRuns = candidateRuns.filter((run) => countedRunIds.has(run.id));
|
|
19250
19361
|
const classificationRows = app.db.select({
|
|
@@ -19283,6 +19394,7 @@ function readCompetitorLandscape(app, projectName, query, selection) {
|
|
|
19283
19394
|
// Share of voice needs one query class behind it. `all` pools branded
|
|
19284
19395
|
// and non-brand, so withhold the ratio and publish the counts instead.
|
|
19285
19396
|
shareOfVoiceEligible: filters.queryClass !== void 0 && filters.queryClass !== "all",
|
|
19397
|
+
...filters.answers === "not-mentioned" ? { shareOfVoiceUnavailableReason: "answer-selection" } : {},
|
|
19286
19398
|
snapshots: selectedSnapshots.map((snapshot, i) => ({
|
|
19287
19399
|
id: snapshot.id,
|
|
19288
19400
|
createdAt: snapshot.createdAt,
|
|
@@ -19372,12 +19484,16 @@ function readCompetitorLandscape(app, projectName, query, selection) {
|
|
|
19372
19484
|
...filters.groupBy !== void 0 ? { groupBy: filters.groupBy } : {},
|
|
19373
19485
|
queryClass: filters.queryClass ?? "all",
|
|
19374
19486
|
location: filters.location ?? null,
|
|
19375
|
-
runId: filters.runId ?? null
|
|
19487
|
+
runId: filters.runId ?? null,
|
|
19488
|
+
...filters.answers === void 0 ? {} : { answers: filters.answers }
|
|
19376
19489
|
},
|
|
19377
19490
|
truncated,
|
|
19378
19491
|
runCount: countedRuns.length,
|
|
19379
19492
|
runIds: pooledRunIds(countedRuns),
|
|
19380
19493
|
countUnits: COMPETITOR_LANDSCAPE_COUNT_UNITS,
|
|
19494
|
+
...filters.answers !== "not-mentioned" ? {} : {
|
|
19495
|
+
answerSelection: { answers: filters.answers, populationSize: eligibleSnapshots.length, answerCount: snapshots.length, unknownMentionAnswers }
|
|
19496
|
+
},
|
|
19381
19497
|
...modelComparison ? { modelComparison } : {}
|
|
19382
19498
|
};
|
|
19383
19499
|
return response;
|
|
@@ -19464,9 +19580,21 @@ function scopeForFrozenPlan(plan, kind, groupKey) {
|
|
|
19464
19580
|
classes.add(assignment.queryClass);
|
|
19465
19581
|
queryClassesByExecution.set(assignment.executionNodeKey, classes);
|
|
19466
19582
|
}
|
|
19583
|
+
const targets = measurementPlanV2ReportTargets(plan);
|
|
19584
|
+
const readMention = createTargetMentionReader(targets, plan.identities.projectBrand.ownedHosts);
|
|
19585
|
+
const targetIdsByExecution = /* @__PURE__ */ new Map();
|
|
19586
|
+
for (const edge of plan.usageEdges) {
|
|
19587
|
+
const ids = targetIdsByExecution.get(edge.executionNodeKey) ?? /* @__PURE__ */ new Set();
|
|
19588
|
+
ids.add(edge.targetKey);
|
|
19589
|
+
targetIdsByExecution.set(edge.executionNodeKey, ids);
|
|
19590
|
+
}
|
|
19467
19591
|
return {
|
|
19468
19592
|
executionNodeKeys,
|
|
19469
19593
|
queryClassesByExecution,
|
|
19594
|
+
mentioned: (answerText, executionId, citedUrls) => {
|
|
19595
|
+
const ids = executionId === null ? [] : [...targetIdsByExecution.get(executionId) ?? []];
|
|
19596
|
+
return readMention(answerText, ids, citedUrls);
|
|
19597
|
+
},
|
|
19470
19598
|
competitors: mergePins(groups.flatMap((group) => group.competitors.map((competitor) => ({
|
|
19471
19599
|
domain: competitor.domain,
|
|
19472
19600
|
label: competitor.label,
|
|
@@ -19550,12 +19678,12 @@ function mergePins(...collections) {
|
|
|
19550
19678
|
|
|
19551
19679
|
// ../api-routes/src/runs.ts
|
|
19552
19680
|
import crypto15 from "crypto";
|
|
19553
|
-
import { and as and19, eq as eq26, asc as asc3, desc as desc10, inArray as inArray12, or as or6, sql as
|
|
19681
|
+
import { and as and19, eq as eq26, asc as asc3, desc as desc10, inArray as inArray12, or as or6, sql as sql11 } from "drizzle-orm";
|
|
19554
19682
|
import { gte as gte4 } from "drizzle-orm";
|
|
19555
19683
|
|
|
19556
19684
|
// ../api-routes/src/run-fill.ts
|
|
19557
19685
|
import crypto14 from "crypto";
|
|
19558
|
-
import { and as and18, desc as desc9, eq as eq25, gt as gt4, inArray as inArray11, isNull as isNull7, ne as ne5, or as or5, sql as
|
|
19686
|
+
import { and as and18, desc as desc9, eq as eq25, gt as gt4, inArray as inArray11, isNull as isNull7, ne as ne5, or as or5, sql as sql10 } from "drizzle-orm";
|
|
19559
19687
|
var ACTIVE_STATUSES = ["queued", "running"];
|
|
19560
19688
|
function normalizeProvider(value) {
|
|
19561
19689
|
return value.trim().toLocaleLowerCase("en");
|
|
@@ -19586,7 +19714,7 @@ function newerFullSweep(db, run) {
|
|
|
19586
19714
|
gt4(runs.createdAt, run.createdAt),
|
|
19587
19715
|
or5(
|
|
19588
19716
|
inArray11(runs.status, ["queued", "running", "completed", "partial"]),
|
|
19589
|
-
|
|
19717
|
+
sql10`EXISTS (SELECT 1 FROM ${querySnapshots} WHERE ${querySnapshots.runId} = ${runs.id})`
|
|
19590
19718
|
)
|
|
19591
19719
|
)).orderBy(desc9(runs.createdAt)).get();
|
|
19592
19720
|
}
|
|
@@ -20006,7 +20134,7 @@ async function runRoutes(app, opts) {
|
|
|
20006
20134
|
});
|
|
20007
20135
|
app.get("/projects/:name/runs/latest", async (request, reply) => {
|
|
20008
20136
|
const project = resolveProject(app.db, request.params.name);
|
|
20009
|
-
const countRow = app.db.select({ count:
|
|
20137
|
+
const countRow = app.db.select({ count: sql11`count(*)` }).from(runs).where(and19(eq26(runs.projectId, project.id), notProbeRun())).get();
|
|
20010
20138
|
const totalRuns = countRow?.count ?? 0;
|
|
20011
20139
|
const latestRun = app.db.select().from(runs).where(and19(eq26(runs.projectId, project.id), notProbeRun())).orderBy(desc10(runs.createdAt), desc10(runs.id)).limit(1).get();
|
|
20012
20140
|
if (!latestRun) {
|
|
@@ -25217,10 +25345,11 @@ async function measurementPropertyEvidenceRoutes(app) {
|
|
|
25217
25345
|
}
|
|
25218
25346
|
|
|
25219
25347
|
// ../api-routes/src/measurement-portfolio-reads.ts
|
|
25348
|
+
import { createHash as createHash6 } from "crypto";
|
|
25220
25349
|
import { and as and25, desc as desc13, eq as eq32, inArray as inArray15, lt as lt5, or as or7 } from "drizzle-orm";
|
|
25221
25350
|
|
|
25222
25351
|
// ../api-routes/src/measurement-question-reads.ts
|
|
25223
|
-
import { and as and24, eq as eq31, getTableColumns, inArray as inArray14, sql as
|
|
25352
|
+
import { and as and24, eq as eq31, getTableColumns, inArray as inArray14, sql as sql12 } from "drizzle-orm";
|
|
25224
25353
|
var DEFAULT_LIMIT2 = 50;
|
|
25225
25354
|
var TERMINAL_STATUSES = /* @__PURE__ */ new Set([
|
|
25226
25355
|
RunStatuses.completed,
|
|
@@ -25339,7 +25468,7 @@ function selectedQuestionSnapshots(db, runId, manifest, scope) {
|
|
|
25339
25468
|
conditions.push(inArray14(querySnapshots.measurementExecutionId, executionIds));
|
|
25340
25469
|
}
|
|
25341
25470
|
if (scope?.provider !== void 0) {
|
|
25342
|
-
conditions.push(
|
|
25471
|
+
conditions.push(sql12`lower(trim(${querySnapshots.provider})) = ${normalizeText(scope.provider)}`);
|
|
25343
25472
|
}
|
|
25344
25473
|
return db.select(measurementQuestionSnapshotColumns).from(querySnapshots).where(and24(...conditions)).all();
|
|
25345
25474
|
}
|
|
@@ -25656,6 +25785,12 @@ function countMetric2(rate2) {
|
|
|
25656
25785
|
if (rate2.numerator === null) return unavailable3(metricReason2(rate2.reason));
|
|
25657
25786
|
return { state: "available", value: rate2.numerator, numerator: rate2.numerator, denominator: rate2.denominator };
|
|
25658
25787
|
}
|
|
25788
|
+
function neverMentionedMetric(rate2, propertyCount) {
|
|
25789
|
+
if (rate2.numerator === null || rate2.denominator === null) return unavailable3(metricReason2(rate2.reason));
|
|
25790
|
+
if (rate2.denominator !== propertyCount) return unavailable3("not_applicable");
|
|
25791
|
+
const neverMentioned = rate2.denominator - rate2.numerator;
|
|
25792
|
+
return { state: "available", value: neverMentioned, numerator: neverMentioned, denominator: rate2.denominator };
|
|
25793
|
+
}
|
|
25659
25794
|
function materializeWithDb(db, active, plan, run) {
|
|
25660
25795
|
return materializeMeasurementQuestionRun(db, active, plan, run);
|
|
25661
25796
|
}
|
|
@@ -25789,7 +25924,8 @@ function answerDomains(answer) {
|
|
|
25789
25924
|
function domainRows(answers, limit) {
|
|
25790
25925
|
const counts2 = /* @__PURE__ */ new Map();
|
|
25791
25926
|
for (const answer of answers) {
|
|
25792
|
-
|
|
25927
|
+
const domains = new Set([...answerDomains(answer)].map((host) => registrableDomain(host) || host));
|
|
25928
|
+
for (const domain of domains) counts2.set(domain, (counts2.get(domain) ?? 0) + 1);
|
|
25793
25929
|
}
|
|
25794
25930
|
const rows = [...counts2].map(([domain, count4]) => ({ domain, answers: count4 })).sort((left, right) => right.answers - left.answers || compareText13(left.domain, right.domain));
|
|
25795
25931
|
return { rows: rows.slice(0, limit), total: rows.length };
|
|
@@ -25845,8 +25981,8 @@ function recommendationRows(target, population) {
|
|
|
25845
25981
|
};
|
|
25846
25982
|
}).sort((left, right) => right.occurrences - left.occurrences || compareText13(left.name, right.name));
|
|
25847
25983
|
}
|
|
25848
|
-
function namedInsteadFields(names) {
|
|
25849
|
-
const returned = names.slice(0,
|
|
25984
|
+
function namedInsteadFields(names, limit = MEASUREMENT_PORTFOLIO_ROW_EVIDENCE_LIMIT) {
|
|
25985
|
+
const returned = names.slice(0, limit);
|
|
25850
25986
|
return {
|
|
25851
25987
|
namedInsteadInAnswerText: returned.map(({ name, occurrences }) => ({ name, answers: occurrences })),
|
|
25852
25988
|
namedInsteadInAnswerTextTotal: names.length,
|
|
@@ -25939,6 +26075,7 @@ function marketRollup(plan, measured, scope, includeNested) {
|
|
|
25939
26075
|
...identity,
|
|
25940
26076
|
propertyCount: group.targetKeys.length,
|
|
25941
26077
|
propertiesMentioned: unavailable3("no_completed_run"),
|
|
26078
|
+
propertiesNeverMentioned: unavailable3("no_completed_run"),
|
|
25942
26079
|
mentionCoverage: unavailable3("no_completed_run"),
|
|
25943
26080
|
citationCoverage: unavailable3("no_completed_run")
|
|
25944
26081
|
};
|
|
@@ -25949,12 +26086,25 @@ function marketRollup(plan, measured, scope, includeNested) {
|
|
|
25949
26086
|
...identity,
|
|
25950
26087
|
propertyCount: targetKeys.length,
|
|
25951
26088
|
propertiesMentioned: countMetric2(overview.propertiesMentioned),
|
|
26089
|
+
propertiesNeverMentioned: neverMentionedMetric(overview.propertiesMentioned, targetKeys.length),
|
|
25952
26090
|
mentionCoverage: coverageMetric2(overview.mentionCoverage),
|
|
25953
26091
|
citationCoverage: coverageMetric2(overview.citationCoverage)
|
|
25954
26092
|
};
|
|
25955
26093
|
}).sort(compareWeakestMarket);
|
|
25956
26094
|
return { markets, totalMarkets: markets.length, marketsTruncated: false };
|
|
25957
26095
|
}
|
|
26096
|
+
function weakestMarkets(markets, queryClass) {
|
|
26097
|
+
const topLevel = markets.filter((market) => market.parentGroupKey === null);
|
|
26098
|
+
const eligible = topLevel.flatMap((market) => market.mentionCoverage.state === "available" && market.citationCoverage.state === "available" ? [{ ...market, mentionCoverage: market.mentionCoverage, citationCoverage: market.citationCoverage }] : []);
|
|
26099
|
+
return {
|
|
26100
|
+
population: "top-level-markets",
|
|
26101
|
+
queryClass,
|
|
26102
|
+
totalMarkets: topLevel.length,
|
|
26103
|
+
eligibleMarketCount: eligible.length,
|
|
26104
|
+
excludedMarketCount: topLevel.length - eligible.length,
|
|
26105
|
+
items: eligible.sort(compareWeakestMarket).slice(0, MEASUREMENT_PORTFOLIO_WEAKEST_MARKET_LIMIT).map(({ groupKey, label, propertyCount, mentionCoverage, citationCoverage }) => ({ groupKey, label, propertyCount, mentionCoverage, citationCoverage }))
|
|
26106
|
+
};
|
|
26107
|
+
}
|
|
25958
26108
|
function compareWeakestMarket(a, b) {
|
|
25959
26109
|
const aUnavailable = a.mentionCoverage.state === "unavailable" || a.citationCoverage.state === "unavailable";
|
|
25960
26110
|
const bUnavailable = b.mentionCoverage.state === "unavailable" || b.citationCoverage.state === "unavailable";
|
|
@@ -25997,9 +26147,182 @@ function mentionRanking(rows, limit) {
|
|
|
25997
26147
|
truncated: eligible.length > limit
|
|
25998
26148
|
};
|
|
25999
26149
|
}
|
|
26150
|
+
function portfolioFilterFingerprint(query) {
|
|
26151
|
+
return createHash6("sha256").update(JSON.stringify([
|
|
26152
|
+
query.groupKey ?? null,
|
|
26153
|
+
query.queryClass,
|
|
26154
|
+
query.provider ?? null,
|
|
26155
|
+
query.location === void 0 ? null : normalizeMeasurementLocation(query.location),
|
|
26156
|
+
query.includeNestedMarkets ?? false,
|
|
26157
|
+
query.answers === "not-mentioned" ? query.answers : null,
|
|
26158
|
+
query.list ?? MeasurementPortfolioLists["weakest-properties"]
|
|
26159
|
+
])).digest("base64url");
|
|
26160
|
+
}
|
|
26161
|
+
function portfolioCursor(query, versionId) {
|
|
26162
|
+
if (query.cursor === void 0) return void 0;
|
|
26163
|
+
if (!query.compact) throw validationError("A portfolio cursor requires compact=true.");
|
|
26164
|
+
let value;
|
|
26165
|
+
try {
|
|
26166
|
+
value = JSON.parse(Buffer.from(query.cursor, "base64url").toString("utf8"));
|
|
26167
|
+
} catch {
|
|
26168
|
+
throw validationError("Invalid measurement portfolio cursor.");
|
|
26169
|
+
}
|
|
26170
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) throw validationError("Invalid measurement portfolio cursor.");
|
|
26171
|
+
const cursor = value;
|
|
26172
|
+
if (cursor.v !== 1 || cursor.runId !== null && typeof cursor.runId !== "string" || cursor.versionId !== versionId || cursor.filters !== portfolioFilterFingerprint(query) || typeof cursor.evidence !== "string" || !Array.isArray(cursor.offsets) || !Number.isSafeInteger(cursor.sourceLimit) || Number(cursor.sourceLimit) < 1 || Number(cursor.sourceLimit) > 50 || cursor.offsets.length !== 7 || cursor.offsets.some((offset) => !Number.isSafeInteger(offset) || Number(offset) < 0) || query.runId !== void 0 && cursor.runId !== query.runId) {
|
|
26173
|
+
throw validationError("The measurement portfolio cursor does not match the run, revision or filters.");
|
|
26174
|
+
}
|
|
26175
|
+
return cursor;
|
|
26176
|
+
}
|
|
26177
|
+
function compactPortfolioPage(response, query, versionId, evidence, cursor) {
|
|
26178
|
+
if (!query.compact) {
|
|
26179
|
+
if (query.list !== void 0) throw validationError("A portfolio list requires compact=true.");
|
|
26180
|
+
return measurementPortfolioSummaryResponseSchema.parse(response);
|
|
26181
|
+
}
|
|
26182
|
+
if (cursor !== void 0 && (cursor.evidence !== evidence || cursor.runId !== response.measurement.displayedRunId)) {
|
|
26183
|
+
throw validationError("The measurement portfolio cursor evidence changed between pages.");
|
|
26184
|
+
}
|
|
26185
|
+
const lists = [
|
|
26186
|
+
MeasurementPortfolioLists["weakest-properties"],
|
|
26187
|
+
MeasurementPortfolioLists["strongest-mentions"],
|
|
26188
|
+
MeasurementPortfolioLists["weakest-mentions"],
|
|
26189
|
+
MeasurementPortfolioLists["excluded-mentions"],
|
|
26190
|
+
MeasurementPortfolioLists.markets,
|
|
26191
|
+
MeasurementPortfolioLists["observed-names"],
|
|
26192
|
+
MeasurementPortfolioLists["cited-domains"]
|
|
26193
|
+
];
|
|
26194
|
+
const paths = [
|
|
26195
|
+
"weakestProperties",
|
|
26196
|
+
"mentionRanking.strongest",
|
|
26197
|
+
"mentionRanking.weakest",
|
|
26198
|
+
"mentionRanking.excluded",
|
|
26199
|
+
"markets",
|
|
26200
|
+
"answerEvidence.observedNames",
|
|
26201
|
+
"answerEvidence.citedDomains"
|
|
26202
|
+
];
|
|
26203
|
+
const pageList = query.list ?? MeasurementPortfolioLists["weakest-properties"];
|
|
26204
|
+
const selected = lists.indexOf(pageList);
|
|
26205
|
+
const arrays = [
|
|
26206
|
+
response.weakestProperties,
|
|
26207
|
+
response.mentionRanking.strongest,
|
|
26208
|
+
response.mentionRanking.weakest,
|
|
26209
|
+
response.mentionRanking.excluded,
|
|
26210
|
+
response.markets,
|
|
26211
|
+
response.answerEvidence?.observedNames ?? [],
|
|
26212
|
+
response.answerEvidence?.citedDomains ?? []
|
|
26213
|
+
];
|
|
26214
|
+
const offsets = cursor?.offsets ?? arrays.map(() => 0);
|
|
26215
|
+
if (arrays.some((rows, index2) => offsets[index2] > rows.length)) throw validationError("Invalid measurement portfolio cursor offset.");
|
|
26216
|
+
const requested = Math.min(query.limit ?? 50, arrays[selected].length - offsets[selected]);
|
|
26217
|
+
const initialSummaryLimit = query.list === void 0 && cursor === void 0 ? MEASUREMENT_PORTFOLIO_DEFAULT_LIMIT : 0;
|
|
26218
|
+
const build = (count4, summaryLimit2) => {
|
|
26219
|
+
const pages = arrays.map((rows, index2) => index2 === selected ? rows.slice(offsets[index2], offsets[index2] + count4) : rows.slice(0, summaryLimit2));
|
|
26220
|
+
const nextOffsets = offsets.map((offset, index2) => index2 === selected ? offset + count4 : offset);
|
|
26221
|
+
const hasMore = nextOffsets[selected] < arrays[selected].length;
|
|
26222
|
+
const omitted = paths.filter((_, index2) => index2 !== selected && pages[index2].length < arrays[index2].length);
|
|
26223
|
+
return {
|
|
26224
|
+
...response,
|
|
26225
|
+
pageList,
|
|
26226
|
+
detailsOmitted: [...response.detailsOmitted ?? [], ...omitted],
|
|
26227
|
+
weakestProperties: pages[0],
|
|
26228
|
+
mentionRanking: {
|
|
26229
|
+
...response.mentionRanking,
|
|
26230
|
+
strongest: pages[1],
|
|
26231
|
+
weakest: pages[2],
|
|
26232
|
+
excluded: pages[3],
|
|
26233
|
+
excludedTotal: response.mentionRanking.excluded.length,
|
|
26234
|
+
truncated: pages[1].length < response.mentionRanking.eligiblePropertyCount || pages[2].length < response.mentionRanking.eligiblePropertyCount
|
|
26235
|
+
},
|
|
26236
|
+
markets: pages[4],
|
|
26237
|
+
marketsTruncated: pages[4].length < response.totalMarkets,
|
|
26238
|
+
truncated: pages[0].length < response.totalProperties,
|
|
26239
|
+
...response.answerEvidence === void 0 ? {} : {
|
|
26240
|
+
answerEvidence: {
|
|
26241
|
+
...response.answerEvidence,
|
|
26242
|
+
observedNames: pages[5],
|
|
26243
|
+
citedDomains: pages[6]
|
|
26244
|
+
}
|
|
26245
|
+
},
|
|
26246
|
+
nextCursor: hasMore ? Buffer.from(JSON.stringify({
|
|
26247
|
+
v: 1,
|
|
26248
|
+
runId: response.measurement.displayedRunId,
|
|
26249
|
+
versionId,
|
|
26250
|
+
filters: portfolioFilterFingerprint(query),
|
|
26251
|
+
evidence,
|
|
26252
|
+
offsets: nextOffsets,
|
|
26253
|
+
sourceLimit: cursor?.sourceLimit ?? query.limit ?? MEASUREMENT_PORTFOLIO_DEFAULT_LIMIT
|
|
26254
|
+
})).toString("base64url") : null
|
|
26255
|
+
};
|
|
26256
|
+
};
|
|
26257
|
+
const fits = (page4) => JSON.stringify(page4, null, 2).length <= 18e3;
|
|
26258
|
+
const initialComplete = build(requested, initialSummaryLimit);
|
|
26259
|
+
if (fits(initialComplete)) return measurementPortfolioSummaryResponseSchema.parse(initialComplete);
|
|
26260
|
+
const minimum = Math.min(1, requested);
|
|
26261
|
+
let summaryLow = 0;
|
|
26262
|
+
let summaryHigh = initialSummaryLimit;
|
|
26263
|
+
let summaryLimit = 0;
|
|
26264
|
+
while (summaryLow <= summaryHigh) {
|
|
26265
|
+
const candidate = Math.floor((summaryLow + summaryHigh) / 2);
|
|
26266
|
+
if (fits(build(minimum, candidate))) {
|
|
26267
|
+
summaryLimit = candidate;
|
|
26268
|
+
summaryLow = candidate + 1;
|
|
26269
|
+
} else summaryHigh = candidate - 1;
|
|
26270
|
+
}
|
|
26271
|
+
const complete = build(requested, summaryLimit);
|
|
26272
|
+
if (fits(complete)) return measurementPortfolioSummaryResponseSchema.parse(complete);
|
|
26273
|
+
let page3 = build(minimum, summaryLimit);
|
|
26274
|
+
if (!fits(page3)) throw validationError("One portfolio row or its metadata exceeds the compact page budget. Read a narrower scope without compact=true.");
|
|
26275
|
+
let low = minimum + 1;
|
|
26276
|
+
let high = requested;
|
|
26277
|
+
while (low <= high) {
|
|
26278
|
+
const count4 = Math.floor((low + high) / 2);
|
|
26279
|
+
const candidate = build(count4, summaryLimit);
|
|
26280
|
+
if (fits(candidate)) {
|
|
26281
|
+
page3 = candidate;
|
|
26282
|
+
low = count4 + 1;
|
|
26283
|
+
} else high = count4 - 1;
|
|
26284
|
+
}
|
|
26285
|
+
return measurementPortfolioSummaryResponseSchema.parse(page3);
|
|
26286
|
+
}
|
|
26287
|
+
function portfolioAnswerEvidence(plan, materialized, filters, targetKeys) {
|
|
26288
|
+
const targets = new Set(targetKeys);
|
|
26289
|
+
const executions = new Set(plan.assignments.filter((assignment) => targets.has(assignment.targetKey) && (filters.queryClass === "all" || assignment.queryClass === filters.queryClass)).map((assignment) => assignment.executionNodeKey));
|
|
26290
|
+
const slots = materialized.input.expectedSlots.filter((slot) => executions.has(slot.executionId) && (filters.provider === void 0 || normalizeText2(slot.provider) === normalizeText2(filters.provider)) && (filters.location === void 0 || normalizeMeasurementLocation(slot.location) === normalizeMeasurementLocation(filters.location)));
|
|
26291
|
+
const mentionsBySlot = /* @__PURE__ */ new Map();
|
|
26292
|
+
for (const answer of materialized.evidence.answers) pushTo(mentionsBySlot, answer.expectedSlotId, answer.mentioned);
|
|
26293
|
+
const selected = [];
|
|
26294
|
+
let unknownMentionAnswers = 0;
|
|
26295
|
+
for (const slot of slots) {
|
|
26296
|
+
const mentions = mentionsBySlot.get(slot.id);
|
|
26297
|
+
if (mentions?.some((mentioned) => mentioned === true)) continue;
|
|
26298
|
+
const observation = materialized.observationsBySlot.get(slot.id);
|
|
26299
|
+
const snapshot = observation === void 0 ? void 0 : materialized.snapshotsById.get(observation.id);
|
|
26300
|
+
if (!snapshot || snapshot.answerText === null || !mentions?.length || mentions.some((mentioned) => mentioned === null)) {
|
|
26301
|
+
unknownMentionAnswers++;
|
|
26302
|
+
continue;
|
|
26303
|
+
}
|
|
26304
|
+
selected.push({ slotId: slot.id, snapshot, citedUrls: observation.citedUrls ?? observation.historicalCitedUrls ?? [] });
|
|
26305
|
+
}
|
|
26306
|
+
const names = observedCompetitorNames(selected.map((answer) => answer.snapshot));
|
|
26307
|
+
const domains = domainRows(selected, Number.POSITIVE_INFINITY);
|
|
26308
|
+
return {
|
|
26309
|
+
answers: "not-mentioned",
|
|
26310
|
+
populationSize: slots.length,
|
|
26311
|
+
answerCount: selected.length,
|
|
26312
|
+
unknownMentionAnswers,
|
|
26313
|
+
observedNames: names,
|
|
26314
|
+
observedNamesTotal: names.length,
|
|
26315
|
+
citedDomains: domains.rows,
|
|
26316
|
+
citedDomainsTotal: domains.total
|
|
26317
|
+
};
|
|
26318
|
+
}
|
|
26000
26319
|
function portfolioResponse(db, active, plan, query, ownDomains) {
|
|
26320
|
+
if ((query.list === MeasurementPortfolioLists["observed-names"] || query.list === MeasurementPortfolioLists["cited-domains"]) && query.answers !== "not-mentioned") {
|
|
26321
|
+
throw validationError("Portfolio evidence lists require answers=not-mentioned.");
|
|
26322
|
+
}
|
|
26001
26323
|
const group = query.groupKey === void 0 ? void 0 : requireGroup2(plan, query.groupKey);
|
|
26002
|
-
const
|
|
26324
|
+
const cursor = portfolioCursor(query, active.version.id);
|
|
26325
|
+
const run = selectMeasurementQuestionRun(db, active.version.projectId, active, query.runId ?? cursor?.runId ?? void 0);
|
|
26003
26326
|
const requestedTargetKeys = group?.targetKeys ?? plan.targets.map((target) => target.stableKey);
|
|
26004
26327
|
const targetKeys = targetKeysForRun(plan, run, requestedTargetKeys, false);
|
|
26005
26328
|
const targets = targetKeys.map((targetKey) => requireTarget3(plan, targetKey));
|
|
@@ -26013,6 +26336,7 @@ function portfolioResponse(db, active, plan, query, ownDomains) {
|
|
|
26013
26336
|
const includeNestedMarkets = query.includeNestedMarkets ?? false;
|
|
26014
26337
|
const locate = propertyLocations(plan);
|
|
26015
26338
|
if (!run) {
|
|
26339
|
+
const markets2 = marketRollup(plan, void 0, group, includeNestedMarkets);
|
|
26016
26340
|
const rows2 = targets.map((target) => ({
|
|
26017
26341
|
...propertyDto2(target),
|
|
26018
26342
|
...locate(target.stableKey),
|
|
@@ -26024,27 +26348,42 @@ function portfolioResponse(db, active, plan, query, ownDomains) {
|
|
|
26024
26348
|
citedDomains: [],
|
|
26025
26349
|
citedDomainsTotal: 0
|
|
26026
26350
|
})).sort(compareWeakest);
|
|
26027
|
-
|
|
26351
|
+
const response2 = {
|
|
26028
26352
|
portfolio: { groupKey: group?.stableKey ?? null, label: group?.label ?? null, measurementScope: null },
|
|
26029
26353
|
measurement,
|
|
26030
26354
|
queryClass: query.queryClass,
|
|
26031
26355
|
engines: [],
|
|
26032
26356
|
metrics: {
|
|
26033
26357
|
propertiesMentioned: unavailable3("no_completed_run"),
|
|
26358
|
+
propertiesNeverMentioned: unavailable3("no_completed_run"),
|
|
26034
26359
|
mentionCoverage: unavailable3("no_completed_run"),
|
|
26035
26360
|
citationCoverage: unavailable3("no_completed_run")
|
|
26036
26361
|
},
|
|
26037
|
-
|
|
26362
|
+
...query.compact && group === void 0 ? { weakestMarkets: weakestMarkets(markets2.markets, query.queryClass) } : {},
|
|
26363
|
+
weakestProperties: query.compact ? rows2.map(({
|
|
26364
|
+
namedInsteadInAnswerText: _names,
|
|
26365
|
+
namedInsteadInAnswerTextTotal: _namesTotal,
|
|
26366
|
+
recommendedInstead: _recommended,
|
|
26367
|
+
recommendedInsteadTotal: _recommendedTotal,
|
|
26368
|
+
recommendedInsteadTruncated: _recommendedTruncated,
|
|
26369
|
+
citedDomains: _domains,
|
|
26370
|
+
citedDomainsTotal: _domainsTotal,
|
|
26371
|
+
...row
|
|
26372
|
+
}) => row) : rows2.slice(0, limit),
|
|
26038
26373
|
tiedAtWeakest: null,
|
|
26039
26374
|
weakestAnswerSources: null,
|
|
26040
|
-
mentionRanking: mentionRanking(rows2, limit),
|
|
26375
|
+
mentionRanking: mentionRanking(rows2, query.compact ? Number.POSITIVE_INFINITY : limit),
|
|
26041
26376
|
// Sorted through the same comparator as the measured branch. Plan order
|
|
26042
26377
|
// is `stableKey`, so emitting it raw put markets in an order the schema
|
|
26043
26378
|
// documents as worst-first and that changes the moment a run lands.
|
|
26044
|
-
...
|
|
26379
|
+
...markets2,
|
|
26045
26380
|
totalProperties: rows2.length,
|
|
26046
|
-
truncated: rows2.length > limit
|
|
26047
|
-
|
|
26381
|
+
truncated: rows2.length > limit,
|
|
26382
|
+
...query.compact || query.answers === "not-mentioned" ? {
|
|
26383
|
+
detailsOmitted: [...query.compact ? ["weakestProperties.answerEvidence"] : [], ...query.answers === "not-mentioned" ? ["answerEvidence.noCompletedRun"] : []]
|
|
26384
|
+
} : {}
|
|
26385
|
+
};
|
|
26386
|
+
return compactPortfolioPage(response2, query, active.version.id, "no-run", cursor);
|
|
26048
26387
|
}
|
|
26049
26388
|
const { materialized, overview, evaluator } = filteredOverviewWithDb(db, active, plan, run, filters, targetKeys);
|
|
26050
26389
|
const readTarget2 = createTargetAnswerReader(plan, materialized, filters);
|
|
@@ -26065,11 +26404,13 @@ function portfolioResponse(db, active, plan, query, ownDomains) {
|
|
|
26065
26404
|
};
|
|
26066
26405
|
}).sort(compareWeakest);
|
|
26067
26406
|
const displayed = ranked.slice(0, limit);
|
|
26068
|
-
const rows = displayed.map(({ target, population, ...row }) => {
|
|
26069
|
-
|
|
26407
|
+
const rows = (query.compact ? ranked : displayed).map(({ target, population, ...row }) => {
|
|
26408
|
+
if (query.compact) return row;
|
|
26409
|
+
const evidenceLimit = query.limit === void 0 ? MEASUREMENT_PORTFOLIO_DEFAULT_ROW_EVIDENCE_LIMIT : MEASUREMENT_PORTFOLIO_ROW_EVIDENCE_LIMIT;
|
|
26410
|
+
const domains = domainRows(population.sourceAnswers, evidenceLimit);
|
|
26070
26411
|
return {
|
|
26071
26412
|
...row,
|
|
26072
|
-
...namedInsteadFields(recommendationRows(target, population)),
|
|
26413
|
+
...namedInsteadFields(recommendationRows(target, population), evidenceLimit),
|
|
26073
26414
|
citedDomains: domains.rows,
|
|
26074
26415
|
citedDomainsTotal: domains.total
|
|
26075
26416
|
};
|
|
@@ -26077,7 +26418,10 @@ function portfolioResponse(db, active, plan, query, ownDomains) {
|
|
|
26077
26418
|
const tie = weakestTie(ranked);
|
|
26078
26419
|
const engines = /* @__PURE__ */ new Set();
|
|
26079
26420
|
for (const population of populations.values()) for (const provider of population.providers) engines.add(provider);
|
|
26080
|
-
|
|
26421
|
+
const answerEvidence = query.answers !== "not-mentioned" ? void 0 : portfolioAnswerEvidence(plan, materialized, filters, targetKeys);
|
|
26422
|
+
const sourceRows = query.compact ? ranked.slice(0, cursor?.sourceLimit ?? limit) : displayed;
|
|
26423
|
+
const markets = marketRollup(plan, { run, evaluator }, group, includeNestedMarkets);
|
|
26424
|
+
const response = {
|
|
26081
26425
|
portfolio: {
|
|
26082
26426
|
groupKey: group?.stableKey ?? null,
|
|
26083
26427
|
label: group?.label ?? null,
|
|
@@ -26088,17 +26432,38 @@ function portfolioResponse(db, active, plan, query, ownDomains) {
|
|
|
26088
26432
|
engines: [...engines].sort(compareText13),
|
|
26089
26433
|
metrics: {
|
|
26090
26434
|
propertiesMentioned: countMetric2(overview.propertiesMentioned),
|
|
26435
|
+
propertiesNeverMentioned: neverMentionedMetric(overview.propertiesMentioned, targetKeys.length),
|
|
26091
26436
|
mentionCoverage: coverageMetric2(overview.mentionCoverage),
|
|
26092
26437
|
citationCoverage: coverageMetric2(overview.citationCoverage)
|
|
26093
26438
|
},
|
|
26439
|
+
...query.compact && group === void 0 ? { weakestMarkets: weakestMarkets(markets.markets, query.queryClass) } : {},
|
|
26094
26440
|
weakestProperties: rows,
|
|
26095
|
-
tiedAtWeakest: tie.summary
|
|
26096
|
-
|
|
26097
|
-
|
|
26098
|
-
|
|
26441
|
+
tiedAtWeakest: query.compact && tie.summary !== null ? {
|
|
26442
|
+
count: tie.summary.count,
|
|
26443
|
+
mentionRate: tie.summary.mentionRate,
|
|
26444
|
+
citationRate: tie.summary.citationRate,
|
|
26445
|
+
note: tie.summary.note
|
|
26446
|
+
} : tie.summary,
|
|
26447
|
+
...query.compact ? {
|
|
26448
|
+
detailsOmitted: ["weakestProperties.answerEvidence", "tiedAtWeakest.byMetro", "tiedAtWeakest.namedInstead"]
|
|
26449
|
+
} : {},
|
|
26450
|
+
weakestAnswerSources: {
|
|
26451
|
+
...weakestAnswerSources([...sourceRows, ...tie.rows], ownDomains),
|
|
26452
|
+
...query.compact ? { basis: "initial-weakest-selection-and-zero-signal-tie" } : {}
|
|
26453
|
+
},
|
|
26454
|
+
mentionRanking: mentionRanking(ranked, query.compact ? Number.POSITIVE_INFINITY : limit),
|
|
26455
|
+
...markets,
|
|
26099
26456
|
totalProperties: ranked.length,
|
|
26100
|
-
truncated: ranked.length > limit
|
|
26101
|
-
|
|
26457
|
+
truncated: ranked.length > limit,
|
|
26458
|
+
...answerEvidence === void 0 ? {} : {
|
|
26459
|
+
answerEvidence: query.compact ? answerEvidence : {
|
|
26460
|
+
...answerEvidence,
|
|
26461
|
+
observedNames: answerEvidence.observedNames.slice(0, MEASUREMENT_PORTFOLIO_TIE_NAMED_INSTEAD_LIMIT),
|
|
26462
|
+
citedDomains: answerEvidence.citedDomains.slice(0, MEASUREMENT_PORTFOLIO_ANSWER_SOURCES_LIMIT)
|
|
26463
|
+
}
|
|
26464
|
+
}
|
|
26465
|
+
};
|
|
26466
|
+
return compactPortfolioPage(response, query, active.version.id, snapshotEvidenceFingerprint([...materialized.snapshotsById.values()]), cursor);
|
|
26102
26467
|
}
|
|
26103
26468
|
function weakestTie(ranked) {
|
|
26104
26469
|
if (ranked.length === 0) return { summary: null, rows: [] };
|
|
@@ -26108,6 +26473,7 @@ function weakestTie(ranked) {
|
|
|
26108
26473
|
}
|
|
26109
26474
|
const mentionRate3 = first.mentionCoverage.value;
|
|
26110
26475
|
const citationRate2 = first.citationCoverage.value;
|
|
26476
|
+
if (mentionRate3 !== 0 || citationRate2 !== 0) return { summary: null, rows: [] };
|
|
26111
26477
|
const rows = ranked.filter((row) => row.mentionCoverage.state === "available" && row.mentionCoverage.value === mentionRate3 && row.citationCoverage.state === "available" && row.citationCoverage.value === citationRate2);
|
|
26112
26478
|
if (rows.length < 2) return { summary: null, rows: [] };
|
|
26113
26479
|
const namedInstead = tieNamedInstead(rows);
|
|
@@ -26654,7 +27020,9 @@ async function measurementPortfolioReadRoutes(app) {
|
|
|
26654
27020
|
async (request) => {
|
|
26655
27021
|
const project = resolveProject(app.db, request.params.name);
|
|
26656
27022
|
const nested = request.query.includeNestedMarkets;
|
|
26657
|
-
const
|
|
27023
|
+
const nestedRaw = nested === "true" || nested === "false" ? { ...request.query, includeNestedMarkets: nested === "true" } : request.query;
|
|
27024
|
+
const compact = request.query.compact;
|
|
27025
|
+
const raw = compact === "true" || compact === "false" ? { ...nestedRaw, compact: compact === "true" } : nestedRaw;
|
|
26658
27026
|
const query = parseLimitQuery(raw, measurementPortfolioSummaryQuerySchema, "Invalid measurement portfolio summary query");
|
|
26659
27027
|
const { active, plan } = activeV2Plan2(app.db, project.id, "Portfolio summary");
|
|
26660
27028
|
return portfolioResponse(app.db, active, plan, query, effectiveDomains(project));
|
|
@@ -26690,7 +27058,7 @@ async function measurementPortfolioReadRoutes(app) {
|
|
|
26690
27058
|
}
|
|
26691
27059
|
|
|
26692
27060
|
// ../api-routes/src/visibility-report.ts
|
|
26693
|
-
import { createHash as
|
|
27061
|
+
import { createHash as createHash8 } from "crypto";
|
|
26694
27062
|
import { and as and26, desc as desc14, eq as eq33, gte as gte5, inArray as inArray16, isNotNull as isNotNull4, isNull as isNull8, lt as lt6, lte as lte6, or as or8 } from "drizzle-orm";
|
|
26695
27063
|
|
|
26696
27064
|
// ../api-routes/src/measurement-scope-options.ts
|
|
@@ -26746,7 +27114,7 @@ function simpleScopeOptions() {
|
|
|
26746
27114
|
}
|
|
26747
27115
|
|
|
26748
27116
|
// ../api-routes/src/visibility-report-reader.ts
|
|
26749
|
-
import { createHash as
|
|
27117
|
+
import { createHash as createHash7 } from "crypto";
|
|
26750
27118
|
var VisibilityReportCursorError = class extends Error {
|
|
26751
27119
|
constructor(message) {
|
|
26752
27120
|
super(message);
|
|
@@ -27149,7 +27517,7 @@ function breakdown(candidates, definition, populationTargetKeys = selectedTarget
|
|
|
27149
27517
|
return { properties, groups };
|
|
27150
27518
|
}
|
|
27151
27519
|
function fingerprint(value) {
|
|
27152
|
-
return
|
|
27520
|
+
return createHash7("sha256").update(JSON.stringify(value)).digest("base64url");
|
|
27153
27521
|
}
|
|
27154
27522
|
function cursorOf2(cursor) {
|
|
27155
27523
|
return Buffer.from(JSON.stringify(cursor), "utf8").toString("base64url");
|
|
@@ -27702,11 +28070,11 @@ function stableSimpleDefinitionId(definition) {
|
|
|
27702
28070
|
// not freeze a competitor set, whereas a newly captured empty list did.
|
|
27703
28071
|
competitors: definition.competitors === void 0 ? null : [...definition.competitors].map((competitor) => ({ domain: competitor.domain, label: competitor.label, aliases: [...competitor.aliases].sort() })).sort((left, right) => left.domain.localeCompare(right.domain, "en"))
|
|
27704
28072
|
};
|
|
27705
|
-
return `simple:${
|
|
28073
|
+
return `simple:${createHash8("sha256").update(JSON.stringify(source)).digest("hex")}`;
|
|
27706
28074
|
}
|
|
27707
28075
|
function simpleModelFingerprint(definition) {
|
|
27708
28076
|
const engines = [...definition.engines].map((engine) => ({ provider: normalizedText3(engine.provider), requestedModel: engine.requestedModel })).sort((left, right) => left.provider.localeCompare(right.provider, "en"));
|
|
27709
|
-
return `simple-model:${
|
|
28077
|
+
return `simple-model:${createHash8("sha256").update(JSON.stringify(engines)).digest("hex")}`;
|
|
27710
28078
|
}
|
|
27711
28079
|
function frozenSimpleRun(run, definition, snapshots, includeEvidence = true) {
|
|
27712
28080
|
const matcher = compileBrandAliases(effectiveBrandNames({
|
|
@@ -31923,19 +32291,19 @@ async function intelligenceRoutes(app) {
|
|
|
31923
32291
|
}
|
|
31924
32292
|
|
|
31925
32293
|
// ../api-routes/src/traffic-analytics.ts
|
|
31926
|
-
import { and as and33, desc as desc19, eq as eq39, gte as gte8, lt as lt8, lte as lte7, ne as ne8, sql as
|
|
32294
|
+
import { and as and33, desc as desc19, eq as eq39, gte as gte8, lt as lt8, lte as lte7, ne as ne8, sql as sql14 } from "drizzle-orm";
|
|
31927
32295
|
|
|
31928
32296
|
// ../api-routes/src/ai-referral-status.ts
|
|
31929
|
-
import { and as and32, sql as
|
|
31930
|
-
var REDIRECT_LIST =
|
|
32297
|
+
import { and as and32, sql as sql13 } from "drizzle-orm";
|
|
32298
|
+
var REDIRECT_LIST = sql13.join([...LOCATION_REDIRECT_STATUSES].map((s) => sql13`${s}`), sql13`, `);
|
|
31931
32299
|
function referralLandedCondition() {
|
|
31932
|
-
return
|
|
32300
|
+
return sql13`${aiReferralEventsHourly.status} NOT IN (${REDIRECT_LIST})`;
|
|
31933
32301
|
}
|
|
31934
32302
|
function countableReferralCondition() {
|
|
31935
32303
|
return and32(nonSubresourceReferralPathCondition(), referralLandedCondition());
|
|
31936
32304
|
}
|
|
31937
32305
|
function nonSubresourceReferralPathCondition() {
|
|
31938
|
-
return
|
|
32306
|
+
return sql13`
|
|
31939
32307
|
LOWER(${aiReferralEventsHourly.landingPathNormalized}) NOT LIKE '/_next/static/%'
|
|
31940
32308
|
AND LOWER(${aiReferralEventsHourly.landingPathNormalized}) NOT LIKE '/assets/%'
|
|
31941
32309
|
AND LOWER(${aiReferralEventsHourly.landingPathNormalized}) NOT LIKE '/static/%'
|
|
@@ -31977,15 +32345,15 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
31977
32345
|
const priorStart = new Date(priorStartMs).toISOString();
|
|
31978
32346
|
const trendStart = new Date(trendStartMs).toISOString();
|
|
31979
32347
|
const firstObservations = [
|
|
31980
|
-
db.select({ first:
|
|
31981
|
-
db.select({ first:
|
|
31982
|
-
db.select({ first:
|
|
32348
|
+
db.select({ first: sql14`MIN(${crawlerEventsHourly.tsHour})` }).from(crawlerEventsHourly).where(eq39(crawlerEventsHourly.projectId, projectId)).get()?.first,
|
|
32349
|
+
db.select({ first: sql14`MIN(${aiUserFetchEventsHourly.tsHour})` }).from(aiUserFetchEventsHourly).where(eq39(aiUserFetchEventsHourly.projectId, projectId)).get()?.first,
|
|
32350
|
+
db.select({ first: sql14`MIN(${aiReferralEventsHourly.tsHour})` }).from(aiReferralEventsHourly).where(eq39(aiReferralEventsHourly.projectId, projectId)).get()?.first
|
|
31983
32351
|
].filter((first) => typeof first === "string" && first.length > 0);
|
|
31984
32352
|
const coverageStart = firstObservations.sort().at(0) ?? null;
|
|
31985
32353
|
const priorWindowComplete = coverageStart !== null && Date.parse(coverageStart) <= priorStartMs;
|
|
31986
32354
|
const recordedChange = (current, prior) => priorWindowComplete ? deltaPercent(current, prior) : null;
|
|
31987
32355
|
const sumVerifiedCrawlers = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
31988
|
-
db.select({ total:
|
|
32356
|
+
db.select({ total: sql14`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
31989
32357
|
and33(
|
|
31990
32358
|
eq39(crawlerEventsHourly.projectId, projectId),
|
|
31991
32359
|
eq39(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
@@ -31995,7 +32363,7 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
31995
32363
|
).get()?.total ?? 0
|
|
31996
32364
|
);
|
|
31997
32365
|
const sumUnverifiedCrawlers = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
31998
|
-
db.select({ total:
|
|
32366
|
+
db.select({ total: sql14`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
31999
32367
|
and33(
|
|
32000
32368
|
eq39(crawlerEventsHourly.projectId, projectId),
|
|
32001
32369
|
ne8(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
@@ -32006,9 +32374,9 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32006
32374
|
);
|
|
32007
32375
|
const sumReferrals = (windowStartIso, windowEndIso, exclusiveEnd = false) => {
|
|
32008
32376
|
const row = db.select({
|
|
32009
|
-
total:
|
|
32010
|
-
paid:
|
|
32011
|
-
organic:
|
|
32377
|
+
total: sql14`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
32378
|
+
paid: sql14`COALESCE(SUM(${aiReferralEventsHourly.paidSessionsOrHits}), 0)`,
|
|
32379
|
+
organic: sql14`COALESCE(SUM(${aiReferralEventsHourly.organicSessionsOrHits}), 0)`
|
|
32012
32380
|
}).from(aiReferralEventsHourly).where(
|
|
32013
32381
|
and33(
|
|
32014
32382
|
eq39(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -32020,7 +32388,7 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32020
32388
|
return aiReferralClassCounts(Number(row?.total ?? 0), Number(row?.paid ?? 0), Number(row?.organic ?? 0));
|
|
32021
32389
|
};
|
|
32022
32390
|
const sumUserFetches = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
32023
|
-
db.select({ total:
|
|
32391
|
+
db.select({ total: sql14`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
32024
32392
|
and33(
|
|
32025
32393
|
eq39(aiUserFetchEventsHourly.projectId, projectId),
|
|
32026
32394
|
gte8(aiUserFetchEventsHourly.tsHour, windowStartIso),
|
|
@@ -32036,7 +32404,7 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32036
32404
|
const userFetchPrior = sumUserFetches(priorStart, headlineStart, true);
|
|
32037
32405
|
const referralCurrent = sumReferrals(headlineStart, headlineEnd);
|
|
32038
32406
|
const referralAllCurrent = Number(
|
|
32039
|
-
db.select({ total:
|
|
32407
|
+
db.select({ total: sql14`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
32040
32408
|
and33(
|
|
32041
32409
|
eq39(aiReferralEventsHourly.projectId, projectId),
|
|
32042
32410
|
nonSubresourceReferralPathCondition(),
|
|
@@ -32050,7 +32418,7 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32050
32418
|
const crawlerByOperatorRows = db.select({
|
|
32051
32419
|
operator: crawlerEventsHourly.operator,
|
|
32052
32420
|
verificationStatus: crawlerEventsHourly.verificationStatus,
|
|
32053
|
-
hits:
|
|
32421
|
+
hits: sql14`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
32054
32422
|
}).from(crawlerEventsHourly).where(
|
|
32055
32423
|
and33(
|
|
32056
32424
|
eq39(crawlerEventsHourly.projectId, projectId),
|
|
@@ -32060,7 +32428,7 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32060
32428
|
).groupBy(crawlerEventsHourly.operator, crawlerEventsHourly.verificationStatus).all();
|
|
32061
32429
|
const crawlerByOperatorPriorRows = db.select({
|
|
32062
32430
|
operator: crawlerEventsHourly.operator,
|
|
32063
|
-
hits:
|
|
32431
|
+
hits: sql14`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
32064
32432
|
}).from(crawlerEventsHourly).where(
|
|
32065
32433
|
and33(
|
|
32066
32434
|
eq39(crawlerEventsHourly.projectId, projectId),
|
|
@@ -32070,7 +32438,7 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32070
32438
|
).groupBy(crawlerEventsHourly.operator).all();
|
|
32071
32439
|
const referralByOperatorRows = db.select({
|
|
32072
32440
|
operator: aiReferralEventsHourly.operator,
|
|
32073
|
-
hits:
|
|
32441
|
+
hits: sql14`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`
|
|
32074
32442
|
}).from(aiReferralEventsHourly).where(
|
|
32075
32443
|
and33(
|
|
32076
32444
|
eq39(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -32081,7 +32449,7 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32081
32449
|
).groupBy(aiReferralEventsHourly.operator).all();
|
|
32082
32450
|
const userFetchByOperatorRows = db.select({
|
|
32083
32451
|
operator: aiUserFetchEventsHourly.operator,
|
|
32084
|
-
hits:
|
|
32452
|
+
hits: sql14`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`
|
|
32085
32453
|
}).from(aiUserFetchEventsHourly).where(
|
|
32086
32454
|
and33(
|
|
32087
32455
|
eq39(aiUserFetchEventsHourly.projectId, projectId),
|
|
@@ -32124,16 +32492,16 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32124
32492
|
);
|
|
32125
32493
|
const topPathsRows = db.select({
|
|
32126
32494
|
path: crawlerEventsHourly.pathNormalized,
|
|
32127
|
-
verifiedHits:
|
|
32128
|
-
unverifiedHits:
|
|
32129
|
-
operators:
|
|
32495
|
+
verifiedHits: sql14`COALESCE(SUM(CASE WHEN ${crawlerEventsHourly.verificationStatus} = ${VerificationStatuses.verified} THEN ${crawlerEventsHourly.hits} ELSE 0 END), 0)`,
|
|
32496
|
+
unverifiedHits: sql14`COALESCE(SUM(CASE WHEN ${crawlerEventsHourly.verificationStatus} <> ${VerificationStatuses.verified} THEN ${crawlerEventsHourly.hits} ELSE 0 END), 0)`,
|
|
32497
|
+
operators: sql14`COUNT(DISTINCT ${crawlerEventsHourly.operator})`
|
|
32130
32498
|
}).from(crawlerEventsHourly).where(
|
|
32131
32499
|
and33(
|
|
32132
32500
|
eq39(crawlerEventsHourly.projectId, projectId),
|
|
32133
32501
|
gte8(crawlerEventsHourly.tsHour, headlineStart),
|
|
32134
32502
|
lte7(crawlerEventsHourly.tsHour, headlineEnd)
|
|
32135
32503
|
)
|
|
32136
|
-
).groupBy(crawlerEventsHourly.pathNormalized).orderBy(desc19(
|
|
32504
|
+
).groupBy(crawlerEventsHourly.pathNormalized).orderBy(desc19(sql14`SUM(${crawlerEventsHourly.hits})`)).limit(SERVER_ACTIVITY_TOP_PATHS_LIMIT).all();
|
|
32137
32505
|
const topCrawledPaths = topPathsRows.map((r) => ({
|
|
32138
32506
|
path: r.path,
|
|
32139
32507
|
verifiedHits: Number(r.verifiedHits),
|
|
@@ -32142,8 +32510,8 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32142
32510
|
}));
|
|
32143
32511
|
const referralProductsRows = db.select({
|
|
32144
32512
|
product: aiReferralEventsHourly.product,
|
|
32145
|
-
arrivals:
|
|
32146
|
-
landingPaths:
|
|
32513
|
+
arrivals: sql14`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
32514
|
+
landingPaths: sql14`COUNT(DISTINCT ${aiReferralEventsHourly.landingPathNormalized})`
|
|
32147
32515
|
}).from(aiReferralEventsHourly).where(
|
|
32148
32516
|
and33(
|
|
32149
32517
|
eq39(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -32151,7 +32519,7 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32151
32519
|
gte8(aiReferralEventsHourly.tsHour, headlineStart),
|
|
32152
32520
|
lte7(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
32153
32521
|
)
|
|
32154
|
-
).groupBy(aiReferralEventsHourly.product).orderBy(desc19(
|
|
32522
|
+
).groupBy(aiReferralEventsHourly.product).orderBy(desc19(sql14`SUM(${aiReferralEventsHourly.sessionsOrHits})`)).all();
|
|
32155
32523
|
const referralProducts = referralProductsRows.map((r) => ({
|
|
32156
32524
|
product: r.product,
|
|
32157
32525
|
arrivals: Number(r.arrivals),
|
|
@@ -32159,8 +32527,8 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32159
32527
|
}));
|
|
32160
32528
|
const topReferralRows = db.select({
|
|
32161
32529
|
path: aiReferralEventsHourly.landingPathNormalized,
|
|
32162
|
-
arrivals:
|
|
32163
|
-
products:
|
|
32530
|
+
arrivals: sql14`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
32531
|
+
products: sql14`COUNT(DISTINCT ${aiReferralEventsHourly.product})`
|
|
32164
32532
|
}).from(aiReferralEventsHourly).where(
|
|
32165
32533
|
and33(
|
|
32166
32534
|
eq39(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -32168,26 +32536,26 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32168
32536
|
gte8(aiReferralEventsHourly.tsHour, headlineStart),
|
|
32169
32537
|
lte7(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
32170
32538
|
)
|
|
32171
|
-
).groupBy(aiReferralEventsHourly.landingPathNormalized).orderBy(desc19(
|
|
32539
|
+
).groupBy(aiReferralEventsHourly.landingPathNormalized).orderBy(desc19(sql14`SUM(${aiReferralEventsHourly.sessionsOrHits})`)).limit(SERVER_ACTIVITY_TOP_PATHS_LIMIT).all();
|
|
32172
32540
|
const topReferralLandingPaths = topReferralRows.map((r) => ({
|
|
32173
32541
|
path: r.path,
|
|
32174
32542
|
arrivals: Number(r.arrivals),
|
|
32175
32543
|
distinctProducts: Number(r.products)
|
|
32176
32544
|
}));
|
|
32177
32545
|
const crawlerTrendRows = db.select({
|
|
32178
|
-
date:
|
|
32179
|
-
verifiedHits:
|
|
32180
|
-
unverifiedHits:
|
|
32546
|
+
date: sql14`SUBSTR(${crawlerEventsHourly.tsHour}, 1, 10)`,
|
|
32547
|
+
verifiedHits: sql14`COALESCE(SUM(CASE WHEN ${crawlerEventsHourly.verificationStatus} = ${VerificationStatuses.verified} THEN ${crawlerEventsHourly.hits} ELSE 0 END), 0)`,
|
|
32548
|
+
unverifiedHits: sql14`COALESCE(SUM(CASE WHEN ${crawlerEventsHourly.verificationStatus} <> ${VerificationStatuses.verified} THEN ${crawlerEventsHourly.hits} ELSE 0 END), 0)`
|
|
32181
32549
|
}).from(crawlerEventsHourly).where(
|
|
32182
32550
|
and33(
|
|
32183
32551
|
eq39(crawlerEventsHourly.projectId, projectId),
|
|
32184
32552
|
gte8(crawlerEventsHourly.tsHour, trendStart),
|
|
32185
32553
|
lte7(crawlerEventsHourly.tsHour, headlineEnd)
|
|
32186
32554
|
)
|
|
32187
|
-
).groupBy(
|
|
32555
|
+
).groupBy(sql14`SUBSTR(${crawlerEventsHourly.tsHour}, 1, 10)`).all();
|
|
32188
32556
|
const referralTrendRows = db.select({
|
|
32189
|
-
date:
|
|
32190
|
-
hits:
|
|
32557
|
+
date: sql14`SUBSTR(${aiReferralEventsHourly.tsHour}, 1, 10)`,
|
|
32558
|
+
hits: sql14`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`
|
|
32191
32559
|
}).from(aiReferralEventsHourly).where(
|
|
32192
32560
|
and33(
|
|
32193
32561
|
eq39(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -32195,17 +32563,17 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32195
32563
|
gte8(aiReferralEventsHourly.tsHour, trendStart),
|
|
32196
32564
|
lte7(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
32197
32565
|
)
|
|
32198
|
-
).groupBy(
|
|
32566
|
+
).groupBy(sql14`SUBSTR(${aiReferralEventsHourly.tsHour}, 1, 10)`).all();
|
|
32199
32567
|
const userFetchTrendRows = db.select({
|
|
32200
|
-
date:
|
|
32201
|
-
hits:
|
|
32568
|
+
date: sql14`SUBSTR(${aiUserFetchEventsHourly.tsHour}, 1, 10)`,
|
|
32569
|
+
hits: sql14`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`
|
|
32202
32570
|
}).from(aiUserFetchEventsHourly).where(
|
|
32203
32571
|
and33(
|
|
32204
32572
|
eq39(aiUserFetchEventsHourly.projectId, projectId),
|
|
32205
32573
|
gte8(aiUserFetchEventsHourly.tsHour, trendStart),
|
|
32206
32574
|
lte7(aiUserFetchEventsHourly.tsHour, headlineEnd)
|
|
32207
32575
|
)
|
|
32208
|
-
).groupBy(
|
|
32576
|
+
).groupBy(sql14`SUBSTR(${aiUserFetchEventsHourly.tsHour}, 1, 10)`).all();
|
|
32209
32577
|
const emptyTrendEntry = () => ({ verifiedCrawlerHits: 0, unverifiedCrawlerHits: 0, userFetchHits: 0, referralArrivals: 0 });
|
|
32210
32578
|
const dailyTrendMap = /* @__PURE__ */ new Map();
|
|
32211
32579
|
for (const r of crawlerTrendRows) {
|
|
@@ -32287,7 +32655,7 @@ async function trafficAnalyticsRoutes(app) {
|
|
|
32287
32655
|
}
|
|
32288
32656
|
|
|
32289
32657
|
// ../api-routes/src/referral-assessment.ts
|
|
32290
|
-
import { and as and34, desc as desc20, eq as eq40, gte as gte9, lte as lte8, ne as ne9, sql as
|
|
32658
|
+
import { and as and34, desc as desc20, eq as eq40, gte as gte9, lte as lte8, ne as ne9, sql as sql15 } from "drizzle-orm";
|
|
32291
32659
|
|
|
32292
32660
|
// ../api-routes/src/ga-ai-referral-aggregation.ts
|
|
32293
32661
|
function normalizeAiTrafficClass(value) {
|
|
@@ -32467,14 +32835,14 @@ function buildReferralAssessment(db, projectName, input) {
|
|
|
32467
32835
|
const limit = query.limit ?? 100;
|
|
32468
32836
|
const sumCounts = (condition) => {
|
|
32469
32837
|
const row = db.select({
|
|
32470
|
-
rows:
|
|
32471
|
-
total:
|
|
32472
|
-
paid:
|
|
32473
|
-
organic:
|
|
32838
|
+
rows: sql15`count(*)`,
|
|
32839
|
+
total: sql15`coalesce(sum(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
32840
|
+
paid: sql15`coalesce(sum(${aiReferralEventsHourly.paidSessionsOrHits}), 0)`,
|
|
32841
|
+
organic: sql15`coalesce(sum(${aiReferralEventsHourly.organicSessionsOrHits}), 0)`
|
|
32474
32842
|
}).from(aiReferralEventsHourly).where(and34(where, condition)).get();
|
|
32475
32843
|
return { rows: Number(row?.rows ?? 0), counts: aiReferralClassCounts(Number(row?.total ?? 0), Number(row?.paid ?? 0), Number(row?.organic ?? 0)) };
|
|
32476
32844
|
};
|
|
32477
|
-
const { rows: observedRows, counts: raw } = sumCounts(
|
|
32845
|
+
const { rows: observedRows, counts: raw } = sumCounts(sql15`1 = 1`);
|
|
32478
32846
|
const landed = sumCounts(referralLandedCondition()).counts;
|
|
32479
32847
|
const countable = sumCounts(countableReferralCondition()).counts;
|
|
32480
32848
|
const difference2 = (whole, part) => aiReferralClassCounts(whole.total - part.total, whole.paid - part.paid, whole.organic - part.organic);
|
|
@@ -32485,18 +32853,18 @@ function buildReferralAssessment(db, projectName, input) {
|
|
|
32485
32853
|
product: aiReferralEventsHourly.product,
|
|
32486
32854
|
landingPathNormalized: aiReferralEventsHourly.landingPathNormalized,
|
|
32487
32855
|
tsHour: aiReferralEventsHourly.tsHour,
|
|
32488
|
-
total:
|
|
32489
|
-
paid:
|
|
32490
|
-
organic:
|
|
32491
|
-
}).from(aiReferralEventsHourly).where(and34(where, countableReferralCondition())).groupBy(aiReferralEventsHourly.sourceId, aiReferralEventsHourly.product, aiReferralEventsHourly.landingPathNormalized, aiReferralEventsHourly.tsHour).having(
|
|
32856
|
+
total: sql15`sum(${aiReferralEventsHourly.sessionsOrHits})`.as("total"),
|
|
32857
|
+
paid: sql15`sum(${aiReferralEventsHourly.paidSessionsOrHits})`.as("paid"),
|
|
32858
|
+
organic: sql15`sum(${aiReferralEventsHourly.organicSessionsOrHits})`.as("organic")
|
|
32859
|
+
}).from(aiReferralEventsHourly).where(and34(where, countableReferralCondition())).groupBy(aiReferralEventsHourly.sourceId, aiReferralEventsHourly.product, aiReferralEventsHourly.landingPathNormalized, aiReferralEventsHourly.tsHour).having(sql15`sum(${aiReferralEventsHourly.sessionsOrHits}) >= ${burstThreshold}`).as("candidates");
|
|
32492
32860
|
const candidateTotals = db.select({
|
|
32493
|
-
groups:
|
|
32494
|
-
total:
|
|
32495
|
-
paid:
|
|
32496
|
-
organic:
|
|
32861
|
+
groups: sql15`count(*)`,
|
|
32862
|
+
total: sql15`coalesce(sum(${candidates.total}), 0)`,
|
|
32863
|
+
paid: sql15`coalesce(sum(${candidates.paid}), 0)`,
|
|
32864
|
+
organic: sql15`coalesce(sum(${candidates.organic}), 0)`
|
|
32497
32865
|
}).from(candidates).get();
|
|
32498
32866
|
const suspected = aiReferralClassCounts(Number(candidateTotals?.total ?? 0), Number(candidateTotals?.paid ?? 0), Number(candidateTotals?.organic ?? 0));
|
|
32499
|
-
const bursts = db.select().from(candidates).orderBy(
|
|
32867
|
+
const bursts = db.select().from(candidates).orderBy(sql15`${candidates.total} desc`, candidates.tsHour, candidates.sourceId, candidates.product, candidates.landingPathNormalized).limit(limit).all().map((row) => ({
|
|
32500
32868
|
sourceId: row.sourceId,
|
|
32501
32869
|
product: row.product,
|
|
32502
32870
|
landingPathNormalized: row.landingPathNormalized,
|
|
@@ -32511,7 +32879,7 @@ function buildReferralAssessment(db, projectName, input) {
|
|
|
32511
32879
|
trafficClass: gaAiReferrals.trafficClass,
|
|
32512
32880
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
32513
32881
|
channelGroup: gaAiReferrals.channelGroup,
|
|
32514
|
-
sessions:
|
|
32882
|
+
sessions: sql15`sum(${gaAiReferrals.sessions})`
|
|
32515
32883
|
}).from(gaAiReferrals).where(and34(eq40(gaAiReferrals.projectId, project.id), gte9(gaAiReferrals.date, query.startDate), lte8(gaAiReferrals.date, query.endDate))).groupBy(gaAiReferrals.date, gaAiReferrals.source, gaAiReferrals.medium, gaAiReferrals.trafficClass, gaAiReferrals.sourceDimension, gaAiReferrals.channelGroup).all();
|
|
32516
32884
|
const gaSessions = gaRows.length ? resolveWinningDimensions(gaRows).reduce((sum, row) => sum + row.paidSessions + row.organicSessions, 0) : latestGaSyncQueriedWindow(db, project.id, query.startDate, query.endDate) ? 0 : null;
|
|
32517
32885
|
const observedRatio = observedRows > 0 && gaSessions !== null && gaSessions > 0 ? Math.round(countable.total / gaSessions * 100) / 100 : null;
|
|
@@ -32554,10 +32922,10 @@ async function referralAssessmentRoutes(app) {
|
|
|
32554
32922
|
}
|
|
32555
32923
|
|
|
32556
32924
|
// ../api-routes/src/organic-evidence.ts
|
|
32557
|
-
import { and as and36, desc as desc21, eq as eq42, gte as gte11, lte as lte10, or as or11, sql as
|
|
32925
|
+
import { and as and36, desc as desc21, eq as eq42, gte as gte11, lte as lte10, or as or11, sql as sql17 } from "drizzle-orm";
|
|
32558
32926
|
|
|
32559
32927
|
// ../api-routes/src/ga-measurement-analysis.ts
|
|
32560
|
-
import { and as and35, eq as eq41, gte as gte10, lte as lte9, or as or10, sql as
|
|
32928
|
+
import { and as and35, eq as eq41, gte as gte10, lte as lte9, or as or10, sql as sql16 } from "drizzle-orm";
|
|
32561
32929
|
var UNKNOWN_LANDING_PAGE = "(not set)";
|
|
32562
32930
|
function addDays(date, offset) {
|
|
32563
32931
|
const value = /* @__PURE__ */ new Date(`${date}T00:00:00.000Z`);
|
|
@@ -32729,24 +33097,24 @@ function buildGaMeasurementAnalysis(db, projectName, options = {}) {
|
|
|
32729
33097
|
const scopedConditions = (hostColumn, normalizedPathColumn, rawPathColumn) => {
|
|
32730
33098
|
const conditions = [];
|
|
32731
33099
|
if (parsedHostScope.data === "marketing") {
|
|
32732
|
-
const lowerHost =
|
|
32733
|
-
const normalizedHost2 =
|
|
33100
|
+
const lowerHost = sql16`lower(${hostColumn})`;
|
|
33101
|
+
const normalizedHost2 = sql16`case when ${lowerHost} like 'www.%' then substr(${lowerHost}, 5) else ${lowerHost} end`;
|
|
32734
33102
|
conditions.push(or10(...marketingHosts.flatMap((host) => [
|
|
32735
|
-
|
|
32736
|
-
|
|
33103
|
+
sql16`${normalizedHost2} = ${host}`,
|
|
33104
|
+
sql16`${normalizedHost2} like ${`%.${host}`}`
|
|
32737
33105
|
])));
|
|
32738
33106
|
}
|
|
32739
33107
|
if (pathPrefix && pathPrefix !== "/") {
|
|
32740
|
-
const rawPath =
|
|
32741
|
-
const landingPath =
|
|
32742
|
-
conditions.push(or10(
|
|
33108
|
+
const rawPath = sql16`case when instr(${rawPathColumn}, '?') = 0 then ${rawPathColumn} else substr(${rawPathColumn}, 1, instr(${rawPathColumn}, '?') - 1) end`;
|
|
33109
|
+
const landingPath = sql16`case when trim(coalesce(${normalizedPathColumn}, '')) = '' then ${rawPath} else ${normalizedPathColumn} end`;
|
|
33110
|
+
conditions.push(or10(sql16`${landingPath} = ${pathPrefix}`, sql16`${landingPath} like ${`${pathPrefix}/%`}`));
|
|
32743
33111
|
}
|
|
32744
33112
|
return conditions;
|
|
32745
33113
|
};
|
|
32746
33114
|
const acquisitionScope = scopedConditions(gaAcquisitionDaily.hostName, gaAcquisitionDaily.landingPageNormalized, gaAcquisitionDaily.landingPage);
|
|
32747
33115
|
const leadLandingScope = scopedConditions(gaLeadEventsDaily.hostName, gaLeadEventsDaily.landingPageNormalized, gaLeadEventsDaily.landingPage);
|
|
32748
|
-
const acquisitionAnchor = db.select({ date:
|
|
32749
|
-
const leadAnchor = db.select({ date:
|
|
33116
|
+
const acquisitionAnchor = db.select({ date: sql16`max(${gaAcquisitionDaily.date})` }).from(gaAcquisitionDaily).where(and35(eq41(gaAcquisitionDaily.projectId, project.id), ...acquisitionScope)).get()?.date ?? null;
|
|
33117
|
+
const leadAnchor = db.select({ date: sql16`max(${gaLeadEventsDaily.date})` }).from(gaLeadEventsDaily).where(and35(eq41(gaLeadEventsDaily.projectId, project.id), or10(
|
|
32750
33118
|
eq41(gaLeadEventsDaily.attributionScope, "channel"),
|
|
32751
33119
|
and35(eq41(gaLeadEventsDaily.attributionScope, "landing-page"), ...leadLandingScope)
|
|
32752
33120
|
))).get()?.date ?? null;
|
|
@@ -32755,7 +33123,7 @@ function buildGaMeasurementAnalysis(db, projectName, options = {}) {
|
|
|
32755
33123
|
const gaStartDate = gaPeriods[0]?.startDate;
|
|
32756
33124
|
const acquisitionRows = gaStartDate ? db.select().from(gaAcquisitionDaily).where(and35(eq41(gaAcquisitionDaily.projectId, project.id), gte10(gaAcquisitionDaily.date, gaStartDate), lte9(gaAcquisitionDaily.date, gaAnchor))).all() : [];
|
|
32757
33125
|
const leadRows = gaStartDate ? db.select().from(gaLeadEventsDaily).where(and35(eq41(gaLeadEventsDaily.projectId, project.id), gte10(gaLeadEventsDaily.date, gaStartDate), lte9(gaLeadEventsDaily.date, gaAnchor))).all() : [];
|
|
32758
|
-
const engagementAnchor = db.select({ date:
|
|
33126
|
+
const engagementAnchor = db.select({ date: sql16`max(${gaDailyTotals.date})` }).from(gaDailyTotals).where(eq41(gaDailyTotals.projectId, project.id)).get()?.date ?? null;
|
|
32759
33127
|
const engagementPeriodWindow = engagementAnchor ? buildPeriods(engagementAnchor, days) : [];
|
|
32760
33128
|
const engagementStartDate = engagementPeriodWindow[0]?.startDate;
|
|
32761
33129
|
const engagementRows = engagementStartDate ? db.select().from(gaDailyTotals).where(and35(
|
|
@@ -32764,15 +33132,15 @@ function buildGaMeasurementAnalysis(db, projectName, options = {}) {
|
|
|
32764
33132
|
lte9(gaDailyTotals.date, engagementAnchor)
|
|
32765
33133
|
)).all() : [];
|
|
32766
33134
|
const engagementAvailableFrom = db.select({
|
|
32767
|
-
date:
|
|
33135
|
+
date: sql16`min(${gaDailyTotals.date})`
|
|
32768
33136
|
}).from(gaDailyTotals).where(and35(
|
|
32769
33137
|
eq41(gaDailyTotals.projectId, project.id),
|
|
32770
33138
|
or10(
|
|
32771
|
-
|
|
32772
|
-
|
|
33139
|
+
sql16`${gaDailyTotals.engagementRate} is not null`,
|
|
33140
|
+
sql16`${gaDailyTotals.newUsers} is not null`
|
|
32773
33141
|
)
|
|
32774
33142
|
)).get()?.date ?? null;
|
|
32775
|
-
const gscAnchor = db.select({ date:
|
|
33143
|
+
const gscAnchor = db.select({ date: sql16`max(${gscDailyTotals.date})` }).from(gscDailyTotals).where(eq41(gscDailyTotals.projectId, project.id)).get()?.date ?? null;
|
|
32776
33144
|
const gscPeriods = gscAnchor ? buildPeriods(gscAnchor, days) : [];
|
|
32777
33145
|
const gscStartDate = gscPeriods[0]?.startDate;
|
|
32778
33146
|
const propertyRows = gscStartDate ? db.select().from(gscDailyTotals).where(and35(eq41(gscDailyTotals.projectId, project.id), gte10(gscDailyTotals.date, gscStartDate), lte9(gscDailyTotals.date, gscAnchor))).all() : [];
|
|
@@ -33099,22 +33467,22 @@ function buildOrganicEvidence(db, projectName, periodDays) {
|
|
|
33099
33467
|
const nativeStartDate = nativeAcquisition.periods[0]?.startDate;
|
|
33100
33468
|
const nativeEndDate = nativeAcquisition.periods.at(-1)?.endDate;
|
|
33101
33469
|
const gscCoverage = aggregateCoverage(db.select({
|
|
33102
|
-
startDate:
|
|
33103
|
-
endDate:
|
|
33104
|
-
observedDays:
|
|
33470
|
+
startDate: sql17`min(${gscDailyTotals.date})`,
|
|
33471
|
+
endDate: sql17`max(${gscDailyTotals.date})`,
|
|
33472
|
+
observedDays: sql17`count(distinct ${gscDailyTotals.date})`
|
|
33105
33473
|
}).from(gscDailyTotals).where(eq42(gscDailyTotals.projectId, project.id)).get());
|
|
33106
33474
|
const legacyGaCoverage = aggregateCoverage(db.select({
|
|
33107
|
-
startDate:
|
|
33108
|
-
endDate:
|
|
33109
|
-
observedDays:
|
|
33475
|
+
startDate: sql17`min(${gaTrafficSnapshots.date})`,
|
|
33476
|
+
endDate: sql17`max(${gaTrafficSnapshots.date})`,
|
|
33477
|
+
observedDays: sql17`count(distinct ${gaTrafficSnapshots.date})`
|
|
33110
33478
|
}).from(gaTrafficSnapshots).where(eq42(gaTrafficSnapshots.projectId, project.id)).get());
|
|
33111
33479
|
const nativeCoverage = nativeAcquisitionActive && normalizedMarketingHosts.length ? aggregateCoverage(db.select({
|
|
33112
|
-
startDate:
|
|
33113
|
-
endDate:
|
|
33114
|
-
observedDays:
|
|
33480
|
+
startDate: sql17`min(${gaAcquisitionDaily.date})`,
|
|
33481
|
+
endDate: sql17`max(${gaAcquisitionDaily.date})`,
|
|
33482
|
+
observedDays: sql17`count(distinct ${gaAcquisitionDaily.date})`
|
|
33115
33483
|
}).from(gaAcquisitionDaily).where(and36(
|
|
33116
33484
|
eq42(gaAcquisitionDaily.projectId, project.id),
|
|
33117
|
-
or11(...normalizedMarketingHosts.map((candidate) =>
|
|
33485
|
+
or11(...normalizedMarketingHosts.map((candidate) => sql17`lower(trim(${gaAcquisitionDaily.hostName})) = ${candidate} or lower(trim(${gaAcquisitionDaily.hostName})) like ${`%.${candidate}`}`))
|
|
33118
33486
|
)).get()) : null;
|
|
33119
33487
|
const gaCoverage = nativeAcquisitionActive ? nativeCoverage : legacyGaCoverage;
|
|
33120
33488
|
const asOfDate = gscCoverage?.endDate ?? gaCoverage?.endDate ?? null;
|
|
@@ -33244,7 +33612,7 @@ function buildOrganicEvidence(db, projectName, periodDays) {
|
|
|
33244
33612
|
const allCrawlers = db.select().from(crawlerEventsHourly).where(and36(eq42(crawlerEventsHourly.projectId, project.id), gte11(crawlerEventsHourly.tsHour, serverStart), lte10(crawlerEventsHourly.tsHour, serverEnd))).all();
|
|
33245
33613
|
const allFetches = db.select().from(aiUserFetchEventsHourly).where(and36(eq42(aiUserFetchEventsHourly.projectId, project.id), gte11(aiUserFetchEventsHourly.tsHour, serverStart), lte10(aiUserFetchEventsHourly.tsHour, serverEnd))).all();
|
|
33246
33614
|
const allReferrals = db.select().from(aiReferralEventsHourly).where(and36(eq42(aiReferralEventsHourly.projectId, project.id), countableReferralCondition(), gte11(aiReferralEventsHourly.tsHour, serverStart), lte10(aiReferralEventsHourly.tsHour, serverEnd))).all();
|
|
33247
|
-
const [serverCoverageRow] = db.all(
|
|
33615
|
+
const [serverCoverageRow] = db.all(sql17`
|
|
33248
33616
|
select
|
|
33249
33617
|
min(day) as startDate,
|
|
33250
33618
|
max(day) as endDate,
|
|
@@ -34636,10 +35004,10 @@ async function resultsExportRoutes(app) {
|
|
|
34636
35004
|
}
|
|
34637
35005
|
|
|
34638
35006
|
// ../api-routes/src/composites.ts
|
|
34639
|
-
import { eq as eq47, and as and41, desc as desc23, sql as
|
|
35007
|
+
import { eq as eq47, and as and41, desc as desc23, sql as sql19, like as like4, or as or12, inArray as inArray23 } from "drizzle-orm";
|
|
34640
35008
|
|
|
34641
35009
|
// ../api-routes/src/gsc-totals.ts
|
|
34642
|
-
import { and as and40, asc as asc7, eq as eq46, sql as
|
|
35010
|
+
import { and as and40, asc as asc7, eq as eq46, sql as sql18, max, min } from "drizzle-orm";
|
|
34643
35011
|
var WINDOW_DAYS = { "7d": 7, "30d": 30, "90d": 90 };
|
|
34644
35012
|
function resolveGscWindowRange(window, latestDataDate, today) {
|
|
34645
35013
|
if (window === "all") {
|
|
@@ -34703,8 +35071,8 @@ function readEarliestGscDataDate(db, projectId) {
|
|
|
34703
35071
|
return [property, dimensioned].filter((d) => d !== null).reduce((earliest, d) => earliest === null || d < earliest ? d : earliest, null);
|
|
34704
35072
|
}
|
|
34705
35073
|
function readEarliestGscQueryDataDate(db, projectId) {
|
|
34706
|
-
const accurate = db.select({ earliest: min(gscQueryDailyTotals.date) }).from(gscQueryDailyTotals).where(and40(eq46(gscQueryDailyTotals.projectId, projectId),
|
|
34707
|
-
const dimensioned = db.select({ earliest: min(gscSearchData.date) }).from(gscSearchData).where(and40(eq46(gscSearchData.projectId, projectId),
|
|
35074
|
+
const accurate = db.select({ earliest: min(gscQueryDailyTotals.date) }).from(gscQueryDailyTotals).where(and40(eq46(gscQueryDailyTotals.projectId, projectId), sql18`${gscQueryDailyTotals.date} <> ''`)).get()?.earliest ?? null;
|
|
35075
|
+
const dimensioned = db.select({ earliest: min(gscSearchData.date) }).from(gscSearchData).where(and40(eq46(gscSearchData.projectId, projectId), sql18`${gscSearchData.date} <> ''`)).get()?.earliest ?? null;
|
|
34708
35076
|
return [accurate, dimensioned].filter((d) => d !== null).reduce((earliest, d) => earliest === null || d < earliest ? d : earliest, null);
|
|
34709
35077
|
}
|
|
34710
35078
|
function readGscDailyTotals(db, projectId, startDate, endDate) {
|
|
@@ -34716,8 +35084,8 @@ function readGscDailyTotals(db, projectId, startDate, endDate) {
|
|
|
34716
35084
|
}).from(gscDailyTotals).where(
|
|
34717
35085
|
and40(
|
|
34718
35086
|
eq46(gscDailyTotals.projectId, projectId),
|
|
34719
|
-
|
|
34720
|
-
|
|
35087
|
+
sql18`${gscDailyTotals.date} >= ${startDate}`,
|
|
35088
|
+
sql18`${gscDailyTotals.date} <= ${endDate}`
|
|
34721
35089
|
)
|
|
34722
35090
|
).orderBy(asc7(gscDailyTotals.date)).all();
|
|
34723
35091
|
return rows.map((r) => {
|
|
@@ -34746,8 +35114,8 @@ function readGscQueryDailyRows(db, projectId, startDate, endDate) {
|
|
|
34746
35114
|
}).from(gscQueryDailyTotals).where(
|
|
34747
35115
|
and40(
|
|
34748
35116
|
eq46(gscQueryDailyTotals.projectId, projectId),
|
|
34749
|
-
|
|
34750
|
-
|
|
35117
|
+
sql18`${gscQueryDailyTotals.date} >= ${startDate}`,
|
|
35118
|
+
sql18`${gscQueryDailyTotals.date} <= ${endDate}`
|
|
34751
35119
|
)
|
|
34752
35120
|
).all();
|
|
34753
35121
|
return rows.map((r) => {
|
|
@@ -34765,16 +35133,16 @@ function readGscQueryDailyFallbackRows(db, projectId, startDate, endDate) {
|
|
|
34765
35133
|
const rows = db.select({
|
|
34766
35134
|
date: gscSearchData.date,
|
|
34767
35135
|
query: gscSearchData.query,
|
|
34768
|
-
clicks:
|
|
34769
|
-
impressions:
|
|
34770
|
-
position:
|
|
35136
|
+
clicks: sql18`COALESCE(SUM(${gscSearchData.clicks}), 0)`,
|
|
35137
|
+
impressions: sql18`COALESCE(SUM(${gscSearchData.impressions}), 0)`,
|
|
35138
|
+
position: sql18`COALESCE(SUM(CAST(${gscSearchData.position} AS REAL) * ${gscSearchData.impressions}) * 1.0 / NULLIF(SUM(${gscSearchData.impressions}), 0), 0)`
|
|
34771
35139
|
}).from(gscSearchData).where(
|
|
34772
35140
|
and40(
|
|
34773
35141
|
eq46(gscSearchData.projectId, projectId),
|
|
34774
|
-
|
|
34775
|
-
|
|
34776
|
-
|
|
34777
|
-
|
|
35142
|
+
sql18`${gscSearchData.date} >= ${startDate}`,
|
|
35143
|
+
sql18`${gscSearchData.date} <= ${endDate}`,
|
|
35144
|
+
sql18`${gscSearchData.query} <> ''`,
|
|
35145
|
+
sql18`${gscSearchData.impressions} > 0`
|
|
34778
35146
|
)
|
|
34779
35147
|
).groupBy(gscSearchData.date, gscSearchData.query).all();
|
|
34780
35148
|
return rows.map((r) => ({
|
|
@@ -34791,7 +35159,7 @@ function readGscQueryTotalsPage(db, projectId, startDate, endDate, limit, offset
|
|
|
34791
35159
|
const bound = (n, floor) => Math.min(Math.max(Math.floor(n), floor), Number.MAX_SAFE_INTEGER);
|
|
34792
35160
|
const pageLimit2 = bound(limit, 1);
|
|
34793
35161
|
const pageOffset = bound(offset, 0);
|
|
34794
|
-
const perQuery =
|
|
35162
|
+
const perQuery = sql18`
|
|
34795
35163
|
WITH accurate AS (
|
|
34796
35164
|
SELECT query, clicks, impressions,
|
|
34797
35165
|
COALESCE(CAST(position AS REAL), 0) AS position
|
|
@@ -34830,7 +35198,7 @@ function readGscQueryTotalsPage(db, projectId, startDate, endDate, limit, offset
|
|
|
34830
35198
|
)
|
|
34831
35199
|
GROUP BY query
|
|
34832
35200
|
)`;
|
|
34833
|
-
const rows = db.all(
|
|
35201
|
+
const rows = db.all(sql18`${perQuery}
|
|
34834
35202
|
SELECT query, clicks, impressions,
|
|
34835
35203
|
CASE WHEN impressions > 0 THEN weighted_position_sum * 1.0 / impressions
|
|
34836
35204
|
ELSE position_sum * 1.0 / days END AS position,
|
|
@@ -34842,7 +35210,7 @@ function readGscQueryTotalsPage(db, projectId, startDate, endDate, limit, offset
|
|
|
34842
35210
|
FROM per_query
|
|
34843
35211
|
ORDER BY clicks DESC, impressions DESC, query ASC
|
|
34844
35212
|
LIMIT ${pageLimit2} OFFSET ${pageOffset}`);
|
|
34845
|
-
const totalMatching = rows.length > 0 ? Number(rows[0].total_matching) : Number(db.get(
|
|
35213
|
+
const totalMatching = rows.length > 0 ? Number(rows[0].total_matching) : Number(db.get(sql18`${perQuery} SELECT COUNT(*) AS total FROM per_query`).total);
|
|
34846
35214
|
return {
|
|
34847
35215
|
rows: rows.map((r) => ({
|
|
34848
35216
|
query: r.query,
|
|
@@ -35092,10 +35460,10 @@ async function compositeRoutes(app, options) {
|
|
|
35092
35460
|
and41(
|
|
35093
35461
|
eq47(queries.projectId, project.id),
|
|
35094
35462
|
or12(
|
|
35095
|
-
|
|
35096
|
-
|
|
35097
|
-
|
|
35098
|
-
|
|
35463
|
+
sql19`${querySnapshots.answerText} LIKE ${pattern} ESCAPE '\\'`,
|
|
35464
|
+
sql19`${querySnapshots.citedDomains} LIKE ${pattern} ESCAPE '\\'`,
|
|
35465
|
+
sql19`${querySnapshots.citedUrls} LIKE ${pattern} ESCAPE '\\'`,
|
|
35466
|
+
sql19`${querySnapshots.rawResponse} LIKE ${pattern} ESCAPE '\\'`,
|
|
35099
35467
|
like4(queries.query, pattern)
|
|
35100
35468
|
)
|
|
35101
35469
|
)
|
|
@@ -35106,8 +35474,8 @@ async function compositeRoutes(app, options) {
|
|
|
35106
35474
|
or12(
|
|
35107
35475
|
like4(insights.title, pattern),
|
|
35108
35476
|
like4(insights.query, pattern),
|
|
35109
|
-
|
|
35110
|
-
|
|
35477
|
+
sql19`${insights.recommendation} LIKE ${pattern} ESCAPE '\\'`,
|
|
35478
|
+
sql19`${insights.cause} LIKE ${pattern} ESCAPE '\\'`
|
|
35111
35479
|
)
|
|
35112
35480
|
)
|
|
35113
35481
|
).orderBy(desc23(insights.createdAt)).limit(limit + 1).all();
|
|
@@ -37889,6 +38257,7 @@ var routeCatalog = [
|
|
|
37889
38257
|
tags: ["measurement-plans"],
|
|
37890
38258
|
parameters: [
|
|
37891
38259
|
nameParameter,
|
|
38260
|
+
{ name: "compact", in: "query", description: "Byte-bounded whole-row page; omits optional heavy detail with explicit metadata. Default false for compatibility.", schema: { type: "boolean" } },
|
|
37892
38261
|
{ name: "scope", in: "query", required: true, description: "Reporting scope.", schema: { type: "string", enum: ["all", "group", "property"] } },
|
|
37893
38262
|
{ name: "groupKey", in: "query", description: 'Group stable key, required when scope is "group".', schema: stringSchema },
|
|
37894
38263
|
{ name: "targetKey", in: "query", description: 'Target stable key, required when scope is "property".', schema: stringSchema },
|
|
@@ -37956,7 +38325,7 @@ var routeCatalog = [
|
|
|
37956
38325
|
path: "/api/v1/projects/{name}/sentiment/compare",
|
|
37957
38326
|
summary: "Compare compatible stored sentiment periods",
|
|
37958
38327
|
tags: ["sentiment"],
|
|
37959
|
-
parameters: [nameParameter, ...sentimentSelectionParameters, ...["fromRunId", "toRunId"].map((name) => ({ name, in: "query", required: true, description: "Exact source run for this period.", schema: stringSchema }))],
|
|
38328
|
+
parameters: [nameParameter, ...sentimentSelectionParameters, ...["fromRunId", "toRunId"].map((name) => ({ name, in: "query", required: true, description: name === "fromRunId" ? "Exact source run or previous-rated to resolve the preceding rated run in this scope. Identical run IDs are refused." : "Exact target run for this period.", schema: stringSchema }))],
|
|
37960
38329
|
responses: { 200: jsonResponse("Matched units, exclusions, and conservative comparison or refusal reasons.", "SentimentComparison"), 400: errorResponse("Invalid comparison selection."), 404: errorResponse("Project or selected source not found.") }
|
|
37961
38330
|
},
|
|
37962
38331
|
{
|
|
@@ -38050,16 +38419,20 @@ var routeCatalog = [
|
|
|
38050
38419
|
method: "get",
|
|
38051
38420
|
path: "/api/v1/projects/{name}/measurement-portfolio-summary",
|
|
38052
38421
|
summary: "Get the weakest measured Properties",
|
|
38053
|
-
description: "Returns a compact, revision-pinned portfolio ranking from stored results, plus a worst-first market roll-up. It defaults to non-brand queries, ranks measured mention coverage before citation coverage, and keeps unavailable rows separate from measured weakness. Every Property row carries its metro, submarkets and query count; weakest rows also carry namedInsteadInAnswerText (names written in the answer text of answers that neither named nor cited the Property, counted by answer) and citedDomains (domains cited by the Property's answers, counted by answer, every engine included). tiedAtWeakest reports Properties sharing the weakest rates; over the whole tie, not just the returned rows, byMetro counts the tied Properties per metro label (a Property in two metros counts in both) and namedInstead ranks the names written in the answer text of the tie's answers that neither named nor cited the Property (distinct answers, never citations). weakestAnswerSources ranks the domains cited across the weakest and tied Properties, each answer once. markets holds one level by default (
|
|
38422
|
+
description: "Returns a compact, revision-pinned portfolio ranking from stored results, plus a worst-first market roll-up. It defaults to non-brand queries, ranks measured mention coverage before citation coverage, and keeps unavailable rows separate from measured weakness. Every Property row carries its metro, submarkets and query count; weakest rows also carry namedInsteadInAnswerText (names written in the answer text of answers that neither named nor cited the Property, counted by answer) and citedDomains (domains cited by the Property's answers, counted by answer, every engine included). tiedAtWeakest reports Properties sharing the weakest rates; over the whole tie, not just the returned rows, byMetro counts the tied Properties per metro label (a Property in two metros counts in both) and namedInstead ranks the names written in the answer text of the tie's answers that neither named nor cited the Property (distinct answers, never citations). weakestAnswerSources ranks the domains cited across the weakest and tied Properties, each answer once. markets holds one level by default (top-level markets, or direct children of the selected group), worst-first; raw reads return all markets and compact reads return bounded summaries or list=markets pages. includeNestedMarkets selects every level. propertiesNeverMentioned counts zero mentions even if a Property was cited. tiedAtWeakest requires both zero mentions and zero citations. Compact nextCursor advances only pageList, default weakest-properties. Other default first-page lists are bounded summaries and later pages omit their row bodies. list selects one complete ranking, market or evidence list without repeating sibling rows; totals stay complete. List, filters, revision, run and evidence are bound to the cursor; limit may change. Every market is scoped to the displayed run, so a market row matches that market read with groupKey; markets may share Properties and never sum to the portfolio totals. It never starts provider work.",
|
|
38054
38423
|
tags: ["measurement-plans"],
|
|
38055
38424
|
parameters: [
|
|
38056
38425
|
nameParameter,
|
|
38426
|
+
{ name: "compact", in: "query", description: "Byte-bounded whole-row page of pageList, default weakest-properties. Other initial lists are bounded summaries. Default false for compatibility.", schema: { type: "boolean" } },
|
|
38427
|
+
{ name: "cursor", in: "query", description: "nextCursor from the preceding compact page; keep every selection unchanged.", schema: stringSchema },
|
|
38428
|
+
{ name: "list", in: "query", description: "Compact page selection identity; enumerate this list only and omit sibling row lists. observed-names and cited-domains require answers=not-mentioned. Keep unchanged across cursors.", schema: { type: "string", enum: ["weakest-properties", "strongest-mentions", "weakest-mentions", "excluded-mentions", "markets", "observed-names", "cited-domains"] } },
|
|
38429
|
+
{ name: "answers", in: "query", description: "Competitor evidence population. not-mentioned excludes every answer naming any targeted Property.", schema: { type: "string", enum: ["all", "not-mentioned"] } },
|
|
38057
38430
|
{ name: "groupKey", in: "query", description: "Optional reporting group stable key.", schema: stringSchema },
|
|
38058
38431
|
{ name: "queryClass", in: "query", description: "Question class. Defaults to non-brand.", schema: { type: "string", enum: ["all", "branded", "non-brand"], default: "non-brand" } },
|
|
38059
38432
|
{ name: "provider", in: "query", description: "Restrict to one answer provider.", schema: stringSchema },
|
|
38060
38433
|
{ name: "location", in: "query", description: "Restrict to one execution location label.", schema: stringSchema },
|
|
38061
38434
|
{ name: "runId", in: "query", description: "Read this completed or partial active-revision run, including a named spot check.", schema: stringSchema },
|
|
38062
|
-
{ name: "limit", in: "query", description: "
|
|
38435
|
+
{ name: "limit", in: "query", description: "Compact selected-list page size: auto-sized up to 50 by default; the byte budget may return fewer whole rows. Raw reads default to 4 Property rows and never cap markets.", schema: { type: "integer", minimum: 1, maximum: 50 } },
|
|
38063
38436
|
{ name: "includeNestedMarkets", in: "query", description: "Return every market in scope at every level. Defaults to false: one level (every top-level market, or every direct child of the selected group).", schema: { type: "boolean" } }
|
|
38064
38437
|
],
|
|
38065
38438
|
responses: {
|
|
@@ -38929,6 +39302,7 @@ var routeCatalog = [
|
|
|
38929
39302
|
tags: ["analytics", "competitors"],
|
|
38930
39303
|
parameters: [
|
|
38931
39304
|
nameParameter,
|
|
39305
|
+
{ name: "answers", in: "query", description: "not-mentioned selects only answers where none of their targeted subjects was named. all or omitted keeps the full population; answerSelection reports population and selected counts.", schema: { type: "string", enum: ["all", "not-mentioned"] } },
|
|
38932
39306
|
analyticsWindowParameter,
|
|
38933
39307
|
competitorLandscapeGroupKeyParameter,
|
|
38934
39308
|
competitorLandscapeScopeParameter,
|
|
@@ -43724,7 +44098,7 @@ var routeCatalog = [
|
|
|
43724
44098
|
method: "get",
|
|
43725
44099
|
path: "/api/v1/projects/{name}/technical-aeo",
|
|
43726
44100
|
summary: "Get the Technical AEO scorecard for a project",
|
|
43727
|
-
description: "Returns the latest completed/partial site-audit, or the historical audit selected by `runId`: aggregate 0\u2013100 score, page counts, the full per-factor scorecard (site-level averages with pass/partial/fail distribution), cross-cutting issues, prioritized fixes, and the delta vs the audit immediately before it. When the project has never been audited, `hasData` is false and the numeric fields are zeroed \u2014 render an onboarding state.",
|
|
44101
|
+
description: "runSelection selects a complete run on the latest UTC scan date, otherwise the largest page sample, then newest; it flags same-date candidates. Factor pagesFailing and pagesPartial are separate; a factor with failing pages is fail. Returns the latest completed/partial site-audit, or the historical audit selected by `runId`: aggregate 0\u2013100 score, page counts, the full per-factor scorecard (site-level averages with pass/partial/fail distribution), cross-cutting issues, prioritized fixes, and the delta vs the audit immediately before it. When the project has never been audited, `hasData` is false and the numeric fields are zeroed \u2014 render an onboarding state.",
|
|
43728
44102
|
tags: ["technical-aeo"],
|
|
43729
44103
|
parameters: [
|
|
43730
44104
|
nameParameter,
|
|
@@ -43773,13 +44147,15 @@ var routeCatalog = [
|
|
|
43773
44147
|
method: "get",
|
|
43774
44148
|
path: "/api/v1/projects/{name}/technical-aeo/crawl",
|
|
43775
44149
|
summary: "Get persisted Technical AEO crawl metadata",
|
|
43776
|
-
description: "Returns
|
|
44150
|
+
description: "Returns a non-probe site-audit crawl on the requested UTC date, the selected run, or the latest scan date. Prefer complete scans, then the largest page sample, then the latest scan on that date. runSelection records same-date ambiguity; inventorySummary reports exact whole-snapshot eligibility and disjoint excluded reasons. `complete` and `termination` qualify budget-limited evidence. `hasCrawlData=false` never synthesizes a graph from legacy scorecard rows; dated no-data reads expose bounded `availableScanDates` with full distinct-date totals and matching month/day candidates to resolve an omitted year; `legacyAuditAvailable` says that the old score/pages/trend data can still be read separately.",
|
|
43777
44151
|
tags: ["technical-aeo"],
|
|
43778
44152
|
parameters: [
|
|
43779
44153
|
nameParameter,
|
|
43780
|
-
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest crawl.", schema: stringSchema }
|
|
44154
|
+
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest crawl.", schema: stringSchema },
|
|
44155
|
+
{ name: "date", in: "query", description: "UTC scan date YYYY-MM-DD, mutually exclusive with runId. Prefer complete, then largest, then latest scans on that exact date; missing data never falls back to another date.", schema: { type: "string", format: "date" } }
|
|
43781
44156
|
],
|
|
43782
44157
|
responses: {
|
|
44158
|
+
400: errorResponse("Invalid date, conflicting runId/date, or invalid cursor identity."),
|
|
43783
44159
|
200: jsonResponse("Persisted crawl summary returned.", "SiteCrawlSummaryDto"),
|
|
43784
44160
|
404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
|
|
43785
44161
|
}
|
|
@@ -43886,11 +44262,12 @@ var routeCatalog = [
|
|
|
43886
44262
|
method: "get",
|
|
43887
44263
|
path: "/api/v1/projects/{name}/technical-aeo/crawl/pages",
|
|
43888
44264
|
summary: "List persisted Technical AEO crawl pages",
|
|
43889
|
-
description: "Cursor-paged canonical crawl nodes for the latest or selected crawl. `inventoryEligible` is Canonry technical-inventory eligibility, not a statement about actual Google index state.",
|
|
44265
|
+
description: "Cursor-paged canonical crawl nodes for the requested UTC date, latest scan date, or selected crawl. inventorySummary gives whole-snapshot eligibility and excluded reasons independent of paging and filters; healthReasonCounts sums the filtered list before paging. Dated no-data reads expose `availableScanDates` for an explicit retry on a known date, never an automatic fallback. Unknown indexability never means canonical-away. `inventoryEligible` is Canonry technical-inventory eligibility, not a statement about actual Google index state.",
|
|
43890
44266
|
tags: ["technical-aeo"],
|
|
43891
44267
|
parameters: [
|
|
43892
44268
|
nameParameter,
|
|
43893
44269
|
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest crawl.", schema: stringSchema },
|
|
44270
|
+
{ name: "date", in: "query", description: "UTC scan date YYYY-MM-DD, mutually exclusive with runId. Prefer complete, then largest, then latest scans on that exact date; missing data never falls back to another date.", schema: { type: "string", format: "date" } },
|
|
43894
44271
|
{ name: "inventoryEligible", in: "query", description: "Filter Canonry technical inventory eligibility (`true` or `false`).", schema: booleanSchema },
|
|
43895
44272
|
{ name: "fetchState", in: "query", description: "Filter by persisted fetch state.", schema: stringSchema },
|
|
43896
44273
|
{ name: "indexabilityState", in: "query", description: "Filter by crawler-derived indexability state; this is not Google index coverage.", schema: stringSchema },
|
|
@@ -43902,6 +44279,7 @@ var routeCatalog = [
|
|
|
43902
44279
|
crawlLimitParameter
|
|
43903
44280
|
],
|
|
43904
44281
|
responses: {
|
|
44282
|
+
400: errorResponse("Invalid date, conflicting runId/date, or invalid cursor identity."),
|
|
43905
44283
|
200: jsonResponse("Crawl pages returned.", "SiteCrawlPagesResponseDto"),
|
|
43906
44284
|
404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
|
|
43907
44285
|
}
|
|
@@ -43986,7 +44364,7 @@ var routeCatalog = [
|
|
|
43986
44364
|
method: "get",
|
|
43987
44365
|
path: "/api/v1/projects/{name}/technical-aeo/runs",
|
|
43988
44366
|
summary: "List Site Health scan history",
|
|
43989
|
-
description: "Returns every non-probe `site-audit` run for the project, newest first, including runs still queued or running and runs that failed. `hasCrawlData` says whether that scan published a page and internal-link crawl: a legacy score-only scan is listed with `hasCrawlData=false` rather than hidden, and the crawl-scoped reads answer it with their no-crawl state instead of a 404.",
|
|
44367
|
+
description: "Returns every non-probe `site-audit` run for the project, newest first, including runs still queued or running and runs that failed. `hasCrawlData` says whether that scan published a page and internal-link crawl: a legacy score-only scan is listed with `hasCrawlData=false` rather than hidden, and the crawl-scoped reads answer it with their no-crawl state instead of a 404. `preferredRunId` identifies the default persisted scan shared by score/crawl reads, independently of the history page limit; an explicit selected run stays authoritative.",
|
|
43990
44368
|
tags: ["technical-aeo"],
|
|
43991
44369
|
parameters: [
|
|
43992
44370
|
nameParameter,
|
|
@@ -45331,7 +45709,7 @@ function formatNotification(row) {
|
|
|
45331
45709
|
|
|
45332
45710
|
// ../api-routes/src/google.ts
|
|
45333
45711
|
import crypto33 from "crypto";
|
|
45334
|
-
import { eq as eq54, and as and45, desc as desc27, sql as
|
|
45712
|
+
import { eq as eq54, and as and45, desc as desc27, sql as sql20, inArray as inArray25 } from "drizzle-orm";
|
|
45335
45713
|
|
|
45336
45714
|
// ../api-routes/src/gsc-period-comparison.ts
|
|
45337
45715
|
var DAY_MS = 864e5;
|
|
@@ -47834,11 +48212,11 @@ async function googleRoutes(app, opts) {
|
|
|
47834
48212
|
const cutoffDate = startDate ? null : resolvedWindow.startDate;
|
|
47835
48213
|
const effectiveEndDate = endDate ?? (daysParam === null ? void 0 : resolvedWindow.endDate ?? void 0);
|
|
47836
48214
|
const conditions = [eq54(gscSearchData.projectId, project.id)];
|
|
47837
|
-
if (startDate) conditions.push(
|
|
47838
|
-
else if (cutoffDate) conditions.push(
|
|
47839
|
-
if (effectiveEndDate) conditions.push(
|
|
47840
|
-
if (query) conditions.push(
|
|
47841
|
-
if (page3) conditions.push(
|
|
48215
|
+
if (startDate) conditions.push(sql20`${gscSearchData.date} >= ${startDate}`);
|
|
48216
|
+
else if (cutoffDate) conditions.push(sql20`${gscSearchData.date} >= ${cutoffDate}`);
|
|
48217
|
+
if (effectiveEndDate) conditions.push(sql20`${gscSearchData.date} <= ${effectiveEndDate}`);
|
|
48218
|
+
if (query) conditions.push(sql20`${gscSearchData.query} LIKE ${"%" + escapeLikePattern(query) + "%"} ESCAPE '\\'`);
|
|
48219
|
+
if (page3) conditions.push(sql20`${gscSearchData.page} LIKE ${"%" + escapeLikePattern(page3) + "%"} ESCAPE '\\'`);
|
|
47842
48220
|
const limitVal = Math.max(parseInt(limit ?? "500", 10) || 0, 1);
|
|
47843
48221
|
const offsetVal = Math.max(parseInt(offset ?? "0", 10) || 0, 0);
|
|
47844
48222
|
const orderColumns = {
|
|
@@ -47847,8 +48225,8 @@ async function googleRoutes(app, opts) {
|
|
|
47847
48225
|
date: [desc27(gscSearchData.date), desc27(gscSearchData.clicks), gscSearchData.query]
|
|
47848
48226
|
}[orderBy];
|
|
47849
48227
|
const rows = app.db.select().from(gscSearchData).where(and45(...conditions)).orderBy(...orderColumns).limit(limitVal).offset(offsetVal).all();
|
|
47850
|
-
const totalMatching = app.db.select({ total:
|
|
47851
|
-
const latestAvailableDate = app.db.select({ latest:
|
|
48228
|
+
const totalMatching = app.db.select({ total: sql20`COUNT(*)` }).from(gscSearchData).where(and45(...conditions)).get()?.total ?? 0;
|
|
48229
|
+
const latestAvailableDate = app.db.select({ latest: sql20`MAX(${gscSearchData.date})` }).from(gscSearchData).where(eq54(gscSearchData.projectId, project.id)).get()?.latest ?? null;
|
|
47852
48230
|
return {
|
|
47853
48231
|
rows: rows.map((r) => ({
|
|
47854
48232
|
date: r.date,
|
|
@@ -47892,12 +48270,12 @@ async function googleRoutes(app, opts) {
|
|
|
47892
48270
|
const fetchStart = priorWindowIsSynced ? priorPeriodStart : windowStart;
|
|
47893
48271
|
const dailyTotals = readGscDailyTotals(app.db, project.id, fetchStart, windowEnd);
|
|
47894
48272
|
const conditions = [eq54(gscSearchData.projectId, project.id)];
|
|
47895
|
-
if (fetchStart) conditions.push(
|
|
47896
|
-
if (endDate) conditions.push(
|
|
48273
|
+
if (fetchStart) conditions.push(sql20`${gscSearchData.date} >= ${fetchStart}`);
|
|
48274
|
+
if (endDate) conditions.push(sql20`${gscSearchData.date} <= ${endDate}`);
|
|
47897
48275
|
const dimensionedRows = app.db.select({
|
|
47898
48276
|
date: gscSearchData.date,
|
|
47899
|
-
clicks:
|
|
47900
|
-
impressions:
|
|
48277
|
+
clicks: sql20`COALESCE(SUM(${gscSearchData.clicks}), 0)`,
|
|
48278
|
+
impressions: sql20`COALESCE(SUM(${gscSearchData.impressions}), 0)`
|
|
47901
48279
|
}).from(gscSearchData).where(and45(...conditions)).groupBy(gscSearchData.date).orderBy(gscSearchData.date).all();
|
|
47902
48280
|
const propertyDates = new Set(dailyTotals.map((d) => d.date));
|
|
47903
48281
|
const comparisonDaily = mergeGscDailyTotalsWithFallback(
|
|
@@ -47995,23 +48373,23 @@ async function googleRoutes(app, opts) {
|
|
|
47995
48373
|
);
|
|
47996
48374
|
const cutoffDate = startDate ? null : resolvedWindow.startDate;
|
|
47997
48375
|
const conditions = [eq54(gscSearchData.projectId, project.id)];
|
|
47998
|
-
if (startDate) conditions.push(
|
|
47999
|
-
else if (cutoffDate) conditions.push(
|
|
48000
|
-
if (endDate) conditions.push(
|
|
48376
|
+
if (startDate) conditions.push(sql20`${gscSearchData.date} >= ${startDate}`);
|
|
48377
|
+
else if (cutoffDate) conditions.push(sql20`${gscSearchData.date} >= ${cutoffDate}`);
|
|
48378
|
+
if (endDate) conditions.push(sql20`${gscSearchData.date} <= ${endDate}`);
|
|
48001
48379
|
const limitVal = Math.max(parseInt(limit ?? "50", 10) || 0, 1);
|
|
48002
48380
|
const rows = app.db.select({
|
|
48003
48381
|
page: gscSearchData.page,
|
|
48004
|
-
clicks:
|
|
48005
|
-
impressions:
|
|
48006
|
-
}).from(gscSearchData).where(and45(...conditions)).groupBy(gscSearchData.page).orderBy(desc27(
|
|
48382
|
+
clicks: sql20`COALESCE(SUM(${gscSearchData.clicks}), 0)`,
|
|
48383
|
+
impressions: sql20`COALESCE(SUM(${gscSearchData.impressions}), 0)`
|
|
48384
|
+
}).from(gscSearchData).where(and45(...conditions)).groupBy(gscSearchData.page).orderBy(desc27(sql20`SUM(${gscSearchData.clicks})`), desc27(sql20`SUM(${gscSearchData.impressions})`)).limit(limitVal).all();
|
|
48007
48385
|
const windowStart = startDate ?? cutoffDate ?? "";
|
|
48008
48386
|
const windowEnd = endDate ?? resolvedWindow.endDate ?? "9999-12-31";
|
|
48009
48387
|
const dailyTotals = readGscDailyTotals(app.db, project.id, windowStart, windowEnd);
|
|
48010
48388
|
const totalClicks = dailyTotals.reduce((sum, d) => sum + d.clicks, 0);
|
|
48011
48389
|
const totalImpressions = dailyTotals.reduce((sum, d) => sum + d.impressions, 0);
|
|
48012
48390
|
const rankedSpan = app.db.select({
|
|
48013
|
-
first:
|
|
48014
|
-
last:
|
|
48391
|
+
first: sql20`MIN(${gscSearchData.date})`,
|
|
48392
|
+
last: sql20`MAX(${gscSearchData.date})`
|
|
48015
48393
|
}).from(gscSearchData).where(and45(...conditions)).get();
|
|
48016
48394
|
const coveredFrom = dailyTotals.length > 0 ? dailyTotals[0].date : null;
|
|
48017
48395
|
const coveredThrough = dailyTotals.length > 0 ? dailyTotals[dailyTotals.length - 1].date : null;
|
|
@@ -49009,8 +49387,8 @@ async function googleRoutes(app, opts) {
|
|
|
49009
49387
|
});
|
|
49010
49388
|
const conditions = [eq54(gbpReviews.projectId, project.id)];
|
|
49011
49389
|
if (locationName) conditions.push(eq54(gbpReviews.locationName, locationName));
|
|
49012
|
-
if (negativeOnly) conditions.push(
|
|
49013
|
-
const total = app.db.select({ n:
|
|
49390
|
+
if (negativeOnly) conditions.push(sql20`${gbpReviews.starRating} BETWEEN 1 AND ${negativeMaxStars}`);
|
|
49391
|
+
const total = app.db.select({ n: sql20`count(*)` }).from(gbpReviews).where(and45(...conditions)).get()?.n ?? 0;
|
|
49014
49392
|
const reviews = app.db.select().from(gbpReviews).where(and45(...conditions)).orderBy(desc27(gbpReviews.updateTime), gbpReviews.reviewName).limit(limit).all().map((r) => ({
|
|
49015
49393
|
locationName: r.locationName,
|
|
49016
49394
|
origin: r.origin,
|
|
@@ -49098,7 +49476,7 @@ function parseReviewLimit(value) {
|
|
|
49098
49476
|
|
|
49099
49477
|
// ../api-routes/src/google-marketing.ts
|
|
49100
49478
|
import crypto34 from "crypto";
|
|
49101
|
-
import { and as and46, desc as desc28, eq as eq55, isNull as isNull10, lt as lt9, ne as ne10, or as or13, sql as
|
|
49479
|
+
import { and as and46, desc as desc28, eq as eq55, isNull as isNull10, lt as lt9, ne as ne10, or as or13, sql as sql21 } from "drizzle-orm";
|
|
49102
49480
|
import { z as z4 } from "zod";
|
|
49103
49481
|
var snapshotPageQuerySchema = z4.object({
|
|
49104
49482
|
limit: z4.coerce.number().int().min(1).max(GOOGLE_MARKETING_STORED_SNAPSHOT_PAGE_MAX).optional(),
|
|
@@ -50049,7 +50427,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
50049
50427
|
selectedCustomerCurrencyCode: null,
|
|
50050
50428
|
selectedCustomerTimeZone: null,
|
|
50051
50429
|
selectedCustomerStatus: null,
|
|
50052
|
-
selectionGeneration:
|
|
50430
|
+
selectionGeneration: sql21`${googleAdsConnections.selectionGeneration} + 1`,
|
|
50053
50431
|
scopes: tokens.scopes,
|
|
50054
50432
|
lastValidatedAt: now,
|
|
50055
50433
|
lastCustomerSnapshotId: null,
|
|
@@ -50080,7 +50458,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
50080
50458
|
selectedContainerPublicId: null,
|
|
50081
50459
|
selectedWorkspaceId: null,
|
|
50082
50460
|
selectedWorkspaceName: null,
|
|
50083
|
-
selectionGeneration:
|
|
50461
|
+
selectionGeneration: sql21`${gtmConnections.selectionGeneration} + 1`,
|
|
50084
50462
|
scopes: tokens.scopes,
|
|
50085
50463
|
lastValidatedAt: now,
|
|
50086
50464
|
lastSnapshotAt: null,
|
|
@@ -50231,7 +50609,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
50231
50609
|
selectedCustomerStatus: customerChanged ? null : existing.selectedCustomerStatus,
|
|
50232
50610
|
// A same-value write still starts a new selection generation. Do not
|
|
50233
50611
|
// let timestamp equality reuse append-only evidence from before it.
|
|
50234
|
-
selectionGeneration:
|
|
50612
|
+
selectionGeneration: sql21`${googleAdsConnections.selectionGeneration} + 1`,
|
|
50235
50613
|
lastValidatedAt: now,
|
|
50236
50614
|
lastCustomerSnapshotId: null,
|
|
50237
50615
|
lastInventorySnapshotAt: null,
|
|
@@ -50275,7 +50653,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
50275
50653
|
selectedContainerPublicId: containerChanged ? null : existing.selectedContainerPublicId,
|
|
50276
50654
|
selectedWorkspaceId,
|
|
50277
50655
|
selectedWorkspaceName: workspaceChanged ? null : existing.selectedWorkspaceName,
|
|
50278
|
-
selectionGeneration:
|
|
50656
|
+
selectionGeneration: sql21`${gtmConnections.selectionGeneration} + 1`,
|
|
50279
50657
|
lastValidatedAt: now,
|
|
50280
50658
|
// Every selection write starts a new evidence generation, including a
|
|
50281
50659
|
// same-value reselection. The timestamp alone cannot order a prior
|
|
@@ -50387,7 +50765,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
50387
50765
|
const limit = query.limit ?? 50;
|
|
50388
50766
|
const rows = app.db.select().from(googleAdsRawSnapshots).where(and46(...conditions)).orderBy(desc28(googleAdsRawSnapshots.capturedAt), desc28(googleAdsRawSnapshots.id)).limit(limit + 1).all();
|
|
50389
50767
|
const page3 = rows.slice(0, limit);
|
|
50390
|
-
const total = app.db.select({ total:
|
|
50768
|
+
const total = app.db.select({ total: sql21`COUNT(*)`.mapWith(Number).as("total") }).from(googleAdsRawSnapshots).where(eq55(googleAdsRawSnapshots.projectId, project.id)).get();
|
|
50391
50769
|
return {
|
|
50392
50770
|
snapshots: page3.map(toGoogleAdsSnapshotMetadata),
|
|
50393
50771
|
nextCursor: rows.length > limit && page3.length > 0 ? encodeSnapshotCursor({ capturedAt: page3.at(-1).capturedAt, id: page3.at(-1).id }) : null,
|
|
@@ -50419,7 +50797,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
50419
50797
|
const limit = query.limit ?? 50;
|
|
50420
50798
|
const rows = app.db.select().from(gtmRawSnapshots).where(and46(...conditions)).orderBy(desc28(gtmRawSnapshots.capturedAt), desc28(gtmRawSnapshots.id)).limit(limit + 1).all();
|
|
50421
50799
|
const page3 = rows.slice(0, limit);
|
|
50422
|
-
const total = app.db.select({ total:
|
|
50800
|
+
const total = app.db.select({ total: sql21`COUNT(*)`.mapWith(Number).as("total") }).from(gtmRawSnapshots).where(eq55(gtmRawSnapshots.projectId, project.id)).get();
|
|
50423
50801
|
return {
|
|
50424
50802
|
snapshots: page3.map(toGtmSnapshotMetadata),
|
|
50425
50803
|
nextCursor: rows.length > limit && page3.length > 0 ? encodeSnapshotCursor({ capturedAt: page3.at(-1).capturedAt, id: page3.at(-1).id }) : null,
|
|
@@ -50454,7 +50832,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
50454
50832
|
selectedCustomerCurrencyCode: null,
|
|
50455
50833
|
selectedCustomerTimeZone: null,
|
|
50456
50834
|
selectedCustomerStatus: null,
|
|
50457
|
-
selectionGeneration:
|
|
50835
|
+
selectionGeneration: sql21`${googleAdsConnections.selectionGeneration} + 1`,
|
|
50458
50836
|
scopes: [],
|
|
50459
50837
|
lastValidatedAt: null,
|
|
50460
50838
|
lastCustomerSnapshotId: null,
|
|
@@ -50493,7 +50871,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
50493
50871
|
selectedContainerPublicId: null,
|
|
50494
50872
|
selectedWorkspaceId: null,
|
|
50495
50873
|
selectedWorkspaceName: null,
|
|
50496
|
-
selectionGeneration:
|
|
50874
|
+
selectionGeneration: sql21`${gtmConnections.selectionGeneration} + 1`,
|
|
50497
50875
|
scopes: [],
|
|
50498
50876
|
lastValidatedAt: null,
|
|
50499
50877
|
lastSnapshotAt: null,
|
|
@@ -50688,7 +51066,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
50688
51066
|
|
|
50689
51067
|
// ../api-routes/src/ads.ts
|
|
50690
51068
|
import crypto37 from "crypto";
|
|
50691
|
-
import { eq as eq57, and as and48, asc as asc11, desc as desc29, gt as gt6, gte as gte14, lt as lt10, lte as lte14, ne as ne11, inArray as inArray26, or as or15, isNull as isNull12, isNotNull as isNotNull7, sql as
|
|
51069
|
+
import { eq as eq57, and as and48, asc as asc11, desc as desc29, gt as gt6, gte as gte14, lt as lt10, lte as lte14, ne as ne11, inArray as inArray26, or as or15, isNull as isNull12, isNotNull as isNotNull7, sql as sql23 } from "drizzle-orm";
|
|
50692
51070
|
|
|
50693
51071
|
// ../api-routes/src/ads-activation.ts
|
|
50694
51072
|
import crypto35 from "crypto";
|
|
@@ -52074,7 +52452,7 @@ async function executeApprovedAdsActivation(dependencies, request) {
|
|
|
52074
52452
|
|
|
52075
52453
|
// ../api-routes/src/ads-activation-routes.ts
|
|
52076
52454
|
import crypto36 from "crypto";
|
|
52077
|
-
import { and as and47, asc as asc10, eq as eq56, gt as gt5, isNotNull as isNotNull6, isNull as isNull11, lte as lte13, or as or14, sql as
|
|
52455
|
+
import { and as and47, asc as asc10, eq as eq56, gt as gt5, isNotNull as isNotNull6, isNull as isNull11, lte as lte13, or as or14, sql as sql22 } from "drizzle-orm";
|
|
52078
52456
|
var MAX_ACTIVATION_GRANT_LIFETIME_MS = 24 * 60 * 60 * 1e3;
|
|
52079
52457
|
var ACTIVATION_LEASE_MS = 5 * 6e4;
|
|
52080
52458
|
var ACTIVATION_REVOCATION_SETTLEMENT_MS = ACTIVATION_LEASE_MS;
|
|
@@ -52646,7 +53024,7 @@ function selectWatchdogOperations(app, nowIso, batchSize, allowLiveRevocationLea
|
|
|
52646
53024
|
),
|
|
52647
53025
|
leaseAvailable
|
|
52648
53026
|
);
|
|
52649
|
-
const freshRevocationPriority =
|
|
53027
|
+
const freshRevocationPriority = sql22`CASE
|
|
52650
53028
|
WHEN ${adsActivationGrants.revocationRequestedAt} IS NOT NULL
|
|
52651
53029
|
AND (
|
|
52652
53030
|
${adsOperations.lastReconciledAt} IS NULL
|
|
@@ -54054,7 +54432,7 @@ function claimOperationForReconciliation(app, row, leaseOwner, now, leaseMs, pol
|
|
|
54054
54432
|
state: AdsOperationStates.reconciling,
|
|
54055
54433
|
leaseOwner,
|
|
54056
54434
|
leaseExpiresAt: leaseExpiresAt2,
|
|
54057
|
-
reconcileAttempts:
|
|
54435
|
+
reconcileAttempts: sql23`${adsOperations.reconcileAttempts} + 1`,
|
|
54058
54436
|
updatedAt: nowIso
|
|
54059
54437
|
}).where(and48(
|
|
54060
54438
|
eq57(adsOperations.id, row.id),
|
|
@@ -56656,7 +57034,7 @@ async function cdpRoutes(app, opts) {
|
|
|
56656
57034
|
|
|
56657
57035
|
// ../api-routes/src/ga.ts
|
|
56658
57036
|
import crypto40 from "crypto";
|
|
56659
|
-
import { eq as eq60, desc as desc31, and as and51, sql as
|
|
57037
|
+
import { eq as eq60, desc as desc31, and as and51, sql as sql24 } from "drizzle-orm";
|
|
56660
57038
|
|
|
56661
57039
|
// ../api-routes/src/ga-session-history.ts
|
|
56662
57040
|
function buildSessionHistory(landingPageDays, dailyTotals) {
|
|
@@ -56729,8 +57107,8 @@ function withSessionShares(rows) {
|
|
|
56729
57107
|
}
|
|
56730
57108
|
function dateRangeConditions(column, range) {
|
|
56731
57109
|
const conditions = [];
|
|
56732
|
-
if (range.startDate) conditions.push(
|
|
56733
|
-
if (range.endDate) conditions.push(
|
|
57110
|
+
if (range.startDate) conditions.push(sql24`${column} >= ${range.startDate}`);
|
|
57111
|
+
if (range.endDate) conditions.push(sql24`${column} <= ${range.endDate}`);
|
|
56734
57112
|
return conditions;
|
|
56735
57113
|
}
|
|
56736
57114
|
var SOCIAL_CHANNEL_GROUPS2 = /* @__PURE__ */ new Set(["Organic Social", "Paid Social"]);
|
|
@@ -56790,8 +57168,8 @@ function persistAcquisitionMeasurement(db, input) {
|
|
|
56790
57168
|
db.transaction((tx) => {
|
|
56791
57169
|
tx.delete(gaAcquisitionDaily).where(and51(
|
|
56792
57170
|
eq60(gaAcquisitionDaily.projectId, input.projectId),
|
|
56793
|
-
|
|
56794
|
-
|
|
57171
|
+
sql24`${gaAcquisitionDaily.date} >= ${startDate}`,
|
|
57172
|
+
sql24`${gaAcquisitionDaily.date} <= ${endDate}`
|
|
56795
57173
|
)).run();
|
|
56796
57174
|
for (const row of input.report.rows) {
|
|
56797
57175
|
tx.insert(gaAcquisitionDaily).values({
|
|
@@ -56832,8 +57210,8 @@ function persistLeadMeasurement(db, input) {
|
|
|
56832
57210
|
db.transaction((tx) => {
|
|
56833
57211
|
tx.delete(gaLeadEventsDaily).where(and51(
|
|
56834
57212
|
eq60(gaLeadEventsDaily.projectId, input.projectId),
|
|
56835
|
-
|
|
56836
|
-
|
|
57213
|
+
sql24`${gaLeadEventsDaily.date} >= ${startDate}`,
|
|
57214
|
+
sql24`${gaLeadEventsDaily.date} <= ${endDate}`
|
|
56837
57215
|
)).run();
|
|
56838
57216
|
for (const row of input.report.rows) {
|
|
56839
57217
|
tx.insert(gaLeadEventsDaily).values({
|
|
@@ -57208,8 +57586,8 @@ async function ga4Routes(app, opts) {
|
|
|
57208
57586
|
tx.delete(gaTrafficSnapshots).where(
|
|
57209
57587
|
and51(
|
|
57210
57588
|
eq60(gaTrafficSnapshots.projectId, project.id),
|
|
57211
|
-
|
|
57212
|
-
|
|
57589
|
+
sql24`${gaTrafficSnapshots.date} >= ${summary3.periodStart}`,
|
|
57590
|
+
sql24`${gaTrafficSnapshots.date} <= ${summary3.periodEnd}`
|
|
57213
57591
|
)
|
|
57214
57592
|
).run();
|
|
57215
57593
|
for (const row of rows) {
|
|
@@ -57231,8 +57609,8 @@ async function ga4Routes(app, opts) {
|
|
|
57231
57609
|
tx.delete(gaDailyTotals).where(
|
|
57232
57610
|
and51(
|
|
57233
57611
|
eq60(gaDailyTotals.projectId, project.id),
|
|
57234
|
-
|
|
57235
|
-
|
|
57612
|
+
sql24`${gaDailyTotals.date} >= ${summary3.periodStart}`,
|
|
57613
|
+
sql24`${gaDailyTotals.date} <= ${summary3.periodEnd}`
|
|
57236
57614
|
)
|
|
57237
57615
|
).run();
|
|
57238
57616
|
for (const row of dailyTotals) {
|
|
@@ -57257,8 +57635,8 @@ async function ga4Routes(app, opts) {
|
|
|
57257
57635
|
tx.delete(gaAiReferrals).where(
|
|
57258
57636
|
and51(
|
|
57259
57637
|
eq60(gaAiReferrals.projectId, project.id),
|
|
57260
|
-
|
|
57261
|
-
|
|
57638
|
+
sql24`${gaAiReferrals.date} >= ${summary3.periodStart}`,
|
|
57639
|
+
sql24`${gaAiReferrals.date} <= ${summary3.periodEnd}`
|
|
57262
57640
|
)
|
|
57263
57641
|
).run();
|
|
57264
57642
|
for (const row of aiReferrals) {
|
|
@@ -57289,8 +57667,8 @@ async function ga4Routes(app, opts) {
|
|
|
57289
57667
|
tx.delete(gaSocialReferrals).where(
|
|
57290
57668
|
and51(
|
|
57291
57669
|
eq60(gaSocialReferrals.projectId, project.id),
|
|
57292
|
-
|
|
57293
|
-
|
|
57670
|
+
sql24`${gaSocialReferrals.date} >= ${summary3.periodStart}`,
|
|
57671
|
+
sql24`${gaSocialReferrals.date} <= ${summary3.periodEnd}`
|
|
57294
57672
|
)
|
|
57295
57673
|
).run();
|
|
57296
57674
|
for (const row of socialReferrals) {
|
|
@@ -57472,9 +57850,9 @@ async function ga4Routes(app, opts) {
|
|
|
57472
57850
|
totalUsers: gaTrafficSummaries.totalUsers
|
|
57473
57851
|
}).from(gaTrafficSummaries).where(eq60(gaTrafficSummaries.projectId, project.id)).get();
|
|
57474
57852
|
const retainedBounds = [
|
|
57475
|
-
app.db.select({ min:
|
|
57476
|
-
app.db.select({ min:
|
|
57477
|
-
app.db.select({ min:
|
|
57853
|
+
app.db.select({ min: sql24`MIN(${gaTrafficSnapshots.date})`, max: sql24`MAX(${gaTrafficSnapshots.date})` }).from(gaTrafficSnapshots).where(eq60(gaTrafficSnapshots.projectId, project.id)).get(),
|
|
57854
|
+
app.db.select({ min: sql24`MIN(${gaAiReferrals.date})`, max: sql24`MAX(${gaAiReferrals.date})` }).from(gaAiReferrals).where(eq60(gaAiReferrals.projectId, project.id)).get(),
|
|
57855
|
+
app.db.select({ min: sql24`MIN(${gaSocialReferrals.date})`, max: sql24`MAX(${gaSocialReferrals.date})` }).from(gaSocialReferrals).where(eq60(gaSocialReferrals.projectId, project.id)).get()
|
|
57478
57856
|
];
|
|
57479
57857
|
const retainedDates = retainedBounds.flatMap((bound) => [bound?.min, bound?.max]).filter((date) => Boolean(date));
|
|
57480
57858
|
const projectSummaryCoversAll = Boolean(
|
|
@@ -57488,9 +57866,9 @@ async function ga4Routes(app, opts) {
|
|
|
57488
57866
|
const socialConditions = [eq60(gaSocialReferrals.projectId, project.id), ...dateRangeConditions(gaSocialReferrals.date, measuredRange)];
|
|
57489
57867
|
const snapshotTotalsRow = !denominatorRow ? (() => {
|
|
57490
57868
|
const summed = app.db.select({
|
|
57491
|
-
totalSessions:
|
|
57492
|
-
totalOrganicSessions:
|
|
57493
|
-
totalUsers:
|
|
57869
|
+
totalSessions: sql24`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)`,
|
|
57870
|
+
totalOrganicSessions: sql24`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)`,
|
|
57871
|
+
totalUsers: sql24`COALESCE(SUM(${gaTrafficSnapshots.users}), 0)`
|
|
57494
57872
|
}).from(gaTrafficSnapshots).where(and51(...snapshotConditions)).get();
|
|
57495
57873
|
return {
|
|
57496
57874
|
totalSessions: summed?.totalSessions ?? 0,
|
|
@@ -57503,35 +57881,35 @@ async function ga4Routes(app, opts) {
|
|
|
57503
57881
|
})() : null;
|
|
57504
57882
|
const summaryRow = denominatorRow ?? snapshotTotalsRow;
|
|
57505
57883
|
const directTotalRow = windowSummaryRow ? { totalDirectSessions: windowSummaryRow.totalDirectSessions } : app.db.select({
|
|
57506
|
-
totalDirectSessions:
|
|
57884
|
+
totalDirectSessions: sql24`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)`
|
|
57507
57885
|
}).from(gaTrafficSnapshots).where(and51(...snapshotConditions)).get();
|
|
57508
57886
|
const rows = app.db.select({
|
|
57509
|
-
landingPage:
|
|
57510
|
-
sessions:
|
|
57511
|
-
organicSessions:
|
|
57512
|
-
directSessions:
|
|
57513
|
-
users:
|
|
57514
|
-
}).from(gaTrafficSnapshots).where(and51(...snapshotConditions)).groupBy(
|
|
57887
|
+
landingPage: sql24`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`,
|
|
57888
|
+
sessions: sql24`SUM(${gaTrafficSnapshots.sessions})`,
|
|
57889
|
+
organicSessions: sql24`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
57890
|
+
directSessions: sql24`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)`,
|
|
57891
|
+
users: sql24`SUM(${gaTrafficSnapshots.users})`
|
|
57892
|
+
}).from(gaTrafficSnapshots).where(and51(...snapshotConditions)).groupBy(sql24`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`).orderBy(sql24`SUM(${gaTrafficSnapshots.sessions}) DESC`).limit(limit).all();
|
|
57515
57893
|
const aiReferralRows = app.db.select({
|
|
57516
57894
|
source: gaAiReferrals.source,
|
|
57517
57895
|
medium: gaAiReferrals.medium,
|
|
57518
57896
|
trafficClass: gaAiReferrals.trafficClass,
|
|
57519
57897
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
57520
|
-
sessions:
|
|
57898
|
+
sessions: sql24`SUM(${gaAiReferrals.sessions})`
|
|
57521
57899
|
}).from(gaAiReferrals).where(and51(...aiConditions)).groupBy(gaAiReferrals.source, gaAiReferrals.medium, gaAiReferrals.trafficClass, gaAiReferrals.sourceDimension).all();
|
|
57522
57900
|
const aiReferralLandingPageRows = app.db.select({
|
|
57523
57901
|
source: gaAiReferrals.source,
|
|
57524
57902
|
medium: gaAiReferrals.medium,
|
|
57525
57903
|
trafficClass: gaAiReferrals.trafficClass,
|
|
57526
57904
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
57527
|
-
landingPage:
|
|
57528
|
-
sessions:
|
|
57905
|
+
landingPage: sql24`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`,
|
|
57906
|
+
sessions: sql24`SUM(${gaAiReferrals.sessions})`
|
|
57529
57907
|
}).from(gaAiReferrals).where(and51(...aiConditions)).groupBy(
|
|
57530
57908
|
gaAiReferrals.source,
|
|
57531
57909
|
gaAiReferrals.medium,
|
|
57532
57910
|
gaAiReferrals.trafficClass,
|
|
57533
57911
|
gaAiReferrals.sourceDimension,
|
|
57534
|
-
|
|
57912
|
+
sql24`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`
|
|
57535
57913
|
).all();
|
|
57536
57914
|
const aiRowsForTotals = app.db.select({
|
|
57537
57915
|
date: gaAiReferrals.date,
|
|
@@ -57557,10 +57935,10 @@ async function ga4Routes(app, opts) {
|
|
|
57557
57935
|
// is GA's COUNT DISTINCT at that grain, so summing over the window's
|
|
57558
57936
|
// dates counts a returning visitor once per day they returned. Same
|
|
57559
57937
|
// conclusion the AI-referral surfaces reached in 4.135.0.
|
|
57560
|
-
sessions:
|
|
57561
|
-
}).from(gaSocialReferrals).where(and51(...socialConditions)).groupBy(gaSocialReferrals.source, gaSocialReferrals.medium, gaSocialReferrals.channelGroup).orderBy(
|
|
57938
|
+
sessions: sql24`SUM(${gaSocialReferrals.sessions})`
|
|
57939
|
+
}).from(gaSocialReferrals).where(and51(...socialConditions)).groupBy(gaSocialReferrals.source, gaSocialReferrals.medium, gaSocialReferrals.channelGroup).orderBy(sql24`SUM(${gaSocialReferrals.sessions}) DESC`).all();
|
|
57562
57940
|
const socialTotals = app.db.select({
|
|
57563
|
-
sessions:
|
|
57941
|
+
sessions: sql24`SUM(${gaSocialReferrals.sessions})`
|
|
57564
57942
|
}).from(gaSocialReferrals).where(and51(...socialConditions)).get();
|
|
57565
57943
|
const latestSync = app.db.select({ syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(eq60(gaTrafficSummaries.projectId, project.id)).orderBy(desc31(gaTrafficSummaries.syncedAt)).limit(1).get();
|
|
57566
57944
|
const total = summaryRow?.totalSessions ?? 0;
|
|
@@ -57681,20 +58059,20 @@ async function ga4Routes(app, opts) {
|
|
|
57681
58059
|
source: gaAiReferrals.source,
|
|
57682
58060
|
medium: gaAiReferrals.medium,
|
|
57683
58061
|
trafficClass: gaAiReferrals.trafficClass,
|
|
57684
|
-
landingPage:
|
|
58062
|
+
landingPage: sql24`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`,
|
|
57685
58063
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
57686
58064
|
// No `users`: ga-ai-referral-aggregation states the rule for this
|
|
57687
58065
|
// table, that there is no grain at which the stored users column may be
|
|
57688
58066
|
// added up. ga4AiReferralDtoSchema.users was deprecated for this in
|
|
57689
58067
|
// 4.135.0; the history endpoint kept emitting one.
|
|
57690
|
-
sessions:
|
|
58068
|
+
sessions: sql24`SUM(${gaAiReferrals.sessions})`
|
|
57691
58069
|
}).from(gaAiReferrals).where(and51(...conditions)).groupBy(
|
|
57692
58070
|
gaAiReferrals.date,
|
|
57693
58071
|
gaAiReferrals.source,
|
|
57694
58072
|
gaAiReferrals.medium,
|
|
57695
58073
|
gaAiReferrals.trafficClass,
|
|
57696
58074
|
gaAiReferrals.sourceDimension,
|
|
57697
|
-
|
|
58075
|
+
sql24`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`
|
|
57698
58076
|
).orderBy(gaAiReferrals.date).all();
|
|
57699
58077
|
return rows;
|
|
57700
58078
|
});
|
|
@@ -57739,10 +58117,10 @@ async function ga4Routes(app, opts) {
|
|
|
57739
58117
|
d.setDate(d.getDate() - n);
|
|
57740
58118
|
return fmt(d);
|
|
57741
58119
|
};
|
|
57742
|
-
const sumSocial = (from, to) => app.db.select({ sessions:
|
|
58120
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql24`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and51(
|
|
57743
58121
|
eq60(gaSocialReferrals.projectId, project.id),
|
|
57744
|
-
|
|
57745
|
-
|
|
58122
|
+
sql24`${gaSocialReferrals.date} >= ${from}`,
|
|
58123
|
+
sql24`${gaSocialReferrals.date} < ${to}`
|
|
57746
58124
|
)).get();
|
|
57747
58125
|
const current7d = sumSocial(daysAgo(7), fmt(today));
|
|
57748
58126
|
const prev7d = sumSocial(daysAgo(14), daysAgo(7));
|
|
@@ -57750,19 +58128,19 @@ async function ga4Routes(app, opts) {
|
|
|
57750
58128
|
const prev30d = sumSocial(daysAgo(60), daysAgo(30));
|
|
57751
58129
|
const sourceCurrent = app.db.select({
|
|
57752
58130
|
source: gaSocialReferrals.source,
|
|
57753
|
-
sessions:
|
|
58131
|
+
sessions: sql24`SUM(${gaSocialReferrals.sessions})`
|
|
57754
58132
|
}).from(gaSocialReferrals).where(and51(
|
|
57755
58133
|
eq60(gaSocialReferrals.projectId, project.id),
|
|
57756
|
-
|
|
57757
|
-
|
|
58134
|
+
sql24`${gaSocialReferrals.date} >= ${daysAgo(7)}`,
|
|
58135
|
+
sql24`${gaSocialReferrals.date} < ${fmt(today)}`
|
|
57758
58136
|
)).groupBy(gaSocialReferrals.source).all();
|
|
57759
58137
|
const sourcePrev = app.db.select({
|
|
57760
58138
|
source: gaSocialReferrals.source,
|
|
57761
|
-
sessions:
|
|
58139
|
+
sessions: sql24`SUM(${gaSocialReferrals.sessions})`
|
|
57762
58140
|
}).from(gaSocialReferrals).where(and51(
|
|
57763
58141
|
eq60(gaSocialReferrals.projectId, project.id),
|
|
57764
|
-
|
|
57765
|
-
|
|
58142
|
+
sql24`${gaSocialReferrals.date} >= ${daysAgo(14)}`,
|
|
58143
|
+
sql24`${gaSocialReferrals.date} < ${daysAgo(7)}`
|
|
57766
58144
|
)).groupBy(gaSocialReferrals.source).all();
|
|
57767
58145
|
const response = {
|
|
57768
58146
|
socialSessions7d: current7d?.sessions ?? 0,
|
|
@@ -57785,16 +58163,16 @@ async function ga4Routes(app, opts) {
|
|
|
57785
58163
|
d.setDate(d.getDate() - n);
|
|
57786
58164
|
return fmt(d);
|
|
57787
58165
|
};
|
|
57788
|
-
const sumTotal = (from, to) => app.db.select({ sessions:
|
|
57789
|
-
const sumOrganic = (from, to) => app.db.select({ sessions:
|
|
57790
|
-
const sumDirect = (from, to) => app.db.select({ sessions:
|
|
57791
|
-
const sumAi = (from, to) => app.db.select({ sessions:
|
|
58166
|
+
const sumTotal = (from, to) => app.db.select({ sessions: sql24`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)` }).from(gaTrafficSnapshots).where(and51(eq60(gaTrafficSnapshots.projectId, project.id), sql24`${gaTrafficSnapshots.date} >= ${from}`, sql24`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
58167
|
+
const sumOrganic = (from, to) => app.db.select({ sessions: sql24`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)` }).from(gaTrafficSnapshots).where(and51(eq60(gaTrafficSnapshots.projectId, project.id), sql24`${gaTrafficSnapshots.date} >= ${from}`, sql24`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
58168
|
+
const sumDirect = (from, to) => app.db.select({ sessions: sql24`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)` }).from(gaTrafficSnapshots).where(and51(eq60(gaTrafficSnapshots.projectId, project.id), sql24`${gaTrafficSnapshots.date} >= ${from}`, sql24`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
58169
|
+
const sumAi = (from, to) => app.db.select({ sessions: sql24`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and51(
|
|
57792
58170
|
eq60(gaAiReferrals.projectId, project.id),
|
|
57793
|
-
|
|
57794
|
-
|
|
58171
|
+
sql24`${gaAiReferrals.date} >= ${from}`,
|
|
58172
|
+
sql24`${gaAiReferrals.date} < ${to}`,
|
|
57795
58173
|
eq60(gaAiReferrals.sourceDimension, "session")
|
|
57796
58174
|
)).get();
|
|
57797
|
-
const sumSocial = (from, to) => app.db.select({ sessions:
|
|
58175
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql24`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and51(eq60(gaSocialReferrals.projectId, project.id), sql24`${gaSocialReferrals.date} >= ${from}`, sql24`${gaSocialReferrals.date} < ${to}`)).get();
|
|
57798
58176
|
const todayStr = fmt(today);
|
|
57799
58177
|
const buildTrend = (sum) => {
|
|
57800
58178
|
const c7 = sum(daysAgo(7), todayStr)?.sessions ?? 0;
|
|
@@ -57803,20 +58181,20 @@ async function ga4Routes(app, opts) {
|
|
|
57803
58181
|
const p30 = sum(daysAgo(60), daysAgo(30))?.sessions ?? 0;
|
|
57804
58182
|
return { sessions7d: c7, sessionsPrev7d: p7, trend7dPct: deltaPercent(c7, p7), sessions30d: c30, sessionsPrev30d: p30, trend30dPct: deltaPercent(c30, p30) };
|
|
57805
58183
|
};
|
|
57806
|
-
const aiSourceCurrent = app.db.select({ source: gaAiReferrals.source, sessions:
|
|
58184
|
+
const aiSourceCurrent = app.db.select({ source: gaAiReferrals.source, sessions: sql24`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and51(
|
|
57807
58185
|
eq60(gaAiReferrals.projectId, project.id),
|
|
57808
|
-
|
|
57809
|
-
|
|
58186
|
+
sql24`${gaAiReferrals.date} >= ${daysAgo(7)}`,
|
|
58187
|
+
sql24`${gaAiReferrals.date} < ${todayStr}`,
|
|
57810
58188
|
eq60(gaAiReferrals.sourceDimension, "session")
|
|
57811
58189
|
)).groupBy(gaAiReferrals.source).all();
|
|
57812
|
-
const aiSourcePrev = app.db.select({ source: gaAiReferrals.source, sessions:
|
|
58190
|
+
const aiSourcePrev = app.db.select({ source: gaAiReferrals.source, sessions: sql24`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and51(
|
|
57813
58191
|
eq60(gaAiReferrals.projectId, project.id),
|
|
57814
|
-
|
|
57815
|
-
|
|
58192
|
+
sql24`${gaAiReferrals.date} >= ${daysAgo(14)}`,
|
|
58193
|
+
sql24`${gaAiReferrals.date} < ${daysAgo(7)}`,
|
|
57816
58194
|
eq60(gaAiReferrals.sourceDimension, "session")
|
|
57817
58195
|
)).groupBy(gaAiReferrals.source).all();
|
|
57818
|
-
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions:
|
|
57819
|
-
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions:
|
|
58196
|
+
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions: sql24`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and51(eq60(gaSocialReferrals.projectId, project.id), sql24`${gaSocialReferrals.date} >= ${daysAgo(7)}`, sql24`${gaSocialReferrals.date} < ${todayStr}`)).groupBy(gaSocialReferrals.source).all();
|
|
58197
|
+
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions: sql24`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and51(eq60(gaSocialReferrals.projectId, project.id), sql24`${gaSocialReferrals.date} >= ${daysAgo(14)}`, sql24`${gaSocialReferrals.date} < ${daysAgo(7)}`)).groupBy(gaSocialReferrals.source).all();
|
|
57820
58198
|
const response = {
|
|
57821
58199
|
total: buildTrend(sumTotal),
|
|
57822
58200
|
organic: buildTrend(sumOrganic),
|
|
@@ -57835,9 +58213,9 @@ async function ga4Routes(app, opts) {
|
|
|
57835
58213
|
const conditions = [eq60(gaTrafficSnapshots.projectId, project.id), ...dateRangeConditions(gaTrafficSnapshots.date, range)];
|
|
57836
58214
|
const rows = app.db.select({
|
|
57837
58215
|
date: gaTrafficSnapshots.date,
|
|
57838
|
-
sessions:
|
|
57839
|
-
organicSessions:
|
|
57840
|
-
users:
|
|
58216
|
+
sessions: sql24`SUM(${gaTrafficSnapshots.sessions})`,
|
|
58217
|
+
organicSessions: sql24`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
58218
|
+
users: sql24`SUM(${gaTrafficSnapshots.users})`
|
|
57841
58219
|
}).from(gaTrafficSnapshots).where(and51(...conditions)).groupBy(gaTrafficSnapshots.date).orderBy(gaTrafficSnapshots.date).all();
|
|
57842
58220
|
const totalConditions = [eq60(gaDailyTotals.projectId, project.id), ...dateRangeConditions(gaDailyTotals.date, range)];
|
|
57843
58221
|
const totals = app.db.select({ date: gaDailyTotals.date, users: gaDailyTotals.users }).from(gaDailyTotals).where(and51(...totalConditions)).all();
|
|
@@ -57855,11 +58233,11 @@ async function ga4Routes(app, opts) {
|
|
|
57855
58233
|
const project = resolveProject(app.db, request.params.name);
|
|
57856
58234
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
57857
58235
|
const trafficPages = app.db.select({
|
|
57858
|
-
landingPage:
|
|
57859
|
-
sessions:
|
|
57860
|
-
organicSessions:
|
|
57861
|
-
users:
|
|
57862
|
-
}).from(gaTrafficSnapshots).where(eq60(gaTrafficSnapshots.projectId, project.id)).groupBy(
|
|
58236
|
+
landingPage: sql24`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`,
|
|
58237
|
+
sessions: sql24`SUM(${gaTrafficSnapshots.sessions})`,
|
|
58238
|
+
organicSessions: sql24`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
58239
|
+
users: sql24`SUM(${gaTrafficSnapshots.users})`
|
|
58240
|
+
}).from(gaTrafficSnapshots).where(eq60(gaTrafficSnapshots.projectId, project.id)).groupBy(sql24`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`).orderBy(sql24`SUM(${gaTrafficSnapshots.sessions}) DESC`).all();
|
|
57863
58241
|
return {
|
|
57864
58242
|
pages: trafficPages.map((r) => ({
|
|
57865
58243
|
landingPage: r.landingPage,
|
|
@@ -59514,7 +59892,7 @@ async function wordpressRoutes(app, opts) {
|
|
|
59514
59892
|
|
|
59515
59893
|
// ../api-routes/src/backlinks.ts
|
|
59516
59894
|
import crypto42 from "crypto";
|
|
59517
|
-
import { and as and53, asc as asc12, desc as desc32, eq as eq61, sql as
|
|
59895
|
+
import { and as and53, asc as asc12, desc as desc32, eq as eq61, sql as sql25 } from "drizzle-orm";
|
|
59518
59896
|
|
|
59519
59897
|
// ../integration-commoncrawl/src/constants.ts
|
|
59520
59898
|
import os2 from "os";
|
|
@@ -59611,7 +59989,7 @@ function parseContentLength(value) {
|
|
|
59611
59989
|
}
|
|
59612
59990
|
|
|
59613
59991
|
// ../integration-commoncrawl/src/downloader.ts
|
|
59614
|
-
import { createHash as
|
|
59992
|
+
import { createHash as createHash9 } from "crypto";
|
|
59615
59993
|
import { createWriteStream } from "fs";
|
|
59616
59994
|
import fs2 from "fs/promises";
|
|
59617
59995
|
import path3 from "path";
|
|
@@ -59637,7 +60015,7 @@ async function downloadFile(opts) {
|
|
|
59637
60015
|
throw new Error(`HTTP ${res.status} ${res.statusText} for ${opts.url}`);
|
|
59638
60016
|
}
|
|
59639
60017
|
const total = parseContentLength2(res.headers.get("content-length"));
|
|
59640
|
-
const hasher =
|
|
60018
|
+
const hasher = createHash9("sha256");
|
|
59641
60019
|
let bytes = 0;
|
|
59642
60020
|
const hashAndCount = new Transform({
|
|
59643
60021
|
transform(chunk, _enc, cb) {
|
|
@@ -59658,7 +60036,7 @@ async function downloadFile(opts) {
|
|
|
59658
60036
|
return { bytes, sha256: sha2564, cached: false, elapsedMs: Date.now() - start };
|
|
59659
60037
|
}
|
|
59660
60038
|
async function hashFile(filePath) {
|
|
59661
|
-
const hasher =
|
|
60039
|
+
const hasher = createHash9("sha256");
|
|
59662
60040
|
const handle = await fs2.open(filePath, "r");
|
|
59663
60041
|
try {
|
|
59664
60042
|
const stream = handle.createReadStream();
|
|
@@ -59804,7 +60182,7 @@ async function queryBacklinks(opts) {
|
|
|
59804
60182
|
const reversed = opts.targets.map(reverseDomain);
|
|
59805
60183
|
const targetList = reversed.map(quote).join(", ");
|
|
59806
60184
|
const limitClause = opts.limitPerTarget ? `QUALIFY row_number() OVER (PARTITION BY t.target_rev_domain ORDER BY v.num_hosts DESC) <= ${Math.floor(opts.limitPerTarget)}` : "";
|
|
59807
|
-
const
|
|
60185
|
+
const sql33 = `
|
|
59808
60186
|
WITH vertices AS (
|
|
59809
60187
|
SELECT * FROM read_csv(
|
|
59810
60188
|
${quote(opts.vertexPath)},
|
|
@@ -59840,7 +60218,7 @@ async function queryBacklinks(opts) {
|
|
|
59840
60218
|
const conn = await instance.connect();
|
|
59841
60219
|
let rows;
|
|
59842
60220
|
try {
|
|
59843
|
-
const reader = await conn.runAndReadAll(
|
|
60221
|
+
const reader = await conn.runAndReadAll(sql33);
|
|
59844
60222
|
rows = reader.getRowObjects();
|
|
59845
60223
|
} finally {
|
|
59846
60224
|
conn.disconnectSync?.();
|
|
@@ -60030,12 +60408,12 @@ function computeFilteredSummary(db, base) {
|
|
|
60030
60408
|
);
|
|
60031
60409
|
const filteredCondition = and53(baseDomainCondition, backlinkCrawlerExclusionClause());
|
|
60032
60410
|
const unfilteredAgg = db.select({
|
|
60033
|
-
count:
|
|
60034
|
-
total:
|
|
60411
|
+
count: sql25`count(*)`,
|
|
60412
|
+
total: sql25`coalesce(sum(${backlinkDomains.numHosts}), 0)`
|
|
60035
60413
|
}).from(backlinkDomains).where(baseDomainCondition).get();
|
|
60036
60414
|
const filteredAgg = db.select({
|
|
60037
|
-
count:
|
|
60038
|
-
total:
|
|
60415
|
+
count: sql25`count(*)`,
|
|
60416
|
+
total: sql25`coalesce(sum(${backlinkDomains.numHosts}), 0)`
|
|
60039
60417
|
}).from(backlinkDomains).where(filteredCondition).get();
|
|
60040
60418
|
const top10Rows = db.select({ numHosts: backlinkDomains.numHosts }).from(backlinkDomains).where(filteredCondition).orderBy(desc32(backlinkDomains.numHosts)).limit(10).all();
|
|
60041
60419
|
const totalLinkingDomains = Number(filteredAgg?.count ?? 0);
|
|
@@ -60061,7 +60439,7 @@ function buildSourceAvailability(db, projectId, source, connected, excludeCrawle
|
|
|
60061
60439
|
const summary3 = db.select().from(backlinkSummaries).where(and53(eq61(backlinkSummaries.projectId, projectId), eq61(backlinkSummaries.source, source))).orderBy(desc32(backlinkSummaries.queriedAt)).limit(1).get();
|
|
60062
60440
|
let totalLinkingDomains = summary3?.totalLinkingDomains ?? 0;
|
|
60063
60441
|
if (summary3 && excludeCrawlers) {
|
|
60064
|
-
const filtered = db.select({ count:
|
|
60442
|
+
const filtered = db.select({ count: sql25`count(*)` }).from(backlinkDomains).where(and53(
|
|
60065
60443
|
eq61(backlinkDomains.projectId, projectId),
|
|
60066
60444
|
eq61(backlinkDomains.source, source),
|
|
60067
60445
|
eq61(backlinkDomains.release, summary3.release),
|
|
@@ -60249,7 +60627,7 @@ async function backlinksRoutes(app, opts) {
|
|
|
60249
60627
|
eq61(backlinkDomains.release, targetRelease)
|
|
60250
60628
|
);
|
|
60251
60629
|
const domainCondition = excludeCrawlers ? and53(baseDomainCondition, backlinkCrawlerExclusionClause()) : baseDomainCondition;
|
|
60252
|
-
const totalRow = app.db.select({ count:
|
|
60630
|
+
const totalRow = app.db.select({ count: sql25`count(*)` }).from(backlinkDomains).where(domainCondition).get();
|
|
60253
60631
|
const rows = app.db.select({
|
|
60254
60632
|
linkingDomain: backlinkDomains.linkingDomain,
|
|
60255
60633
|
numHosts: backlinkDomains.numHosts,
|
|
@@ -60300,7 +60678,7 @@ import crypto45 from "crypto";
|
|
|
60300
60678
|
import { isIP } from "net";
|
|
60301
60679
|
import { isDeepStrictEqual as isDeepStrictEqual2 } from "util";
|
|
60302
60680
|
import { Agent as UndiciAgent } from "undici";
|
|
60303
|
-
import { and as and57, desc as desc33, eq as eq65, gte as gte16, inArray as inArray27, lte as lte17, sql as
|
|
60681
|
+
import { and as and57, desc as desc33, eq as eq65, gte as gte16, inArray as inArray27, lte as lte17, sql as sql28 } from "drizzle-orm";
|
|
60304
60682
|
|
|
60305
60683
|
// ../api-routes/src/cloudflare-worker-version.ts
|
|
60306
60684
|
var CURRENT_CLOUDFLARE_WORKER_VERSION = "1.0.1";
|
|
@@ -74318,7 +74696,7 @@ var perplexitybot_default = {
|
|
|
74318
74696
|
};
|
|
74319
74697
|
|
|
74320
74698
|
// ../integration-traffic/src/ip-range-manifest.ts
|
|
74321
|
-
import { createHash as
|
|
74699
|
+
import { createHash as createHash10 } from "crypto";
|
|
74322
74700
|
function isRecord2(value) {
|
|
74323
74701
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
74324
74702
|
}
|
|
@@ -74410,7 +74788,7 @@ function canonicalPrefixKey(entry) {
|
|
|
74410
74788
|
function ipRangeManifestContentHash(value) {
|
|
74411
74789
|
const prefixes = isRecord2(value) ? value.prefixes : void 0;
|
|
74412
74790
|
const canonical = Array.isArray(prefixes) ? prefixes.map(canonicalPrefixKey).sort() : [`invalid-prefixes:${stableJson2(prefixes)}`];
|
|
74413
|
-
return
|
|
74791
|
+
return createHash10("sha256").update(JSON.stringify(canonical)).digest("hex");
|
|
74414
74792
|
}
|
|
74415
74793
|
function validateIpRangeManifestPayload(value) {
|
|
74416
74794
|
if (!isRecord2(value)) return { ok: false, error: "response must be a JSON object" };
|
|
@@ -76970,10 +77348,10 @@ async function ackCloudflareQueueMessages(client, options) {
|
|
|
76970
77348
|
|
|
76971
77349
|
// ../api-routes/src/traffic-event-ingest.ts
|
|
76972
77350
|
import crypto44 from "crypto";
|
|
76973
|
-
import { and as and55, eq as eq63, gte as gte15, lt as lt12, lte as lte15, sql as
|
|
77351
|
+
import { and as and55, eq as eq63, gte as gte15, lt as lt12, lte as lte15, sql as sql27 } from "drizzle-orm";
|
|
76974
77352
|
|
|
76975
77353
|
// ../api-routes/src/raw-event-sample-retention.ts
|
|
76976
|
-
import { and as and54, eq as eq62, lt as lt11, sql as
|
|
77354
|
+
import { and as and54, eq as eq62, lt as lt11, sql as sql26 } from "drizzle-orm";
|
|
76977
77355
|
var RAW_EVENT_SAMPLE_RETENTION_MS = 30 * 24 * 60 * 6e4;
|
|
76978
77356
|
var RAW_EVENT_SAMPLE_RETENTION_SWEEP_INTERVAL_MS = 24 * 60 * 6e4;
|
|
76979
77357
|
var CANONICAL_UTC_TIMESTAMP_GLOB = "[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9][0-9][0-9]Z";
|
|
@@ -76984,16 +77362,16 @@ function rawEventSampleRetentionCutoff(referenceAt) {
|
|
|
76984
77362
|
}
|
|
76985
77363
|
function deleteExpiredRawEventSamples(db, cutoff, sourceId) {
|
|
76986
77364
|
const sourceScope = sourceId === void 0 ? void 0 : eq62(rawEventSamples.sourceId, sourceId);
|
|
76987
|
-
db.delete(rawEventSamples).where(and54(sourceScope,
|
|
77365
|
+
db.delete(rawEventSamples).where(and54(sourceScope, sql26`julianday(${rawEventSamples.ts}) IS NULL`)).run();
|
|
76988
77366
|
db.delete(rawEventSamples).where(and54(
|
|
76989
77367
|
sourceScope,
|
|
76990
|
-
|
|
77368
|
+
sql26`${rawEventSamples.ts} GLOB ${CANONICAL_UTC_TIMESTAMP_GLOB}`,
|
|
76991
77369
|
lt11(rawEventSamples.ts, cutoff)
|
|
76992
77370
|
)).run();
|
|
76993
77371
|
db.delete(rawEventSamples).where(and54(
|
|
76994
77372
|
sourceScope,
|
|
76995
|
-
|
|
76996
|
-
|
|
77373
|
+
sql26`${rawEventSamples.ts} NOT GLOB ${CANONICAL_UTC_TIMESTAMP_GLOB}`,
|
|
77374
|
+
sql26`julianday(${rawEventSamples.ts}) < julianday(${cutoff})`
|
|
76997
77375
|
)).run();
|
|
76998
77376
|
}
|
|
76999
77377
|
function enforceRawEventSampleRetention(db, sourceId, referenceAt) {
|
|
@@ -77039,7 +77417,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
77039
77417
|
let aiReferralHits = 0;
|
|
77040
77418
|
let unknownHits = 0;
|
|
77041
77419
|
opts.db.transaction((tx) => {
|
|
77042
|
-
tx.update(trafficSources).set({ updatedAt:
|
|
77420
|
+
tx.update(trafficSources).set({ updatedAt: sql27`${trafficSources.updatedAt}` }).where(eq63(trafficSources.id, opts.sourceId)).run();
|
|
77043
77421
|
const source = tx.select().from(trafficSources).where(eq63(trafficSources.id, opts.sourceId)).get();
|
|
77044
77422
|
opts.validateSource(source);
|
|
77045
77423
|
const rawSampleCutoff = enforceRawEventSampleRetention(
|
|
@@ -77096,7 +77474,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
77096
77474
|
crawlerEventsHourly.status
|
|
77097
77475
|
],
|
|
77098
77476
|
set: {
|
|
77099
|
-
hits:
|
|
77477
|
+
hits: sql27`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
77100
77478
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
77101
77479
|
updatedAt: opts.receivedAt
|
|
77102
77480
|
}
|
|
@@ -77127,7 +77505,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
77127
77505
|
crawlerVerificationManifestsHourly.manifestId
|
|
77128
77506
|
],
|
|
77129
77507
|
set: {
|
|
77130
|
-
hits:
|
|
77508
|
+
hits: sql27`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
77131
77509
|
manifestJson: bucket.verificationManifest,
|
|
77132
77510
|
updatedAt: opts.receivedAt
|
|
77133
77511
|
}
|
|
@@ -77161,7 +77539,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
77161
77539
|
aiUserFetchEventsHourly.status
|
|
77162
77540
|
],
|
|
77163
77541
|
set: {
|
|
77164
|
-
hits:
|
|
77542
|
+
hits: sql27`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
77165
77543
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
77166
77544
|
updatedAt: opts.receivedAt
|
|
77167
77545
|
}
|
|
@@ -77192,7 +77570,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
77192
77570
|
aiUserFetchVerificationManifestsHourly.manifestId
|
|
77193
77571
|
],
|
|
77194
77572
|
set: {
|
|
77195
|
-
hits:
|
|
77573
|
+
hits: sql27`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
77196
77574
|
manifestJson: bucket.verificationManifest,
|
|
77197
77575
|
updatedAt: opts.receivedAt
|
|
77198
77576
|
}
|
|
@@ -77230,9 +77608,9 @@ function writeTrafficEventBatch(opts) {
|
|
|
77230
77608
|
aiReferralEventsHourly.status
|
|
77231
77609
|
],
|
|
77232
77610
|
set: {
|
|
77233
|
-
sessionsOrHits:
|
|
77234
|
-
paidSessionsOrHits:
|
|
77235
|
-
organicSessionsOrHits:
|
|
77611
|
+
sessionsOrHits: sql27`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
77612
|
+
paidSessionsOrHits: sql27`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
|
|
77613
|
+
organicSessionsOrHits: sql27`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
|
|
77236
77614
|
updatedAt: opts.receivedAt
|
|
77237
77615
|
}
|
|
77238
77616
|
}).run();
|
|
@@ -77247,7 +77625,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
77247
77625
|
if (!hour || hourlySampleLimit === 0) continue;
|
|
77248
77626
|
let remaining = remainingSamplesByHour.get(hour.start);
|
|
77249
77627
|
if (remaining === void 0) {
|
|
77250
|
-
const existing = tx.select({ total:
|
|
77628
|
+
const existing = tx.select({ total: sql27`COUNT(*)` }).from(rawEventSamples).where(and55(
|
|
77251
77629
|
eq63(rawEventSamples.sourceId, opts.sourceId),
|
|
77252
77630
|
gte15(rawEventSamples.ts, hour.start),
|
|
77253
77631
|
lt12(rawEventSamples.ts, hour.end)
|
|
@@ -77573,7 +77951,7 @@ function crawlerVerificationProvenance(db, projectId, events) {
|
|
|
77573
77951
|
manifestId: crawlerVerificationManifestsHourly.manifestId,
|
|
77574
77952
|
manifest: crawlerVerificationManifestsHourly.manifestJson,
|
|
77575
77953
|
hits: crawlerVerificationManifestsHourly.hits
|
|
77576
|
-
}).from(crawlerVerificationManifestsHourly).where(
|
|
77954
|
+
}).from(crawlerVerificationManifestsHourly).where(sql28`(
|
|
77577
77955
|
${crawlerVerificationManifestsHourly.projectId},
|
|
77578
77956
|
${crawlerVerificationManifestsHourly.sourceId},
|
|
77579
77957
|
${crawlerVerificationManifestsHourly.tsHour},
|
|
@@ -77607,7 +77985,7 @@ function aiUserFetchVerificationProvenance(db, projectId, events) {
|
|
|
77607
77985
|
manifestId: aiUserFetchVerificationManifestsHourly.manifestId,
|
|
77608
77986
|
manifest: aiUserFetchVerificationManifestsHourly.manifestJson,
|
|
77609
77987
|
hits: aiUserFetchVerificationManifestsHourly.hits
|
|
77610
|
-
}).from(aiUserFetchVerificationManifestsHourly).where(
|
|
77988
|
+
}).from(aiUserFetchVerificationManifestsHourly).where(sql28`(
|
|
77611
77989
|
${aiUserFetchVerificationManifestsHourly.projectId},
|
|
77612
77990
|
${aiUserFetchVerificationManifestsHourly.sourceId},
|
|
77613
77991
|
${aiUserFetchVerificationManifestsHourly.tsHour},
|
|
@@ -77871,7 +78249,7 @@ async function runBackfillTask(options) {
|
|
|
77871
78249
|
crawlerEventsHourly.status
|
|
77872
78250
|
],
|
|
77873
78251
|
set: {
|
|
77874
|
-
hits:
|
|
78252
|
+
hits: sql28`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
77875
78253
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
77876
78254
|
updatedAt: finishedAt
|
|
77877
78255
|
}
|
|
@@ -77902,7 +78280,7 @@ async function runBackfillTask(options) {
|
|
|
77902
78280
|
crawlerVerificationManifestsHourly.manifestId
|
|
77903
78281
|
],
|
|
77904
78282
|
set: {
|
|
77905
|
-
hits:
|
|
78283
|
+
hits: sql28`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
77906
78284
|
manifestJson: bucket.verificationManifest,
|
|
77907
78285
|
updatedAt: finishedAt
|
|
77908
78286
|
}
|
|
@@ -77935,7 +78313,7 @@ async function runBackfillTask(options) {
|
|
|
77935
78313
|
aiUserFetchEventsHourly.status
|
|
77936
78314
|
],
|
|
77937
78315
|
set: {
|
|
77938
|
-
hits:
|
|
78316
|
+
hits: sql28`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
77939
78317
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
77940
78318
|
updatedAt: finishedAt
|
|
77941
78319
|
}
|
|
@@ -77966,7 +78344,7 @@ async function runBackfillTask(options) {
|
|
|
77966
78344
|
aiUserFetchVerificationManifestsHourly.manifestId
|
|
77967
78345
|
],
|
|
77968
78346
|
set: {
|
|
77969
|
-
hits:
|
|
78347
|
+
hits: sql28`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
77970
78348
|
manifestJson: bucket.verificationManifest,
|
|
77971
78349
|
updatedAt: finishedAt
|
|
77972
78350
|
}
|
|
@@ -79571,7 +79949,7 @@ async function trafficRoutes(app, opts) {
|
|
|
79571
79949
|
crawlerEventsHourly.status
|
|
79572
79950
|
],
|
|
79573
79951
|
set: {
|
|
79574
|
-
hits:
|
|
79952
|
+
hits: sql28`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
79575
79953
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
79576
79954
|
updatedAt: finishedAt
|
|
79577
79955
|
}
|
|
@@ -79602,7 +79980,7 @@ async function trafficRoutes(app, opts) {
|
|
|
79602
79980
|
crawlerVerificationManifestsHourly.manifestId
|
|
79603
79981
|
],
|
|
79604
79982
|
set: {
|
|
79605
|
-
hits:
|
|
79983
|
+
hits: sql28`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
79606
79984
|
manifestJson: bucket.verificationManifest,
|
|
79607
79985
|
updatedAt: finishedAt
|
|
79608
79986
|
}
|
|
@@ -79636,7 +80014,7 @@ async function trafficRoutes(app, opts) {
|
|
|
79636
80014
|
aiUserFetchEventsHourly.status
|
|
79637
80015
|
],
|
|
79638
80016
|
set: {
|
|
79639
|
-
hits:
|
|
80017
|
+
hits: sql28`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
79640
80018
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
79641
80019
|
updatedAt: finishedAt
|
|
79642
80020
|
}
|
|
@@ -79667,7 +80045,7 @@ async function trafficRoutes(app, opts) {
|
|
|
79667
80045
|
aiUserFetchVerificationManifestsHourly.manifestId
|
|
79668
80046
|
],
|
|
79669
80047
|
set: {
|
|
79670
|
-
hits:
|
|
80048
|
+
hits: sql28`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
79671
80049
|
manifestJson: bucket.verificationManifest,
|
|
79672
80050
|
updatedAt: finishedAt
|
|
79673
80051
|
}
|
|
@@ -79708,9 +80086,9 @@ async function trafficRoutes(app, opts) {
|
|
|
79708
80086
|
// colliding bucket adds its paid and organic arrivals to the ones
|
|
79709
80087
|
// already there rather than overwriting a single label.
|
|
79710
80088
|
set: {
|
|
79711
|
-
sessionsOrHits:
|
|
79712
|
-
paidSessionsOrHits:
|
|
79713
|
-
organicSessionsOrHits:
|
|
80089
|
+
sessionsOrHits: sql28`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
80090
|
+
paidSessionsOrHits: sql28`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
|
|
80091
|
+
organicSessionsOrHits: sql28`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
|
|
79714
80092
|
updatedAt: finishedAt
|
|
79715
80093
|
}
|
|
79716
80094
|
}).run();
|
|
@@ -80003,7 +80381,7 @@ async function trafficRoutes(app, opts) {
|
|
|
80003
80381
|
function buildSourceDetail(projectId, row, since) {
|
|
80004
80382
|
const crawlerPathRows = app.db.select({
|
|
80005
80383
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
80006
|
-
hits:
|
|
80384
|
+
hits: sql28`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
80007
80385
|
}).from(crawlerEventsHourly).where(
|
|
80008
80386
|
and57(
|
|
80009
80387
|
eq65(crawlerEventsHourly.sourceId, row.id),
|
|
@@ -80014,23 +80392,23 @@ async function trafficRoutes(app, opts) {
|
|
|
80014
80392
|
crawlerPathRows.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
|
|
80015
80393
|
);
|
|
80016
80394
|
const crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
80017
|
-
const aiUserFetchTotals = app.db.select({ total:
|
|
80395
|
+
const aiUserFetchTotals = app.db.select({ total: sql28`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
80018
80396
|
and57(
|
|
80019
80397
|
eq65(aiUserFetchEventsHourly.sourceId, row.id),
|
|
80020
80398
|
gte16(aiUserFetchEventsHourly.tsHour, since)
|
|
80021
80399
|
)
|
|
80022
80400
|
).get();
|
|
80023
80401
|
const aiTotals = app.db.select({
|
|
80024
|
-
total:
|
|
80025
|
-
nonhop:
|
|
80026
|
-
landed:
|
|
80402
|
+
total: sql28`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
80403
|
+
nonhop: sql28`COALESCE(SUM(CASE WHEN ${referralLandedCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`,
|
|
80404
|
+
landed: sql28`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`
|
|
80027
80405
|
}).from(aiReferralEventsHourly).where(
|
|
80028
80406
|
and57(
|
|
80029
80407
|
eq65(aiReferralEventsHourly.sourceId, row.id),
|
|
80030
80408
|
gte16(aiReferralEventsHourly.tsHour, since)
|
|
80031
80409
|
)
|
|
80032
80410
|
).get();
|
|
80033
|
-
const sampleTotals = app.db.select({ total:
|
|
80411
|
+
const sampleTotals = app.db.select({ total: sql28`COUNT(*)` }).from(rawEventSamples).where(
|
|
80034
80412
|
and57(
|
|
80035
80413
|
eq65(rawEventSamples.sourceId, row.id),
|
|
80036
80414
|
gte16(rawEventSamples.ts, since)
|
|
@@ -80219,15 +80597,15 @@ async function trafficRoutes(app, opts) {
|
|
|
80219
80597
|
const crawlerWhere = and57(...crawlerFilters);
|
|
80220
80598
|
const pathTotals = app.db.select({
|
|
80221
80599
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
80222
|
-
hits:
|
|
80223
|
-
rows:
|
|
80600
|
+
hits: sql28`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
80601
|
+
rows: sql28`COUNT(*)`
|
|
80224
80602
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerEventsHourly.pathNormalized).all();
|
|
80225
80603
|
crawlerSegments = segmentCrawlerHits(
|
|
80226
80604
|
pathTotals.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
|
|
80227
80605
|
);
|
|
80228
80606
|
crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
80229
80607
|
totalEventRows += pathTotals.reduce((sum, row) => sum + Number(row.rows), 0);
|
|
80230
|
-
const crawlerSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
80608
|
+
const crawlerSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql28`substr(${crawlerEventsHourly.tsHour}, 1, 10)` : crawlerEventsHourly.tsHour;
|
|
80231
80609
|
const distinctPaths = pathTotals.map((r) => ({ pathNormalized: r.pathNormalized }));
|
|
80232
80610
|
const contentPaths = [];
|
|
80233
80611
|
const infraPaths = [];
|
|
@@ -80239,12 +80617,12 @@ async function trafficRoutes(app, opts) {
|
|
|
80239
80617
|
const listed = useContentList ? contentPaths : infraPaths;
|
|
80240
80618
|
const IN_LIST_CAP = 2e4;
|
|
80241
80619
|
const listFitsInSql = listed.length <= IN_LIST_CAP;
|
|
80242
|
-
const contentExpr = listed.length === 0 ? useContentList ?
|
|
80620
|
+
const contentExpr = listed.length === 0 ? useContentList ? sql28`0` : sql28`${crawlerEventsHourly.hits}` : useContentList ? sql28`CASE WHEN ${inArray27(crawlerEventsHourly.pathNormalized, listed)} THEN ${crawlerEventsHourly.hits} ELSE 0 END` : sql28`CASE WHEN ${inArray27(crawlerEventsHourly.pathNormalized, listed)} THEN 0 ELSE ${crawlerEventsHourly.hits} END`;
|
|
80243
80621
|
if (listFitsInSql) {
|
|
80244
80622
|
const crawlerSeries = app.db.select({
|
|
80245
80623
|
bucket: crawlerSeriesBucket,
|
|
80246
|
-
hits:
|
|
80247
|
-
content:
|
|
80624
|
+
hits: sql28`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
80625
|
+
content: sql28`COALESCE(SUM(${contentExpr}), 0)`
|
|
80248
80626
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerSeriesBucket).all();
|
|
80249
80627
|
for (const row of crawlerSeries) {
|
|
80250
80628
|
const point = trafficSeriesPoint(seriesByBucket, String(row.bucket));
|
|
@@ -80256,7 +80634,7 @@ async function trafficRoutes(app, opts) {
|
|
|
80256
80634
|
const grouped = app.db.select({
|
|
80257
80635
|
bucket: crawlerSeriesBucket,
|
|
80258
80636
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
80259
|
-
hits:
|
|
80637
|
+
hits: sql28`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
80260
80638
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerSeriesBucket, crawlerEventsHourly.pathNormalized).all();
|
|
80261
80639
|
for (const row of grouped) {
|
|
80262
80640
|
const point = trafficSeriesPoint(seriesByBucket, String(row.bucket));
|
|
@@ -80293,15 +80671,15 @@ async function trafficRoutes(app, opts) {
|
|
|
80293
80671
|
if (sourceIdParam) userFetchFilters.push(eq65(aiUserFetchEventsHourly.sourceId, sourceIdParam));
|
|
80294
80672
|
const userFetchWhere = and57(...userFetchFilters);
|
|
80295
80673
|
const total = app.db.select({
|
|
80296
|
-
total:
|
|
80297
|
-
rows:
|
|
80674
|
+
total: sql28`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`,
|
|
80675
|
+
rows: sql28`COUNT(*)`
|
|
80298
80676
|
}).from(aiUserFetchEventsHourly).where(userFetchWhere).get();
|
|
80299
80677
|
aiUserFetchTotal = Number(total?.total ?? 0);
|
|
80300
80678
|
totalEventRows += Number(total?.rows ?? 0);
|
|
80301
|
-
const userFetchSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
80679
|
+
const userFetchSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql28`substr(${aiUserFetchEventsHourly.tsHour}, 1, 10)` : aiUserFetchEventsHourly.tsHour;
|
|
80302
80680
|
const userFetchSeries = app.db.select({
|
|
80303
80681
|
bucket: userFetchSeriesBucket,
|
|
80304
|
-
hits:
|
|
80682
|
+
hits: sql28`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`
|
|
80305
80683
|
}).from(aiUserFetchEventsHourly).where(userFetchWhere).groupBy(userFetchSeriesBucket).all();
|
|
80306
80684
|
for (const row of userFetchSeries) {
|
|
80307
80685
|
trafficSeriesPoint(seriesByBucket, row.bucket).aiUserFetchHits = Number(row.hits);
|
|
@@ -80332,12 +80710,12 @@ async function trafficRoutes(app, opts) {
|
|
|
80332
80710
|
if (sourceIdParam) aiFilters.push(eq65(aiReferralEventsHourly.sourceId, sourceIdParam));
|
|
80333
80711
|
const aiWhere = and57(...aiFilters);
|
|
80334
80712
|
const total = app.db.select({
|
|
80335
|
-
total:
|
|
80336
|
-
nonhop:
|
|
80337
|
-
landed:
|
|
80338
|
-
paid:
|
|
80339
|
-
organic:
|
|
80340
|
-
rows:
|
|
80713
|
+
total: sql28`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
80714
|
+
nonhop: sql28`COALESCE(SUM(CASE WHEN ${referralLandedCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`,
|
|
80715
|
+
landed: sql28`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`,
|
|
80716
|
+
paid: sql28`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.paidSessionsOrHits} ELSE 0 END), 0)`,
|
|
80717
|
+
organic: sql28`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.organicSessionsOrHits} ELSE 0 END), 0)`,
|
|
80718
|
+
rows: sql28`COUNT(*)`
|
|
80341
80719
|
}).from(aiReferralEventsHourly).where(aiWhere).get();
|
|
80342
80720
|
aiReferralTotalHits = Number(total?.total ?? 0);
|
|
80343
80721
|
aiReferralNonHopHits = Number(total?.nonhop ?? 0);
|
|
@@ -80348,11 +80726,11 @@ async function trafficRoutes(app, opts) {
|
|
|
80348
80726
|
Number(total?.organic ?? 0)
|
|
80349
80727
|
);
|
|
80350
80728
|
totalEventRows += Number(total?.rows ?? 0);
|
|
80351
|
-
const referralSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
80729
|
+
const referralSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql28`substr(${aiReferralEventsHourly.tsHour}, 1, 10)` : aiReferralEventsHourly.tsHour;
|
|
80352
80730
|
const referralSeries = app.db.select({
|
|
80353
80731
|
bucket: referralSeriesBucket,
|
|
80354
|
-
hits:
|
|
80355
|
-
landed:
|
|
80732
|
+
hits: sql28`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
80733
|
+
landed: sql28`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`
|
|
80356
80734
|
}).from(aiReferralEventsHourly).where(aiWhere).groupBy(referralSeriesBucket).all();
|
|
80357
80735
|
for (const row of referralSeries) {
|
|
80358
80736
|
const point = trafficSeriesPoint(seriesByBucket, row.bucket);
|
|
@@ -80407,9 +80785,9 @@ async function trafficRoutes(app, opts) {
|
|
|
80407
80785
|
const points = completeTrafficSeries(since, until, granularity, seriesByBucket);
|
|
80408
80786
|
const coverageScope = (col, srcCol) => sourceIdParam ? and57(eq65(col, project.id), eq65(srcCol, sourceIdParam)) : eq65(col, project.id);
|
|
80409
80787
|
const firsts = [
|
|
80410
|
-
app.db.select({ v:
|
|
80411
|
-
app.db.select({ v:
|
|
80412
|
-
app.db.select({ v:
|
|
80788
|
+
app.db.select({ v: sql28`MIN(${crawlerEventsHourly.tsHour})` }).from(crawlerEventsHourly).where(coverageScope(crawlerEventsHourly.projectId, crawlerEventsHourly.sourceId)).get()?.v,
|
|
80789
|
+
app.db.select({ v: sql28`MIN(${aiUserFetchEventsHourly.tsHour})` }).from(aiUserFetchEventsHourly).where(coverageScope(aiUserFetchEventsHourly.projectId, aiUserFetchEventsHourly.sourceId)).get()?.v,
|
|
80790
|
+
app.db.select({ v: sql28`MIN(${aiReferralEventsHourly.tsHour})` }).from(aiReferralEventsHourly).where(coverageScope(aiReferralEventsHourly.projectId, aiReferralEventsHourly.sourceId)).get()?.v
|
|
80413
80791
|
].filter((v) => typeof v === "string" && v.length > 0);
|
|
80414
80792
|
const coverageStart = firsts.length ? firsts.slice().sort()[0] : null;
|
|
80415
80793
|
const bucketStartMs = (bucket) => Date.parse(
|
|
@@ -83117,7 +83495,7 @@ var RUNTIME_STATE_CHECKS = [
|
|
|
83117
83495
|
];
|
|
83118
83496
|
|
|
83119
83497
|
// ../api-routes/src/doctor/checks/traffic-source.ts
|
|
83120
|
-
import { and as and65, eq as eq77, gte as gte19, inArray as inArray30, ne as ne14, sql as
|
|
83498
|
+
import { and as and65, eq as eq77, gte as gte19, inArray as inArray30, ne as ne14, sql as sql29 } from "drizzle-orm";
|
|
83121
83499
|
var RECENT_DATA_WARN_DAYS = 7;
|
|
83122
83500
|
var RECENT_DATA_FAIL_DAYS = 30;
|
|
83123
83501
|
function isCloudflareDirectPush(source) {
|
|
@@ -83262,7 +83640,7 @@ var recentDataCheck = {
|
|
|
83262
83640
|
const warnCutoff = new Date(now.getTime() - RECENT_DATA_WARN_DAYS * 24 * 60 * 6e4).toISOString();
|
|
83263
83641
|
const failCutoff = new Date(now.getTime() - RECENT_DATA_FAIL_DAYS * 24 * 60 * 6e4).toISOString();
|
|
83264
83642
|
const recentCrawlers = Number(
|
|
83265
|
-
ctx.db.select({ total:
|
|
83643
|
+
ctx.db.select({ total: sql29`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
83266
83644
|
and65(
|
|
83267
83645
|
eq77(crawlerEventsHourly.projectId, ctx.project.id),
|
|
83268
83646
|
inArray30(crawlerEventsHourly.sourceId, activeSourceIds),
|
|
@@ -83271,7 +83649,7 @@ var recentDataCheck = {
|
|
|
83271
83649
|
).get()?.total ?? 0
|
|
83272
83650
|
);
|
|
83273
83651
|
const recentReferrals = Number(
|
|
83274
|
-
ctx.db.select({ total:
|
|
83652
|
+
ctx.db.select({ total: sql29`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
83275
83653
|
and65(
|
|
83276
83654
|
eq77(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
83277
83655
|
inArray30(aiReferralEventsHourly.sourceId, activeSourceIds),
|
|
@@ -83280,7 +83658,7 @@ var recentDataCheck = {
|
|
|
83280
83658
|
).get()?.total ?? 0
|
|
83281
83659
|
);
|
|
83282
83660
|
const recentUserFetches = Number(
|
|
83283
|
-
ctx.db.select({ total:
|
|
83661
|
+
ctx.db.select({ total: sql29`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
83284
83662
|
and65(
|
|
83285
83663
|
eq77(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
83286
83664
|
inArray30(aiUserFetchEventsHourly.sourceId, activeSourceIds),
|
|
@@ -83306,7 +83684,7 @@ var recentDataCheck = {
|
|
|
83306
83684
|
};
|
|
83307
83685
|
}
|
|
83308
83686
|
const olderCrawlers = Number(
|
|
83309
|
-
ctx.db.select({ total:
|
|
83687
|
+
ctx.db.select({ total: sql29`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
83310
83688
|
and65(
|
|
83311
83689
|
eq77(crawlerEventsHourly.projectId, ctx.project.id),
|
|
83312
83690
|
inArray30(crawlerEventsHourly.sourceId, activeSourceIds),
|
|
@@ -83315,7 +83693,7 @@ var recentDataCheck = {
|
|
|
83315
83693
|
).get()?.total ?? 0
|
|
83316
83694
|
);
|
|
83317
83695
|
const olderReferrals = Number(
|
|
83318
|
-
ctx.db.select({ total:
|
|
83696
|
+
ctx.db.select({ total: sql29`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
83319
83697
|
and65(
|
|
83320
83698
|
eq77(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
83321
83699
|
inArray30(aiReferralEventsHourly.sourceId, activeSourceIds),
|
|
@@ -83324,7 +83702,7 @@ var recentDataCheck = {
|
|
|
83324
83702
|
).get()?.total ?? 0
|
|
83325
83703
|
);
|
|
83326
83704
|
const olderUserFetches = Number(
|
|
83327
|
-
ctx.db.select({ total:
|
|
83705
|
+
ctx.db.select({ total: sql29`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
83328
83706
|
and65(
|
|
83329
83707
|
eq77(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
83330
83708
|
inArray30(aiUserFetchEventsHourly.sourceId, activeSourceIds),
|
|
@@ -85088,7 +85466,7 @@ function dedupeStrings(input) {
|
|
|
85088
85466
|
|
|
85089
85467
|
// ../api-routes/src/technical-aeo.ts
|
|
85090
85468
|
import crypto49 from "crypto";
|
|
85091
|
-
import { and as and67, asc as asc14, count as count2, desc as desc36, eq as eq80, inArray as inArray32, isNotNull as isNotNull8, isNull as isNull15, lt as lt13, or as or18, sql as
|
|
85469
|
+
import { and as and67, asc as asc14, count as count2, desc as desc36, eq as eq80, gte as gte21, inArray as inArray32, isNotNull as isNotNull8, isNull as isNull15, lt as lt13, or as or18, sql as sql30 } from "drizzle-orm";
|
|
85092
85470
|
import { alias } from "drizzle-orm/sqlite-core";
|
|
85093
85471
|
var FETCHED_SITE_CRAWL_STATES = /* @__PURE__ */ new Set([
|
|
85094
85472
|
...SiteCrawlFetchedStates,
|
|
@@ -85170,6 +85548,26 @@ function decodeCursor(value) {
|
|
|
85170
85548
|
return 0;
|
|
85171
85549
|
}
|
|
85172
85550
|
}
|
|
85551
|
+
function parseCrawlDate(value, runId) {
|
|
85552
|
+
if (value === void 0) return void 0;
|
|
85553
|
+
const parsed = calendarDateSchema.safeParse(value);
|
|
85554
|
+
if (!parsed.success) throw validationError('"date" must be a calendar date as YYYY-MM-DD');
|
|
85555
|
+
if (runId !== void 0) throw validationError('"date" and "runId" are mutually exclusive');
|
|
85556
|
+
return parsed.data;
|
|
85557
|
+
}
|
|
85558
|
+
function decodeCrawlPageCursor(value, selectionKey) {
|
|
85559
|
+
if (typeof value !== "string" || value.length === 0 || value.length > 256) return 0;
|
|
85560
|
+
let parsed;
|
|
85561
|
+
try {
|
|
85562
|
+
parsed = JSON.parse(Buffer.from(value, "base64url").toString("utf8"));
|
|
85563
|
+
} catch {
|
|
85564
|
+
return 0;
|
|
85565
|
+
}
|
|
85566
|
+
if (parsed?.selectionKey !== void 0 && parsed.selectionKey !== selectionKey) {
|
|
85567
|
+
throw validationError("Crawl page cursor no longer matches the selected scan and filters; restart pagination");
|
|
85568
|
+
}
|
|
85569
|
+
return decodeCursor(value);
|
|
85570
|
+
}
|
|
85173
85571
|
function normalizeParentPath(value) {
|
|
85174
85572
|
if (typeof value !== "string" || value.trim() === "") return "/";
|
|
85175
85573
|
const bare = value.trim().split(/[?#]/, 1)[0] ?? "/";
|
|
@@ -85214,7 +85612,8 @@ function mapCrawlPage(row) {
|
|
|
85214
85612
|
outboundOccurrences: row.outboundOccurrences,
|
|
85215
85613
|
linkScoreRaw: row.linkScoreRaw,
|
|
85216
85614
|
linkScoreNormalized: row.linkScoreNormalized,
|
|
85217
|
-
healthState: deriveSiteHealthState(row)
|
|
85615
|
+
healthState: deriveSiteHealthState(row),
|
|
85616
|
+
healthReason: deriveSiteHealthReason(row)
|
|
85218
85617
|
};
|
|
85219
85618
|
}
|
|
85220
85619
|
function withRecordedShare(factor) {
|
|
@@ -85310,7 +85709,7 @@ function linkKindFilter(linkKind) {
|
|
|
85310
85709
|
}
|
|
85311
85710
|
var graphSourceNode = alias(siteCrawlGraphNodes, "site_crawl_graph_source_node");
|
|
85312
85711
|
var graphTargetNode = alias(siteCrawlGraphNodes, "site_crawl_graph_target_node");
|
|
85313
|
-
var PAGE_CHANGE_PREDICATE =
|
|
85712
|
+
var PAGE_CHANGE_PREDICATE = sql30`(
|
|
85314
85713
|
current.url IS NOT previous.url
|
|
85315
85714
|
OR current.final_url IS NOT previous.final_url
|
|
85316
85715
|
OR current.path IS NOT previous.path
|
|
@@ -85332,7 +85731,7 @@ var PAGE_CHANGE_PREDICATE = sql29`(
|
|
|
85332
85731
|
OR current.link_score_raw IS NOT previous.link_score_raw
|
|
85333
85732
|
OR current.link_score_normalized IS NOT previous.link_score_normalized
|
|
85334
85733
|
)`;
|
|
85335
|
-
var LINK_CHANGE_PREDICATE =
|
|
85734
|
+
var LINK_CHANGE_PREDICATE = sql30`(
|
|
85336
85735
|
current.source_node_key IS NOT previous.source_node_key
|
|
85337
85736
|
OR current.source_url IS NOT previous.source_url
|
|
85338
85737
|
OR current.target_node_key IS NOT previous.target_node_key
|
|
@@ -85408,21 +85807,191 @@ function changedFields(before, after, fields) {
|
|
|
85408
85807
|
return fields.filter((field) => valueChanged(before[field], after[field])).map(String);
|
|
85409
85808
|
}
|
|
85410
85809
|
async function technicalAeoRoutes(app, opts) {
|
|
85411
|
-
const
|
|
85810
|
+
const visibleCrawlFilters = (projectId) => [
|
|
85811
|
+
eq80(siteCrawlSnapshots.projectId, projectId),
|
|
85812
|
+
eq80(runs.projectId, projectId),
|
|
85813
|
+
eq80(runs.kind, RunKinds["site-audit"]),
|
|
85814
|
+
inArray32(runs.status, SURFACEABLE_STATUSES),
|
|
85815
|
+
notProbeRun()
|
|
85816
|
+
];
|
|
85817
|
+
const availableScanDatesFor = (projectId, requestedDate) => {
|
|
85818
|
+
if (!requestedDate) return void 0;
|
|
85819
|
+
const date = sql30`substr(${siteCrawlSnapshots.createdAt}, 1, 10)`;
|
|
85820
|
+
const filters = visibleCrawlFilters(projectId);
|
|
85821
|
+
const allDates = and67(...filters);
|
|
85822
|
+
const matchingDates = and67(...filters, eq80(sql30`substr(${siteCrawlSnapshots.createdAt}, 6, 5)`, requestedDate.slice(5)));
|
|
85823
|
+
const readDates = (where) => app.db.select({ date }).from(siteCrawlSnapshots).innerJoin(runs, eq80(siteCrawlSnapshots.runId, runs.id)).where(where).groupBy(date).orderBy(desc36(date)).limit(10).all().map((row) => row.date);
|
|
85824
|
+
const countDates = (where) => app.db.select({ value: sql30`count(distinct ${date})` }).from(siteCrawlSnapshots).innerJoin(runs, eq80(siteCrawlSnapshots.runId, runs.id)).where(where).get()?.value ?? 0;
|
|
85825
|
+
return {
|
|
85826
|
+
recentDates: readDates(allDates),
|
|
85827
|
+
totalDates: countDates(allDates),
|
|
85828
|
+
matchingMonthDayDates: readDates(matchingDates),
|
|
85829
|
+
matchingMonthDayTotal: countDates(matchingDates)
|
|
85830
|
+
};
|
|
85831
|
+
};
|
|
85832
|
+
const resolveCrawl = (projectId, runId, requestedDate) => {
|
|
85833
|
+
const filters = visibleCrawlFilters(projectId);
|
|
85834
|
+
if (requestedDate) {
|
|
85835
|
+
const nextDate2 = /* @__PURE__ */ new Date(`${requestedDate}T00:00:00.000Z`);
|
|
85836
|
+
nextDate2.setUTCDate(nextDate2.getUTCDate() + 1);
|
|
85837
|
+
filters.push(gte21(siteCrawlSnapshots.createdAt, requestedDate), lt13(siteCrawlSnapshots.createdAt, nextDate2.toISOString()));
|
|
85838
|
+
}
|
|
85839
|
+
const selected = app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq80(siteCrawlSnapshots.runId, runs.id)).where(and67(...filters, runId ? eq80(siteCrawlSnapshots.runId, runId) : void 0)).orderBy(
|
|
85840
|
+
desc36(sql30`substr(${siteCrawlSnapshots.createdAt}, 1, 10)`),
|
|
85841
|
+
desc36(siteCrawlSnapshots.complete),
|
|
85842
|
+
desc36(siteCrawlSnapshots.pagesFetched),
|
|
85843
|
+
desc36(siteCrawlSnapshots.createdAt),
|
|
85844
|
+
desc36(siteCrawlSnapshots.runId)
|
|
85845
|
+
).limit(1).get();
|
|
85846
|
+
if (!selected) return void 0;
|
|
85847
|
+
const date = selected.snapshot.createdAt.slice(0, 10);
|
|
85848
|
+
const nextDate = /* @__PURE__ */ new Date(`${date}T00:00:00.000Z`);
|
|
85849
|
+
nextDate.setUTCDate(nextDate.getUTCDate() + 1);
|
|
85850
|
+
const sameDate = and67(
|
|
85851
|
+
...filters,
|
|
85852
|
+
gte21(siteCrawlSnapshots.createdAt, date),
|
|
85853
|
+
lt13(siteCrawlSnapshots.createdAt, nextDate.toISOString())
|
|
85854
|
+
);
|
|
85855
|
+
const sameDateRunCount = app.db.select({ value: count2() }).from(siteCrawlSnapshots).innerJoin(runs, eq80(siteCrawlSnapshots.runId, runs.id)).where(sameDate).get()?.value ?? 0;
|
|
85856
|
+
const candidates = app.db.select({
|
|
85857
|
+
runId: siteCrawlSnapshots.runId,
|
|
85858
|
+
createdAt: siteCrawlSnapshots.createdAt,
|
|
85859
|
+
complete: siteCrawlSnapshots.complete,
|
|
85860
|
+
pages: siteCrawlSnapshots.pagesFetched
|
|
85861
|
+
}).from(siteCrawlSnapshots).innerJoin(runs, eq80(siteCrawlSnapshots.runId, runs.id)).where(sameDate).orderBy(
|
|
85862
|
+
desc36(siteCrawlSnapshots.complete),
|
|
85863
|
+
desc36(siteCrawlSnapshots.pagesFetched),
|
|
85864
|
+
desc36(siteCrawlSnapshots.createdAt),
|
|
85865
|
+
desc36(siteCrawlSnapshots.runId)
|
|
85866
|
+
).limit(10).all();
|
|
85867
|
+
const runSelection = {
|
|
85868
|
+
reason: runId ? "explicit-run" : requestedDate ? selected.snapshot.complete ? "requested-date-complete" : "requested-date-most-pages" : selected.snapshot.complete ? "latest-date-complete" : "latest-date-most-pages",
|
|
85869
|
+
date,
|
|
85870
|
+
sameDateRunCount,
|
|
85871
|
+
ambiguousDate: sameDateRunCount > 1,
|
|
85872
|
+
candidates,
|
|
85873
|
+
candidatesTruncated: sameDateRunCount > candidates.length
|
|
85874
|
+
};
|
|
85875
|
+
return { ...selected, runSelection };
|
|
85876
|
+
};
|
|
85877
|
+
const resolveAudit = (projectId, runId) => {
|
|
85878
|
+
const crawl = resolveCrawl(projectId, runId);
|
|
85879
|
+
const selectedRunId = runId ?? crawl?.snapshot.runId;
|
|
85412
85880
|
const filters = [
|
|
85413
|
-
eq80(
|
|
85881
|
+
eq80(siteAuditSnapshots.projectId, projectId),
|
|
85414
85882
|
eq80(runs.projectId, projectId),
|
|
85415
85883
|
eq80(runs.kind, RunKinds["site-audit"]),
|
|
85416
85884
|
inArray32(runs.status, SURFACEABLE_STATUSES),
|
|
85417
85885
|
notProbeRun()
|
|
85418
85886
|
];
|
|
85419
|
-
|
|
85420
|
-
|
|
85887
|
+
const complete = sql30`case when ${runs.status} = ${RunStatuses.completed} then 1 else 0 end`;
|
|
85888
|
+
const selected = app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs, eq80(siteAuditSnapshots.runId, runs.id)).where(and67(...filters, selectedRunId ? eq80(siteAuditSnapshots.runId, selectedRunId) : void 0)).orderBy(
|
|
85889
|
+
desc36(sql30`substr(${siteAuditSnapshots.createdAt}, 1, 10)`),
|
|
85890
|
+
desc36(complete),
|
|
85891
|
+
desc36(siteAuditSnapshots.pagesAudited),
|
|
85892
|
+
desc36(siteAuditSnapshots.createdAt),
|
|
85893
|
+
desc36(siteAuditSnapshots.runId)
|
|
85894
|
+
).limit(1).get();
|
|
85895
|
+
if (!selected) return void 0;
|
|
85896
|
+
if (crawl) return { ...selected, runSelection: crawl.runSelection };
|
|
85897
|
+
const date = selected.snap.createdAt.slice(0, 10);
|
|
85898
|
+
const nextDate = /* @__PURE__ */ new Date(`${date}T00:00:00.000Z`);
|
|
85899
|
+
nextDate.setUTCDate(nextDate.getUTCDate() + 1);
|
|
85900
|
+
const sameDate = and67(
|
|
85901
|
+
...filters,
|
|
85902
|
+
gte21(siteAuditSnapshots.createdAt, date),
|
|
85903
|
+
lt13(siteAuditSnapshots.createdAt, nextDate.toISOString())
|
|
85904
|
+
);
|
|
85905
|
+
const sameDateRunCount = app.db.select({ value: count2() }).from(siteAuditSnapshots).innerJoin(runs, eq80(siteAuditSnapshots.runId, runs.id)).where(sameDate).get()?.value ?? 0;
|
|
85906
|
+
const candidates = app.db.select({
|
|
85907
|
+
runId: siteAuditSnapshots.runId,
|
|
85908
|
+
createdAt: siteAuditSnapshots.createdAt,
|
|
85909
|
+
complete,
|
|
85910
|
+
pages: siteAuditSnapshots.pagesAudited
|
|
85911
|
+
}).from(siteAuditSnapshots).innerJoin(runs, eq80(siteAuditSnapshots.runId, runs.id)).where(sameDate).orderBy(
|
|
85912
|
+
desc36(complete),
|
|
85913
|
+
desc36(siteAuditSnapshots.pagesAudited),
|
|
85914
|
+
desc36(siteAuditSnapshots.createdAt),
|
|
85915
|
+
desc36(siteAuditSnapshots.runId)
|
|
85916
|
+
).limit(10).all().map((candidate) => ({ ...candidate, complete: Boolean(candidate.complete) }));
|
|
85917
|
+
const runSelection = {
|
|
85918
|
+
reason: runId ? "explicit-run" : selected.runStatus === RunStatuses.completed ? "latest-date-complete" : "latest-date-most-pages",
|
|
85919
|
+
date,
|
|
85920
|
+
sameDateRunCount,
|
|
85921
|
+
ambiguousDate: sameDateRunCount > 1,
|
|
85922
|
+
candidates,
|
|
85923
|
+
candidatesTruncated: sameDateRunCount > candidates.length
|
|
85924
|
+
};
|
|
85925
|
+
return { ...selected, runSelection };
|
|
85421
85926
|
};
|
|
85422
85927
|
const detailScopeFor = (projectId, snapshot) => snapshot.detailsAvailable && snapshot.attemptId ? { projectId, runId: snapshot.runId, attemptId: snapshot.attemptId } : null;
|
|
85423
|
-
const
|
|
85424
|
-
|
|
85425
|
-
|
|
85928
|
+
const crawlReasonGroups = (where) => {
|
|
85929
|
+
const canonicalOther = sql30`case when ${siteCrawlPages.canonicalNodeKey} is not null
|
|
85930
|
+
and ${siteCrawlPages.canonicalNodeKey} <> ${siteCrawlPages.nodeKey} then 1 else 0 end`;
|
|
85931
|
+
const canonicalReason = sql30`exists (select 1 from json_each(${siteCrawlPages.indexabilityReasons})
|
|
85932
|
+
where value = ${SiteCrawlIndexabilityReasons.canonicalToOther})`;
|
|
85933
|
+
const groups = app.db.select({
|
|
85934
|
+
fetchState: siteCrawlPages.fetchState,
|
|
85935
|
+
indexabilityState: siteCrawlPages.indexabilityState,
|
|
85936
|
+
inventoryEligible: siteCrawlPages.inventoryEligible,
|
|
85937
|
+
canonicalOther,
|
|
85938
|
+
canonicalReason,
|
|
85939
|
+
pages: count2(),
|
|
85940
|
+
exampleUrl: sql30`min(${siteCrawlPages.url})`
|
|
85941
|
+
}).from(siteCrawlPages).where(where).groupBy(
|
|
85942
|
+
siteCrawlPages.fetchState,
|
|
85943
|
+
siteCrawlPages.indexabilityState,
|
|
85944
|
+
siteCrawlPages.inventoryEligible,
|
|
85945
|
+
canonicalOther,
|
|
85946
|
+
canonicalReason
|
|
85947
|
+
).limit(257).all();
|
|
85948
|
+
if (groups.length > 256) throw validationError("Persisted crawl has too many distinct health-state variants to summarize");
|
|
85949
|
+
return groups.map((group) => ({
|
|
85950
|
+
inventoryEligible: group.inventoryEligible,
|
|
85951
|
+
healthReason: deriveSiteHealthReason({
|
|
85952
|
+
fetchState: group.fetchState,
|
|
85953
|
+
indexabilityState: group.indexabilityState,
|
|
85954
|
+
indexabilityReasons: group.canonicalReason ? [SiteCrawlIndexabilityReasons.canonicalToOther] : [],
|
|
85955
|
+
// Preserve only the identity inequality tested by the canonical helper.
|
|
85956
|
+
nodeKey: "source",
|
|
85957
|
+
canonicalNodeKey: group.canonicalOther ? "other" : null
|
|
85958
|
+
}),
|
|
85959
|
+
pages: group.pages,
|
|
85960
|
+
exampleUrl: group.exampleUrl
|
|
85961
|
+
}));
|
|
85962
|
+
};
|
|
85963
|
+
const reasonCounts = (groups) => {
|
|
85964
|
+
const counts2 = /* @__PURE__ */ new Map();
|
|
85965
|
+
for (const group of groups) {
|
|
85966
|
+
const existing = counts2.get(group.healthReason);
|
|
85967
|
+
if (existing) {
|
|
85968
|
+
existing.pages += group.pages;
|
|
85969
|
+
if (group.exampleUrl < existing.exampleUrl) existing.exampleUrl = group.exampleUrl;
|
|
85970
|
+
} else counts2.set(group.healthReason, { healthReason: group.healthReason, pages: group.pages, exampleUrl: group.exampleUrl });
|
|
85971
|
+
}
|
|
85972
|
+
return [...counts2.values()].sort((a, b) => a.healthReason.localeCompare(b.healthReason));
|
|
85973
|
+
};
|
|
85974
|
+
const inventorySummaryFor = (scope) => {
|
|
85975
|
+
if (!scope) return null;
|
|
85976
|
+
const groups = crawlReasonGroups(and67(
|
|
85977
|
+
eq80(siteCrawlPages.projectId, scope.projectId),
|
|
85978
|
+
eq80(siteCrawlPages.runId, scope.runId),
|
|
85979
|
+
eq80(siteCrawlPages.attemptId, scope.attemptId)
|
|
85980
|
+
));
|
|
85981
|
+
const total = groups.reduce((sum, group) => sum + group.pages, 0);
|
|
85982
|
+
const eligible = groups.reduce((sum, group) => sum + (group.inventoryEligible ? group.pages : 0), 0);
|
|
85983
|
+
return {
|
|
85984
|
+
scope: "selected-snapshot",
|
|
85985
|
+
total,
|
|
85986
|
+
eligible,
|
|
85987
|
+
excluded: total - eligible,
|
|
85988
|
+
excludedReasons: reasonCounts(groups.filter((group) => !group.inventoryEligible))
|
|
85989
|
+
};
|
|
85990
|
+
};
|
|
85991
|
+
const completenessOf = (target) => ({
|
|
85992
|
+
complete: target.snapshot.complete,
|
|
85993
|
+
termination: target.snapshot.termination,
|
|
85994
|
+
runSelection: target.runSelection
|
|
85426
85995
|
});
|
|
85427
85996
|
const NO_CRAWL_COMPLETENESS = { complete: false, termination: null };
|
|
85428
85997
|
const pageInScope = (scope, selector) => {
|
|
@@ -85462,6 +86031,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85462
86031
|
hasCrawlData: false,
|
|
85463
86032
|
legacyAuditAvailable,
|
|
85464
86033
|
runId: null,
|
|
86034
|
+
inventorySummary: null,
|
|
85465
86035
|
runStatus: null,
|
|
85466
86036
|
requestedRootUrl: null,
|
|
85467
86037
|
rootUrl: null,
|
|
@@ -85481,16 +86051,42 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85481
86051
|
inArray32(runs.status, SURFACEABLE_STATUSES),
|
|
85482
86052
|
notProbeRun()
|
|
85483
86053
|
];
|
|
85484
|
-
const
|
|
85485
|
-
const latest = app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs, eq80(siteAuditSnapshots.runId, runs.id)).where(and67(...targetFilters)).orderBy(desc36(siteAuditSnapshots.createdAt)).limit(1).get();
|
|
86054
|
+
const latest = resolveAudit(project.id, request.query.runId);
|
|
85486
86055
|
if (!latest) {
|
|
85487
|
-
|
|
85488
|
-
|
|
86056
|
+
const crawl = resolveCrawl(project.id, request.query.runId);
|
|
86057
|
+
if (!crawl && request.query.runId) throw notFound("Site audit run", request.query.runId);
|
|
86058
|
+
return crawl ? {
|
|
86059
|
+
...emptyScore(project.name),
|
|
86060
|
+
runId: crawl.snapshot.runId,
|
|
86061
|
+
runStatus: crawl.runStatus,
|
|
86062
|
+
runSelection: crawl.runSelection
|
|
86063
|
+
} : emptyScore(project.name);
|
|
85489
86064
|
}
|
|
85490
86065
|
const snap = latest.snap;
|
|
85491
86066
|
const previous = app.db.select({ snap: siteAuditSnapshots }).from(siteAuditSnapshots).innerJoin(runs, eq80(siteAuditSnapshots.runId, runs.id)).where(and67(...baseFilters, lt13(siteAuditSnapshots.createdAt, snap.createdAt))).orderBy(desc36(siteAuditSnapshots.createdAt)).limit(1).get()?.snap ?? null;
|
|
85492
86067
|
const deltaScore = previous ? snap.aggregateScore - previous.aggregateScore : null;
|
|
85493
86068
|
const trend = deltaScore == null ? null : deltaScore > 0 ? SiteAuditTrendDirections.up : deltaScore < 0 ? SiteAuditTrendDirections.down : SiteAuditTrendDirections.flat;
|
|
86069
|
+
const affectedPageRanking = {
|
|
86070
|
+
basis: "pages-below-pass",
|
|
86071
|
+
scope: "audited-pages",
|
|
86072
|
+
pagesAudited: snap.pagesAudited,
|
|
86073
|
+
items: [],
|
|
86074
|
+
unavailableFactorIds: []
|
|
86075
|
+
};
|
|
86076
|
+
for (const factor of snap.factorAverages) {
|
|
86077
|
+
if (!Number.isInteger(factor.pagesFailing) || factor.pagesFailing < 0 || !Number.isInteger(factor.pagesPartial) || factor.pagesPartial < 0) {
|
|
86078
|
+
affectedPageRanking.unavailableFactorIds.push(factor.id);
|
|
86079
|
+
continue;
|
|
86080
|
+
}
|
|
86081
|
+
affectedPageRanking.items.push({
|
|
86082
|
+
factorId: factor.id,
|
|
86083
|
+
pagesBelowPass: factor.pagesFailing + factor.pagesPartial,
|
|
86084
|
+
pagesFailing: factor.pagesFailing,
|
|
86085
|
+
pagesPartial: factor.pagesPartial
|
|
86086
|
+
});
|
|
86087
|
+
}
|
|
86088
|
+
affectedPageRanking.items.sort((a, b) => b.pagesBelowPass - a.pagesBelowPass || a.factorId.localeCompare(b.factorId));
|
|
86089
|
+
affectedPageRanking.unavailableFactorIds.sort();
|
|
85494
86090
|
return {
|
|
85495
86091
|
project: project.name,
|
|
85496
86092
|
hasData: true,
|
|
@@ -85507,30 +86103,33 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85507
86103
|
trend,
|
|
85508
86104
|
previousScore: previous?.aggregateScore ?? null,
|
|
85509
86105
|
previousAuditedAt: previous?.auditedAt ?? null,
|
|
85510
|
-
|
|
86106
|
+
runSelection: latest.runSelection,
|
|
86107
|
+
factors: snap.factorAverages.map((factor) => ({
|
|
86108
|
+
...withRecordedShare(factor),
|
|
86109
|
+
status: factorStatusFromDistribution(factor)
|
|
86110
|
+
})),
|
|
86111
|
+
affectedPageRanking,
|
|
85511
86112
|
crossCuttingIssues: snap.crossCuttingIssues,
|
|
85512
86113
|
prioritizedFixes: snap.prioritizedFixes
|
|
85513
86114
|
};
|
|
85514
86115
|
});
|
|
85515
86116
|
app.get("/projects/:name/technical-aeo/pages", async (request) => {
|
|
85516
86117
|
const project = resolveProject(app.db, request.params.name);
|
|
85517
|
-
const
|
|
85518
|
-
eq80(siteAuditSnapshots.projectId, project.id),
|
|
85519
|
-
eq80(runs.projectId, project.id),
|
|
85520
|
-
eq80(runs.kind, RunKinds["site-audit"]),
|
|
85521
|
-
inArray32(runs.status, SURFACEABLE_STATUSES),
|
|
85522
|
-
notProbeRun()
|
|
85523
|
-
];
|
|
85524
|
-
if (request.query.runId) targetFilters.push(eq80(siteAuditSnapshots.runId, request.query.runId));
|
|
85525
|
-
const latest = app.db.select({ runId: siteAuditSnapshots.runId, auditedAt: siteAuditSnapshots.auditedAt }).from(siteAuditSnapshots).innerJoin(runs, eq80(siteAuditSnapshots.runId, runs.id)).where(and67(...targetFilters)).orderBy(desc36(siteAuditSnapshots.createdAt)).limit(1).get();
|
|
85526
|
-
if (!latest && request.query.runId) {
|
|
85527
|
-
throw notFound("Site audit run", request.query.runId);
|
|
85528
|
-
}
|
|
86118
|
+
const latest = resolveAudit(project.id, request.query.runId);
|
|
85529
86119
|
if (!latest) {
|
|
85530
|
-
|
|
86120
|
+
const crawl = resolveCrawl(project.id, request.query.runId);
|
|
86121
|
+
if (!crawl && request.query.runId) throw notFound("Site audit run", request.query.runId);
|
|
86122
|
+
return {
|
|
86123
|
+
project: project.name,
|
|
86124
|
+
runId: crawl?.snapshot.runId ?? null,
|
|
86125
|
+
auditedAt: null,
|
|
86126
|
+
total: 0,
|
|
86127
|
+
pages: [],
|
|
86128
|
+
...crawl ? { runSelection: crawl.runSelection } : {}
|
|
86129
|
+
};
|
|
85531
86130
|
}
|
|
85532
86131
|
const statusFilter = request.query.status === "success" || request.query.status === "error" ? request.query.status : null;
|
|
85533
|
-
const conds = [eq80(siteAuditPages.projectId, project.id), eq80(siteAuditPages.runId, latest.runId)];
|
|
86132
|
+
const conds = [eq80(siteAuditPages.projectId, project.id), eq80(siteAuditPages.runId, latest.snap.runId)];
|
|
85534
86133
|
if (statusFilter) conds.push(eq80(siteAuditPages.status, statusFilter));
|
|
85535
86134
|
const where = and67(...conds);
|
|
85536
86135
|
const totalRow = app.db.select({ value: count2() }).from(siteAuditPages).where(where).get();
|
|
@@ -85546,7 +86145,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85546
86145
|
error: row.error,
|
|
85547
86146
|
factors: row.factors.map(withRecordedShare)
|
|
85548
86147
|
}));
|
|
85549
|
-
return { project: project.name, runId: latest.runId, auditedAt: latest.auditedAt, total, pages };
|
|
86148
|
+
return { project: project.name, runId: latest.snap.runId, auditedAt: latest.snap.auditedAt, runSelection: latest.runSelection, total, pages };
|
|
85550
86149
|
});
|
|
85551
86150
|
app.get("/projects/:name/technical-aeo/trend", async (request) => {
|
|
85552
86151
|
const project = resolveProject(app.db, request.params.name);
|
|
@@ -85567,11 +86166,16 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85567
86166
|
});
|
|
85568
86167
|
app.get("/projects/:name/technical-aeo/crawl", async (request) => {
|
|
85569
86168
|
const project = resolveProject(app.db, request.params.name);
|
|
85570
|
-
const
|
|
86169
|
+
const requestedDate = parseCrawlDate(request.query.date, request.query.runId);
|
|
86170
|
+
const target = resolveCrawl(project.id, request.query.runId, requestedDate);
|
|
85571
86171
|
const legacyAuditAvailable = hasLegacyAudit(project.id);
|
|
85572
86172
|
if (!target) {
|
|
85573
86173
|
assertKnownAuditRun(project.id, request.query.runId);
|
|
85574
|
-
return
|
|
86174
|
+
return {
|
|
86175
|
+
...emptyCrawlSummary(project.name, legacyAuditAvailable),
|
|
86176
|
+
requestedDate,
|
|
86177
|
+
availableScanDates: availableScanDatesFor(project.id, requestedDate)
|
|
86178
|
+
};
|
|
85575
86179
|
}
|
|
85576
86180
|
const snapshot = target.snapshot;
|
|
85577
86181
|
const deadLinks = !snapshot.checkDeadLinks || snapshot.deadLinkState === "disabled" ? { state: "disabled" } : snapshot.deadLinkState === "complete" ? { state: "complete", checked: snapshot.deadLinksChecked, found: snapshot.deadLinksFound, unverified: snapshot.deadLinksUnverified } : snapshot.deadLinkState === "partial" ? { state: "partial", checked: snapshot.deadLinksChecked, found: snapshot.deadLinksFound, unverified: snapshot.deadLinksUnverified } : { state: "unavailable" };
|
|
@@ -85580,6 +86184,8 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85580
86184
|
hasCrawlData: true,
|
|
85581
86185
|
legacyAuditAvailable,
|
|
85582
86186
|
runId: snapshot.runId,
|
|
86187
|
+
requestedDate,
|
|
86188
|
+
inventorySummary: inventorySummaryFor(detailScopeFor(project.id, snapshot)),
|
|
85583
86189
|
runStatus: target.runStatus,
|
|
85584
86190
|
requestedRootUrl: snapshot.requestedRootUrl,
|
|
85585
86191
|
rootUrl: snapshot.rootUrl,
|
|
@@ -85589,8 +86195,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85589
86195
|
indexabilityVersion: snapshot.indexabilityVersion,
|
|
85590
86196
|
linkScoreVersion: snapshot.linkScoreVersion,
|
|
85591
86197
|
effectiveOptions: snapshot.effectiveOptions,
|
|
85592
|
-
|
|
85593
|
-
termination: snapshot.termination,
|
|
86198
|
+
...completenessOf(target),
|
|
85594
86199
|
detailsAvailable: snapshot.detailsAvailable,
|
|
85595
86200
|
counts: {
|
|
85596
86201
|
pagesDiscovered: snapshot.pagesDiscovered,
|
|
@@ -85636,7 +86241,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85636
86241
|
project: project.name,
|
|
85637
86242
|
hasCrawlData: true,
|
|
85638
86243
|
runId: snapshot.runId,
|
|
85639
|
-
...completenessOf(
|
|
86244
|
+
...completenessOf(target),
|
|
85640
86245
|
rootNodeKey: null,
|
|
85641
86246
|
layout: { state: "unavailable", version: null, reason: "details-unavailable" },
|
|
85642
86247
|
templateDetection,
|
|
@@ -85666,7 +86271,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85666
86271
|
project: project.name,
|
|
85667
86272
|
hasCrawlData: true,
|
|
85668
86273
|
runId: snapshot.runId,
|
|
85669
|
-
...completenessOf(
|
|
86274
|
+
...completenessOf(target),
|
|
85670
86275
|
rootNodeKey,
|
|
85671
86276
|
layout: { state: "unavailable", version: null, reason: "legacy-snapshot" },
|
|
85672
86277
|
templateDetection,
|
|
@@ -85689,7 +86294,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85689
86294
|
project: project.name,
|
|
85690
86295
|
hasCrawlData: true,
|
|
85691
86296
|
runId: snapshot.runId,
|
|
85692
|
-
...completenessOf(
|
|
86297
|
+
...completenessOf(target),
|
|
85693
86298
|
rootNodeKey,
|
|
85694
86299
|
layout: {
|
|
85695
86300
|
state: "unavailable",
|
|
@@ -85741,7 +86346,8 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85741
86346
|
)).where(and67(...graphScope, lt13(siteCrawlGraphNodes.sampleRank, maxNodes))).orderBy(asc14(siteCrawlGraphNodes.sampleRank)).all();
|
|
85742
86347
|
const nodes = nodeRows.map(({ indexabilityReasons, canonicalNodeKey, ...row }) => ({
|
|
85743
86348
|
...row,
|
|
85744
|
-
healthState: deriveSiteHealthState({ ...row, indexabilityReasons, canonicalNodeKey })
|
|
86349
|
+
healthState: deriveSiteHealthState({ ...row, indexabilityReasons, canonicalNodeKey }),
|
|
86350
|
+
healthReason: deriveSiteHealthReason({ ...row, indexabilityReasons, canonicalNodeKey })
|
|
85745
86351
|
}));
|
|
85746
86352
|
const graphLinkKindFilter = linkKind === SiteHealthLinkKinds.all ? void 0 : eq80(siteCrawlGraphEdges.isTemplate, linkKind === SiteHealthLinkKinds.template);
|
|
85747
86353
|
const edges = app.db.select({
|
|
@@ -85779,7 +86385,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85779
86385
|
project: project.name,
|
|
85780
86386
|
hasCrawlData: true,
|
|
85781
86387
|
runId: snapshot.runId,
|
|
85782
|
-
...completenessOf(
|
|
86388
|
+
...completenessOf(target),
|
|
85783
86389
|
rootNodeKey,
|
|
85784
86390
|
layout: {
|
|
85785
86391
|
state: "ready",
|
|
@@ -85936,8 +86542,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85936
86542
|
project: project.name,
|
|
85937
86543
|
hasCrawlData: true,
|
|
85938
86544
|
runId: snapshot.runId,
|
|
85939
|
-
|
|
85940
|
-
termination: snapshot.termination,
|
|
86545
|
+
...completenessOf(target),
|
|
85941
86546
|
state: "ready",
|
|
85942
86547
|
focusNodeKey: focus.nodeKey,
|
|
85943
86548
|
focusUrl: focus.url,
|
|
@@ -85988,8 +86593,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85988
86593
|
state: "details-unavailable",
|
|
85989
86594
|
from: null,
|
|
85990
86595
|
to: null,
|
|
85991
|
-
|
|
85992
|
-
termination: snapshot.termination,
|
|
86596
|
+
...completenessOf(target),
|
|
85993
86597
|
maxDepth,
|
|
85994
86598
|
visitedNodes: 0,
|
|
85995
86599
|
nodes: [],
|
|
@@ -86051,8 +86655,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86051
86655
|
return {
|
|
86052
86656
|
project: project.name,
|
|
86053
86657
|
runId: snapshot.runId,
|
|
86054
|
-
|
|
86055
|
-
termination: snapshot.termination,
|
|
86658
|
+
...completenessOf(target),
|
|
86056
86659
|
state: truncated || frontier.length > 0 ? "truncated" : "unreachable",
|
|
86057
86660
|
from: reference(fromPage),
|
|
86058
86661
|
to: reference(toPage),
|
|
@@ -86081,8 +86684,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86081
86684
|
return {
|
|
86082
86685
|
project: project.name,
|
|
86083
86686
|
runId: snapshot.runId,
|
|
86084
|
-
|
|
86085
|
-
termination: snapshot.termination,
|
|
86687
|
+
...completenessOf(target),
|
|
86086
86688
|
state: "found",
|
|
86087
86689
|
from: reference(fromPage),
|
|
86088
86690
|
to: reference(toPage),
|
|
@@ -86205,9 +86807,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86205
86807
|
const summary4 = { pages: zero2(), links: zero2() };
|
|
86206
86808
|
if (scopeFilter !== "links") {
|
|
86207
86809
|
if (includeAdded || includeChanged) {
|
|
86208
|
-
const addedExpression = includeAdded ?
|
|
86209
|
-
const changedExpression = includeChanged ?
|
|
86210
|
-
const [counts2 = { added: 0, changed: 0 }] = app.db.all(
|
|
86810
|
+
const addedExpression = includeAdded ? sql30`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql30`0`;
|
|
86811
|
+
const changedExpression = includeChanged ? sql30`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${PAGE_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql30`0`;
|
|
86812
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql30`
|
|
86211
86813
|
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
86212
86814
|
FROM site_crawl_pages AS current
|
|
86213
86815
|
LEFT JOIN site_crawl_pages AS previous
|
|
@@ -86223,7 +86825,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86223
86825
|
summary4.pages.changed = Number(counts2.changed);
|
|
86224
86826
|
}
|
|
86225
86827
|
if (includeRemoved) {
|
|
86226
|
-
const [counts2 = { removed: 0 }] = app.db.all(
|
|
86828
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql30`
|
|
86227
86829
|
SELECT COUNT(*) AS removed
|
|
86228
86830
|
FROM site_crawl_pages AS previous
|
|
86229
86831
|
LEFT JOIN site_crawl_pages AS current
|
|
@@ -86241,9 +86843,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86241
86843
|
}
|
|
86242
86844
|
if (scopeFilter !== "pages") {
|
|
86243
86845
|
if (includeAdded || includeChanged) {
|
|
86244
|
-
const addedExpression = includeAdded ?
|
|
86245
|
-
const changedExpression = includeChanged ?
|
|
86246
|
-
const [counts2 = { added: 0, changed: 0 }] = app.db.all(
|
|
86846
|
+
const addedExpression = includeAdded ? sql30`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql30`0`;
|
|
86847
|
+
const changedExpression = includeChanged ? sql30`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${LINK_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql30`0`;
|
|
86848
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql30`
|
|
86247
86849
|
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
86248
86850
|
FROM site_crawl_edges AS current
|
|
86249
86851
|
LEFT JOIN site_crawl_edges AS previous
|
|
@@ -86261,7 +86863,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86261
86863
|
summary4.links.changed = Number(counts2.changed);
|
|
86262
86864
|
}
|
|
86263
86865
|
if (includeRemoved) {
|
|
86264
|
-
const [counts2 = { removed: 0 }] = app.db.all(
|
|
86866
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql30`
|
|
86265
86867
|
SELECT COUNT(*) AS removed
|
|
86266
86868
|
FROM site_crawl_edges AS previous
|
|
86267
86869
|
LEFT JOIN site_crawl_edges AS current
|
|
@@ -86291,48 +86893,48 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86291
86893
|
const includeRemoved = changeFilter === "all" || changeFilter === "removed";
|
|
86292
86894
|
const table = entity === "page" ? "site_crawl_pages" : "site_crawl_edges";
|
|
86293
86895
|
const keyColumn = entity === "page" ? "node_key" : "edge_key";
|
|
86294
|
-
const internalCurrent = entity === "link" ?
|
|
86295
|
-
const internalPreviousJoin = entity === "link" ?
|
|
86296
|
-
const internalPrevious = entity === "link" ?
|
|
86297
|
-
const internalCurrentJoin = entity === "link" ?
|
|
86896
|
+
const internalCurrent = entity === "link" ? sql30`AND current.internal = 1` : sql30``;
|
|
86897
|
+
const internalPreviousJoin = entity === "link" ? sql30`AND previous.internal = 1` : sql30``;
|
|
86898
|
+
const internalPrevious = entity === "link" ? sql30`AND previous.internal = 1` : sql30``;
|
|
86899
|
+
const internalCurrentJoin = entity === "link" ? sql30`AND current.internal = 1` : sql30``;
|
|
86298
86900
|
const difference2 = entity === "page" ? PAGE_CHANGE_PREDICATE : LINK_CHANGE_PREDICATE;
|
|
86299
|
-
const afterKind = changeFilter === "added" ?
|
|
86300
|
-
const afterRows = includeAfter ? app.db.all(
|
|
86901
|
+
const afterKind = changeFilter === "added" ? sql30`previous.id IS NULL` : changeFilter === "changed" ? sql30`previous.id IS NOT NULL AND ${difference2}` : sql30`previous.id IS NULL OR (previous.id IS NOT NULL AND ${difference2})`;
|
|
86902
|
+
const afterRows = includeAfter ? app.db.all(sql30`
|
|
86301
86903
|
SELECT ${entity} AS entity,
|
|
86302
86904
|
CASE WHEN previous.id IS NULL THEN 'added' ELSE 'changed' END AS change,
|
|
86303
|
-
current.${
|
|
86304
|
-
FROM ${
|
|
86305
|
-
LEFT JOIN ${
|
|
86905
|
+
current.${sql30.raw(keyColumn)} AS key
|
|
86906
|
+
FROM ${sql30.raw(table)} AS current
|
|
86907
|
+
LEFT JOIN ${sql30.raw(table)} AS previous
|
|
86306
86908
|
ON previous.project_id = ${beforeScope.projectId}
|
|
86307
86909
|
AND previous.run_id = ${beforeScope.runId}
|
|
86308
86910
|
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
86309
86911
|
${internalPreviousJoin}
|
|
86310
|
-
AND previous.${
|
|
86912
|
+
AND previous.${sql30.raw(keyColumn)} = current.${sql30.raw(keyColumn)}
|
|
86311
86913
|
WHERE current.project_id = ${afterScope.projectId}
|
|
86312
86914
|
AND current.run_id = ${afterScope.runId}
|
|
86313
86915
|
AND current.attempt_id = ${afterScope.attemptId}
|
|
86314
86916
|
${internalCurrent}
|
|
86315
|
-
AND current.${
|
|
86917
|
+
AND current.${sql30.raw(keyColumn)} > ${afterKey}
|
|
86316
86918
|
AND (${afterKind})
|
|
86317
|
-
ORDER BY current.${
|
|
86919
|
+
ORDER BY current.${sql30.raw(keyColumn)} ASC
|
|
86318
86920
|
LIMIT ${wanted2}
|
|
86319
86921
|
`) : [];
|
|
86320
|
-
const removedRows = includeRemoved ? app.db.all(
|
|
86321
|
-
SELECT ${entity} AS entity, 'removed' AS change, previous.${
|
|
86322
|
-
FROM ${
|
|
86323
|
-
LEFT JOIN ${
|
|
86922
|
+
const removedRows = includeRemoved ? app.db.all(sql30`
|
|
86923
|
+
SELECT ${entity} AS entity, 'removed' AS change, previous.${sql30.raw(keyColumn)} AS key
|
|
86924
|
+
FROM ${sql30.raw(table)} AS previous
|
|
86925
|
+
LEFT JOIN ${sql30.raw(table)} AS current
|
|
86324
86926
|
ON current.project_id = ${afterScope.projectId}
|
|
86325
86927
|
AND current.run_id = ${afterScope.runId}
|
|
86326
86928
|
AND current.attempt_id = ${afterScope.attemptId}
|
|
86327
86929
|
${internalCurrentJoin}
|
|
86328
|
-
AND current.${
|
|
86930
|
+
AND current.${sql30.raw(keyColumn)} = previous.${sql30.raw(keyColumn)}
|
|
86329
86931
|
WHERE previous.project_id = ${beforeScope.projectId}
|
|
86330
86932
|
AND previous.run_id = ${beforeScope.runId}
|
|
86331
86933
|
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
86332
86934
|
${internalPrevious}
|
|
86333
|
-
AND previous.${
|
|
86935
|
+
AND previous.${sql30.raw(keyColumn)} > ${afterKey}
|
|
86334
86936
|
AND current.id IS NULL
|
|
86335
|
-
ORDER BY previous.${
|
|
86937
|
+
ORDER BY previous.${sql30.raw(keyColumn)} ASC
|
|
86336
86938
|
LIMIT ${wanted2}
|
|
86337
86939
|
`) : [];
|
|
86338
86940
|
const merged = [];
|
|
@@ -86456,8 +87058,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86456
87058
|
const provenance = {
|
|
86457
87059
|
project: project.name,
|
|
86458
87060
|
runId: snapshot.runId,
|
|
86459
|
-
|
|
86460
|
-
termination: snapshot.termination
|
|
87061
|
+
...completenessOf(target)
|
|
86461
87062
|
};
|
|
86462
87063
|
const scope = detailScopeFor(project.id, snapshot);
|
|
86463
87064
|
if (!scope) return { state: "details-unavailable", ...provenance };
|
|
@@ -86488,15 +87089,44 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86488
87089
|
});
|
|
86489
87090
|
app.get("/projects/:name/technical-aeo/crawl/pages", async (request) => {
|
|
86490
87091
|
const project = resolveProject(app.db, request.params.name);
|
|
86491
|
-
const
|
|
87092
|
+
const requestedDate = parseCrawlDate(request.query.date, request.query.runId);
|
|
87093
|
+
const target = resolveCrawl(project.id, request.query.runId, requestedDate);
|
|
86492
87094
|
if (!target) {
|
|
86493
87095
|
assertKnownAuditRun(project.id, request.query.runId);
|
|
86494
|
-
|
|
87096
|
+
decodeCrawlPageCursor(request.query.cursor, null);
|
|
87097
|
+
return {
|
|
87098
|
+
project: project.name,
|
|
87099
|
+
hasCrawlData: false,
|
|
87100
|
+
runId: null,
|
|
87101
|
+
requestedDate,
|
|
87102
|
+
inventorySummary: null,
|
|
87103
|
+
healthReasonCounts: null,
|
|
87104
|
+
availableScanDates: availableScanDatesFor(project.id, requestedDate),
|
|
87105
|
+
...NO_CRAWL_COMPLETENESS,
|
|
87106
|
+
total: 0,
|
|
87107
|
+
nextCursor: null,
|
|
87108
|
+
healthStateFilter: null,
|
|
87109
|
+
pages: []
|
|
87110
|
+
};
|
|
86495
87111
|
}
|
|
86496
87112
|
const snapshot = target.snapshot;
|
|
86497
87113
|
if (!snapshot.detailsAvailable || !snapshot.attemptId) {
|
|
86498
|
-
|
|
87114
|
+
decodeCrawlPageCursor(request.query.cursor, null);
|
|
87115
|
+
return {
|
|
87116
|
+
project: project.name,
|
|
87117
|
+
hasCrawlData: true,
|
|
87118
|
+
runId: snapshot.runId,
|
|
87119
|
+
requestedDate,
|
|
87120
|
+
inventorySummary: null,
|
|
87121
|
+
healthReasonCounts: null,
|
|
87122
|
+
...completenessOf(target),
|
|
87123
|
+
total: 0,
|
|
87124
|
+
nextCursor: null,
|
|
87125
|
+
healthStateFilter: null,
|
|
87126
|
+
pages: []
|
|
87127
|
+
};
|
|
86499
87128
|
}
|
|
87129
|
+
const inventorySummary = inventorySummaryFor(detailScopeFor(project.id, snapshot));
|
|
86500
87130
|
const filters = [
|
|
86501
87131
|
eq80(siteCrawlPages.projectId, project.id),
|
|
86502
87132
|
eq80(siteCrawlPages.runId, snapshot.runId),
|
|
@@ -86510,8 +87140,22 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86510
87140
|
if (request.query.auditState) filters.push(eq80(siteCrawlPages.auditState, request.query.auditState));
|
|
86511
87141
|
const healthState = parseSiteHealthState(request.query.healthState);
|
|
86512
87142
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
86513
|
-
const
|
|
86514
|
-
const
|
|
87143
|
+
const sort = ["score-desc", "score-asc", "path"].includes(request.query.sort ?? "") ? request.query.sort : "url";
|
|
87144
|
+
const selectionKey = crypto49.createHash("sha256").update(JSON.stringify({
|
|
87145
|
+
projectId: project.id,
|
|
87146
|
+
runId: snapshot.runId,
|
|
87147
|
+
attemptId: snapshot.attemptId,
|
|
87148
|
+
requestedDate: requestedDate ?? null,
|
|
87149
|
+
nodeKey: request.query.nodeKey || null,
|
|
87150
|
+
inventoryEligible,
|
|
87151
|
+
fetchState: request.query.fetchState || null,
|
|
87152
|
+
indexabilityState: request.query.indexabilityState || null,
|
|
87153
|
+
auditState: request.query.auditState || null,
|
|
87154
|
+
healthState,
|
|
87155
|
+
sort
|
|
87156
|
+
})).digest("base64url");
|
|
87157
|
+
const offset = decodeCrawlPageCursor(request.query.cursor, selectionKey);
|
|
87158
|
+
const orderBy = sort === "score-desc" ? [desc36(siteCrawlPages.auditScore), asc14(siteCrawlPages.nodeKey)] : sort === "score-asc" ? [asc14(siteCrawlPages.auditScore), asc14(siteCrawlPages.nodeKey)] : sort === "path" ? [asc14(siteCrawlPages.path), asc14(siteCrawlPages.nodeKey)] : [asc14(siteCrawlPages.url), asc14(siteCrawlPages.nodeKey)];
|
|
86515
87159
|
let healthStateFilter = null;
|
|
86516
87160
|
if (healthState) {
|
|
86517
87161
|
const legacyRow = app.db.select({ id: siteCrawlPages.id }).from(siteCrawlPages).where(and67(
|
|
@@ -86528,7 +87172,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86528
87172
|
project: project.name,
|
|
86529
87173
|
hasCrawlData: true,
|
|
86530
87174
|
runId: snapshot.runId,
|
|
86531
|
-
|
|
87175
|
+
requestedDate,
|
|
87176
|
+
inventorySummary,
|
|
87177
|
+
healthReasonCounts: null,
|
|
87178
|
+
...completenessOf(target),
|
|
86532
87179
|
total: 0,
|
|
86533
87180
|
nextCursor: null,
|
|
86534
87181
|
healthStateFilter,
|
|
@@ -86543,9 +87190,12 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86543
87190
|
project: project.name,
|
|
86544
87191
|
hasCrawlData: true,
|
|
86545
87192
|
runId: snapshot.runId,
|
|
86546
|
-
|
|
87193
|
+
requestedDate,
|
|
87194
|
+
inventorySummary,
|
|
87195
|
+
healthReasonCounts: reasonCounts(crawlReasonGroups(where)),
|
|
87196
|
+
...completenessOf(target),
|
|
86547
87197
|
total,
|
|
86548
|
-
nextCursor: nextOffset < total ?
|
|
87198
|
+
nextCursor: nextOffset < total ? Buffer.from(JSON.stringify({ offset: nextOffset, selectionKey })).toString("base64url") : null,
|
|
86549
87199
|
healthStateFilter,
|
|
86550
87200
|
pages: rows.map(mapCrawlPage)
|
|
86551
87201
|
};
|
|
@@ -86560,7 +87210,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86560
87210
|
}
|
|
86561
87211
|
const snapshot = target.snapshot;
|
|
86562
87212
|
if (!snapshot.detailsAvailable || !snapshot.attemptId) {
|
|
86563
|
-
return { project: project.name, hasCrawlData: true, runId: snapshot.runId, ...completenessOf(
|
|
87213
|
+
return { project: project.name, hasCrawlData: true, runId: snapshot.runId, ...completenessOf(target), parentPath, nextCursor: null, children: [] };
|
|
86564
87214
|
}
|
|
86565
87215
|
const sourceRows = app.db.select({
|
|
86566
87216
|
path: siteCrawlPages.path,
|
|
@@ -86605,7 +87255,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86605
87255
|
project: project.name,
|
|
86606
87256
|
hasCrawlData: true,
|
|
86607
87257
|
runId: snapshot.runId,
|
|
86608
|
-
...completenessOf(
|
|
87258
|
+
...completenessOf(target),
|
|
86609
87259
|
parentPath,
|
|
86610
87260
|
nextCursor: hasMore ? encodeCursor2(offset + visible.length) : null,
|
|
86611
87261
|
children: visible.map(([path8, child]) => ({
|
|
@@ -86643,7 +87293,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86643
87293
|
project: project.name,
|
|
86644
87294
|
hasCrawlData: true,
|
|
86645
87295
|
runId: snapshot.runId,
|
|
86646
|
-
...completenessOf(
|
|
87296
|
+
...completenessOf(target),
|
|
86647
87297
|
total: 0,
|
|
86648
87298
|
nextCursor: null,
|
|
86649
87299
|
templateDetection,
|
|
@@ -86672,7 +87322,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86672
87322
|
project: project.name,
|
|
86673
87323
|
hasCrawlData: true,
|
|
86674
87324
|
runId: snapshot.runId,
|
|
86675
|
-
...completenessOf(
|
|
87325
|
+
...completenessOf(target),
|
|
86676
87326
|
total,
|
|
86677
87327
|
nextCursor: nextOffset < total ? encodeCursor2(nextOffset) : null,
|
|
86678
87328
|
templateDetection,
|
|
@@ -86711,7 +87361,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86711
87361
|
project: project.name,
|
|
86712
87362
|
hasCrawlData: true,
|
|
86713
87363
|
runId: snapshot.runId,
|
|
86714
|
-
...completenessOf(
|
|
87364
|
+
...completenessOf(target),
|
|
86715
87365
|
nodeKey: request.query.nodeKey ?? null,
|
|
86716
87366
|
url: request.query.url ?? null,
|
|
86717
87367
|
templateDetection,
|
|
@@ -86738,7 +87388,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86738
87388
|
project: project.name,
|
|
86739
87389
|
hasCrawlData: true,
|
|
86740
87390
|
runId: snapshot.runId,
|
|
86741
|
-
...completenessOf(
|
|
87391
|
+
...completenessOf(target),
|
|
86742
87392
|
nodeKey: request.query.nodeKey ?? null,
|
|
86743
87393
|
url: request.query.url ?? null,
|
|
86744
87394
|
templateDetection,
|
|
@@ -86759,13 +87409,13 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86759
87409
|
}
|
|
86760
87410
|
const snapshot = target.snapshot;
|
|
86761
87411
|
if (!snapshot.checkDeadLinks || snapshot.deadLinkState === "disabled") {
|
|
86762
|
-
return { project: project.name, runId: snapshot.runId, state: "disabled", checkDeadLinks: false };
|
|
87412
|
+
return { project: project.name, runId: snapshot.runId, state: "disabled", checkDeadLinks: false, runSelection: target.runSelection };
|
|
86763
87413
|
}
|
|
86764
87414
|
if (snapshot.deadLinkState === "unavailable") {
|
|
86765
|
-
return { project: project.name, runId: snapshot.runId, state: "unavailable", legacyAuditAvailable };
|
|
87415
|
+
return { project: project.name, runId: snapshot.runId, state: "unavailable", legacyAuditAvailable, runSelection: target.runSelection };
|
|
86766
87416
|
}
|
|
86767
87417
|
if (!snapshot.attemptId) {
|
|
86768
|
-
return { project: project.name, runId: snapshot.runId, state: "partial", checkDeadLinks: true, checked: snapshot.deadLinksChecked, found: snapshot.deadLinksFound, unverified: snapshot.deadLinksUnverified, total: 0, nextCursor: null, deadLinks: [] };
|
|
87418
|
+
return { project: project.name, runId: snapshot.runId, state: "partial", checkDeadLinks: true, checked: snapshot.deadLinksChecked, found: snapshot.deadLinksFound, unverified: snapshot.deadLinksUnverified, total: 0, nextCursor: null, deadLinks: [], runSelection: target.runSelection };
|
|
86769
87419
|
}
|
|
86770
87420
|
const where = and67(
|
|
86771
87421
|
eq80(siteCrawlFindings.projectId, project.id),
|
|
@@ -86781,7 +87431,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86781
87431
|
// a fabricated broken link shown to a client is not worth the saved
|
|
86782
87432
|
// predicate. Applied to `total` as well as the rows, or the count would
|
|
86783
87433
|
// promise a page the list cannot fill.
|
|
86784
|
-
isNotNull8(
|
|
87434
|
+
isNotNull8(sql30`json_extract(${siteCrawlFindings.evidence}, '$.statusCode')`)
|
|
86785
87435
|
);
|
|
86786
87436
|
const total = app.db.select({ value: count2() }).from(siteCrawlFindings).where(where).get()?.value ?? 0;
|
|
86787
87437
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
@@ -86801,6 +87451,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86801
87451
|
return {
|
|
86802
87452
|
project: project.name,
|
|
86803
87453
|
runId: snapshot.runId,
|
|
87454
|
+
runSelection: target.runSelection,
|
|
86804
87455
|
state,
|
|
86805
87456
|
checkDeadLinks: true,
|
|
86806
87457
|
checked: snapshot.deadLinksChecked,
|
|
@@ -86836,6 +87487,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86836
87487
|
)).all().map((row) => row.runId));
|
|
86837
87488
|
return {
|
|
86838
87489
|
project: project.name,
|
|
87490
|
+
preferredRunId: resolveCrawl(project.id)?.snapshot.runId ?? resolveAudit(project.id)?.snap.runId ?? null,
|
|
86839
87491
|
scans: rows.map((row) => ({
|
|
86840
87492
|
...row,
|
|
86841
87493
|
status: row.status,
|
|
@@ -87026,7 +87678,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87026
87678
|
// ../api-routes/src/research.ts
|
|
87027
87679
|
import crypto50 from "crypto";
|
|
87028
87680
|
import { z as z5 } from "zod";
|
|
87029
|
-
import { and as and68, count as count3, desc as desc37, eq as eq81, gte as
|
|
87681
|
+
import { and as and68, count as count3, desc as desc37, eq as eq81, gte as gte22, lt as lt14, or as or19, sql as sql31 } from "drizzle-orm";
|
|
87030
87682
|
var sameLocation = (a, b) => a.label === b.label && a.city === b.city && a.region === b.region && a.country === b.country && a.timezone === b.timezone;
|
|
87031
87683
|
var BATCH_RECEIPT_PREFIX = "__canonry_research_batch__:";
|
|
87032
87684
|
async function researchRoutes(app, opts) {
|
|
@@ -87086,9 +87738,9 @@ async function researchRoutes(app, opts) {
|
|
|
87086
87738
|
const { start, end, date } = utcDayBounds(now);
|
|
87087
87739
|
const used = tx.select({ value: count3() }).from(researchRuns).where(and68(
|
|
87088
87740
|
eq81(researchRuns.projectId, project.id),
|
|
87089
|
-
|
|
87741
|
+
gte22(researchRuns.createdAt, start),
|
|
87090
87742
|
lt14(researchRuns.createdAt, end),
|
|
87091
|
-
|
|
87743
|
+
sql31`(json_extract(${researchRuns.initiatedBy}, '$.role') = ${UserRoles.viewer} OR json_extract(${researchRuns.initiatedBy}, '$.limited') = 1)`
|
|
87092
87744
|
)).get()?.value ?? 0;
|
|
87093
87745
|
if (used >= viewerDailyRunLimit) {
|
|
87094
87746
|
throw researchDailyLimitExceeded(project.name, viewerDailyRunLimit, date);
|
|
@@ -87141,9 +87793,9 @@ async function researchRoutes(app, opts) {
|
|
|
87141
87793
|
const { start, end, date } = utcDayBounds(now);
|
|
87142
87794
|
const used = tx.select({ value: count3() }).from(researchRuns).where(and68(
|
|
87143
87795
|
eq81(researchRuns.projectId, project.id),
|
|
87144
|
-
|
|
87796
|
+
gte22(researchRuns.createdAt, start),
|
|
87145
87797
|
lt14(researchRuns.createdAt, end),
|
|
87146
|
-
|
|
87798
|
+
sql31`(json_extract(${researchRuns.initiatedBy}, '$.role') = ${UserRoles.viewer} OR json_extract(${researchRuns.initiatedBy}, '$.limited') = 1)`
|
|
87147
87799
|
)).get()?.value ?? 0;
|
|
87148
87800
|
if (used + prepared.length > viewerDailyRunLimit) throw researchDailyLimitExceeded(project.name, viewerDailyRunLimit, date);
|
|
87149
87801
|
}
|
|
@@ -88407,7 +89059,7 @@ function computeCitedCompetitorDomains(citedDomains, competitorDomains) {
|
|
|
88407
89059
|
var RECURRENCE_LOOKBACK_RUNS = 5;
|
|
88408
89060
|
var HISTORY_WINDOW_RUNS = Math.max(PERSISTENT_GAP_THRESHOLD, 5);
|
|
88409
89061
|
var log = createLogger("IntelligenceService");
|
|
88410
|
-
var runChronologyKey =
|
|
89062
|
+
var runChronologyKey = sql32`coalesce(${runs.finishedAt}, ${runs.createdAt})`;
|
|
88411
89063
|
function analyzableRunPredicates(projectId) {
|
|
88412
89064
|
return [
|
|
88413
89065
|
eq84(runs.projectId, projectId),
|
|
@@ -88475,7 +89127,7 @@ var IntelligenceService = class {
|
|
|
88475
89127
|
// MEASUREMENT: only runs that actually wrote snapshots — see above.
|
|
88476
89128
|
// Index seek on idx_snapshots_run, short-circuited on the first row.
|
|
88477
89129
|
exists2(
|
|
88478
|
-
this.db.select({ one:
|
|
89130
|
+
this.db.select({ one: sql32`1` }).from(querySnapshots).where(eq84(querySnapshots.runId, runs.id))
|
|
88479
89131
|
)
|
|
88480
89132
|
)
|
|
88481
89133
|
).orderBy(desc38(runs.finishedAt), desc38(runs.createdAt)).limit(HISTORY_WINDOW_RUNS).all();
|
|
@@ -88561,7 +89213,7 @@ var IntelligenceService = class {
|
|
|
88561
89213
|
const selected = this.db.select().from(gbpLocations).where(and71(
|
|
88562
89214
|
eq84(gbpLocations.projectId, projectId),
|
|
88563
89215
|
eq84(gbpLocations.selected, true),
|
|
88564
|
-
|
|
89216
|
+
gte23(gbpLocations.syncedAt, windowStart),
|
|
88565
89217
|
lte20(gbpLocations.syncedAt, windowEnd)
|
|
88566
89218
|
)).all();
|
|
88567
89219
|
if (selected.length === 0) {
|
|
@@ -88941,7 +89593,7 @@ var IntelligenceService = class {
|
|
|
88941
89593
|
position: gscSearchData.position
|
|
88942
89594
|
}).from(gscSearchData).where(and71(
|
|
88943
89595
|
eq84(gscSearchData.projectId, projectId),
|
|
88944
|
-
|
|
89596
|
+
gte23(gscSearchData.date, window.startDate),
|
|
88945
89597
|
lte20(gscSearchData.date, window.endDate)
|
|
88946
89598
|
)).all();
|
|
88947
89599
|
const byDay = /* @__PURE__ */ new Map();
|