@canonry/canonry 7.7.0 → 7.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agent-workspace/skills/aero/SKILL.md +43 -17
- package/assets/agent-workspace/skills/aero/references/agent-operations.md +4 -1
- package/assets/agent-workspace/skills/aero/references/portfolio-analysis.md +15 -7
- package/assets/agent-workspace/skills/canonry/SKILL.md +4 -1
- package/assets/assets/{AuditHistoryPanel-qumC-I56.js → AuditHistoryPanel-D0nukTwD.js} +1 -1
- package/assets/assets/{BacklinksPage-BNyYcY5w.js → BacklinksPage-B525ij0g.js} +1 -1
- package/assets/assets/{HistoryPage-CQfOM6NA.js → HistoryPage-DvupqQDM.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-D0LTQHL7.js → MeasurementPropertyPage-CwDbi1XN.js} +1 -1
- package/assets/assets/{ProjectPage-QQErJAT8.js → ProjectPage-DmZZ2C7q.js} +1 -1
- package/assets/assets/{RunsPage-CniAGDVF.js → RunsPage-DHTvHcGm.js} +1 -1
- package/assets/assets/{SettingsPage-BPsrE4ao.js → SettingsPage-BwAFzB7A.js} +1 -1
- package/assets/assets/SiteHealthSection-hi54eSIt.js +4 -0
- package/assets/assets/{TrafficPage-DXKk92zy.js → TrafficPage-BOcXASAM.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-4POLIA_c.js → TrafficSourceDetailPage-DTq0cN6g.js} +1 -1
- package/assets/assets/{extract-error-message-C6t0hB2w.js → extract-error-message-BOTDI6g5.js} +1 -1
- package/assets/assets/index-BLJfGYUw.js +86 -0
- package/assets/assets/{index-HWSQgVum.css → index-Rcbm8bK1.css} +1 -1
- package/assets/assets/{react-sigma_core.esm.min-D7GjzWYV.js → react-sigma_core.esm.min-BEBdJNtD.js} +1 -1
- package/assets/assets/{v2-overview-adapter-ChE5k_dV.js → v2-overview-adapter-BPZ-H3VY.js} +1 -1
- package/assets/index.html +2 -2
- package/dist/{chunk-ODEAE5T5.js → chunk-AAMKTL6B.js} +3 -3
- package/dist/{chunk-WJJTKZKK.js → chunk-DO7XPDG5.js} +5977 -5743
- package/dist/{chunk-75RSUOHZ.js → chunk-KRFTA4I7.js} +1126 -475
- package/dist/{chunk-WBTYXLXL.js → chunk-LO56QXPO.js} +5 -4
- package/dist/{chunk-LFUSODDL.js → chunk-NU62UT2S.js} +444 -158
- package/dist/{chunk-JJXTHJ4E.js → chunk-T2V2PESH.js} +4 -3
- package/dist/{chunk-3DB43KT2.js → chunk-ZLB7WX5D.js} +26 -33
- package/dist/cli.js +67 -28
- package/dist/{demo-server-UWBPY743.js → demo-server-KDMG3APY.js} +4 -4
- package/dist/index.js +6 -6
- package/dist/{intelligence-service-LOH37GPJ.js → intelligence-service-LAXJFE3U.js} +2 -2
- package/dist/mcp.js +4 -4
- package/package.json +9 -9
- package/assets/assets/SiteHealthSection-zW3ykBCu.js +0 -4
- package/assets/assets/index-DBQ-8jB1.js +0 -86
|
@@ -88,9 +88,11 @@ import {
|
|
|
88
88
|
MEASUREMENT_PLAN_V2_SCHEMA_VERSION,
|
|
89
89
|
MEASUREMENT_PORTFOLIO_ANSWER_SOURCES_LIMIT,
|
|
90
90
|
MEASUREMENT_PORTFOLIO_DEFAULT_LIMIT,
|
|
91
|
+
MEASUREMENT_PORTFOLIO_DEFAULT_ROW_EVIDENCE_LIMIT,
|
|
91
92
|
MEASUREMENT_PORTFOLIO_ROW_EVIDENCE_LIMIT,
|
|
92
93
|
MEASUREMENT_PORTFOLIO_TIE_NAMED_INSTEAD_LIMIT,
|
|
93
94
|
MEASUREMENT_PORTFOLIO_TIE_NOTE,
|
|
95
|
+
MEASUREMENT_PORTFOLIO_WEAKEST_MARKET_LIMIT,
|
|
94
96
|
MEASUREMENT_PROPERTY_CITED_DOMAINS_LIMIT,
|
|
95
97
|
MIN_BRAND_ALIAS_KEY_LENGTH,
|
|
96
98
|
MIN_DOMAIN_BRAND_KEY_LENGTH,
|
|
@@ -98,6 +100,7 @@ import {
|
|
|
98
100
|
MODEL_ATTRIBUTION_EVENT_LIMIT,
|
|
99
101
|
MeasurementEvidenceShapes,
|
|
100
102
|
MeasurementPlanValidationError,
|
|
103
|
+
MeasurementPortfolioLists,
|
|
101
104
|
MeasurementRunScopeError,
|
|
102
105
|
OPERATIONAL_LOG_FIELDS_HEADER,
|
|
103
106
|
OPERATIONAL_LOG_OPT_IN_CONTEXT_FIELDS,
|
|
@@ -143,6 +146,7 @@ import {
|
|
|
143
146
|
SentimentOutcomes,
|
|
144
147
|
SiteAuditTrendDirections,
|
|
145
148
|
SiteCrawlFetchedStates,
|
|
149
|
+
SiteCrawlIndexabilityReasons,
|
|
146
150
|
SiteHealthLinkKinds,
|
|
147
151
|
SiteHealthTemplateDetections,
|
|
148
152
|
TrafficEventConfidences,
|
|
@@ -250,6 +254,7 @@ import {
|
|
|
250
254
|
buildMeasurementRunManifestV1,
|
|
251
255
|
businessProfileReviewsCover,
|
|
252
256
|
calendarDateRange,
|
|
257
|
+
calendarDateSchema,
|
|
253
258
|
calendarMonthBounds,
|
|
254
259
|
calendarRecurrenceSchema,
|
|
255
260
|
canonicalMeasurementExecutionIdentityJson,
|
|
@@ -326,6 +331,7 @@ import {
|
|
|
326
331
|
deriveCitedUrlCandidates,
|
|
327
332
|
deriveConversionTrackingIntegrityStatus,
|
|
328
333
|
deriveGoogleAdsEffectiveGoalGraph,
|
|
334
|
+
deriveSiteHealthReason,
|
|
329
335
|
deriveSiteHealthState,
|
|
330
336
|
deriveWinnabilityClass,
|
|
331
337
|
describeCompetitorAliasRejection,
|
|
@@ -355,6 +361,7 @@ import {
|
|
|
355
361
|
extractAnswerMentions,
|
|
356
362
|
extractDomainsFromText,
|
|
357
363
|
extractRecommendedCompetitors,
|
|
364
|
+
factorStatusFromDistribution,
|
|
358
365
|
factorStatusFromScore,
|
|
359
366
|
feedbackAcceptedDtoSchema,
|
|
360
367
|
feedbackSubmissionSchema,
|
|
@@ -800,10 +807,10 @@ import {
|
|
|
800
807
|
wordpressSchemaDeployResultDtoSchema,
|
|
801
808
|
wordpressSchemaStatusResultDtoSchema,
|
|
802
809
|
wordpressStatusDtoSchema
|
|
803
|
-
} from "./chunk-
|
|
810
|
+
} from "./chunk-DO7XPDG5.js";
|
|
804
811
|
|
|
805
812
|
// src/intelligence-service.ts
|
|
806
|
-
import { eq as eq84, desc as desc38, asc as asc15, and as and71, ne as ne15, or as or20, inArray as inArray34, gte as
|
|
813
|
+
import { eq as eq84, desc as desc38, asc as asc15, and as and71, ne as ne15, or as or20, inArray as inArray34, gte as gte23, lte as lte20, isNull as isNull16, sql as sql32, exists as exists2 } from "drizzle-orm";
|
|
807
814
|
|
|
808
815
|
// ../db/src/client.ts
|
|
809
816
|
import { mkdirSync } from "fs";
|
|
@@ -10290,7 +10297,7 @@ function buildShareOfVoiceFrame(input) {
|
|
|
10290
10297
|
const competitorMentions = selected.reduce((sum, row) => sum + row.mentions, 0);
|
|
10291
10298
|
const denominator = input.projectMentions + competitorMentions;
|
|
10292
10299
|
const basis = input.tracked ? "tracked" : input.competitors.length > 0 ? "observed" : null;
|
|
10293
|
-
const reason = !input.classSelected ? "select-query-class" : basis === null ? "no-competitors" : !input.tracked && domains.length < MIN_OBSERVED_COMPETITORS ? "insufficient-observed" : input.answeredResults === 0 ? "no-answers" : denominator === 0 ? "no-mentions" : null;
|
|
10300
|
+
const reason = input.unavailableReason ?? (!input.classSelected ? "select-query-class" : basis === null ? "no-competitors" : !input.tracked && domains.length < MIN_OBSERVED_COMPETITORS ? "insufficient-observed" : input.answeredResults === 0 ? "no-answers" : denominator === 0 ? "no-mentions" : null);
|
|
10294
10301
|
return {
|
|
10295
10302
|
basis,
|
|
10296
10303
|
reason,
|
|
@@ -10458,6 +10465,7 @@ function buildCompetitorLandscapeHistory(options) {
|
|
|
10458
10465
|
const frame = buildShareOfVoiceFrame({
|
|
10459
10466
|
tracked: pinned.size > 0,
|
|
10460
10467
|
classSelected: options.shareOfVoiceEligible,
|
|
10468
|
+
unavailableReason: options.shareOfVoiceUnavailableReason,
|
|
10461
10469
|
projectMentions: projectRow.mentionCount,
|
|
10462
10470
|
answeredResults,
|
|
10463
10471
|
competitors: competitiveRows.map((row) => ({ domain: row.identity.domain, mentions: row.mentionCount }))
|
|
@@ -13525,18 +13533,12 @@ function sentimentSentenceSpans(text2) {
|
|
|
13525
13533
|
}
|
|
13526
13534
|
return spans.map((span, index2) => ({ id: `s${index2 + 1}`, ...span }));
|
|
13527
13535
|
}
|
|
13528
|
-
function
|
|
13536
|
+
function sentimentClassifierIdentity(source) {
|
|
13529
13537
|
const edge = source.edges[0];
|
|
13530
13538
|
const subject = { id: source.subject.key, displayName: source.subject.name, aliases: source.subject.aliases, qualifiedAliases: source.subject.identityAliases, urls: source.subject.urls, mentionNotApplicable: source.subject.mentionNotApplicable };
|
|
13531
13539
|
return {
|
|
13532
|
-
sourceSnapshotId: source.snapshotId,
|
|
13533
|
-
sourceText: source.sourceText,
|
|
13534
|
-
sourceTextHash: sentimentHash(source.sourceText),
|
|
13535
13540
|
subject,
|
|
13536
|
-
subjectHash: sentimentHash({ subject, context: edge.context }),
|
|
13537
13541
|
language: source.language,
|
|
13538
|
-
definition,
|
|
13539
|
-
sentences: sentimentSentenceSpans(source.sourceText),
|
|
13540
13542
|
context: {
|
|
13541
13543
|
queryId: edge.queryKey,
|
|
13542
13544
|
queryText: edge.queryText,
|
|
@@ -13551,12 +13553,27 @@ function sentimentClassifierInput(source, definition) {
|
|
|
13551
13553
|
}
|
|
13552
13554
|
};
|
|
13553
13555
|
}
|
|
13556
|
+
function sentimentSourceHashes(source, identity = sentimentClassifierIdentity(source)) {
|
|
13557
|
+
return { sourceTextHash: sentimentHash(source.sourceText), subjectHash: sentimentHash({ subject: identity.subject, context: source.edges[0].context }) };
|
|
13558
|
+
}
|
|
13559
|
+
function sentimentClassifierInput(source, definition) {
|
|
13560
|
+
const identity = sentimentClassifierIdentity(source);
|
|
13561
|
+
return {
|
|
13562
|
+
sourceSnapshotId: source.snapshotId,
|
|
13563
|
+
sourceText: source.sourceText,
|
|
13564
|
+
...identity,
|
|
13565
|
+
...sentimentSourceHashes(source, identity),
|
|
13566
|
+
definition,
|
|
13567
|
+
sentences: sentimentSentenceSpans(source.sourceText)
|
|
13568
|
+
};
|
|
13569
|
+
}
|
|
13554
13570
|
|
|
13555
13571
|
// ../api-routes/src/sentiment-service.ts
|
|
13556
13572
|
var DISCLOSURE = "Experimental English sentiment classification. Enabling sends each stored answer's text to TypeSafe with the frozen subject identity (name, aliases, qualified aliases and URLs), the tracked query text and query class, the answer engine with its requested and served models, the location, and internal query, subject, Property, group and market identifiers.";
|
|
13557
13573
|
var FULL_ROWS = { include: /* @__PURE__ */ new Set(["assessments", "locations"]), limit: null, after: null };
|
|
13558
13574
|
var COMPACT_ROWS = { include: /* @__PURE__ */ new Set(), limit: null, after: null };
|
|
13559
13575
|
var OPERATIONAL_OUTCOMES = /* @__PURE__ */ new Set([SentimentOutcomes.pending, SentimentOutcomes.running, SentimentOutcomes["waiting-to-retry"], SentimentOutcomes.failed, SentimentOutcomes.canceled]);
|
|
13576
|
+
var PREVIOUS_RATED_CANDIDATE_LIMIT = 50;
|
|
13560
13577
|
var RATED_OUTCOMES = /* @__PURE__ */ new Set([SentimentOutcomes.favorable, SentimentOutcomes.mixed, SentimentOutcomes.unfavorable]);
|
|
13561
13578
|
var SentimentService = class {
|
|
13562
13579
|
constructor(db, options) {
|
|
@@ -13761,11 +13778,11 @@ var SentimentService = class {
|
|
|
13761
13778
|
}
|
|
13762
13779
|
items(projectId, selection, source, withInput = false) {
|
|
13763
13780
|
if (!selection.runId || !selection.evaluationDefinitionId) return [];
|
|
13764
|
-
|
|
13781
|
+
this.definition(selection.evaluationDefinitionId);
|
|
13765
13782
|
const filter = sourceFilter(selection);
|
|
13766
13783
|
const allowed = new Map(source.assessments.map((item2) => {
|
|
13767
|
-
const
|
|
13768
|
-
return [`${item2.snapshotId}:${
|
|
13784
|
+
const hashes = sentimentSourceHashes(item2);
|
|
13785
|
+
return [`${item2.snapshotId}:${hashes.sourceTextHash}:${hashes.subjectHash}`, item2];
|
|
13769
13786
|
}));
|
|
13770
13787
|
const scope = and4(eq9(sentimentWorkItems.projectId, projectId), eq9(sentimentWorkItems.runId, selection.runId), eq9(sentimentWorkItems.evaluationDefinitionId, selection.evaluationDefinitionId));
|
|
13771
13788
|
const rows = this.db.select({
|
|
@@ -13792,7 +13809,7 @@ var SentimentService = class {
|
|
|
13792
13809
|
if (!inputs) return [item2];
|
|
13793
13810
|
const input = storedSentimentClassifierInputSchema.parse(inputs.get(work.id));
|
|
13794
13811
|
const primary = edges[0];
|
|
13795
|
-
const projected =
|
|
13812
|
+
const projected = sentimentClassifierIdentity({ ...member, edges }).context;
|
|
13796
13813
|
const context = { ...input.context, ...projected, queryText: primary.queryText };
|
|
13797
13814
|
const evidence = { assessmentId: work.id, runId: work.runId, sourceSnapshotId: work.snapshotId, sourceText: input.sourceText, sourceTextHash: work.sourceTextHash, subject: input.subject, subjectHash: work.subjectHash, context, evaluationDefinitionId: work.evaluationDefinitionId, outcome: item2.outcome, conclusion: classification?.kind === "classified" ? classification.conclusion : [], complaint: classification?.kind === "classified" ? classification.complaint : null, returnedModel: item2.returnedModel, reason: item2.reason };
|
|
13798
13815
|
return [{ ...item2, input, evidence }];
|
|
@@ -13928,7 +13945,8 @@ var SentimentService = class {
|
|
|
13928
13945
|
return { state: result.state, reason: result.reason, provisional: result.provisional, coverage: result.coverage, score: result.score, selection: result.selection, runIds: result.selection.runIds ?? (result.selection.runId ? [result.selection.runId] : []) };
|
|
13929
13946
|
};
|
|
13930
13947
|
const branded = headline("branded"), nonBrand = headline("non-brand");
|
|
13931
|
-
|
|
13948
|
+
const pooledOverall = { ...overallSentiment(reads, !configured), pooled: true };
|
|
13949
|
+
return { configured, branded, nonBrand, headlineQueryClass: "branded", pooledOverall, overall: pooledOverall };
|
|
13932
13950
|
}
|
|
13933
13951
|
evidence(projectId, query, limit, cursor) {
|
|
13934
13952
|
const { assessmentId, outcome: outcome2, ...sourceSelection } = query;
|
|
@@ -13949,13 +13967,52 @@ var SentimentService = class {
|
|
|
13949
13967
|
const page3 = items.slice(0, limit);
|
|
13950
13968
|
return { state, selection, items: page3.map((item2) => item2.evidence), nextCursor: items.length > limit ? this.sign({ projectId, selection, fingerprint: read.fingerprint, after: page3.at(-1).work.id }) : null };
|
|
13951
13969
|
}
|
|
13970
|
+
previousRatedRun(projectId, query, toRunId) {
|
|
13971
|
+
const target = this.db.select().from(runs).where(and4(eq9(runs.projectId, projectId), eq9(runs.id, toRunId))).get();
|
|
13972
|
+
if (!target) throw notFound("Run", toRunId);
|
|
13973
|
+
const selected = this.resolve(projectId, { ...query, runId: toRunId });
|
|
13974
|
+
if (!selected.source.assessments.length) {
|
|
13975
|
+
const incomplete = Boolean(selected.source.skipped["incomplete-run"]);
|
|
13976
|
+
throw validationError(incomplete ? "The target run is incomplete; no sentiment trend can be selected." : "The target run has no eligible sentiment population for this selection.", { reason: incomplete ? "target-incomplete" : "target-population-unavailable" });
|
|
13977
|
+
}
|
|
13978
|
+
const filter = sourceFilter(query);
|
|
13979
|
+
const population = (source) => new Set(source.assessments.map((member) => comparisonUnit(sentimentClassifierIdentity({ ...member, edges: member.edges.filter((edge) => matchesSentimentEdge(edge, filter)) }))));
|
|
13980
|
+
const targetUnits = population(selected.source);
|
|
13981
|
+
const requestedVersion = query.revision === void 0 ? void 0 : this.db.select({ id: measurementPlanVersions.id }).from(measurementPlanVersions).where(and4(eq9(measurementPlanVersions.projectId, projectId), eq9(measurementPlanVersions.revision, query.revision))).get()?.id ?? "__missing-revision__";
|
|
13982
|
+
const candidates = this.db.selectDistinct({ id: runs.id, createdAt: runs.createdAt }).from(runs).innerJoin(sentimentWorkItems, and4(eq9(sentimentWorkItems.runId, runs.id), eq9(sentimentWorkItems.projectId, projectId))).innerJoin(sentimentResults, eq9(sentimentResults.workItemId, sentimentWorkItems.id)).where(and4(
|
|
13983
|
+
eq9(runs.projectId, projectId),
|
|
13984
|
+
eq9(runs.kind, target.kind),
|
|
13985
|
+
notProbeRun(),
|
|
13986
|
+
inArray3(sentimentResults.outcome, [...RATED_OUTCOMES]),
|
|
13987
|
+
query.evaluationDefinitionId ? eq9(sentimentWorkItems.evaluationDefinitionId, query.evaluationDefinitionId) : void 0,
|
|
13988
|
+
query.queryClass ? sql6`exists (select 1 from json_each(${sentimentWorkItems.edges}) as edge where json_extract(edge.value, '$.queryClass') = ${query.queryClass})` : void 0,
|
|
13989
|
+
requestedVersion ? eq9(runs.measurementPlanVersionId, requestedVersion) : void 0,
|
|
13990
|
+
or2(lt2(runs.createdAt, target.createdAt), and4(eq9(runs.createdAt, target.createdAt), lt2(runs.id, target.id))),
|
|
13991
|
+
target.measurementPlanVersionId ? isNotNull2(runs.measurementPlanVersionId) : isNull2(runs.measurementPlanVersionId)
|
|
13992
|
+
)).orderBy(desc3(runs.createdAt), desc3(runs.id)).limit(PREVIOUS_RATED_CANDIDATE_LIMIT + 1).all();
|
|
13993
|
+
let newestRatedRunId;
|
|
13994
|
+
for (const candidate of candidates.slice(0, PREVIOUS_RATED_CANDIDATE_LIMIT)) {
|
|
13995
|
+
const read = this.resolve(projectId, { ...query, runId: candidate.id });
|
|
13996
|
+
if (!read.source.assessments.length) continue;
|
|
13997
|
+
const candidateUnits = population(read.source);
|
|
13998
|
+
if (candidateUnits.size !== targetUnits.size || [...targetUnits].some((key) => !candidateUnits.has(key))) {
|
|
13999
|
+
newestRatedRunId ??= candidate.id;
|
|
14000
|
+
continue;
|
|
14001
|
+
}
|
|
14002
|
+
if (this.items(projectId, read.selection, read.source).some((item2) => RATED_OUTCOMES.has(item2.outcome))) return candidate.id;
|
|
14003
|
+
}
|
|
14004
|
+
if (candidates.length > PREVIOUS_RATED_CANDIDATE_LIMIT) throw validationError("No compatible rated predecessor was found within the bounded search; select an older run explicitly.", { reason: "previous-rated-search-limit", candidateLimit: PREVIOUS_RATED_CANDIDATE_LIMIT, ...newestRatedRunId ? { newestRatedRunId } : {} });
|
|
14005
|
+
if (newestRatedRunId) throw validationError("Earlier rated runs exist, but their selected sentiment population differs from the target.", { reason: "previous-rated-population-changed", newestRatedRunId });
|
|
14006
|
+
throw validationError("No previous rated run exists for the selected sentiment scope.", { reason: "previous-rated-run-unavailable" });
|
|
14007
|
+
}
|
|
13952
14008
|
compare(projectId, query, fromRunId, toRunId) {
|
|
13953
14009
|
if (query.runIds) throw validationError("Comparison requires one explicit run per period.");
|
|
14010
|
+
if (fromRunId === toRunId) throw validationError("Sentiment comparison requires two distinct runs.", { reason: "same-run" });
|
|
14011
|
+
if (fromRunId === "previous-rated") fromRunId = this.previousRatedRun(projectId, query, toRunId);
|
|
13954
14012
|
const from = this.resolve(projectId, { ...query, runId: fromRunId }), to = this.resolve(projectId, { ...query, runId: toRunId });
|
|
13955
14013
|
const fromItems = this.items(projectId, from.selection, from.source, true), toItems = this.items(projectId, to.selection, to.source, true);
|
|
13956
|
-
const unit = (item2) => sentimentHash({ query: item2.evidence.context.queryId, queryText: item2.evidence.context.queryText, provider: item2.input.context.provider, subject: item2.input.subject, context: item2.evidence.context.locationContext, language: item2.input.language, usageEdges: item2.evidence.context.usageEdges });
|
|
13957
14014
|
const units = (items) => {
|
|
13958
|
-
const byItem = new Map(items.map((item2) => [item2,
|
|
14015
|
+
const byItem = new Map(items.map((item2) => [item2, comparisonUnit(item2.input, item2.evidence.context)])), byUnit = /* @__PURE__ */ new Map();
|
|
13959
14016
|
for (const [item2, key] of byItem) if (!byUnit.has(key)) byUnit.set(key, item2);
|
|
13960
14017
|
return { byItem, byUnit };
|
|
13961
14018
|
};
|
|
@@ -13988,6 +14045,9 @@ var SentimentService = class {
|
|
|
13988
14045
|
return { from: fromSummary, to: toSummary, verdict, favorableRateDelta: verdict && fromSummary.score.favorableRate !== null && toSummary.score.favorableRate !== null ? toSummary.score.favorableRate - fromSummary.score.favorableRate : null, refusalReasons: [...new Set(refusalReasons)], commonUnits: common.size, excludedFrom: expectedFrom - fromSummary.coverage.selected, excludedTo: expectedTo - toSummary.coverage.selected, changedScope, method: "wilson-independent-v1", limitation: SENTIMENT_INTERVAL_LIMITATION };
|
|
13989
14046
|
}
|
|
13990
14047
|
};
|
|
14048
|
+
function comparisonUnit(input, context = input.context) {
|
|
14049
|
+
return sentimentHash({ query: context.queryId, queryText: context.queryText, provider: input.context.provider, subject: input.subject, context: context.locationContext, language: input.language, usageEdges: context.usageEdges });
|
|
14050
|
+
}
|
|
13991
14051
|
var jobAttemptJoin = and4(eq9(sentimentJobItems.workItemId, sentimentAttempts.workItemId), eq9(sentimentJobItems.projectId, sentimentAttempts.projectId));
|
|
13992
14052
|
function emptySource() {
|
|
13993
14053
|
return { assessments: [], runIds: [], sourceCoverage: { expected: 0, completed: 0 }, sourceSlotIds: { expected: [], completed: [] }, skipped: {} };
|
|
@@ -15529,6 +15589,24 @@ function resolveMentionIdentity(text2, targets, aliases, citedTargetIds, ambigui
|
|
|
15529
15589
|
}
|
|
15530
15590
|
return state;
|
|
15531
15591
|
}
|
|
15592
|
+
function createTargetMentionReader(targets, ownedHosts = []) {
|
|
15593
|
+
const compiledAliases = compiledMentionAliases(targets);
|
|
15594
|
+
const mentionableIds = new Set(compiledAliases.map((alias2) => alias2.targetId));
|
|
15595
|
+
const aliases = indexMentionAliases(compiledAliases);
|
|
15596
|
+
const ambiguityPatterns = new Map(targets.map((target) => [target.id, identityAmbiguityPatterns(target.aliases)]));
|
|
15597
|
+
const routes = compiledTargetRoutes(targets);
|
|
15598
|
+
const normalizedOwnedHosts = ownedHosts.map(normalizedHost);
|
|
15599
|
+
return (answerText, targetIds, citedUrls = []) => {
|
|
15600
|
+
if (answerText === null || targetIds.length === 0) return null;
|
|
15601
|
+
const citedTargetIds = new Set(citedUrls.flatMap((url) => {
|
|
15602
|
+
const source = classifySourceAttribution(url, routes, normalizedOwnedHosts);
|
|
15603
|
+
return source.classification === "matched" ? source.matchedTargetIds : [];
|
|
15604
|
+
}));
|
|
15605
|
+
const state = resolveMentionIdentity(answerProseForMentions(answerText), targets, aliases, citedTargetIds, ambiguityPatterns);
|
|
15606
|
+
if (targetIds.some((id) => state.mentioned.has(id))) return true;
|
|
15607
|
+
return targetIds.some((id) => !mentionableIds.has(id) || state.unknown.has(id)) ? null : false;
|
|
15608
|
+
};
|
|
15609
|
+
}
|
|
15532
15610
|
function observationSource(observation) {
|
|
15533
15611
|
if (observation.citedUrls !== null) {
|
|
15534
15612
|
return {
|
|
@@ -16253,6 +16331,10 @@ function historicalEvidence(rawResponse) {
|
|
|
16253
16331
|
return { urls: [], complete: false };
|
|
16254
16332
|
}
|
|
16255
16333
|
}
|
|
16334
|
+
function measurementSnapshotSources(snapshot) {
|
|
16335
|
+
if (snapshot.citedUrls !== null) return { urls: snapshot.citedUrls, complete: snapshot.captureStatus === "complete", historical: false };
|
|
16336
|
+
return { ...historicalEvidence(snapshot.rawResponse), historical: true };
|
|
16337
|
+
}
|
|
16256
16338
|
function slotLocation(slot) {
|
|
16257
16339
|
return slot.context?.label ?? null;
|
|
16258
16340
|
}
|
|
@@ -16301,7 +16383,7 @@ function observationInputs(manifest, snapshots, legacy) {
|
|
|
16301
16383
|
if (!supportsRequestedContext(snapshot, slot)) return [];
|
|
16302
16384
|
validateSupportedLocation(snapshot, slot);
|
|
16303
16385
|
}
|
|
16304
|
-
const
|
|
16386
|
+
const sources = measurementSnapshotSources(snapshot);
|
|
16305
16387
|
return [{
|
|
16306
16388
|
id: snapshot.id,
|
|
16307
16389
|
executionId: snapshot.measurementExecutionId,
|
|
@@ -16309,12 +16391,9 @@ function observationInputs(manifest, snapshots, legacy) {
|
|
|
16309
16391
|
provider: snapshot.provider.trim().toLocaleLowerCase("en"),
|
|
16310
16392
|
location: snapshot.location,
|
|
16311
16393
|
answerText: snapshot.answerText,
|
|
16312
|
-
citedUrls:
|
|
16313
|
-
citedUrlsComplete:
|
|
16314
|
-
...
|
|
16315
|
-
const historical = historicalEvidence(snapshot.rawResponse);
|
|
16316
|
-
return { historicalCitedUrls: historical.urls, historicalCitedUrlsComplete: historical.complete };
|
|
16317
|
-
})() : {}
|
|
16394
|
+
citedUrls: sources.historical ? null : sources.urls,
|
|
16395
|
+
citedUrlsComplete: !sources.historical && sources.complete,
|
|
16396
|
+
...sources.historical ? { historicalCitedUrls: sources.urls, historicalCitedUrlsComplete: sources.complete } : {}
|
|
16318
16397
|
}];
|
|
16319
16398
|
});
|
|
16320
16399
|
}
|
|
@@ -16349,6 +16428,15 @@ function reportInput(revision, plan, manifest, snapshots, legacy) {
|
|
|
16349
16428
|
function v2UsageEdgeId(edge) {
|
|
16350
16429
|
return `target:${edge.targetKey}:${edge.queryId}:${edge.executionNodeKey}`;
|
|
16351
16430
|
}
|
|
16431
|
+
function measurementPlanV2ReportTargets(plan) {
|
|
16432
|
+
return plan.targets.map((target) => ({
|
|
16433
|
+
id: target.stableKey,
|
|
16434
|
+
label: target.label,
|
|
16435
|
+
aliases: target.mentionNotApplicable ? [] : target.aliases,
|
|
16436
|
+
...target.identityAliases === void 0 ? {} : { identityAliases: target.identityAliases },
|
|
16437
|
+
urls: target.urlMatchers.map((matcher, index2) => matcherInput(target.stableKey, matcher, index2))
|
|
16438
|
+
}));
|
|
16439
|
+
}
|
|
16352
16440
|
function buildMeasurementPlanV2ReportInput(revision, plan, manifest, snapshots) {
|
|
16353
16441
|
const classByAssignment = new Map(plan.assignments.map((assignment) => [
|
|
16354
16442
|
`${assignment.targetKey}:${assignment.queryId}`,
|
|
@@ -16366,15 +16454,7 @@ function buildMeasurementPlanV2ReportInput(revision, plan, manifest, snapshots)
|
|
|
16366
16454
|
ownedHosts: plan.identities.projectBrand.ownedHosts,
|
|
16367
16455
|
projectBrandNames: plan.identities.projectBrand.names,
|
|
16368
16456
|
projectDomain: plan.identities.projectBrand.canonicalHost,
|
|
16369
|
-
targets: plan
|
|
16370
|
-
id: target.stableKey,
|
|
16371
|
-
label: target.label,
|
|
16372
|
-
// The revision already decided this Property cannot be mentioned. Feeding
|
|
16373
|
-
// its aliases in anyway would turn "not applicable" into a 0% reading.
|
|
16374
|
-
aliases: target.mentionNotApplicable ? [] : target.aliases,
|
|
16375
|
-
...target.identityAliases === void 0 ? {} : { identityAliases: target.identityAliases },
|
|
16376
|
-
urls: target.urlMatchers.map((matcher, index2) => matcherInput(target.stableKey, matcher, index2))
|
|
16377
|
-
})),
|
|
16457
|
+
targets: measurementPlanV2ReportTargets(plan),
|
|
16378
16458
|
groups: plan.groups.map((group) => ({
|
|
16379
16459
|
id: group.stableKey,
|
|
16380
16460
|
label: group.label,
|
|
@@ -18358,7 +18438,7 @@ function parseBody(schema, value, message) {
|
|
|
18358
18438
|
}
|
|
18359
18439
|
|
|
18360
18440
|
// ../api-routes/src/competitor-landscape.ts
|
|
18361
|
-
import { and as and17, desc as desc8, eq as eq24, gte as gte3, inArray as inArray10, isNull as isNull6 } from "drizzle-orm";
|
|
18441
|
+
import { and as and17, desc as desc8, eq as eq24, gte as gte3, inArray as inArray10, isNull as isNull6, sql as sql9 } from "drizzle-orm";
|
|
18362
18442
|
|
|
18363
18443
|
// ../api-routes/src/mention-share-inputs.ts
|
|
18364
18444
|
import { eq as eq22 } from "drizzle-orm";
|
|
@@ -18494,7 +18574,11 @@ function createMeasurementOverviewCache(maxEntries = MEASUREMENT_OVERVIEW_CACHE_
|
|
|
18494
18574
|
};
|
|
18495
18575
|
}
|
|
18496
18576
|
function parseOverviewQuery(raw) {
|
|
18497
|
-
const candidate = {
|
|
18577
|
+
const candidate = {
|
|
18578
|
+
...raw,
|
|
18579
|
+
...raw.limit === void 0 ? {} : { limit: Number(raw.limit) },
|
|
18580
|
+
...raw.compact === "true" || raw.compact === "false" ? { compact: raw.compact === "true" } : {}
|
|
18581
|
+
};
|
|
18498
18582
|
const parsed = measurementOverviewQuerySchema.safeParse(candidate);
|
|
18499
18583
|
if (!parsed.success) throw validationError("Invalid measurement overview query", { issues: parsed.error.issues });
|
|
18500
18584
|
return parsed.data;
|
|
@@ -18564,7 +18648,8 @@ function overviewFilterFingerprint(query) {
|
|
|
18564
18648
|
location: query.location === void 0 ? null : normalizedText(query.location),
|
|
18565
18649
|
from: query.from ?? null,
|
|
18566
18650
|
to: query.to ?? null,
|
|
18567
|
-
search: query.search === void 0 ? null : normalizedText(query.search)
|
|
18651
|
+
search: query.search === void 0 ? null : normalizedText(query.search),
|
|
18652
|
+
compact: query.compact ?? false
|
|
18568
18653
|
};
|
|
18569
18654
|
return createHash4("sha256").update(JSON.stringify(filters)).digest("base64url");
|
|
18570
18655
|
}
|
|
@@ -18726,12 +18811,19 @@ function pageOf(rows, query, displayedRunId, activePlanVersionId, evidenceFinger
|
|
|
18726
18811
|
if (index2 < 0) throw validationError("The measurement overview cursor does not belong to this result set.");
|
|
18727
18812
|
offset = index2 + 1;
|
|
18728
18813
|
}
|
|
18729
|
-
const
|
|
18814
|
+
const selected = ordered.slice(offset, offset + limit);
|
|
18815
|
+
const items = query.compact ? selected.map((row) => ({ ...row, providers: [] })) : selected;
|
|
18816
|
+
if (query.compact) {
|
|
18817
|
+
while (JSON.stringify(items, null, 2).length > 12e3 && items.length > 1) items.pop();
|
|
18818
|
+
if (JSON.stringify(items, null, 2).length > 12e3) {
|
|
18819
|
+
throw validationError("One measurement Property row exceeds the compact page budget. Read a narrower scope without compact=true.");
|
|
18820
|
+
}
|
|
18821
|
+
}
|
|
18730
18822
|
const last = items.at(-1);
|
|
18731
18823
|
return {
|
|
18732
18824
|
page: {
|
|
18733
18825
|
items,
|
|
18734
|
-
nextCursor: last === void 0 || ordered.at(offset +
|
|
18826
|
+
nextCursor: last === void 0 || ordered.at(offset + items.length) === void 0 ? null : cursorOf(
|
|
18735
18827
|
last,
|
|
18736
18828
|
sort,
|
|
18737
18829
|
displayedRunId,
|
|
@@ -19098,6 +19190,16 @@ async function measurementOverviewRoutes(app, options = {}) {
|
|
|
19098
19190
|
if (!active) throw notFound("Active measurement plan", project.name);
|
|
19099
19191
|
const scope = resolveScope(active.plan, query);
|
|
19100
19192
|
const response = active.plan.schemaVersion === MEASUREMENT_PLAN_V2_SCHEMA_VERSION ? planV2Overview(app.db, project.id, active, active.plan, query, scope, cache) : planV1Overview(app.db, project.id, active, query, scope);
|
|
19193
|
+
if (query.compact) {
|
|
19194
|
+
response.detailsOmitted = ["properties.providers", ...response.detailsOmitted ?? []];
|
|
19195
|
+
if (response.namedShareOfVoice !== void 0) {
|
|
19196
|
+
delete response.namedShareOfVoice;
|
|
19197
|
+
response.detailsOmitted.push("namedShareOfVoice");
|
|
19198
|
+
}
|
|
19199
|
+
}
|
|
19200
|
+
if (query.compact && JSON.stringify(response, null, 2).length > 18e3) {
|
|
19201
|
+
throw validationError("Measurement overview metadata exceeds the compact page budget. Read a narrower scope without compact=true.");
|
|
19202
|
+
}
|
|
19101
19203
|
return measurementOverviewResponseSchema.parse(response);
|
|
19102
19204
|
}
|
|
19103
19205
|
);
|
|
@@ -19192,7 +19294,7 @@ function readCompetitorLandscape(app, projectName, query, selection) {
|
|
|
19192
19294
|
throw validationError("Invalid competitor landscape query", { issues: parsed.error.issues });
|
|
19193
19295
|
}
|
|
19194
19296
|
const filters = parsed.data;
|
|
19195
|
-
const classSplitByDefault = filters.scope === void 0 && filters.groupKey === void 0 && filters.queryClass !== void 0 && filters.queryClass !== "all";
|
|
19297
|
+
const classSplitByDefault = filters.scope === void 0 && filters.groupKey === void 0 && (filters.queryClass !== void 0 && filters.queryClass !== "all" || filters.answers === "not-mentioned");
|
|
19196
19298
|
if ((selection?.autoAdvanced || classSplitByDefault) && activeMeasurementPlan(app.db, project.id)?.plan.schemaVersion === 2) filters.scope = "all-markets";
|
|
19197
19299
|
const window = parseWindow(filters.window);
|
|
19198
19300
|
const cutoff = windowCutoff(window);
|
|
@@ -19226,6 +19328,7 @@ function readCompetitorLandscape(app, projectName, query, selection) {
|
|
|
19226
19328
|
citedDomains: querySnapshots.citedDomains,
|
|
19227
19329
|
citedUrls: querySnapshots.citedUrls,
|
|
19228
19330
|
captureStatus: querySnapshots.captureStatus,
|
|
19331
|
+
rawResponse: advanced && filters.answers === "not-mentioned" ? sql9`case when ${querySnapshots.citedUrls} is null then ${querySnapshots.rawResponse} else null end` : sql9`null`,
|
|
19229
19332
|
location: querySnapshots.location,
|
|
19230
19333
|
measurementExecutionId: querySnapshots.measurementExecutionId,
|
|
19231
19334
|
createdAt: querySnapshots.createdAt
|
|
@@ -19245,7 +19348,14 @@ function readCompetitorLandscape(app, projectName, query, selection) {
|
|
|
19245
19348
|
const run = runById.get(snapshot.runId);
|
|
19246
19349
|
return run !== void 0 && run.trigger !== "probe" && run.status !== RunStatuses.completed && run.status !== RunStatuses.partial;
|
|
19247
19350
|
}).length;
|
|
19248
|
-
const
|
|
19351
|
+
const eligibleSnapshots = inScope.filter((snapshot) => eligibleRuns.has(snapshot.runId));
|
|
19352
|
+
let unknownMentionAnswers = 0;
|
|
19353
|
+
const simpleMentionInputs = !advanced && filters.answers === "not-mentioned" ? buildMentionShareInputs({ project, competitors: [], snapshots: eligibleSnapshots, queryTextById }) : void 0;
|
|
19354
|
+
const snapshots = filters.answers !== "not-mentioned" ? eligibleSnapshots : eligibleSnapshots.filter((snapshot, index2) => {
|
|
19355
|
+
const mentioned = advanced ? advanced.runScopes.get(snapshot.runId)?.mentioned(snapshot.answerText, snapshot.measurementExecutionId, measurementSnapshotSources(snapshot).urls) ?? null : snapshot.answerText === null || !simpleMentionInputs.classified ? null : simpleMentionInputs.snapshots[index2].projectMentioned;
|
|
19356
|
+
if (mentioned === null) unknownMentionAnswers++;
|
|
19357
|
+
return mentioned === false;
|
|
19358
|
+
});
|
|
19249
19359
|
const countedRunIds = new Set(snapshots.map((snapshot) => snapshot.runId));
|
|
19250
19360
|
const countedRuns = candidateRuns.filter((run) => countedRunIds.has(run.id));
|
|
19251
19361
|
const classificationRows = app.db.select({
|
|
@@ -19284,6 +19394,7 @@ function readCompetitorLandscape(app, projectName, query, selection) {
|
|
|
19284
19394
|
// Share of voice needs one query class behind it. `all` pools branded
|
|
19285
19395
|
// and non-brand, so withhold the ratio and publish the counts instead.
|
|
19286
19396
|
shareOfVoiceEligible: filters.queryClass !== void 0 && filters.queryClass !== "all",
|
|
19397
|
+
...filters.answers === "not-mentioned" ? { shareOfVoiceUnavailableReason: "answer-selection" } : {},
|
|
19287
19398
|
snapshots: selectedSnapshots.map((snapshot, i) => ({
|
|
19288
19399
|
id: snapshot.id,
|
|
19289
19400
|
createdAt: snapshot.createdAt,
|
|
@@ -19373,12 +19484,16 @@ function readCompetitorLandscape(app, projectName, query, selection) {
|
|
|
19373
19484
|
...filters.groupBy !== void 0 ? { groupBy: filters.groupBy } : {},
|
|
19374
19485
|
queryClass: filters.queryClass ?? "all",
|
|
19375
19486
|
location: filters.location ?? null,
|
|
19376
|
-
runId: filters.runId ?? null
|
|
19487
|
+
runId: filters.runId ?? null,
|
|
19488
|
+
...filters.answers === void 0 ? {} : { answers: filters.answers }
|
|
19377
19489
|
},
|
|
19378
19490
|
truncated,
|
|
19379
19491
|
runCount: countedRuns.length,
|
|
19380
19492
|
runIds: pooledRunIds(countedRuns),
|
|
19381
19493
|
countUnits: COMPETITOR_LANDSCAPE_COUNT_UNITS,
|
|
19494
|
+
...filters.answers !== "not-mentioned" ? {} : {
|
|
19495
|
+
answerSelection: { answers: filters.answers, populationSize: eligibleSnapshots.length, answerCount: snapshots.length, unknownMentionAnswers }
|
|
19496
|
+
},
|
|
19382
19497
|
...modelComparison ? { modelComparison } : {}
|
|
19383
19498
|
};
|
|
19384
19499
|
return response;
|
|
@@ -19465,9 +19580,21 @@ function scopeForFrozenPlan(plan, kind, groupKey) {
|
|
|
19465
19580
|
classes.add(assignment.queryClass);
|
|
19466
19581
|
queryClassesByExecution.set(assignment.executionNodeKey, classes);
|
|
19467
19582
|
}
|
|
19583
|
+
const targets = measurementPlanV2ReportTargets(plan);
|
|
19584
|
+
const readMention = createTargetMentionReader(targets, plan.identities.projectBrand.ownedHosts);
|
|
19585
|
+
const targetIdsByExecution = /* @__PURE__ */ new Map();
|
|
19586
|
+
for (const edge of plan.usageEdges) {
|
|
19587
|
+
const ids = targetIdsByExecution.get(edge.executionNodeKey) ?? /* @__PURE__ */ new Set();
|
|
19588
|
+
ids.add(edge.targetKey);
|
|
19589
|
+
targetIdsByExecution.set(edge.executionNodeKey, ids);
|
|
19590
|
+
}
|
|
19468
19591
|
return {
|
|
19469
19592
|
executionNodeKeys,
|
|
19470
19593
|
queryClassesByExecution,
|
|
19594
|
+
mentioned: (answerText, executionId, citedUrls) => {
|
|
19595
|
+
const ids = executionId === null ? [] : [...targetIdsByExecution.get(executionId) ?? []];
|
|
19596
|
+
return readMention(answerText, ids, citedUrls);
|
|
19597
|
+
},
|
|
19471
19598
|
competitors: mergePins(groups.flatMap((group) => group.competitors.map((competitor) => ({
|
|
19472
19599
|
domain: competitor.domain,
|
|
19473
19600
|
label: competitor.label,
|
|
@@ -19551,12 +19678,12 @@ function mergePins(...collections) {
|
|
|
19551
19678
|
|
|
19552
19679
|
// ../api-routes/src/runs.ts
|
|
19553
19680
|
import crypto15 from "crypto";
|
|
19554
|
-
import { and as and19, eq as eq26, asc as asc3, desc as desc10, inArray as inArray12, or as or6, sql as
|
|
19681
|
+
import { and as and19, eq as eq26, asc as asc3, desc as desc10, inArray as inArray12, or as or6, sql as sql11 } from "drizzle-orm";
|
|
19555
19682
|
import { gte as gte4 } from "drizzle-orm";
|
|
19556
19683
|
|
|
19557
19684
|
// ../api-routes/src/run-fill.ts
|
|
19558
19685
|
import crypto14 from "crypto";
|
|
19559
|
-
import { and as and18, desc as desc9, eq as eq25, gt as gt4, inArray as inArray11, isNull as isNull7, ne as ne5, or as or5, sql as
|
|
19686
|
+
import { and as and18, desc as desc9, eq as eq25, gt as gt4, inArray as inArray11, isNull as isNull7, ne as ne5, or as or5, sql as sql10 } from "drizzle-orm";
|
|
19560
19687
|
var ACTIVE_STATUSES = ["queued", "running"];
|
|
19561
19688
|
function normalizeProvider(value) {
|
|
19562
19689
|
return value.trim().toLocaleLowerCase("en");
|
|
@@ -19587,7 +19714,7 @@ function newerFullSweep(db, run) {
|
|
|
19587
19714
|
gt4(runs.createdAt, run.createdAt),
|
|
19588
19715
|
or5(
|
|
19589
19716
|
inArray11(runs.status, ["queued", "running", "completed", "partial"]),
|
|
19590
|
-
|
|
19717
|
+
sql10`EXISTS (SELECT 1 FROM ${querySnapshots} WHERE ${querySnapshots.runId} = ${runs.id})`
|
|
19591
19718
|
)
|
|
19592
19719
|
)).orderBy(desc9(runs.createdAt)).get();
|
|
19593
19720
|
}
|
|
@@ -20007,7 +20134,7 @@ async function runRoutes(app, opts) {
|
|
|
20007
20134
|
});
|
|
20008
20135
|
app.get("/projects/:name/runs/latest", async (request, reply) => {
|
|
20009
20136
|
const project = resolveProject(app.db, request.params.name);
|
|
20010
|
-
const countRow = app.db.select({ count:
|
|
20137
|
+
const countRow = app.db.select({ count: sql11`count(*)` }).from(runs).where(and19(eq26(runs.projectId, project.id), notProbeRun())).get();
|
|
20011
20138
|
const totalRuns = countRow?.count ?? 0;
|
|
20012
20139
|
const latestRun = app.db.select().from(runs).where(and19(eq26(runs.projectId, project.id), notProbeRun())).orderBy(desc10(runs.createdAt), desc10(runs.id)).limit(1).get();
|
|
20013
20140
|
if (!latestRun) {
|
|
@@ -25218,10 +25345,11 @@ async function measurementPropertyEvidenceRoutes(app) {
|
|
|
25218
25345
|
}
|
|
25219
25346
|
|
|
25220
25347
|
// ../api-routes/src/measurement-portfolio-reads.ts
|
|
25348
|
+
import { createHash as createHash6 } from "crypto";
|
|
25221
25349
|
import { and as and25, desc as desc13, eq as eq32, inArray as inArray15, lt as lt5, or as or7 } from "drizzle-orm";
|
|
25222
25350
|
|
|
25223
25351
|
// ../api-routes/src/measurement-question-reads.ts
|
|
25224
|
-
import { and as and24, eq as eq31, getTableColumns, inArray as inArray14, sql as
|
|
25352
|
+
import { and as and24, eq as eq31, getTableColumns, inArray as inArray14, sql as sql12 } from "drizzle-orm";
|
|
25225
25353
|
var DEFAULT_LIMIT2 = 50;
|
|
25226
25354
|
var TERMINAL_STATUSES = /* @__PURE__ */ new Set([
|
|
25227
25355
|
RunStatuses.completed,
|
|
@@ -25340,7 +25468,7 @@ function selectedQuestionSnapshots(db, runId, manifest, scope) {
|
|
|
25340
25468
|
conditions.push(inArray14(querySnapshots.measurementExecutionId, executionIds));
|
|
25341
25469
|
}
|
|
25342
25470
|
if (scope?.provider !== void 0) {
|
|
25343
|
-
conditions.push(
|
|
25471
|
+
conditions.push(sql12`lower(trim(${querySnapshots.provider})) = ${normalizeText(scope.provider)}`);
|
|
25344
25472
|
}
|
|
25345
25473
|
return db.select(measurementQuestionSnapshotColumns).from(querySnapshots).where(and24(...conditions)).all();
|
|
25346
25474
|
}
|
|
@@ -25657,6 +25785,12 @@ function countMetric2(rate2) {
|
|
|
25657
25785
|
if (rate2.numerator === null) return unavailable3(metricReason2(rate2.reason));
|
|
25658
25786
|
return { state: "available", value: rate2.numerator, numerator: rate2.numerator, denominator: rate2.denominator };
|
|
25659
25787
|
}
|
|
25788
|
+
function neverMentionedMetric(rate2, propertyCount) {
|
|
25789
|
+
if (rate2.numerator === null || rate2.denominator === null) return unavailable3(metricReason2(rate2.reason));
|
|
25790
|
+
if (rate2.denominator !== propertyCount) return unavailable3("not_applicable");
|
|
25791
|
+
const neverMentioned = rate2.denominator - rate2.numerator;
|
|
25792
|
+
return { state: "available", value: neverMentioned, numerator: neverMentioned, denominator: rate2.denominator };
|
|
25793
|
+
}
|
|
25660
25794
|
function materializeWithDb(db, active, plan, run) {
|
|
25661
25795
|
return materializeMeasurementQuestionRun(db, active, plan, run);
|
|
25662
25796
|
}
|
|
@@ -25790,7 +25924,8 @@ function answerDomains(answer) {
|
|
|
25790
25924
|
function domainRows(answers, limit) {
|
|
25791
25925
|
const counts2 = /* @__PURE__ */ new Map();
|
|
25792
25926
|
for (const answer of answers) {
|
|
25793
|
-
|
|
25927
|
+
const domains = new Set([...answerDomains(answer)].map((host) => registrableDomain(host) || host));
|
|
25928
|
+
for (const domain of domains) counts2.set(domain, (counts2.get(domain) ?? 0) + 1);
|
|
25794
25929
|
}
|
|
25795
25930
|
const rows = [...counts2].map(([domain, count4]) => ({ domain, answers: count4 })).sort((left, right) => right.answers - left.answers || compareText13(left.domain, right.domain));
|
|
25796
25931
|
return { rows: rows.slice(0, limit), total: rows.length };
|
|
@@ -25846,8 +25981,8 @@ function recommendationRows(target, population) {
|
|
|
25846
25981
|
};
|
|
25847
25982
|
}).sort((left, right) => right.occurrences - left.occurrences || compareText13(left.name, right.name));
|
|
25848
25983
|
}
|
|
25849
|
-
function namedInsteadFields(names) {
|
|
25850
|
-
const returned = names.slice(0,
|
|
25984
|
+
function namedInsteadFields(names, limit = MEASUREMENT_PORTFOLIO_ROW_EVIDENCE_LIMIT) {
|
|
25985
|
+
const returned = names.slice(0, limit);
|
|
25851
25986
|
return {
|
|
25852
25987
|
namedInsteadInAnswerText: returned.map(({ name, occurrences }) => ({ name, answers: occurrences })),
|
|
25853
25988
|
namedInsteadInAnswerTextTotal: names.length,
|
|
@@ -25940,6 +26075,7 @@ function marketRollup(plan, measured, scope, includeNested) {
|
|
|
25940
26075
|
...identity,
|
|
25941
26076
|
propertyCount: group.targetKeys.length,
|
|
25942
26077
|
propertiesMentioned: unavailable3("no_completed_run"),
|
|
26078
|
+
propertiesNeverMentioned: unavailable3("no_completed_run"),
|
|
25943
26079
|
mentionCoverage: unavailable3("no_completed_run"),
|
|
25944
26080
|
citationCoverage: unavailable3("no_completed_run")
|
|
25945
26081
|
};
|
|
@@ -25950,12 +26086,25 @@ function marketRollup(plan, measured, scope, includeNested) {
|
|
|
25950
26086
|
...identity,
|
|
25951
26087
|
propertyCount: targetKeys.length,
|
|
25952
26088
|
propertiesMentioned: countMetric2(overview.propertiesMentioned),
|
|
26089
|
+
propertiesNeverMentioned: neverMentionedMetric(overview.propertiesMentioned, targetKeys.length),
|
|
25953
26090
|
mentionCoverage: coverageMetric2(overview.mentionCoverage),
|
|
25954
26091
|
citationCoverage: coverageMetric2(overview.citationCoverage)
|
|
25955
26092
|
};
|
|
25956
26093
|
}).sort(compareWeakestMarket);
|
|
25957
26094
|
return { markets, totalMarkets: markets.length, marketsTruncated: false };
|
|
25958
26095
|
}
|
|
26096
|
+
function weakestMarkets(markets, queryClass) {
|
|
26097
|
+
const topLevel = markets.filter((market) => market.parentGroupKey === null);
|
|
26098
|
+
const eligible = topLevel.flatMap((market) => market.mentionCoverage.state === "available" && market.citationCoverage.state === "available" ? [{ ...market, mentionCoverage: market.mentionCoverage, citationCoverage: market.citationCoverage }] : []);
|
|
26099
|
+
return {
|
|
26100
|
+
population: "top-level-markets",
|
|
26101
|
+
queryClass,
|
|
26102
|
+
totalMarkets: topLevel.length,
|
|
26103
|
+
eligibleMarketCount: eligible.length,
|
|
26104
|
+
excludedMarketCount: topLevel.length - eligible.length,
|
|
26105
|
+
items: eligible.sort(compareWeakestMarket).slice(0, MEASUREMENT_PORTFOLIO_WEAKEST_MARKET_LIMIT).map(({ groupKey, label, propertyCount, mentionCoverage, citationCoverage }) => ({ groupKey, label, propertyCount, mentionCoverage, citationCoverage }))
|
|
26106
|
+
};
|
|
26107
|
+
}
|
|
25959
26108
|
function compareWeakestMarket(a, b) {
|
|
25960
26109
|
const aUnavailable = a.mentionCoverage.state === "unavailable" || a.citationCoverage.state === "unavailable";
|
|
25961
26110
|
const bUnavailable = b.mentionCoverage.state === "unavailable" || b.citationCoverage.state === "unavailable";
|
|
@@ -25998,9 +26147,182 @@ function mentionRanking(rows, limit) {
|
|
|
25998
26147
|
truncated: eligible.length > limit
|
|
25999
26148
|
};
|
|
26000
26149
|
}
|
|
26150
|
+
function portfolioFilterFingerprint(query) {
|
|
26151
|
+
return createHash6("sha256").update(JSON.stringify([
|
|
26152
|
+
query.groupKey ?? null,
|
|
26153
|
+
query.queryClass,
|
|
26154
|
+
query.provider ?? null,
|
|
26155
|
+
query.location === void 0 ? null : normalizeMeasurementLocation(query.location),
|
|
26156
|
+
query.includeNestedMarkets ?? false,
|
|
26157
|
+
query.answers === "not-mentioned" ? query.answers : null,
|
|
26158
|
+
query.list ?? MeasurementPortfolioLists["weakest-properties"]
|
|
26159
|
+
])).digest("base64url");
|
|
26160
|
+
}
|
|
26161
|
+
function portfolioCursor(query, versionId) {
|
|
26162
|
+
if (query.cursor === void 0) return void 0;
|
|
26163
|
+
if (!query.compact) throw validationError("A portfolio cursor requires compact=true.");
|
|
26164
|
+
let value;
|
|
26165
|
+
try {
|
|
26166
|
+
value = JSON.parse(Buffer.from(query.cursor, "base64url").toString("utf8"));
|
|
26167
|
+
} catch {
|
|
26168
|
+
throw validationError("Invalid measurement portfolio cursor.");
|
|
26169
|
+
}
|
|
26170
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) throw validationError("Invalid measurement portfolio cursor.");
|
|
26171
|
+
const cursor = value;
|
|
26172
|
+
if (cursor.v !== 1 || cursor.runId !== null && typeof cursor.runId !== "string" || cursor.versionId !== versionId || cursor.filters !== portfolioFilterFingerprint(query) || typeof cursor.evidence !== "string" || !Array.isArray(cursor.offsets) || !Number.isSafeInteger(cursor.sourceLimit) || Number(cursor.sourceLimit) < 1 || Number(cursor.sourceLimit) > 50 || cursor.offsets.length !== 7 || cursor.offsets.some((offset) => !Number.isSafeInteger(offset) || Number(offset) < 0) || query.runId !== void 0 && cursor.runId !== query.runId) {
|
|
26173
|
+
throw validationError("The measurement portfolio cursor does not match the run, revision or filters.");
|
|
26174
|
+
}
|
|
26175
|
+
return cursor;
|
|
26176
|
+
}
|
|
26177
|
+
function compactPortfolioPage(response, query, versionId, evidence, cursor) {
|
|
26178
|
+
if (!query.compact) {
|
|
26179
|
+
if (query.list !== void 0) throw validationError("A portfolio list requires compact=true.");
|
|
26180
|
+
return measurementPortfolioSummaryResponseSchema.parse(response);
|
|
26181
|
+
}
|
|
26182
|
+
if (cursor !== void 0 && (cursor.evidence !== evidence || cursor.runId !== response.measurement.displayedRunId)) {
|
|
26183
|
+
throw validationError("The measurement portfolio cursor evidence changed between pages.");
|
|
26184
|
+
}
|
|
26185
|
+
const lists = [
|
|
26186
|
+
MeasurementPortfolioLists["weakest-properties"],
|
|
26187
|
+
MeasurementPortfolioLists["strongest-mentions"],
|
|
26188
|
+
MeasurementPortfolioLists["weakest-mentions"],
|
|
26189
|
+
MeasurementPortfolioLists["excluded-mentions"],
|
|
26190
|
+
MeasurementPortfolioLists.markets,
|
|
26191
|
+
MeasurementPortfolioLists["observed-names"],
|
|
26192
|
+
MeasurementPortfolioLists["cited-domains"]
|
|
26193
|
+
];
|
|
26194
|
+
const paths = [
|
|
26195
|
+
"weakestProperties",
|
|
26196
|
+
"mentionRanking.strongest",
|
|
26197
|
+
"mentionRanking.weakest",
|
|
26198
|
+
"mentionRanking.excluded",
|
|
26199
|
+
"markets",
|
|
26200
|
+
"answerEvidence.observedNames",
|
|
26201
|
+
"answerEvidence.citedDomains"
|
|
26202
|
+
];
|
|
26203
|
+
const pageList = query.list ?? MeasurementPortfolioLists["weakest-properties"];
|
|
26204
|
+
const selected = lists.indexOf(pageList);
|
|
26205
|
+
const arrays = [
|
|
26206
|
+
response.weakestProperties,
|
|
26207
|
+
response.mentionRanking.strongest,
|
|
26208
|
+
response.mentionRanking.weakest,
|
|
26209
|
+
response.mentionRanking.excluded,
|
|
26210
|
+
response.markets,
|
|
26211
|
+
response.answerEvidence?.observedNames ?? [],
|
|
26212
|
+
response.answerEvidence?.citedDomains ?? []
|
|
26213
|
+
];
|
|
26214
|
+
const offsets = cursor?.offsets ?? arrays.map(() => 0);
|
|
26215
|
+
if (arrays.some((rows, index2) => offsets[index2] > rows.length)) throw validationError("Invalid measurement portfolio cursor offset.");
|
|
26216
|
+
const requested = Math.min(query.limit ?? 50, arrays[selected].length - offsets[selected]);
|
|
26217
|
+
const initialSummaryLimit = query.list === void 0 && cursor === void 0 ? MEASUREMENT_PORTFOLIO_DEFAULT_LIMIT : 0;
|
|
26218
|
+
const build = (count4, summaryLimit2) => {
|
|
26219
|
+
const pages = arrays.map((rows, index2) => index2 === selected ? rows.slice(offsets[index2], offsets[index2] + count4) : rows.slice(0, summaryLimit2));
|
|
26220
|
+
const nextOffsets = offsets.map((offset, index2) => index2 === selected ? offset + count4 : offset);
|
|
26221
|
+
const hasMore = nextOffsets[selected] < arrays[selected].length;
|
|
26222
|
+
const omitted = paths.filter((_, index2) => index2 !== selected && pages[index2].length < arrays[index2].length);
|
|
26223
|
+
return {
|
|
26224
|
+
...response,
|
|
26225
|
+
pageList,
|
|
26226
|
+
detailsOmitted: [...response.detailsOmitted ?? [], ...omitted],
|
|
26227
|
+
weakestProperties: pages[0],
|
|
26228
|
+
mentionRanking: {
|
|
26229
|
+
...response.mentionRanking,
|
|
26230
|
+
strongest: pages[1],
|
|
26231
|
+
weakest: pages[2],
|
|
26232
|
+
excluded: pages[3],
|
|
26233
|
+
excludedTotal: response.mentionRanking.excluded.length,
|
|
26234
|
+
truncated: pages[1].length < response.mentionRanking.eligiblePropertyCount || pages[2].length < response.mentionRanking.eligiblePropertyCount
|
|
26235
|
+
},
|
|
26236
|
+
markets: pages[4],
|
|
26237
|
+
marketsTruncated: pages[4].length < response.totalMarkets,
|
|
26238
|
+
truncated: pages[0].length < response.totalProperties,
|
|
26239
|
+
...response.answerEvidence === void 0 ? {} : {
|
|
26240
|
+
answerEvidence: {
|
|
26241
|
+
...response.answerEvidence,
|
|
26242
|
+
observedNames: pages[5],
|
|
26243
|
+
citedDomains: pages[6]
|
|
26244
|
+
}
|
|
26245
|
+
},
|
|
26246
|
+
nextCursor: hasMore ? Buffer.from(JSON.stringify({
|
|
26247
|
+
v: 1,
|
|
26248
|
+
runId: response.measurement.displayedRunId,
|
|
26249
|
+
versionId,
|
|
26250
|
+
filters: portfolioFilterFingerprint(query),
|
|
26251
|
+
evidence,
|
|
26252
|
+
offsets: nextOffsets,
|
|
26253
|
+
sourceLimit: cursor?.sourceLimit ?? query.limit ?? MEASUREMENT_PORTFOLIO_DEFAULT_LIMIT
|
|
26254
|
+
})).toString("base64url") : null
|
|
26255
|
+
};
|
|
26256
|
+
};
|
|
26257
|
+
const fits = (page4) => JSON.stringify(page4, null, 2).length <= 18e3;
|
|
26258
|
+
const initialComplete = build(requested, initialSummaryLimit);
|
|
26259
|
+
if (fits(initialComplete)) return measurementPortfolioSummaryResponseSchema.parse(initialComplete);
|
|
26260
|
+
const minimum = Math.min(1, requested);
|
|
26261
|
+
let summaryLow = 0;
|
|
26262
|
+
let summaryHigh = initialSummaryLimit;
|
|
26263
|
+
let summaryLimit = 0;
|
|
26264
|
+
while (summaryLow <= summaryHigh) {
|
|
26265
|
+
const candidate = Math.floor((summaryLow + summaryHigh) / 2);
|
|
26266
|
+
if (fits(build(minimum, candidate))) {
|
|
26267
|
+
summaryLimit = candidate;
|
|
26268
|
+
summaryLow = candidate + 1;
|
|
26269
|
+
} else summaryHigh = candidate - 1;
|
|
26270
|
+
}
|
|
26271
|
+
const complete = build(requested, summaryLimit);
|
|
26272
|
+
if (fits(complete)) return measurementPortfolioSummaryResponseSchema.parse(complete);
|
|
26273
|
+
let page3 = build(minimum, summaryLimit);
|
|
26274
|
+
if (!fits(page3)) throw validationError("One portfolio row or its metadata exceeds the compact page budget. Read a narrower scope without compact=true.");
|
|
26275
|
+
let low = minimum + 1;
|
|
26276
|
+
let high = requested;
|
|
26277
|
+
while (low <= high) {
|
|
26278
|
+
const count4 = Math.floor((low + high) / 2);
|
|
26279
|
+
const candidate = build(count4, summaryLimit);
|
|
26280
|
+
if (fits(candidate)) {
|
|
26281
|
+
page3 = candidate;
|
|
26282
|
+
low = count4 + 1;
|
|
26283
|
+
} else high = count4 - 1;
|
|
26284
|
+
}
|
|
26285
|
+
return measurementPortfolioSummaryResponseSchema.parse(page3);
|
|
26286
|
+
}
|
|
26287
|
+
function portfolioAnswerEvidence(plan, materialized, filters, targetKeys) {
|
|
26288
|
+
const targets = new Set(targetKeys);
|
|
26289
|
+
const executions = new Set(plan.assignments.filter((assignment) => targets.has(assignment.targetKey) && (filters.queryClass === "all" || assignment.queryClass === filters.queryClass)).map((assignment) => assignment.executionNodeKey));
|
|
26290
|
+
const slots = materialized.input.expectedSlots.filter((slot) => executions.has(slot.executionId) && (filters.provider === void 0 || normalizeText2(slot.provider) === normalizeText2(filters.provider)) && (filters.location === void 0 || normalizeMeasurementLocation(slot.location) === normalizeMeasurementLocation(filters.location)));
|
|
26291
|
+
const mentionsBySlot = /* @__PURE__ */ new Map();
|
|
26292
|
+
for (const answer of materialized.evidence.answers) pushTo(mentionsBySlot, answer.expectedSlotId, answer.mentioned);
|
|
26293
|
+
const selected = [];
|
|
26294
|
+
let unknownMentionAnswers = 0;
|
|
26295
|
+
for (const slot of slots) {
|
|
26296
|
+
const mentions = mentionsBySlot.get(slot.id);
|
|
26297
|
+
if (mentions?.some((mentioned) => mentioned === true)) continue;
|
|
26298
|
+
const observation = materialized.observationsBySlot.get(slot.id);
|
|
26299
|
+
const snapshot = observation === void 0 ? void 0 : materialized.snapshotsById.get(observation.id);
|
|
26300
|
+
if (!snapshot || snapshot.answerText === null || !mentions?.length || mentions.some((mentioned) => mentioned === null)) {
|
|
26301
|
+
unknownMentionAnswers++;
|
|
26302
|
+
continue;
|
|
26303
|
+
}
|
|
26304
|
+
selected.push({ slotId: slot.id, snapshot, citedUrls: observation.citedUrls ?? observation.historicalCitedUrls ?? [] });
|
|
26305
|
+
}
|
|
26306
|
+
const names = observedCompetitorNames(selected.map((answer) => answer.snapshot));
|
|
26307
|
+
const domains = domainRows(selected, Number.POSITIVE_INFINITY);
|
|
26308
|
+
return {
|
|
26309
|
+
answers: "not-mentioned",
|
|
26310
|
+
populationSize: slots.length,
|
|
26311
|
+
answerCount: selected.length,
|
|
26312
|
+
unknownMentionAnswers,
|
|
26313
|
+
observedNames: names,
|
|
26314
|
+
observedNamesTotal: names.length,
|
|
26315
|
+
citedDomains: domains.rows,
|
|
26316
|
+
citedDomainsTotal: domains.total
|
|
26317
|
+
};
|
|
26318
|
+
}
|
|
26001
26319
|
function portfolioResponse(db, active, plan, query, ownDomains) {
|
|
26320
|
+
if ((query.list === MeasurementPortfolioLists["observed-names"] || query.list === MeasurementPortfolioLists["cited-domains"]) && query.answers !== "not-mentioned") {
|
|
26321
|
+
throw validationError("Portfolio evidence lists require answers=not-mentioned.");
|
|
26322
|
+
}
|
|
26002
26323
|
const group = query.groupKey === void 0 ? void 0 : requireGroup2(plan, query.groupKey);
|
|
26003
|
-
const
|
|
26324
|
+
const cursor = portfolioCursor(query, active.version.id);
|
|
26325
|
+
const run = selectMeasurementQuestionRun(db, active.version.projectId, active, query.runId ?? cursor?.runId ?? void 0);
|
|
26004
26326
|
const requestedTargetKeys = group?.targetKeys ?? plan.targets.map((target) => target.stableKey);
|
|
26005
26327
|
const targetKeys = targetKeysForRun(plan, run, requestedTargetKeys, false);
|
|
26006
26328
|
const targets = targetKeys.map((targetKey) => requireTarget3(plan, targetKey));
|
|
@@ -26014,6 +26336,7 @@ function portfolioResponse(db, active, plan, query, ownDomains) {
|
|
|
26014
26336
|
const includeNestedMarkets = query.includeNestedMarkets ?? false;
|
|
26015
26337
|
const locate = propertyLocations(plan);
|
|
26016
26338
|
if (!run) {
|
|
26339
|
+
const markets2 = marketRollup(plan, void 0, group, includeNestedMarkets);
|
|
26017
26340
|
const rows2 = targets.map((target) => ({
|
|
26018
26341
|
...propertyDto2(target),
|
|
26019
26342
|
...locate(target.stableKey),
|
|
@@ -26025,27 +26348,42 @@ function portfolioResponse(db, active, plan, query, ownDomains) {
|
|
|
26025
26348
|
citedDomains: [],
|
|
26026
26349
|
citedDomainsTotal: 0
|
|
26027
26350
|
})).sort(compareWeakest);
|
|
26028
|
-
|
|
26351
|
+
const response2 = {
|
|
26029
26352
|
portfolio: { groupKey: group?.stableKey ?? null, label: group?.label ?? null, measurementScope: null },
|
|
26030
26353
|
measurement,
|
|
26031
26354
|
queryClass: query.queryClass,
|
|
26032
26355
|
engines: [],
|
|
26033
26356
|
metrics: {
|
|
26034
26357
|
propertiesMentioned: unavailable3("no_completed_run"),
|
|
26358
|
+
propertiesNeverMentioned: unavailable3("no_completed_run"),
|
|
26035
26359
|
mentionCoverage: unavailable3("no_completed_run"),
|
|
26036
26360
|
citationCoverage: unavailable3("no_completed_run")
|
|
26037
26361
|
},
|
|
26038
|
-
|
|
26362
|
+
...query.compact && group === void 0 ? { weakestMarkets: weakestMarkets(markets2.markets, query.queryClass) } : {},
|
|
26363
|
+
weakestProperties: query.compact ? rows2.map(({
|
|
26364
|
+
namedInsteadInAnswerText: _names,
|
|
26365
|
+
namedInsteadInAnswerTextTotal: _namesTotal,
|
|
26366
|
+
recommendedInstead: _recommended,
|
|
26367
|
+
recommendedInsteadTotal: _recommendedTotal,
|
|
26368
|
+
recommendedInsteadTruncated: _recommendedTruncated,
|
|
26369
|
+
citedDomains: _domains,
|
|
26370
|
+
citedDomainsTotal: _domainsTotal,
|
|
26371
|
+
...row
|
|
26372
|
+
}) => row) : rows2.slice(0, limit),
|
|
26039
26373
|
tiedAtWeakest: null,
|
|
26040
26374
|
weakestAnswerSources: null,
|
|
26041
|
-
mentionRanking: mentionRanking(rows2, limit),
|
|
26375
|
+
mentionRanking: mentionRanking(rows2, query.compact ? Number.POSITIVE_INFINITY : limit),
|
|
26042
26376
|
// Sorted through the same comparator as the measured branch. Plan order
|
|
26043
26377
|
// is `stableKey`, so emitting it raw put markets in an order the schema
|
|
26044
26378
|
// documents as worst-first and that changes the moment a run lands.
|
|
26045
|
-
...
|
|
26379
|
+
...markets2,
|
|
26046
26380
|
totalProperties: rows2.length,
|
|
26047
|
-
truncated: rows2.length > limit
|
|
26048
|
-
|
|
26381
|
+
truncated: rows2.length > limit,
|
|
26382
|
+
...query.compact || query.answers === "not-mentioned" ? {
|
|
26383
|
+
detailsOmitted: [...query.compact ? ["weakestProperties.answerEvidence"] : [], ...query.answers === "not-mentioned" ? ["answerEvidence.noCompletedRun"] : []]
|
|
26384
|
+
} : {}
|
|
26385
|
+
};
|
|
26386
|
+
return compactPortfolioPage(response2, query, active.version.id, "no-run", cursor);
|
|
26049
26387
|
}
|
|
26050
26388
|
const { materialized, overview, evaluator } = filteredOverviewWithDb(db, active, plan, run, filters, targetKeys);
|
|
26051
26389
|
const readTarget2 = createTargetAnswerReader(plan, materialized, filters);
|
|
@@ -26066,11 +26404,13 @@ function portfolioResponse(db, active, plan, query, ownDomains) {
|
|
|
26066
26404
|
};
|
|
26067
26405
|
}).sort(compareWeakest);
|
|
26068
26406
|
const displayed = ranked.slice(0, limit);
|
|
26069
|
-
const rows = displayed.map(({ target, population, ...row }) => {
|
|
26070
|
-
|
|
26407
|
+
const rows = (query.compact ? ranked : displayed).map(({ target, population, ...row }) => {
|
|
26408
|
+
if (query.compact) return row;
|
|
26409
|
+
const evidenceLimit = query.limit === void 0 ? MEASUREMENT_PORTFOLIO_DEFAULT_ROW_EVIDENCE_LIMIT : MEASUREMENT_PORTFOLIO_ROW_EVIDENCE_LIMIT;
|
|
26410
|
+
const domains = domainRows(population.sourceAnswers, evidenceLimit);
|
|
26071
26411
|
return {
|
|
26072
26412
|
...row,
|
|
26073
|
-
...namedInsteadFields(recommendationRows(target, population)),
|
|
26413
|
+
...namedInsteadFields(recommendationRows(target, population), evidenceLimit),
|
|
26074
26414
|
citedDomains: domains.rows,
|
|
26075
26415
|
citedDomainsTotal: domains.total
|
|
26076
26416
|
};
|
|
@@ -26078,7 +26418,10 @@ function portfolioResponse(db, active, plan, query, ownDomains) {
|
|
|
26078
26418
|
const tie = weakestTie(ranked);
|
|
26079
26419
|
const engines = /* @__PURE__ */ new Set();
|
|
26080
26420
|
for (const population of populations.values()) for (const provider of population.providers) engines.add(provider);
|
|
26081
|
-
|
|
26421
|
+
const answerEvidence = query.answers !== "not-mentioned" ? void 0 : portfolioAnswerEvidence(plan, materialized, filters, targetKeys);
|
|
26422
|
+
const sourceRows = query.compact ? ranked.slice(0, cursor?.sourceLimit ?? limit) : displayed;
|
|
26423
|
+
const markets = marketRollup(plan, { run, evaluator }, group, includeNestedMarkets);
|
|
26424
|
+
const response = {
|
|
26082
26425
|
portfolio: {
|
|
26083
26426
|
groupKey: group?.stableKey ?? null,
|
|
26084
26427
|
label: group?.label ?? null,
|
|
@@ -26089,17 +26432,38 @@ function portfolioResponse(db, active, plan, query, ownDomains) {
|
|
|
26089
26432
|
engines: [...engines].sort(compareText13),
|
|
26090
26433
|
metrics: {
|
|
26091
26434
|
propertiesMentioned: countMetric2(overview.propertiesMentioned),
|
|
26435
|
+
propertiesNeverMentioned: neverMentionedMetric(overview.propertiesMentioned, targetKeys.length),
|
|
26092
26436
|
mentionCoverage: coverageMetric2(overview.mentionCoverage),
|
|
26093
26437
|
citationCoverage: coverageMetric2(overview.citationCoverage)
|
|
26094
26438
|
},
|
|
26439
|
+
...query.compact && group === void 0 ? { weakestMarkets: weakestMarkets(markets.markets, query.queryClass) } : {},
|
|
26095
26440
|
weakestProperties: rows,
|
|
26096
|
-
tiedAtWeakest: tie.summary
|
|
26097
|
-
|
|
26098
|
-
|
|
26099
|
-
|
|
26441
|
+
tiedAtWeakest: query.compact && tie.summary !== null ? {
|
|
26442
|
+
count: tie.summary.count,
|
|
26443
|
+
mentionRate: tie.summary.mentionRate,
|
|
26444
|
+
citationRate: tie.summary.citationRate,
|
|
26445
|
+
note: tie.summary.note
|
|
26446
|
+
} : tie.summary,
|
|
26447
|
+
...query.compact ? {
|
|
26448
|
+
detailsOmitted: ["weakestProperties.answerEvidence", "tiedAtWeakest.byMetro", "tiedAtWeakest.namedInstead"]
|
|
26449
|
+
} : {},
|
|
26450
|
+
weakestAnswerSources: {
|
|
26451
|
+
...weakestAnswerSources([...sourceRows, ...tie.rows], ownDomains),
|
|
26452
|
+
...query.compact ? { basis: "initial-weakest-selection-and-zero-signal-tie" } : {}
|
|
26453
|
+
},
|
|
26454
|
+
mentionRanking: mentionRanking(ranked, query.compact ? Number.POSITIVE_INFINITY : limit),
|
|
26455
|
+
...markets,
|
|
26100
26456
|
totalProperties: ranked.length,
|
|
26101
|
-
truncated: ranked.length > limit
|
|
26102
|
-
|
|
26457
|
+
truncated: ranked.length > limit,
|
|
26458
|
+
...answerEvidence === void 0 ? {} : {
|
|
26459
|
+
answerEvidence: query.compact ? answerEvidence : {
|
|
26460
|
+
...answerEvidence,
|
|
26461
|
+
observedNames: answerEvidence.observedNames.slice(0, MEASUREMENT_PORTFOLIO_TIE_NAMED_INSTEAD_LIMIT),
|
|
26462
|
+
citedDomains: answerEvidence.citedDomains.slice(0, MEASUREMENT_PORTFOLIO_ANSWER_SOURCES_LIMIT)
|
|
26463
|
+
}
|
|
26464
|
+
}
|
|
26465
|
+
};
|
|
26466
|
+
return compactPortfolioPage(response, query, active.version.id, snapshotEvidenceFingerprint([...materialized.snapshotsById.values()]), cursor);
|
|
26103
26467
|
}
|
|
26104
26468
|
function weakestTie(ranked) {
|
|
26105
26469
|
if (ranked.length === 0) return { summary: null, rows: [] };
|
|
@@ -26109,6 +26473,7 @@ function weakestTie(ranked) {
|
|
|
26109
26473
|
}
|
|
26110
26474
|
const mentionRate3 = first.mentionCoverage.value;
|
|
26111
26475
|
const citationRate2 = first.citationCoverage.value;
|
|
26476
|
+
if (mentionRate3 !== 0 || citationRate2 !== 0) return { summary: null, rows: [] };
|
|
26112
26477
|
const rows = ranked.filter((row) => row.mentionCoverage.state === "available" && row.mentionCoverage.value === mentionRate3 && row.citationCoverage.state === "available" && row.citationCoverage.value === citationRate2);
|
|
26113
26478
|
if (rows.length < 2) return { summary: null, rows: [] };
|
|
26114
26479
|
const namedInstead = tieNamedInstead(rows);
|
|
@@ -26655,7 +27020,9 @@ async function measurementPortfolioReadRoutes(app) {
|
|
|
26655
27020
|
async (request) => {
|
|
26656
27021
|
const project = resolveProject(app.db, request.params.name);
|
|
26657
27022
|
const nested = request.query.includeNestedMarkets;
|
|
26658
|
-
const
|
|
27023
|
+
const nestedRaw = nested === "true" || nested === "false" ? { ...request.query, includeNestedMarkets: nested === "true" } : request.query;
|
|
27024
|
+
const compact = request.query.compact;
|
|
27025
|
+
const raw = compact === "true" || compact === "false" ? { ...nestedRaw, compact: compact === "true" } : nestedRaw;
|
|
26659
27026
|
const query = parseLimitQuery(raw, measurementPortfolioSummaryQuerySchema, "Invalid measurement portfolio summary query");
|
|
26660
27027
|
const { active, plan } = activeV2Plan2(app.db, project.id, "Portfolio summary");
|
|
26661
27028
|
return portfolioResponse(app.db, active, plan, query, effectiveDomains(project));
|
|
@@ -26691,7 +27058,7 @@ async function measurementPortfolioReadRoutes(app) {
|
|
|
26691
27058
|
}
|
|
26692
27059
|
|
|
26693
27060
|
// ../api-routes/src/visibility-report.ts
|
|
26694
|
-
import { createHash as
|
|
27061
|
+
import { createHash as createHash8 } from "crypto";
|
|
26695
27062
|
import { and as and26, desc as desc14, eq as eq33, gte as gte5, inArray as inArray16, isNotNull as isNotNull4, isNull as isNull8, lt as lt6, lte as lte6, or as or8 } from "drizzle-orm";
|
|
26696
27063
|
|
|
26697
27064
|
// ../api-routes/src/measurement-scope-options.ts
|
|
@@ -26747,7 +27114,7 @@ function simpleScopeOptions() {
|
|
|
26747
27114
|
}
|
|
26748
27115
|
|
|
26749
27116
|
// ../api-routes/src/visibility-report-reader.ts
|
|
26750
|
-
import { createHash as
|
|
27117
|
+
import { createHash as createHash7 } from "crypto";
|
|
26751
27118
|
var VisibilityReportCursorError = class extends Error {
|
|
26752
27119
|
constructor(message) {
|
|
26753
27120
|
super(message);
|
|
@@ -27150,7 +27517,7 @@ function breakdown(candidates, definition, populationTargetKeys = selectedTarget
|
|
|
27150
27517
|
return { properties, groups };
|
|
27151
27518
|
}
|
|
27152
27519
|
function fingerprint(value) {
|
|
27153
|
-
return
|
|
27520
|
+
return createHash7("sha256").update(JSON.stringify(value)).digest("base64url");
|
|
27154
27521
|
}
|
|
27155
27522
|
function cursorOf2(cursor) {
|
|
27156
27523
|
return Buffer.from(JSON.stringify(cursor), "utf8").toString("base64url");
|
|
@@ -27703,11 +28070,11 @@ function stableSimpleDefinitionId(definition) {
|
|
|
27703
28070
|
// not freeze a competitor set, whereas a newly captured empty list did.
|
|
27704
28071
|
competitors: definition.competitors === void 0 ? null : [...definition.competitors].map((competitor) => ({ domain: competitor.domain, label: competitor.label, aliases: [...competitor.aliases].sort() })).sort((left, right) => left.domain.localeCompare(right.domain, "en"))
|
|
27705
28072
|
};
|
|
27706
|
-
return `simple:${
|
|
28073
|
+
return `simple:${createHash8("sha256").update(JSON.stringify(source)).digest("hex")}`;
|
|
27707
28074
|
}
|
|
27708
28075
|
function simpleModelFingerprint(definition) {
|
|
27709
28076
|
const engines = [...definition.engines].map((engine) => ({ provider: normalizedText3(engine.provider), requestedModel: engine.requestedModel })).sort((left, right) => left.provider.localeCompare(right.provider, "en"));
|
|
27710
|
-
return `simple-model:${
|
|
28077
|
+
return `simple-model:${createHash8("sha256").update(JSON.stringify(engines)).digest("hex")}`;
|
|
27711
28078
|
}
|
|
27712
28079
|
function frozenSimpleRun(run, definition, snapshots, includeEvidence = true) {
|
|
27713
28080
|
const matcher = compileBrandAliases(effectiveBrandNames({
|
|
@@ -31924,19 +32291,19 @@ async function intelligenceRoutes(app) {
|
|
|
31924
32291
|
}
|
|
31925
32292
|
|
|
31926
32293
|
// ../api-routes/src/traffic-analytics.ts
|
|
31927
|
-
import { and as and33, desc as desc19, eq as eq39, gte as gte8, lt as lt8, lte as lte7, ne as ne8, sql as
|
|
32294
|
+
import { and as and33, desc as desc19, eq as eq39, gte as gte8, lt as lt8, lte as lte7, ne as ne8, sql as sql14 } from "drizzle-orm";
|
|
31928
32295
|
|
|
31929
32296
|
// ../api-routes/src/ai-referral-status.ts
|
|
31930
|
-
import { and as and32, sql as
|
|
31931
|
-
var REDIRECT_LIST =
|
|
32297
|
+
import { and as and32, sql as sql13 } from "drizzle-orm";
|
|
32298
|
+
var REDIRECT_LIST = sql13.join([...LOCATION_REDIRECT_STATUSES].map((s) => sql13`${s}`), sql13`, `);
|
|
31932
32299
|
function referralLandedCondition() {
|
|
31933
|
-
return
|
|
32300
|
+
return sql13`${aiReferralEventsHourly.status} NOT IN (${REDIRECT_LIST})`;
|
|
31934
32301
|
}
|
|
31935
32302
|
function countableReferralCondition() {
|
|
31936
32303
|
return and32(nonSubresourceReferralPathCondition(), referralLandedCondition());
|
|
31937
32304
|
}
|
|
31938
32305
|
function nonSubresourceReferralPathCondition() {
|
|
31939
|
-
return
|
|
32306
|
+
return sql13`
|
|
31940
32307
|
LOWER(${aiReferralEventsHourly.landingPathNormalized}) NOT LIKE '/_next/static/%'
|
|
31941
32308
|
AND LOWER(${aiReferralEventsHourly.landingPathNormalized}) NOT LIKE '/assets/%'
|
|
31942
32309
|
AND LOWER(${aiReferralEventsHourly.landingPathNormalized}) NOT LIKE '/static/%'
|
|
@@ -31978,15 +32345,15 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
31978
32345
|
const priorStart = new Date(priorStartMs).toISOString();
|
|
31979
32346
|
const trendStart = new Date(trendStartMs).toISOString();
|
|
31980
32347
|
const firstObservations = [
|
|
31981
|
-
db.select({ first:
|
|
31982
|
-
db.select({ first:
|
|
31983
|
-
db.select({ first:
|
|
32348
|
+
db.select({ first: sql14`MIN(${crawlerEventsHourly.tsHour})` }).from(crawlerEventsHourly).where(eq39(crawlerEventsHourly.projectId, projectId)).get()?.first,
|
|
32349
|
+
db.select({ first: sql14`MIN(${aiUserFetchEventsHourly.tsHour})` }).from(aiUserFetchEventsHourly).where(eq39(aiUserFetchEventsHourly.projectId, projectId)).get()?.first,
|
|
32350
|
+
db.select({ first: sql14`MIN(${aiReferralEventsHourly.tsHour})` }).from(aiReferralEventsHourly).where(eq39(aiReferralEventsHourly.projectId, projectId)).get()?.first
|
|
31984
32351
|
].filter((first) => typeof first === "string" && first.length > 0);
|
|
31985
32352
|
const coverageStart = firstObservations.sort().at(0) ?? null;
|
|
31986
32353
|
const priorWindowComplete = coverageStart !== null && Date.parse(coverageStart) <= priorStartMs;
|
|
31987
32354
|
const recordedChange = (current, prior) => priorWindowComplete ? deltaPercent(current, prior) : null;
|
|
31988
32355
|
const sumVerifiedCrawlers = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
31989
|
-
db.select({ total:
|
|
32356
|
+
db.select({ total: sql14`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
31990
32357
|
and33(
|
|
31991
32358
|
eq39(crawlerEventsHourly.projectId, projectId),
|
|
31992
32359
|
eq39(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
@@ -31996,7 +32363,7 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
31996
32363
|
).get()?.total ?? 0
|
|
31997
32364
|
);
|
|
31998
32365
|
const sumUnverifiedCrawlers = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
31999
|
-
db.select({ total:
|
|
32366
|
+
db.select({ total: sql14`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
32000
32367
|
and33(
|
|
32001
32368
|
eq39(crawlerEventsHourly.projectId, projectId),
|
|
32002
32369
|
ne8(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
@@ -32007,9 +32374,9 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32007
32374
|
);
|
|
32008
32375
|
const sumReferrals = (windowStartIso, windowEndIso, exclusiveEnd = false) => {
|
|
32009
32376
|
const row = db.select({
|
|
32010
|
-
total:
|
|
32011
|
-
paid:
|
|
32012
|
-
organic:
|
|
32377
|
+
total: sql14`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
32378
|
+
paid: sql14`COALESCE(SUM(${aiReferralEventsHourly.paidSessionsOrHits}), 0)`,
|
|
32379
|
+
organic: sql14`COALESCE(SUM(${aiReferralEventsHourly.organicSessionsOrHits}), 0)`
|
|
32013
32380
|
}).from(aiReferralEventsHourly).where(
|
|
32014
32381
|
and33(
|
|
32015
32382
|
eq39(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -32021,7 +32388,7 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32021
32388
|
return aiReferralClassCounts(Number(row?.total ?? 0), Number(row?.paid ?? 0), Number(row?.organic ?? 0));
|
|
32022
32389
|
};
|
|
32023
32390
|
const sumUserFetches = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
32024
|
-
db.select({ total:
|
|
32391
|
+
db.select({ total: sql14`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
32025
32392
|
and33(
|
|
32026
32393
|
eq39(aiUserFetchEventsHourly.projectId, projectId),
|
|
32027
32394
|
gte8(aiUserFetchEventsHourly.tsHour, windowStartIso),
|
|
@@ -32037,7 +32404,7 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32037
32404
|
const userFetchPrior = sumUserFetches(priorStart, headlineStart, true);
|
|
32038
32405
|
const referralCurrent = sumReferrals(headlineStart, headlineEnd);
|
|
32039
32406
|
const referralAllCurrent = Number(
|
|
32040
|
-
db.select({ total:
|
|
32407
|
+
db.select({ total: sql14`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
32041
32408
|
and33(
|
|
32042
32409
|
eq39(aiReferralEventsHourly.projectId, projectId),
|
|
32043
32410
|
nonSubresourceReferralPathCondition(),
|
|
@@ -32051,7 +32418,7 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32051
32418
|
const crawlerByOperatorRows = db.select({
|
|
32052
32419
|
operator: crawlerEventsHourly.operator,
|
|
32053
32420
|
verificationStatus: crawlerEventsHourly.verificationStatus,
|
|
32054
|
-
hits:
|
|
32421
|
+
hits: sql14`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
32055
32422
|
}).from(crawlerEventsHourly).where(
|
|
32056
32423
|
and33(
|
|
32057
32424
|
eq39(crawlerEventsHourly.projectId, projectId),
|
|
@@ -32061,7 +32428,7 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32061
32428
|
).groupBy(crawlerEventsHourly.operator, crawlerEventsHourly.verificationStatus).all();
|
|
32062
32429
|
const crawlerByOperatorPriorRows = db.select({
|
|
32063
32430
|
operator: crawlerEventsHourly.operator,
|
|
32064
|
-
hits:
|
|
32431
|
+
hits: sql14`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
32065
32432
|
}).from(crawlerEventsHourly).where(
|
|
32066
32433
|
and33(
|
|
32067
32434
|
eq39(crawlerEventsHourly.projectId, projectId),
|
|
@@ -32071,7 +32438,7 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32071
32438
|
).groupBy(crawlerEventsHourly.operator).all();
|
|
32072
32439
|
const referralByOperatorRows = db.select({
|
|
32073
32440
|
operator: aiReferralEventsHourly.operator,
|
|
32074
|
-
hits:
|
|
32441
|
+
hits: sql14`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`
|
|
32075
32442
|
}).from(aiReferralEventsHourly).where(
|
|
32076
32443
|
and33(
|
|
32077
32444
|
eq39(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -32082,7 +32449,7 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32082
32449
|
).groupBy(aiReferralEventsHourly.operator).all();
|
|
32083
32450
|
const userFetchByOperatorRows = db.select({
|
|
32084
32451
|
operator: aiUserFetchEventsHourly.operator,
|
|
32085
|
-
hits:
|
|
32452
|
+
hits: sql14`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`
|
|
32086
32453
|
}).from(aiUserFetchEventsHourly).where(
|
|
32087
32454
|
and33(
|
|
32088
32455
|
eq39(aiUserFetchEventsHourly.projectId, projectId),
|
|
@@ -32125,16 +32492,16 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32125
32492
|
);
|
|
32126
32493
|
const topPathsRows = db.select({
|
|
32127
32494
|
path: crawlerEventsHourly.pathNormalized,
|
|
32128
|
-
verifiedHits:
|
|
32129
|
-
unverifiedHits:
|
|
32130
|
-
operators:
|
|
32495
|
+
verifiedHits: sql14`COALESCE(SUM(CASE WHEN ${crawlerEventsHourly.verificationStatus} = ${VerificationStatuses.verified} THEN ${crawlerEventsHourly.hits} ELSE 0 END), 0)`,
|
|
32496
|
+
unverifiedHits: sql14`COALESCE(SUM(CASE WHEN ${crawlerEventsHourly.verificationStatus} <> ${VerificationStatuses.verified} THEN ${crawlerEventsHourly.hits} ELSE 0 END), 0)`,
|
|
32497
|
+
operators: sql14`COUNT(DISTINCT ${crawlerEventsHourly.operator})`
|
|
32131
32498
|
}).from(crawlerEventsHourly).where(
|
|
32132
32499
|
and33(
|
|
32133
32500
|
eq39(crawlerEventsHourly.projectId, projectId),
|
|
32134
32501
|
gte8(crawlerEventsHourly.tsHour, headlineStart),
|
|
32135
32502
|
lte7(crawlerEventsHourly.tsHour, headlineEnd)
|
|
32136
32503
|
)
|
|
32137
|
-
).groupBy(crawlerEventsHourly.pathNormalized).orderBy(desc19(
|
|
32504
|
+
).groupBy(crawlerEventsHourly.pathNormalized).orderBy(desc19(sql14`SUM(${crawlerEventsHourly.hits})`)).limit(SERVER_ACTIVITY_TOP_PATHS_LIMIT).all();
|
|
32138
32505
|
const topCrawledPaths = topPathsRows.map((r) => ({
|
|
32139
32506
|
path: r.path,
|
|
32140
32507
|
verifiedHits: Number(r.verifiedHits),
|
|
@@ -32143,8 +32510,8 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32143
32510
|
}));
|
|
32144
32511
|
const referralProductsRows = db.select({
|
|
32145
32512
|
product: aiReferralEventsHourly.product,
|
|
32146
|
-
arrivals:
|
|
32147
|
-
landingPaths:
|
|
32513
|
+
arrivals: sql14`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
32514
|
+
landingPaths: sql14`COUNT(DISTINCT ${aiReferralEventsHourly.landingPathNormalized})`
|
|
32148
32515
|
}).from(aiReferralEventsHourly).where(
|
|
32149
32516
|
and33(
|
|
32150
32517
|
eq39(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -32152,7 +32519,7 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32152
32519
|
gte8(aiReferralEventsHourly.tsHour, headlineStart),
|
|
32153
32520
|
lte7(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
32154
32521
|
)
|
|
32155
|
-
).groupBy(aiReferralEventsHourly.product).orderBy(desc19(
|
|
32522
|
+
).groupBy(aiReferralEventsHourly.product).orderBy(desc19(sql14`SUM(${aiReferralEventsHourly.sessionsOrHits})`)).all();
|
|
32156
32523
|
const referralProducts = referralProductsRows.map((r) => ({
|
|
32157
32524
|
product: r.product,
|
|
32158
32525
|
arrivals: Number(r.arrivals),
|
|
@@ -32160,8 +32527,8 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32160
32527
|
}));
|
|
32161
32528
|
const topReferralRows = db.select({
|
|
32162
32529
|
path: aiReferralEventsHourly.landingPathNormalized,
|
|
32163
|
-
arrivals:
|
|
32164
|
-
products:
|
|
32530
|
+
arrivals: sql14`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
32531
|
+
products: sql14`COUNT(DISTINCT ${aiReferralEventsHourly.product})`
|
|
32165
32532
|
}).from(aiReferralEventsHourly).where(
|
|
32166
32533
|
and33(
|
|
32167
32534
|
eq39(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -32169,26 +32536,26 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32169
32536
|
gte8(aiReferralEventsHourly.tsHour, headlineStart),
|
|
32170
32537
|
lte7(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
32171
32538
|
)
|
|
32172
|
-
).groupBy(aiReferralEventsHourly.landingPathNormalized).orderBy(desc19(
|
|
32539
|
+
).groupBy(aiReferralEventsHourly.landingPathNormalized).orderBy(desc19(sql14`SUM(${aiReferralEventsHourly.sessionsOrHits})`)).limit(SERVER_ACTIVITY_TOP_PATHS_LIMIT).all();
|
|
32173
32540
|
const topReferralLandingPaths = topReferralRows.map((r) => ({
|
|
32174
32541
|
path: r.path,
|
|
32175
32542
|
arrivals: Number(r.arrivals),
|
|
32176
32543
|
distinctProducts: Number(r.products)
|
|
32177
32544
|
}));
|
|
32178
32545
|
const crawlerTrendRows = db.select({
|
|
32179
|
-
date:
|
|
32180
|
-
verifiedHits:
|
|
32181
|
-
unverifiedHits:
|
|
32546
|
+
date: sql14`SUBSTR(${crawlerEventsHourly.tsHour}, 1, 10)`,
|
|
32547
|
+
verifiedHits: sql14`COALESCE(SUM(CASE WHEN ${crawlerEventsHourly.verificationStatus} = ${VerificationStatuses.verified} THEN ${crawlerEventsHourly.hits} ELSE 0 END), 0)`,
|
|
32548
|
+
unverifiedHits: sql14`COALESCE(SUM(CASE WHEN ${crawlerEventsHourly.verificationStatus} <> ${VerificationStatuses.verified} THEN ${crawlerEventsHourly.hits} ELSE 0 END), 0)`
|
|
32182
32549
|
}).from(crawlerEventsHourly).where(
|
|
32183
32550
|
and33(
|
|
32184
32551
|
eq39(crawlerEventsHourly.projectId, projectId),
|
|
32185
32552
|
gte8(crawlerEventsHourly.tsHour, trendStart),
|
|
32186
32553
|
lte7(crawlerEventsHourly.tsHour, headlineEnd)
|
|
32187
32554
|
)
|
|
32188
|
-
).groupBy(
|
|
32555
|
+
).groupBy(sql14`SUBSTR(${crawlerEventsHourly.tsHour}, 1, 10)`).all();
|
|
32189
32556
|
const referralTrendRows = db.select({
|
|
32190
|
-
date:
|
|
32191
|
-
hits:
|
|
32557
|
+
date: sql14`SUBSTR(${aiReferralEventsHourly.tsHour}, 1, 10)`,
|
|
32558
|
+
hits: sql14`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`
|
|
32192
32559
|
}).from(aiReferralEventsHourly).where(
|
|
32193
32560
|
and33(
|
|
32194
32561
|
eq39(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -32196,17 +32563,17 @@ function readTrafficActivity(db, projectId, windowDays2) {
|
|
|
32196
32563
|
gte8(aiReferralEventsHourly.tsHour, trendStart),
|
|
32197
32564
|
lte7(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
32198
32565
|
)
|
|
32199
|
-
).groupBy(
|
|
32566
|
+
).groupBy(sql14`SUBSTR(${aiReferralEventsHourly.tsHour}, 1, 10)`).all();
|
|
32200
32567
|
const userFetchTrendRows = db.select({
|
|
32201
|
-
date:
|
|
32202
|
-
hits:
|
|
32568
|
+
date: sql14`SUBSTR(${aiUserFetchEventsHourly.tsHour}, 1, 10)`,
|
|
32569
|
+
hits: sql14`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`
|
|
32203
32570
|
}).from(aiUserFetchEventsHourly).where(
|
|
32204
32571
|
and33(
|
|
32205
32572
|
eq39(aiUserFetchEventsHourly.projectId, projectId),
|
|
32206
32573
|
gte8(aiUserFetchEventsHourly.tsHour, trendStart),
|
|
32207
32574
|
lte7(aiUserFetchEventsHourly.tsHour, headlineEnd)
|
|
32208
32575
|
)
|
|
32209
|
-
).groupBy(
|
|
32576
|
+
).groupBy(sql14`SUBSTR(${aiUserFetchEventsHourly.tsHour}, 1, 10)`).all();
|
|
32210
32577
|
const emptyTrendEntry = () => ({ verifiedCrawlerHits: 0, unverifiedCrawlerHits: 0, userFetchHits: 0, referralArrivals: 0 });
|
|
32211
32578
|
const dailyTrendMap = /* @__PURE__ */ new Map();
|
|
32212
32579
|
for (const r of crawlerTrendRows) {
|
|
@@ -32288,7 +32655,7 @@ async function trafficAnalyticsRoutes(app) {
|
|
|
32288
32655
|
}
|
|
32289
32656
|
|
|
32290
32657
|
// ../api-routes/src/referral-assessment.ts
|
|
32291
|
-
import { and as and34, desc as desc20, eq as eq40, gte as gte9, lte as lte8, ne as ne9, sql as
|
|
32658
|
+
import { and as and34, desc as desc20, eq as eq40, gte as gte9, lte as lte8, ne as ne9, sql as sql15 } from "drizzle-orm";
|
|
32292
32659
|
|
|
32293
32660
|
// ../api-routes/src/ga-ai-referral-aggregation.ts
|
|
32294
32661
|
function normalizeAiTrafficClass(value) {
|
|
@@ -32468,14 +32835,14 @@ function buildReferralAssessment(db, projectName, input) {
|
|
|
32468
32835
|
const limit = query.limit ?? 100;
|
|
32469
32836
|
const sumCounts = (condition) => {
|
|
32470
32837
|
const row = db.select({
|
|
32471
|
-
rows:
|
|
32472
|
-
total:
|
|
32473
|
-
paid:
|
|
32474
|
-
organic:
|
|
32838
|
+
rows: sql15`count(*)`,
|
|
32839
|
+
total: sql15`coalesce(sum(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
32840
|
+
paid: sql15`coalesce(sum(${aiReferralEventsHourly.paidSessionsOrHits}), 0)`,
|
|
32841
|
+
organic: sql15`coalesce(sum(${aiReferralEventsHourly.organicSessionsOrHits}), 0)`
|
|
32475
32842
|
}).from(aiReferralEventsHourly).where(and34(where, condition)).get();
|
|
32476
32843
|
return { rows: Number(row?.rows ?? 0), counts: aiReferralClassCounts(Number(row?.total ?? 0), Number(row?.paid ?? 0), Number(row?.organic ?? 0)) };
|
|
32477
32844
|
};
|
|
32478
|
-
const { rows: observedRows, counts: raw } = sumCounts(
|
|
32845
|
+
const { rows: observedRows, counts: raw } = sumCounts(sql15`1 = 1`);
|
|
32479
32846
|
const landed = sumCounts(referralLandedCondition()).counts;
|
|
32480
32847
|
const countable = sumCounts(countableReferralCondition()).counts;
|
|
32481
32848
|
const difference2 = (whole, part) => aiReferralClassCounts(whole.total - part.total, whole.paid - part.paid, whole.organic - part.organic);
|
|
@@ -32486,18 +32853,18 @@ function buildReferralAssessment(db, projectName, input) {
|
|
|
32486
32853
|
product: aiReferralEventsHourly.product,
|
|
32487
32854
|
landingPathNormalized: aiReferralEventsHourly.landingPathNormalized,
|
|
32488
32855
|
tsHour: aiReferralEventsHourly.tsHour,
|
|
32489
|
-
total:
|
|
32490
|
-
paid:
|
|
32491
|
-
organic:
|
|
32492
|
-
}).from(aiReferralEventsHourly).where(and34(where, countableReferralCondition())).groupBy(aiReferralEventsHourly.sourceId, aiReferralEventsHourly.product, aiReferralEventsHourly.landingPathNormalized, aiReferralEventsHourly.tsHour).having(
|
|
32856
|
+
total: sql15`sum(${aiReferralEventsHourly.sessionsOrHits})`.as("total"),
|
|
32857
|
+
paid: sql15`sum(${aiReferralEventsHourly.paidSessionsOrHits})`.as("paid"),
|
|
32858
|
+
organic: sql15`sum(${aiReferralEventsHourly.organicSessionsOrHits})`.as("organic")
|
|
32859
|
+
}).from(aiReferralEventsHourly).where(and34(where, countableReferralCondition())).groupBy(aiReferralEventsHourly.sourceId, aiReferralEventsHourly.product, aiReferralEventsHourly.landingPathNormalized, aiReferralEventsHourly.tsHour).having(sql15`sum(${aiReferralEventsHourly.sessionsOrHits}) >= ${burstThreshold}`).as("candidates");
|
|
32493
32860
|
const candidateTotals = db.select({
|
|
32494
|
-
groups:
|
|
32495
|
-
total:
|
|
32496
|
-
paid:
|
|
32497
|
-
organic:
|
|
32861
|
+
groups: sql15`count(*)`,
|
|
32862
|
+
total: sql15`coalesce(sum(${candidates.total}), 0)`,
|
|
32863
|
+
paid: sql15`coalesce(sum(${candidates.paid}), 0)`,
|
|
32864
|
+
organic: sql15`coalesce(sum(${candidates.organic}), 0)`
|
|
32498
32865
|
}).from(candidates).get();
|
|
32499
32866
|
const suspected = aiReferralClassCounts(Number(candidateTotals?.total ?? 0), Number(candidateTotals?.paid ?? 0), Number(candidateTotals?.organic ?? 0));
|
|
32500
|
-
const bursts = db.select().from(candidates).orderBy(
|
|
32867
|
+
const bursts = db.select().from(candidates).orderBy(sql15`${candidates.total} desc`, candidates.tsHour, candidates.sourceId, candidates.product, candidates.landingPathNormalized).limit(limit).all().map((row) => ({
|
|
32501
32868
|
sourceId: row.sourceId,
|
|
32502
32869
|
product: row.product,
|
|
32503
32870
|
landingPathNormalized: row.landingPathNormalized,
|
|
@@ -32512,7 +32879,7 @@ function buildReferralAssessment(db, projectName, input) {
|
|
|
32512
32879
|
trafficClass: gaAiReferrals.trafficClass,
|
|
32513
32880
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
32514
32881
|
channelGroup: gaAiReferrals.channelGroup,
|
|
32515
|
-
sessions:
|
|
32882
|
+
sessions: sql15`sum(${gaAiReferrals.sessions})`
|
|
32516
32883
|
}).from(gaAiReferrals).where(and34(eq40(gaAiReferrals.projectId, project.id), gte9(gaAiReferrals.date, query.startDate), lte8(gaAiReferrals.date, query.endDate))).groupBy(gaAiReferrals.date, gaAiReferrals.source, gaAiReferrals.medium, gaAiReferrals.trafficClass, gaAiReferrals.sourceDimension, gaAiReferrals.channelGroup).all();
|
|
32517
32884
|
const gaSessions = gaRows.length ? resolveWinningDimensions(gaRows).reduce((sum, row) => sum + row.paidSessions + row.organicSessions, 0) : latestGaSyncQueriedWindow(db, project.id, query.startDate, query.endDate) ? 0 : null;
|
|
32518
32885
|
const observedRatio = observedRows > 0 && gaSessions !== null && gaSessions > 0 ? Math.round(countable.total / gaSessions * 100) / 100 : null;
|
|
@@ -32555,10 +32922,10 @@ async function referralAssessmentRoutes(app) {
|
|
|
32555
32922
|
}
|
|
32556
32923
|
|
|
32557
32924
|
// ../api-routes/src/organic-evidence.ts
|
|
32558
|
-
import { and as and36, desc as desc21, eq as eq42, gte as gte11, lte as lte10, or as or11, sql as
|
|
32925
|
+
import { and as and36, desc as desc21, eq as eq42, gte as gte11, lte as lte10, or as or11, sql as sql17 } from "drizzle-orm";
|
|
32559
32926
|
|
|
32560
32927
|
// ../api-routes/src/ga-measurement-analysis.ts
|
|
32561
|
-
import { and as and35, eq as eq41, gte as gte10, lte as lte9, or as or10, sql as
|
|
32928
|
+
import { and as and35, eq as eq41, gte as gte10, lte as lte9, or as or10, sql as sql16 } from "drizzle-orm";
|
|
32562
32929
|
var UNKNOWN_LANDING_PAGE = "(not set)";
|
|
32563
32930
|
function addDays(date, offset) {
|
|
32564
32931
|
const value = /* @__PURE__ */ new Date(`${date}T00:00:00.000Z`);
|
|
@@ -32730,24 +33097,24 @@ function buildGaMeasurementAnalysis(db, projectName, options = {}) {
|
|
|
32730
33097
|
const scopedConditions = (hostColumn, normalizedPathColumn, rawPathColumn) => {
|
|
32731
33098
|
const conditions = [];
|
|
32732
33099
|
if (parsedHostScope.data === "marketing") {
|
|
32733
|
-
const lowerHost =
|
|
32734
|
-
const normalizedHost2 =
|
|
33100
|
+
const lowerHost = sql16`lower(${hostColumn})`;
|
|
33101
|
+
const normalizedHost2 = sql16`case when ${lowerHost} like 'www.%' then substr(${lowerHost}, 5) else ${lowerHost} end`;
|
|
32735
33102
|
conditions.push(or10(...marketingHosts.flatMap((host) => [
|
|
32736
|
-
|
|
32737
|
-
|
|
33103
|
+
sql16`${normalizedHost2} = ${host}`,
|
|
33104
|
+
sql16`${normalizedHost2} like ${`%.${host}`}`
|
|
32738
33105
|
])));
|
|
32739
33106
|
}
|
|
32740
33107
|
if (pathPrefix && pathPrefix !== "/") {
|
|
32741
|
-
const rawPath =
|
|
32742
|
-
const landingPath =
|
|
32743
|
-
conditions.push(or10(
|
|
33108
|
+
const rawPath = sql16`case when instr(${rawPathColumn}, '?') = 0 then ${rawPathColumn} else substr(${rawPathColumn}, 1, instr(${rawPathColumn}, '?') - 1) end`;
|
|
33109
|
+
const landingPath = sql16`case when trim(coalesce(${normalizedPathColumn}, '')) = '' then ${rawPath} else ${normalizedPathColumn} end`;
|
|
33110
|
+
conditions.push(or10(sql16`${landingPath} = ${pathPrefix}`, sql16`${landingPath} like ${`${pathPrefix}/%`}`));
|
|
32744
33111
|
}
|
|
32745
33112
|
return conditions;
|
|
32746
33113
|
};
|
|
32747
33114
|
const acquisitionScope = scopedConditions(gaAcquisitionDaily.hostName, gaAcquisitionDaily.landingPageNormalized, gaAcquisitionDaily.landingPage);
|
|
32748
33115
|
const leadLandingScope = scopedConditions(gaLeadEventsDaily.hostName, gaLeadEventsDaily.landingPageNormalized, gaLeadEventsDaily.landingPage);
|
|
32749
|
-
const acquisitionAnchor = db.select({ date:
|
|
32750
|
-
const leadAnchor = db.select({ date:
|
|
33116
|
+
const acquisitionAnchor = db.select({ date: sql16`max(${gaAcquisitionDaily.date})` }).from(gaAcquisitionDaily).where(and35(eq41(gaAcquisitionDaily.projectId, project.id), ...acquisitionScope)).get()?.date ?? null;
|
|
33117
|
+
const leadAnchor = db.select({ date: sql16`max(${gaLeadEventsDaily.date})` }).from(gaLeadEventsDaily).where(and35(eq41(gaLeadEventsDaily.projectId, project.id), or10(
|
|
32751
33118
|
eq41(gaLeadEventsDaily.attributionScope, "channel"),
|
|
32752
33119
|
and35(eq41(gaLeadEventsDaily.attributionScope, "landing-page"), ...leadLandingScope)
|
|
32753
33120
|
))).get()?.date ?? null;
|
|
@@ -32756,7 +33123,7 @@ function buildGaMeasurementAnalysis(db, projectName, options = {}) {
|
|
|
32756
33123
|
const gaStartDate = gaPeriods[0]?.startDate;
|
|
32757
33124
|
const acquisitionRows = gaStartDate ? db.select().from(gaAcquisitionDaily).where(and35(eq41(gaAcquisitionDaily.projectId, project.id), gte10(gaAcquisitionDaily.date, gaStartDate), lte9(gaAcquisitionDaily.date, gaAnchor))).all() : [];
|
|
32758
33125
|
const leadRows = gaStartDate ? db.select().from(gaLeadEventsDaily).where(and35(eq41(gaLeadEventsDaily.projectId, project.id), gte10(gaLeadEventsDaily.date, gaStartDate), lte9(gaLeadEventsDaily.date, gaAnchor))).all() : [];
|
|
32759
|
-
const engagementAnchor = db.select({ date:
|
|
33126
|
+
const engagementAnchor = db.select({ date: sql16`max(${gaDailyTotals.date})` }).from(gaDailyTotals).where(eq41(gaDailyTotals.projectId, project.id)).get()?.date ?? null;
|
|
32760
33127
|
const engagementPeriodWindow = engagementAnchor ? buildPeriods(engagementAnchor, days) : [];
|
|
32761
33128
|
const engagementStartDate = engagementPeriodWindow[0]?.startDate;
|
|
32762
33129
|
const engagementRows = engagementStartDate ? db.select().from(gaDailyTotals).where(and35(
|
|
@@ -32765,15 +33132,15 @@ function buildGaMeasurementAnalysis(db, projectName, options = {}) {
|
|
|
32765
33132
|
lte9(gaDailyTotals.date, engagementAnchor)
|
|
32766
33133
|
)).all() : [];
|
|
32767
33134
|
const engagementAvailableFrom = db.select({
|
|
32768
|
-
date:
|
|
33135
|
+
date: sql16`min(${gaDailyTotals.date})`
|
|
32769
33136
|
}).from(gaDailyTotals).where(and35(
|
|
32770
33137
|
eq41(gaDailyTotals.projectId, project.id),
|
|
32771
33138
|
or10(
|
|
32772
|
-
|
|
32773
|
-
|
|
33139
|
+
sql16`${gaDailyTotals.engagementRate} is not null`,
|
|
33140
|
+
sql16`${gaDailyTotals.newUsers} is not null`
|
|
32774
33141
|
)
|
|
32775
33142
|
)).get()?.date ?? null;
|
|
32776
|
-
const gscAnchor = db.select({ date:
|
|
33143
|
+
const gscAnchor = db.select({ date: sql16`max(${gscDailyTotals.date})` }).from(gscDailyTotals).where(eq41(gscDailyTotals.projectId, project.id)).get()?.date ?? null;
|
|
32777
33144
|
const gscPeriods = gscAnchor ? buildPeriods(gscAnchor, days) : [];
|
|
32778
33145
|
const gscStartDate = gscPeriods[0]?.startDate;
|
|
32779
33146
|
const propertyRows = gscStartDate ? db.select().from(gscDailyTotals).where(and35(eq41(gscDailyTotals.projectId, project.id), gte10(gscDailyTotals.date, gscStartDate), lte9(gscDailyTotals.date, gscAnchor))).all() : [];
|
|
@@ -33100,22 +33467,22 @@ function buildOrganicEvidence(db, projectName, periodDays) {
|
|
|
33100
33467
|
const nativeStartDate = nativeAcquisition.periods[0]?.startDate;
|
|
33101
33468
|
const nativeEndDate = nativeAcquisition.periods.at(-1)?.endDate;
|
|
33102
33469
|
const gscCoverage = aggregateCoverage(db.select({
|
|
33103
|
-
startDate:
|
|
33104
|
-
endDate:
|
|
33105
|
-
observedDays:
|
|
33470
|
+
startDate: sql17`min(${gscDailyTotals.date})`,
|
|
33471
|
+
endDate: sql17`max(${gscDailyTotals.date})`,
|
|
33472
|
+
observedDays: sql17`count(distinct ${gscDailyTotals.date})`
|
|
33106
33473
|
}).from(gscDailyTotals).where(eq42(gscDailyTotals.projectId, project.id)).get());
|
|
33107
33474
|
const legacyGaCoverage = aggregateCoverage(db.select({
|
|
33108
|
-
startDate:
|
|
33109
|
-
endDate:
|
|
33110
|
-
observedDays:
|
|
33475
|
+
startDate: sql17`min(${gaTrafficSnapshots.date})`,
|
|
33476
|
+
endDate: sql17`max(${gaTrafficSnapshots.date})`,
|
|
33477
|
+
observedDays: sql17`count(distinct ${gaTrafficSnapshots.date})`
|
|
33111
33478
|
}).from(gaTrafficSnapshots).where(eq42(gaTrafficSnapshots.projectId, project.id)).get());
|
|
33112
33479
|
const nativeCoverage = nativeAcquisitionActive && normalizedMarketingHosts.length ? aggregateCoverage(db.select({
|
|
33113
|
-
startDate:
|
|
33114
|
-
endDate:
|
|
33115
|
-
observedDays:
|
|
33480
|
+
startDate: sql17`min(${gaAcquisitionDaily.date})`,
|
|
33481
|
+
endDate: sql17`max(${gaAcquisitionDaily.date})`,
|
|
33482
|
+
observedDays: sql17`count(distinct ${gaAcquisitionDaily.date})`
|
|
33116
33483
|
}).from(gaAcquisitionDaily).where(and36(
|
|
33117
33484
|
eq42(gaAcquisitionDaily.projectId, project.id),
|
|
33118
|
-
or11(...normalizedMarketingHosts.map((candidate) =>
|
|
33485
|
+
or11(...normalizedMarketingHosts.map((candidate) => sql17`lower(trim(${gaAcquisitionDaily.hostName})) = ${candidate} or lower(trim(${gaAcquisitionDaily.hostName})) like ${`%.${candidate}`}`))
|
|
33119
33486
|
)).get()) : null;
|
|
33120
33487
|
const gaCoverage = nativeAcquisitionActive ? nativeCoverage : legacyGaCoverage;
|
|
33121
33488
|
const asOfDate = gscCoverage?.endDate ?? gaCoverage?.endDate ?? null;
|
|
@@ -33245,7 +33612,7 @@ function buildOrganicEvidence(db, projectName, periodDays) {
|
|
|
33245
33612
|
const allCrawlers = db.select().from(crawlerEventsHourly).where(and36(eq42(crawlerEventsHourly.projectId, project.id), gte11(crawlerEventsHourly.tsHour, serverStart), lte10(crawlerEventsHourly.tsHour, serverEnd))).all();
|
|
33246
33613
|
const allFetches = db.select().from(aiUserFetchEventsHourly).where(and36(eq42(aiUserFetchEventsHourly.projectId, project.id), gte11(aiUserFetchEventsHourly.tsHour, serverStart), lte10(aiUserFetchEventsHourly.tsHour, serverEnd))).all();
|
|
33247
33614
|
const allReferrals = db.select().from(aiReferralEventsHourly).where(and36(eq42(aiReferralEventsHourly.projectId, project.id), countableReferralCondition(), gte11(aiReferralEventsHourly.tsHour, serverStart), lte10(aiReferralEventsHourly.tsHour, serverEnd))).all();
|
|
33248
|
-
const [serverCoverageRow] = db.all(
|
|
33615
|
+
const [serverCoverageRow] = db.all(sql17`
|
|
33249
33616
|
select
|
|
33250
33617
|
min(day) as startDate,
|
|
33251
33618
|
max(day) as endDate,
|
|
@@ -34637,10 +35004,10 @@ async function resultsExportRoutes(app) {
|
|
|
34637
35004
|
}
|
|
34638
35005
|
|
|
34639
35006
|
// ../api-routes/src/composites.ts
|
|
34640
|
-
import { eq as eq47, and as and41, desc as desc23, sql as
|
|
35007
|
+
import { eq as eq47, and as and41, desc as desc23, sql as sql19, like as like4, or as or12, inArray as inArray23 } from "drizzle-orm";
|
|
34641
35008
|
|
|
34642
35009
|
// ../api-routes/src/gsc-totals.ts
|
|
34643
|
-
import { and as and40, asc as asc7, eq as eq46, sql as
|
|
35010
|
+
import { and as and40, asc as asc7, eq as eq46, sql as sql18, max, min } from "drizzle-orm";
|
|
34644
35011
|
var WINDOW_DAYS = { "7d": 7, "30d": 30, "90d": 90 };
|
|
34645
35012
|
function resolveGscWindowRange(window, latestDataDate, today) {
|
|
34646
35013
|
if (window === "all") {
|
|
@@ -34704,8 +35071,8 @@ function readEarliestGscDataDate(db, projectId) {
|
|
|
34704
35071
|
return [property, dimensioned].filter((d) => d !== null).reduce((earliest, d) => earliest === null || d < earliest ? d : earliest, null);
|
|
34705
35072
|
}
|
|
34706
35073
|
function readEarliestGscQueryDataDate(db, projectId) {
|
|
34707
|
-
const accurate = db.select({ earliest: min(gscQueryDailyTotals.date) }).from(gscQueryDailyTotals).where(and40(eq46(gscQueryDailyTotals.projectId, projectId),
|
|
34708
|
-
const dimensioned = db.select({ earliest: min(gscSearchData.date) }).from(gscSearchData).where(and40(eq46(gscSearchData.projectId, projectId),
|
|
35074
|
+
const accurate = db.select({ earliest: min(gscQueryDailyTotals.date) }).from(gscQueryDailyTotals).where(and40(eq46(gscQueryDailyTotals.projectId, projectId), sql18`${gscQueryDailyTotals.date} <> ''`)).get()?.earliest ?? null;
|
|
35075
|
+
const dimensioned = db.select({ earliest: min(gscSearchData.date) }).from(gscSearchData).where(and40(eq46(gscSearchData.projectId, projectId), sql18`${gscSearchData.date} <> ''`)).get()?.earliest ?? null;
|
|
34709
35076
|
return [accurate, dimensioned].filter((d) => d !== null).reduce((earliest, d) => earliest === null || d < earliest ? d : earliest, null);
|
|
34710
35077
|
}
|
|
34711
35078
|
function readGscDailyTotals(db, projectId, startDate, endDate) {
|
|
@@ -34717,8 +35084,8 @@ function readGscDailyTotals(db, projectId, startDate, endDate) {
|
|
|
34717
35084
|
}).from(gscDailyTotals).where(
|
|
34718
35085
|
and40(
|
|
34719
35086
|
eq46(gscDailyTotals.projectId, projectId),
|
|
34720
|
-
|
|
34721
|
-
|
|
35087
|
+
sql18`${gscDailyTotals.date} >= ${startDate}`,
|
|
35088
|
+
sql18`${gscDailyTotals.date} <= ${endDate}`
|
|
34722
35089
|
)
|
|
34723
35090
|
).orderBy(asc7(gscDailyTotals.date)).all();
|
|
34724
35091
|
return rows.map((r) => {
|
|
@@ -34747,8 +35114,8 @@ function readGscQueryDailyRows(db, projectId, startDate, endDate) {
|
|
|
34747
35114
|
}).from(gscQueryDailyTotals).where(
|
|
34748
35115
|
and40(
|
|
34749
35116
|
eq46(gscQueryDailyTotals.projectId, projectId),
|
|
34750
|
-
|
|
34751
|
-
|
|
35117
|
+
sql18`${gscQueryDailyTotals.date} >= ${startDate}`,
|
|
35118
|
+
sql18`${gscQueryDailyTotals.date} <= ${endDate}`
|
|
34752
35119
|
)
|
|
34753
35120
|
).all();
|
|
34754
35121
|
return rows.map((r) => {
|
|
@@ -34766,16 +35133,16 @@ function readGscQueryDailyFallbackRows(db, projectId, startDate, endDate) {
|
|
|
34766
35133
|
const rows = db.select({
|
|
34767
35134
|
date: gscSearchData.date,
|
|
34768
35135
|
query: gscSearchData.query,
|
|
34769
|
-
clicks:
|
|
34770
|
-
impressions:
|
|
34771
|
-
position:
|
|
35136
|
+
clicks: sql18`COALESCE(SUM(${gscSearchData.clicks}), 0)`,
|
|
35137
|
+
impressions: sql18`COALESCE(SUM(${gscSearchData.impressions}), 0)`,
|
|
35138
|
+
position: sql18`COALESCE(SUM(CAST(${gscSearchData.position} AS REAL) * ${gscSearchData.impressions}) * 1.0 / NULLIF(SUM(${gscSearchData.impressions}), 0), 0)`
|
|
34772
35139
|
}).from(gscSearchData).where(
|
|
34773
35140
|
and40(
|
|
34774
35141
|
eq46(gscSearchData.projectId, projectId),
|
|
34775
|
-
|
|
34776
|
-
|
|
34777
|
-
|
|
34778
|
-
|
|
35142
|
+
sql18`${gscSearchData.date} >= ${startDate}`,
|
|
35143
|
+
sql18`${gscSearchData.date} <= ${endDate}`,
|
|
35144
|
+
sql18`${gscSearchData.query} <> ''`,
|
|
35145
|
+
sql18`${gscSearchData.impressions} > 0`
|
|
34779
35146
|
)
|
|
34780
35147
|
).groupBy(gscSearchData.date, gscSearchData.query).all();
|
|
34781
35148
|
return rows.map((r) => ({
|
|
@@ -34792,7 +35159,7 @@ function readGscQueryTotalsPage(db, projectId, startDate, endDate, limit, offset
|
|
|
34792
35159
|
const bound = (n, floor) => Math.min(Math.max(Math.floor(n), floor), Number.MAX_SAFE_INTEGER);
|
|
34793
35160
|
const pageLimit2 = bound(limit, 1);
|
|
34794
35161
|
const pageOffset = bound(offset, 0);
|
|
34795
|
-
const perQuery =
|
|
35162
|
+
const perQuery = sql18`
|
|
34796
35163
|
WITH accurate AS (
|
|
34797
35164
|
SELECT query, clicks, impressions,
|
|
34798
35165
|
COALESCE(CAST(position AS REAL), 0) AS position
|
|
@@ -34831,7 +35198,7 @@ function readGscQueryTotalsPage(db, projectId, startDate, endDate, limit, offset
|
|
|
34831
35198
|
)
|
|
34832
35199
|
GROUP BY query
|
|
34833
35200
|
)`;
|
|
34834
|
-
const rows = db.all(
|
|
35201
|
+
const rows = db.all(sql18`${perQuery}
|
|
34835
35202
|
SELECT query, clicks, impressions,
|
|
34836
35203
|
CASE WHEN impressions > 0 THEN weighted_position_sum * 1.0 / impressions
|
|
34837
35204
|
ELSE position_sum * 1.0 / days END AS position,
|
|
@@ -34843,7 +35210,7 @@ function readGscQueryTotalsPage(db, projectId, startDate, endDate, limit, offset
|
|
|
34843
35210
|
FROM per_query
|
|
34844
35211
|
ORDER BY clicks DESC, impressions DESC, query ASC
|
|
34845
35212
|
LIMIT ${pageLimit2} OFFSET ${pageOffset}`);
|
|
34846
|
-
const totalMatching = rows.length > 0 ? Number(rows[0].total_matching) : Number(db.get(
|
|
35213
|
+
const totalMatching = rows.length > 0 ? Number(rows[0].total_matching) : Number(db.get(sql18`${perQuery} SELECT COUNT(*) AS total FROM per_query`).total);
|
|
34847
35214
|
return {
|
|
34848
35215
|
rows: rows.map((r) => ({
|
|
34849
35216
|
query: r.query,
|
|
@@ -35093,10 +35460,10 @@ async function compositeRoutes(app, options) {
|
|
|
35093
35460
|
and41(
|
|
35094
35461
|
eq47(queries.projectId, project.id),
|
|
35095
35462
|
or12(
|
|
35096
|
-
|
|
35097
|
-
|
|
35098
|
-
|
|
35099
|
-
|
|
35463
|
+
sql19`${querySnapshots.answerText} LIKE ${pattern} ESCAPE '\\'`,
|
|
35464
|
+
sql19`${querySnapshots.citedDomains} LIKE ${pattern} ESCAPE '\\'`,
|
|
35465
|
+
sql19`${querySnapshots.citedUrls} LIKE ${pattern} ESCAPE '\\'`,
|
|
35466
|
+
sql19`${querySnapshots.rawResponse} LIKE ${pattern} ESCAPE '\\'`,
|
|
35100
35467
|
like4(queries.query, pattern)
|
|
35101
35468
|
)
|
|
35102
35469
|
)
|
|
@@ -35107,8 +35474,8 @@ async function compositeRoutes(app, options) {
|
|
|
35107
35474
|
or12(
|
|
35108
35475
|
like4(insights.title, pattern),
|
|
35109
35476
|
like4(insights.query, pattern),
|
|
35110
|
-
|
|
35111
|
-
|
|
35477
|
+
sql19`${insights.recommendation} LIKE ${pattern} ESCAPE '\\'`,
|
|
35478
|
+
sql19`${insights.cause} LIKE ${pattern} ESCAPE '\\'`
|
|
35112
35479
|
)
|
|
35113
35480
|
)
|
|
35114
35481
|
).orderBy(desc23(insights.createdAt)).limit(limit + 1).all();
|
|
@@ -37890,6 +38257,7 @@ var routeCatalog = [
|
|
|
37890
38257
|
tags: ["measurement-plans"],
|
|
37891
38258
|
parameters: [
|
|
37892
38259
|
nameParameter,
|
|
38260
|
+
{ name: "compact", in: "query", description: "Byte-bounded whole-row page; omits optional heavy detail with explicit metadata. Default false for compatibility.", schema: { type: "boolean" } },
|
|
37893
38261
|
{ name: "scope", in: "query", required: true, description: "Reporting scope.", schema: { type: "string", enum: ["all", "group", "property"] } },
|
|
37894
38262
|
{ name: "groupKey", in: "query", description: 'Group stable key, required when scope is "group".', schema: stringSchema },
|
|
37895
38263
|
{ name: "targetKey", in: "query", description: 'Target stable key, required when scope is "property".', schema: stringSchema },
|
|
@@ -37957,7 +38325,7 @@ var routeCatalog = [
|
|
|
37957
38325
|
path: "/api/v1/projects/{name}/sentiment/compare",
|
|
37958
38326
|
summary: "Compare compatible stored sentiment periods",
|
|
37959
38327
|
tags: ["sentiment"],
|
|
37960
|
-
parameters: [nameParameter, ...sentimentSelectionParameters, ...["fromRunId", "toRunId"].map((name) => ({ name, in: "query", required: true, description: "Exact source run for this period.", schema: stringSchema }))],
|
|
38328
|
+
parameters: [nameParameter, ...sentimentSelectionParameters, ...["fromRunId", "toRunId"].map((name) => ({ name, in: "query", required: true, description: name === "fromRunId" ? "Exact source run or previous-rated to resolve the preceding rated run in this scope. Identical run IDs are refused." : "Exact target run for this period.", schema: stringSchema }))],
|
|
37961
38329
|
responses: { 200: jsonResponse("Matched units, exclusions, and conservative comparison or refusal reasons.", "SentimentComparison"), 400: errorResponse("Invalid comparison selection."), 404: errorResponse("Project or selected source not found.") }
|
|
37962
38330
|
},
|
|
37963
38331
|
{
|
|
@@ -38051,16 +38419,20 @@ var routeCatalog = [
|
|
|
38051
38419
|
method: "get",
|
|
38052
38420
|
path: "/api/v1/projects/{name}/measurement-portfolio-summary",
|
|
38053
38421
|
summary: "Get the weakest measured Properties",
|
|
38054
|
-
description: "Returns a compact, revision-pinned portfolio ranking from stored results, plus a worst-first market roll-up. It defaults to non-brand queries, ranks measured mention coverage before citation coverage, and keeps unavailable rows separate from measured weakness. Every Property row carries its metro, submarkets and query count; weakest rows also carry namedInsteadInAnswerText (names written in the answer text of answers that neither named nor cited the Property, counted by answer) and citedDomains (domains cited by the Property's answers, counted by answer, every engine included). tiedAtWeakest reports Properties sharing the weakest rates; over the whole tie, not just the returned rows, byMetro counts the tied Properties per metro label (a Property in two metros counts in both) and namedInstead ranks the names written in the answer text of the tie's answers that neither named nor cited the Property (distinct answers, never citations). weakestAnswerSources ranks the domains cited across the weakest and tied Properties, each answer once. markets holds one level by default (
|
|
38422
|
+
description: "Returns a compact, revision-pinned portfolio ranking from stored results, plus a worst-first market roll-up. It defaults to non-brand queries, ranks measured mention coverage before citation coverage, and keeps unavailable rows separate from measured weakness. Every Property row carries its metro, submarkets and query count; weakest rows also carry namedInsteadInAnswerText (names written in the answer text of answers that neither named nor cited the Property, counted by answer) and citedDomains (domains cited by the Property's answers, counted by answer, every engine included). tiedAtWeakest reports Properties sharing the weakest rates; over the whole tie, not just the returned rows, byMetro counts the tied Properties per metro label (a Property in two metros counts in both) and namedInstead ranks the names written in the answer text of the tie's answers that neither named nor cited the Property (distinct answers, never citations). weakestAnswerSources ranks the domains cited across the weakest and tied Properties, each answer once. markets holds one level by default (top-level markets, or direct children of the selected group), worst-first; raw reads return all markets and compact reads return bounded summaries or list=markets pages. includeNestedMarkets selects every level. propertiesNeverMentioned counts zero mentions even if a Property was cited. tiedAtWeakest requires both zero mentions and zero citations. Compact nextCursor advances only pageList, default weakest-properties. Other default first-page lists are bounded summaries and later pages omit their row bodies. list selects one complete ranking, market or evidence list without repeating sibling rows; totals stay complete. List, filters, revision, run and evidence are bound to the cursor; limit may change. Every market is scoped to the displayed run, so a market row matches that market read with groupKey; markets may share Properties and never sum to the portfolio totals. It never starts provider work.",
|
|
38055
38423
|
tags: ["measurement-plans"],
|
|
38056
38424
|
parameters: [
|
|
38057
38425
|
nameParameter,
|
|
38426
|
+
{ name: "compact", in: "query", description: "Byte-bounded whole-row page of pageList, default weakest-properties. Other initial lists are bounded summaries. Default false for compatibility.", schema: { type: "boolean" } },
|
|
38427
|
+
{ name: "cursor", in: "query", description: "nextCursor from the preceding compact page; keep every selection unchanged.", schema: stringSchema },
|
|
38428
|
+
{ name: "list", in: "query", description: "Compact page selection identity; enumerate this list only and omit sibling row lists. observed-names and cited-domains require answers=not-mentioned. Keep unchanged across cursors.", schema: { type: "string", enum: ["weakest-properties", "strongest-mentions", "weakest-mentions", "excluded-mentions", "markets", "observed-names", "cited-domains"] } },
|
|
38429
|
+
{ name: "answers", in: "query", description: "Competitor evidence population. not-mentioned excludes every answer naming any targeted Property.", schema: { type: "string", enum: ["all", "not-mentioned"] } },
|
|
38058
38430
|
{ name: "groupKey", in: "query", description: "Optional reporting group stable key.", schema: stringSchema },
|
|
38059
38431
|
{ name: "queryClass", in: "query", description: "Question class. Defaults to non-brand.", schema: { type: "string", enum: ["all", "branded", "non-brand"], default: "non-brand" } },
|
|
38060
38432
|
{ name: "provider", in: "query", description: "Restrict to one answer provider.", schema: stringSchema },
|
|
38061
38433
|
{ name: "location", in: "query", description: "Restrict to one execution location label.", schema: stringSchema },
|
|
38062
38434
|
{ name: "runId", in: "query", description: "Read this completed or partial active-revision run, including a named spot check.", schema: stringSchema },
|
|
38063
|
-
{ name: "limit", in: "query", description: "
|
|
38435
|
+
{ name: "limit", in: "query", description: "Compact selected-list page size: auto-sized up to 50 by default; the byte budget may return fewer whole rows. Raw reads default to 4 Property rows and never cap markets.", schema: { type: "integer", minimum: 1, maximum: 50 } },
|
|
38064
38436
|
{ name: "includeNestedMarkets", in: "query", description: "Return every market in scope at every level. Defaults to false: one level (every top-level market, or every direct child of the selected group).", schema: { type: "boolean" } }
|
|
38065
38437
|
],
|
|
38066
38438
|
responses: {
|
|
@@ -38930,6 +39302,7 @@ var routeCatalog = [
|
|
|
38930
39302
|
tags: ["analytics", "competitors"],
|
|
38931
39303
|
parameters: [
|
|
38932
39304
|
nameParameter,
|
|
39305
|
+
{ name: "answers", in: "query", description: "not-mentioned selects only answers where none of their targeted subjects was named. all or omitted keeps the full population; answerSelection reports population and selected counts.", schema: { type: "string", enum: ["all", "not-mentioned"] } },
|
|
38933
39306
|
analyticsWindowParameter,
|
|
38934
39307
|
competitorLandscapeGroupKeyParameter,
|
|
38935
39308
|
competitorLandscapeScopeParameter,
|
|
@@ -43725,7 +44098,7 @@ var routeCatalog = [
|
|
|
43725
44098
|
method: "get",
|
|
43726
44099
|
path: "/api/v1/projects/{name}/technical-aeo",
|
|
43727
44100
|
summary: "Get the Technical AEO scorecard for a project",
|
|
43728
|
-
description: "Returns the latest completed/partial site-audit, or the historical audit selected by `runId`: aggregate 0\u2013100 score, page counts, the full per-factor scorecard (site-level averages with pass/partial/fail distribution), cross-cutting issues, prioritized fixes, and the delta vs the audit immediately before it. When the project has never been audited, `hasData` is false and the numeric fields are zeroed \u2014 render an onboarding state.",
|
|
44101
|
+
description: "runSelection selects a complete run on the latest UTC scan date, otherwise the largest page sample, then newest; it flags same-date candidates. Factor pagesFailing and pagesPartial are separate; a factor with failing pages is fail. Returns the latest completed/partial site-audit, or the historical audit selected by `runId`: aggregate 0\u2013100 score, page counts, the full per-factor scorecard (site-level averages with pass/partial/fail distribution), cross-cutting issues, prioritized fixes, and the delta vs the audit immediately before it. When the project has never been audited, `hasData` is false and the numeric fields are zeroed \u2014 render an onboarding state.",
|
|
43729
44102
|
tags: ["technical-aeo"],
|
|
43730
44103
|
parameters: [
|
|
43731
44104
|
nameParameter,
|
|
@@ -43774,13 +44147,15 @@ var routeCatalog = [
|
|
|
43774
44147
|
method: "get",
|
|
43775
44148
|
path: "/api/v1/projects/{name}/technical-aeo/crawl",
|
|
43776
44149
|
summary: "Get persisted Technical AEO crawl metadata",
|
|
43777
|
-
description: "Returns
|
|
44150
|
+
description: "Returns a non-probe site-audit crawl on the requested UTC date, the selected run, or the latest scan date. Prefer complete scans, then the largest page sample, then the latest scan on that date. runSelection records same-date ambiguity; inventorySummary reports exact whole-snapshot eligibility and disjoint excluded reasons. `complete` and `termination` qualify budget-limited evidence. `hasCrawlData=false` never synthesizes a graph from legacy scorecard rows; dated no-data reads expose bounded `availableScanDates` with full distinct-date totals and matching month/day candidates to resolve an omitted year; `legacyAuditAvailable` says that the old score/pages/trend data can still be read separately.",
|
|
43778
44151
|
tags: ["technical-aeo"],
|
|
43779
44152
|
parameters: [
|
|
43780
44153
|
nameParameter,
|
|
43781
|
-
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest crawl.", schema: stringSchema }
|
|
44154
|
+
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest crawl.", schema: stringSchema },
|
|
44155
|
+
{ name: "date", in: "query", description: "UTC scan date YYYY-MM-DD, mutually exclusive with runId. Prefer complete, then largest, then latest scans on that exact date; missing data never falls back to another date.", schema: { type: "string", format: "date" } }
|
|
43782
44156
|
],
|
|
43783
44157
|
responses: {
|
|
44158
|
+
400: errorResponse("Invalid date, conflicting runId/date, or invalid cursor identity."),
|
|
43784
44159
|
200: jsonResponse("Persisted crawl summary returned.", "SiteCrawlSummaryDto"),
|
|
43785
44160
|
404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
|
|
43786
44161
|
}
|
|
@@ -43887,11 +44262,12 @@ var routeCatalog = [
|
|
|
43887
44262
|
method: "get",
|
|
43888
44263
|
path: "/api/v1/projects/{name}/technical-aeo/crawl/pages",
|
|
43889
44264
|
summary: "List persisted Technical AEO crawl pages",
|
|
43890
|
-
description: "Cursor-paged canonical crawl nodes for the latest or selected crawl. `inventoryEligible` is Canonry technical-inventory eligibility, not a statement about actual Google index state.",
|
|
44265
|
+
description: "Cursor-paged canonical crawl nodes for the requested UTC date, latest scan date, or selected crawl. inventorySummary gives whole-snapshot eligibility and excluded reasons independent of paging and filters; healthReasonCounts sums the filtered list before paging. Dated no-data reads expose `availableScanDates` for an explicit retry on a known date, never an automatic fallback. Unknown indexability never means canonical-away. `inventoryEligible` is Canonry technical-inventory eligibility, not a statement about actual Google index state.",
|
|
43891
44266
|
tags: ["technical-aeo"],
|
|
43892
44267
|
parameters: [
|
|
43893
44268
|
nameParameter,
|
|
43894
44269
|
{ name: "runId", in: "query", description: "Historical site-audit run ID. Omit for the latest crawl.", schema: stringSchema },
|
|
44270
|
+
{ name: "date", in: "query", description: "UTC scan date YYYY-MM-DD, mutually exclusive with runId. Prefer complete, then largest, then latest scans on that exact date; missing data never falls back to another date.", schema: { type: "string", format: "date" } },
|
|
43895
44271
|
{ name: "inventoryEligible", in: "query", description: "Filter Canonry technical inventory eligibility (`true` or `false`).", schema: booleanSchema },
|
|
43896
44272
|
{ name: "fetchState", in: "query", description: "Filter by persisted fetch state.", schema: stringSchema },
|
|
43897
44273
|
{ name: "indexabilityState", in: "query", description: "Filter by crawler-derived indexability state; this is not Google index coverage.", schema: stringSchema },
|
|
@@ -43903,6 +44279,7 @@ var routeCatalog = [
|
|
|
43903
44279
|
crawlLimitParameter
|
|
43904
44280
|
],
|
|
43905
44281
|
responses: {
|
|
44282
|
+
400: errorResponse("Invalid date, conflicting runId/date, or invalid cursor identity."),
|
|
43906
44283
|
200: jsonResponse("Crawl pages returned.", "SiteCrawlPagesResponseDto"),
|
|
43907
44284
|
404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
|
|
43908
44285
|
}
|
|
@@ -43987,7 +44364,7 @@ var routeCatalog = [
|
|
|
43987
44364
|
method: "get",
|
|
43988
44365
|
path: "/api/v1/projects/{name}/technical-aeo/runs",
|
|
43989
44366
|
summary: "List Site Health scan history",
|
|
43990
|
-
description: "Returns every non-probe `site-audit` run for the project, newest first, including runs still queued or running and runs that failed. `hasCrawlData` says whether that scan published a page and internal-link crawl: a legacy score-only scan is listed with `hasCrawlData=false` rather than hidden, and the crawl-scoped reads answer it with their no-crawl state instead of a 404.",
|
|
44367
|
+
description: "Returns every non-probe `site-audit` run for the project, newest first, including runs still queued or running and runs that failed. `hasCrawlData` says whether that scan published a page and internal-link crawl: a legacy score-only scan is listed with `hasCrawlData=false` rather than hidden, and the crawl-scoped reads answer it with their no-crawl state instead of a 404. `preferredRunId` identifies the default persisted scan shared by score/crawl reads, independently of the history page limit; an explicit selected run stays authoritative.",
|
|
43991
44368
|
tags: ["technical-aeo"],
|
|
43992
44369
|
parameters: [
|
|
43993
44370
|
nameParameter,
|
|
@@ -45332,7 +45709,7 @@ function formatNotification(row) {
|
|
|
45332
45709
|
|
|
45333
45710
|
// ../api-routes/src/google.ts
|
|
45334
45711
|
import crypto33 from "crypto";
|
|
45335
|
-
import { eq as eq54, and as and45, desc as desc27, sql as
|
|
45712
|
+
import { eq as eq54, and as and45, desc as desc27, sql as sql20, inArray as inArray25 } from "drizzle-orm";
|
|
45336
45713
|
|
|
45337
45714
|
// ../api-routes/src/gsc-period-comparison.ts
|
|
45338
45715
|
var DAY_MS = 864e5;
|
|
@@ -47835,11 +48212,11 @@ async function googleRoutes(app, opts) {
|
|
|
47835
48212
|
const cutoffDate = startDate ? null : resolvedWindow.startDate;
|
|
47836
48213
|
const effectiveEndDate = endDate ?? (daysParam === null ? void 0 : resolvedWindow.endDate ?? void 0);
|
|
47837
48214
|
const conditions = [eq54(gscSearchData.projectId, project.id)];
|
|
47838
|
-
if (startDate) conditions.push(
|
|
47839
|
-
else if (cutoffDate) conditions.push(
|
|
47840
|
-
if (effectiveEndDate) conditions.push(
|
|
47841
|
-
if (query) conditions.push(
|
|
47842
|
-
if (page3) conditions.push(
|
|
48215
|
+
if (startDate) conditions.push(sql20`${gscSearchData.date} >= ${startDate}`);
|
|
48216
|
+
else if (cutoffDate) conditions.push(sql20`${gscSearchData.date} >= ${cutoffDate}`);
|
|
48217
|
+
if (effectiveEndDate) conditions.push(sql20`${gscSearchData.date} <= ${effectiveEndDate}`);
|
|
48218
|
+
if (query) conditions.push(sql20`${gscSearchData.query} LIKE ${"%" + escapeLikePattern(query) + "%"} ESCAPE '\\'`);
|
|
48219
|
+
if (page3) conditions.push(sql20`${gscSearchData.page} LIKE ${"%" + escapeLikePattern(page3) + "%"} ESCAPE '\\'`);
|
|
47843
48220
|
const limitVal = Math.max(parseInt(limit ?? "500", 10) || 0, 1);
|
|
47844
48221
|
const offsetVal = Math.max(parseInt(offset ?? "0", 10) || 0, 0);
|
|
47845
48222
|
const orderColumns = {
|
|
@@ -47848,8 +48225,8 @@ async function googleRoutes(app, opts) {
|
|
|
47848
48225
|
date: [desc27(gscSearchData.date), desc27(gscSearchData.clicks), gscSearchData.query]
|
|
47849
48226
|
}[orderBy];
|
|
47850
48227
|
const rows = app.db.select().from(gscSearchData).where(and45(...conditions)).orderBy(...orderColumns).limit(limitVal).offset(offsetVal).all();
|
|
47851
|
-
const totalMatching = app.db.select({ total:
|
|
47852
|
-
const latestAvailableDate = app.db.select({ latest:
|
|
48228
|
+
const totalMatching = app.db.select({ total: sql20`COUNT(*)` }).from(gscSearchData).where(and45(...conditions)).get()?.total ?? 0;
|
|
48229
|
+
const latestAvailableDate = app.db.select({ latest: sql20`MAX(${gscSearchData.date})` }).from(gscSearchData).where(eq54(gscSearchData.projectId, project.id)).get()?.latest ?? null;
|
|
47853
48230
|
return {
|
|
47854
48231
|
rows: rows.map((r) => ({
|
|
47855
48232
|
date: r.date,
|
|
@@ -47893,12 +48270,12 @@ async function googleRoutes(app, opts) {
|
|
|
47893
48270
|
const fetchStart = priorWindowIsSynced ? priorPeriodStart : windowStart;
|
|
47894
48271
|
const dailyTotals = readGscDailyTotals(app.db, project.id, fetchStart, windowEnd);
|
|
47895
48272
|
const conditions = [eq54(gscSearchData.projectId, project.id)];
|
|
47896
|
-
if (fetchStart) conditions.push(
|
|
47897
|
-
if (endDate) conditions.push(
|
|
48273
|
+
if (fetchStart) conditions.push(sql20`${gscSearchData.date} >= ${fetchStart}`);
|
|
48274
|
+
if (endDate) conditions.push(sql20`${gscSearchData.date} <= ${endDate}`);
|
|
47898
48275
|
const dimensionedRows = app.db.select({
|
|
47899
48276
|
date: gscSearchData.date,
|
|
47900
|
-
clicks:
|
|
47901
|
-
impressions:
|
|
48277
|
+
clicks: sql20`COALESCE(SUM(${gscSearchData.clicks}), 0)`,
|
|
48278
|
+
impressions: sql20`COALESCE(SUM(${gscSearchData.impressions}), 0)`
|
|
47902
48279
|
}).from(gscSearchData).where(and45(...conditions)).groupBy(gscSearchData.date).orderBy(gscSearchData.date).all();
|
|
47903
48280
|
const propertyDates = new Set(dailyTotals.map((d) => d.date));
|
|
47904
48281
|
const comparisonDaily = mergeGscDailyTotalsWithFallback(
|
|
@@ -47996,23 +48373,23 @@ async function googleRoutes(app, opts) {
|
|
|
47996
48373
|
);
|
|
47997
48374
|
const cutoffDate = startDate ? null : resolvedWindow.startDate;
|
|
47998
48375
|
const conditions = [eq54(gscSearchData.projectId, project.id)];
|
|
47999
|
-
if (startDate) conditions.push(
|
|
48000
|
-
else if (cutoffDate) conditions.push(
|
|
48001
|
-
if (endDate) conditions.push(
|
|
48376
|
+
if (startDate) conditions.push(sql20`${gscSearchData.date} >= ${startDate}`);
|
|
48377
|
+
else if (cutoffDate) conditions.push(sql20`${gscSearchData.date} >= ${cutoffDate}`);
|
|
48378
|
+
if (endDate) conditions.push(sql20`${gscSearchData.date} <= ${endDate}`);
|
|
48002
48379
|
const limitVal = Math.max(parseInt(limit ?? "50", 10) || 0, 1);
|
|
48003
48380
|
const rows = app.db.select({
|
|
48004
48381
|
page: gscSearchData.page,
|
|
48005
|
-
clicks:
|
|
48006
|
-
impressions:
|
|
48007
|
-
}).from(gscSearchData).where(and45(...conditions)).groupBy(gscSearchData.page).orderBy(desc27(
|
|
48382
|
+
clicks: sql20`COALESCE(SUM(${gscSearchData.clicks}), 0)`,
|
|
48383
|
+
impressions: sql20`COALESCE(SUM(${gscSearchData.impressions}), 0)`
|
|
48384
|
+
}).from(gscSearchData).where(and45(...conditions)).groupBy(gscSearchData.page).orderBy(desc27(sql20`SUM(${gscSearchData.clicks})`), desc27(sql20`SUM(${gscSearchData.impressions})`)).limit(limitVal).all();
|
|
48008
48385
|
const windowStart = startDate ?? cutoffDate ?? "";
|
|
48009
48386
|
const windowEnd = endDate ?? resolvedWindow.endDate ?? "9999-12-31";
|
|
48010
48387
|
const dailyTotals = readGscDailyTotals(app.db, project.id, windowStart, windowEnd);
|
|
48011
48388
|
const totalClicks = dailyTotals.reduce((sum, d) => sum + d.clicks, 0);
|
|
48012
48389
|
const totalImpressions = dailyTotals.reduce((sum, d) => sum + d.impressions, 0);
|
|
48013
48390
|
const rankedSpan = app.db.select({
|
|
48014
|
-
first:
|
|
48015
|
-
last:
|
|
48391
|
+
first: sql20`MIN(${gscSearchData.date})`,
|
|
48392
|
+
last: sql20`MAX(${gscSearchData.date})`
|
|
48016
48393
|
}).from(gscSearchData).where(and45(...conditions)).get();
|
|
48017
48394
|
const coveredFrom = dailyTotals.length > 0 ? dailyTotals[0].date : null;
|
|
48018
48395
|
const coveredThrough = dailyTotals.length > 0 ? dailyTotals[dailyTotals.length - 1].date : null;
|
|
@@ -49010,8 +49387,8 @@ async function googleRoutes(app, opts) {
|
|
|
49010
49387
|
});
|
|
49011
49388
|
const conditions = [eq54(gbpReviews.projectId, project.id)];
|
|
49012
49389
|
if (locationName) conditions.push(eq54(gbpReviews.locationName, locationName));
|
|
49013
|
-
if (negativeOnly) conditions.push(
|
|
49014
|
-
const total = app.db.select({ n:
|
|
49390
|
+
if (negativeOnly) conditions.push(sql20`${gbpReviews.starRating} BETWEEN 1 AND ${negativeMaxStars}`);
|
|
49391
|
+
const total = app.db.select({ n: sql20`count(*)` }).from(gbpReviews).where(and45(...conditions)).get()?.n ?? 0;
|
|
49015
49392
|
const reviews = app.db.select().from(gbpReviews).where(and45(...conditions)).orderBy(desc27(gbpReviews.updateTime), gbpReviews.reviewName).limit(limit).all().map((r) => ({
|
|
49016
49393
|
locationName: r.locationName,
|
|
49017
49394
|
origin: r.origin,
|
|
@@ -49099,7 +49476,7 @@ function parseReviewLimit(value) {
|
|
|
49099
49476
|
|
|
49100
49477
|
// ../api-routes/src/google-marketing.ts
|
|
49101
49478
|
import crypto34 from "crypto";
|
|
49102
|
-
import { and as and46, desc as desc28, eq as eq55, isNull as isNull10, lt as lt9, ne as ne10, or as or13, sql as
|
|
49479
|
+
import { and as and46, desc as desc28, eq as eq55, isNull as isNull10, lt as lt9, ne as ne10, or as or13, sql as sql21 } from "drizzle-orm";
|
|
49103
49480
|
import { z as z4 } from "zod";
|
|
49104
49481
|
var snapshotPageQuerySchema = z4.object({
|
|
49105
49482
|
limit: z4.coerce.number().int().min(1).max(GOOGLE_MARKETING_STORED_SNAPSHOT_PAGE_MAX).optional(),
|
|
@@ -50050,7 +50427,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
50050
50427
|
selectedCustomerCurrencyCode: null,
|
|
50051
50428
|
selectedCustomerTimeZone: null,
|
|
50052
50429
|
selectedCustomerStatus: null,
|
|
50053
|
-
selectionGeneration:
|
|
50430
|
+
selectionGeneration: sql21`${googleAdsConnections.selectionGeneration} + 1`,
|
|
50054
50431
|
scopes: tokens.scopes,
|
|
50055
50432
|
lastValidatedAt: now,
|
|
50056
50433
|
lastCustomerSnapshotId: null,
|
|
@@ -50081,7 +50458,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
50081
50458
|
selectedContainerPublicId: null,
|
|
50082
50459
|
selectedWorkspaceId: null,
|
|
50083
50460
|
selectedWorkspaceName: null,
|
|
50084
|
-
selectionGeneration:
|
|
50461
|
+
selectionGeneration: sql21`${gtmConnections.selectionGeneration} + 1`,
|
|
50085
50462
|
scopes: tokens.scopes,
|
|
50086
50463
|
lastValidatedAt: now,
|
|
50087
50464
|
lastSnapshotAt: null,
|
|
@@ -50232,7 +50609,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
50232
50609
|
selectedCustomerStatus: customerChanged ? null : existing.selectedCustomerStatus,
|
|
50233
50610
|
// A same-value write still starts a new selection generation. Do not
|
|
50234
50611
|
// let timestamp equality reuse append-only evidence from before it.
|
|
50235
|
-
selectionGeneration:
|
|
50612
|
+
selectionGeneration: sql21`${googleAdsConnections.selectionGeneration} + 1`,
|
|
50236
50613
|
lastValidatedAt: now,
|
|
50237
50614
|
lastCustomerSnapshotId: null,
|
|
50238
50615
|
lastInventorySnapshotAt: null,
|
|
@@ -50276,7 +50653,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
50276
50653
|
selectedContainerPublicId: containerChanged ? null : existing.selectedContainerPublicId,
|
|
50277
50654
|
selectedWorkspaceId,
|
|
50278
50655
|
selectedWorkspaceName: workspaceChanged ? null : existing.selectedWorkspaceName,
|
|
50279
|
-
selectionGeneration:
|
|
50656
|
+
selectionGeneration: sql21`${gtmConnections.selectionGeneration} + 1`,
|
|
50280
50657
|
lastValidatedAt: now,
|
|
50281
50658
|
// Every selection write starts a new evidence generation, including a
|
|
50282
50659
|
// same-value reselection. The timestamp alone cannot order a prior
|
|
@@ -50388,7 +50765,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
50388
50765
|
const limit = query.limit ?? 50;
|
|
50389
50766
|
const rows = app.db.select().from(googleAdsRawSnapshots).where(and46(...conditions)).orderBy(desc28(googleAdsRawSnapshots.capturedAt), desc28(googleAdsRawSnapshots.id)).limit(limit + 1).all();
|
|
50390
50767
|
const page3 = rows.slice(0, limit);
|
|
50391
|
-
const total = app.db.select({ total:
|
|
50768
|
+
const total = app.db.select({ total: sql21`COUNT(*)`.mapWith(Number).as("total") }).from(googleAdsRawSnapshots).where(eq55(googleAdsRawSnapshots.projectId, project.id)).get();
|
|
50392
50769
|
return {
|
|
50393
50770
|
snapshots: page3.map(toGoogleAdsSnapshotMetadata),
|
|
50394
50771
|
nextCursor: rows.length > limit && page3.length > 0 ? encodeSnapshotCursor({ capturedAt: page3.at(-1).capturedAt, id: page3.at(-1).id }) : null,
|
|
@@ -50420,7 +50797,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
50420
50797
|
const limit = query.limit ?? 50;
|
|
50421
50798
|
const rows = app.db.select().from(gtmRawSnapshots).where(and46(...conditions)).orderBy(desc28(gtmRawSnapshots.capturedAt), desc28(gtmRawSnapshots.id)).limit(limit + 1).all();
|
|
50422
50799
|
const page3 = rows.slice(0, limit);
|
|
50423
|
-
const total = app.db.select({ total:
|
|
50800
|
+
const total = app.db.select({ total: sql21`COUNT(*)`.mapWith(Number).as("total") }).from(gtmRawSnapshots).where(eq55(gtmRawSnapshots.projectId, project.id)).get();
|
|
50424
50801
|
return {
|
|
50425
50802
|
snapshots: page3.map(toGtmSnapshotMetadata),
|
|
50426
50803
|
nextCursor: rows.length > limit && page3.length > 0 ? encodeSnapshotCursor({ capturedAt: page3.at(-1).capturedAt, id: page3.at(-1).id }) : null,
|
|
@@ -50455,7 +50832,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
50455
50832
|
selectedCustomerCurrencyCode: null,
|
|
50456
50833
|
selectedCustomerTimeZone: null,
|
|
50457
50834
|
selectedCustomerStatus: null,
|
|
50458
|
-
selectionGeneration:
|
|
50835
|
+
selectionGeneration: sql21`${googleAdsConnections.selectionGeneration} + 1`,
|
|
50459
50836
|
scopes: [],
|
|
50460
50837
|
lastValidatedAt: null,
|
|
50461
50838
|
lastCustomerSnapshotId: null,
|
|
@@ -50494,7 +50871,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
50494
50871
|
selectedContainerPublicId: null,
|
|
50495
50872
|
selectedWorkspaceId: null,
|
|
50496
50873
|
selectedWorkspaceName: null,
|
|
50497
|
-
selectionGeneration:
|
|
50874
|
+
selectionGeneration: sql21`${gtmConnections.selectionGeneration} + 1`,
|
|
50498
50875
|
scopes: [],
|
|
50499
50876
|
lastValidatedAt: null,
|
|
50500
50877
|
lastSnapshotAt: null,
|
|
@@ -50689,7 +51066,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
50689
51066
|
|
|
50690
51067
|
// ../api-routes/src/ads.ts
|
|
50691
51068
|
import crypto37 from "crypto";
|
|
50692
|
-
import { eq as eq57, and as and48, asc as asc11, desc as desc29, gt as gt6, gte as gte14, lt as lt10, lte as lte14, ne as ne11, inArray as inArray26, or as or15, isNull as isNull12, isNotNull as isNotNull7, sql as
|
|
51069
|
+
import { eq as eq57, and as and48, asc as asc11, desc as desc29, gt as gt6, gte as gte14, lt as lt10, lte as lte14, ne as ne11, inArray as inArray26, or as or15, isNull as isNull12, isNotNull as isNotNull7, sql as sql23 } from "drizzle-orm";
|
|
50693
51070
|
|
|
50694
51071
|
// ../api-routes/src/ads-activation.ts
|
|
50695
51072
|
import crypto35 from "crypto";
|
|
@@ -52075,7 +52452,7 @@ async function executeApprovedAdsActivation(dependencies, request) {
|
|
|
52075
52452
|
|
|
52076
52453
|
// ../api-routes/src/ads-activation-routes.ts
|
|
52077
52454
|
import crypto36 from "crypto";
|
|
52078
|
-
import { and as and47, asc as asc10, eq as eq56, gt as gt5, isNotNull as isNotNull6, isNull as isNull11, lte as lte13, or as or14, sql as
|
|
52455
|
+
import { and as and47, asc as asc10, eq as eq56, gt as gt5, isNotNull as isNotNull6, isNull as isNull11, lte as lte13, or as or14, sql as sql22 } from "drizzle-orm";
|
|
52079
52456
|
var MAX_ACTIVATION_GRANT_LIFETIME_MS = 24 * 60 * 60 * 1e3;
|
|
52080
52457
|
var ACTIVATION_LEASE_MS = 5 * 6e4;
|
|
52081
52458
|
var ACTIVATION_REVOCATION_SETTLEMENT_MS = ACTIVATION_LEASE_MS;
|
|
@@ -52647,7 +53024,7 @@ function selectWatchdogOperations(app, nowIso, batchSize, allowLiveRevocationLea
|
|
|
52647
53024
|
),
|
|
52648
53025
|
leaseAvailable
|
|
52649
53026
|
);
|
|
52650
|
-
const freshRevocationPriority =
|
|
53027
|
+
const freshRevocationPriority = sql22`CASE
|
|
52651
53028
|
WHEN ${adsActivationGrants.revocationRequestedAt} IS NOT NULL
|
|
52652
53029
|
AND (
|
|
52653
53030
|
${adsOperations.lastReconciledAt} IS NULL
|
|
@@ -54055,7 +54432,7 @@ function claimOperationForReconciliation(app, row, leaseOwner, now, leaseMs, pol
|
|
|
54055
54432
|
state: AdsOperationStates.reconciling,
|
|
54056
54433
|
leaseOwner,
|
|
54057
54434
|
leaseExpiresAt: leaseExpiresAt2,
|
|
54058
|
-
reconcileAttempts:
|
|
54435
|
+
reconcileAttempts: sql23`${adsOperations.reconcileAttempts} + 1`,
|
|
54059
54436
|
updatedAt: nowIso
|
|
54060
54437
|
}).where(and48(
|
|
54061
54438
|
eq57(adsOperations.id, row.id),
|
|
@@ -56657,7 +57034,7 @@ async function cdpRoutes(app, opts) {
|
|
|
56657
57034
|
|
|
56658
57035
|
// ../api-routes/src/ga.ts
|
|
56659
57036
|
import crypto40 from "crypto";
|
|
56660
|
-
import { eq as eq60, desc as desc31, and as and51, sql as
|
|
57037
|
+
import { eq as eq60, desc as desc31, and as and51, sql as sql24 } from "drizzle-orm";
|
|
56661
57038
|
|
|
56662
57039
|
// ../api-routes/src/ga-session-history.ts
|
|
56663
57040
|
function buildSessionHistory(landingPageDays, dailyTotals) {
|
|
@@ -56730,8 +57107,8 @@ function withSessionShares(rows) {
|
|
|
56730
57107
|
}
|
|
56731
57108
|
function dateRangeConditions(column, range) {
|
|
56732
57109
|
const conditions = [];
|
|
56733
|
-
if (range.startDate) conditions.push(
|
|
56734
|
-
if (range.endDate) conditions.push(
|
|
57110
|
+
if (range.startDate) conditions.push(sql24`${column} >= ${range.startDate}`);
|
|
57111
|
+
if (range.endDate) conditions.push(sql24`${column} <= ${range.endDate}`);
|
|
56735
57112
|
return conditions;
|
|
56736
57113
|
}
|
|
56737
57114
|
var SOCIAL_CHANNEL_GROUPS2 = /* @__PURE__ */ new Set(["Organic Social", "Paid Social"]);
|
|
@@ -56791,8 +57168,8 @@ function persistAcquisitionMeasurement(db, input) {
|
|
|
56791
57168
|
db.transaction((tx) => {
|
|
56792
57169
|
tx.delete(gaAcquisitionDaily).where(and51(
|
|
56793
57170
|
eq60(gaAcquisitionDaily.projectId, input.projectId),
|
|
56794
|
-
|
|
56795
|
-
|
|
57171
|
+
sql24`${gaAcquisitionDaily.date} >= ${startDate}`,
|
|
57172
|
+
sql24`${gaAcquisitionDaily.date} <= ${endDate}`
|
|
56796
57173
|
)).run();
|
|
56797
57174
|
for (const row of input.report.rows) {
|
|
56798
57175
|
tx.insert(gaAcquisitionDaily).values({
|
|
@@ -56833,8 +57210,8 @@ function persistLeadMeasurement(db, input) {
|
|
|
56833
57210
|
db.transaction((tx) => {
|
|
56834
57211
|
tx.delete(gaLeadEventsDaily).where(and51(
|
|
56835
57212
|
eq60(gaLeadEventsDaily.projectId, input.projectId),
|
|
56836
|
-
|
|
56837
|
-
|
|
57213
|
+
sql24`${gaLeadEventsDaily.date} >= ${startDate}`,
|
|
57214
|
+
sql24`${gaLeadEventsDaily.date} <= ${endDate}`
|
|
56838
57215
|
)).run();
|
|
56839
57216
|
for (const row of input.report.rows) {
|
|
56840
57217
|
tx.insert(gaLeadEventsDaily).values({
|
|
@@ -57209,8 +57586,8 @@ async function ga4Routes(app, opts) {
|
|
|
57209
57586
|
tx.delete(gaTrafficSnapshots).where(
|
|
57210
57587
|
and51(
|
|
57211
57588
|
eq60(gaTrafficSnapshots.projectId, project.id),
|
|
57212
|
-
|
|
57213
|
-
|
|
57589
|
+
sql24`${gaTrafficSnapshots.date} >= ${summary3.periodStart}`,
|
|
57590
|
+
sql24`${gaTrafficSnapshots.date} <= ${summary3.periodEnd}`
|
|
57214
57591
|
)
|
|
57215
57592
|
).run();
|
|
57216
57593
|
for (const row of rows) {
|
|
@@ -57232,8 +57609,8 @@ async function ga4Routes(app, opts) {
|
|
|
57232
57609
|
tx.delete(gaDailyTotals).where(
|
|
57233
57610
|
and51(
|
|
57234
57611
|
eq60(gaDailyTotals.projectId, project.id),
|
|
57235
|
-
|
|
57236
|
-
|
|
57612
|
+
sql24`${gaDailyTotals.date} >= ${summary3.periodStart}`,
|
|
57613
|
+
sql24`${gaDailyTotals.date} <= ${summary3.periodEnd}`
|
|
57237
57614
|
)
|
|
57238
57615
|
).run();
|
|
57239
57616
|
for (const row of dailyTotals) {
|
|
@@ -57258,8 +57635,8 @@ async function ga4Routes(app, opts) {
|
|
|
57258
57635
|
tx.delete(gaAiReferrals).where(
|
|
57259
57636
|
and51(
|
|
57260
57637
|
eq60(gaAiReferrals.projectId, project.id),
|
|
57261
|
-
|
|
57262
|
-
|
|
57638
|
+
sql24`${gaAiReferrals.date} >= ${summary3.periodStart}`,
|
|
57639
|
+
sql24`${gaAiReferrals.date} <= ${summary3.periodEnd}`
|
|
57263
57640
|
)
|
|
57264
57641
|
).run();
|
|
57265
57642
|
for (const row of aiReferrals) {
|
|
@@ -57290,8 +57667,8 @@ async function ga4Routes(app, opts) {
|
|
|
57290
57667
|
tx.delete(gaSocialReferrals).where(
|
|
57291
57668
|
and51(
|
|
57292
57669
|
eq60(gaSocialReferrals.projectId, project.id),
|
|
57293
|
-
|
|
57294
|
-
|
|
57670
|
+
sql24`${gaSocialReferrals.date} >= ${summary3.periodStart}`,
|
|
57671
|
+
sql24`${gaSocialReferrals.date} <= ${summary3.periodEnd}`
|
|
57295
57672
|
)
|
|
57296
57673
|
).run();
|
|
57297
57674
|
for (const row of socialReferrals) {
|
|
@@ -57473,9 +57850,9 @@ async function ga4Routes(app, opts) {
|
|
|
57473
57850
|
totalUsers: gaTrafficSummaries.totalUsers
|
|
57474
57851
|
}).from(gaTrafficSummaries).where(eq60(gaTrafficSummaries.projectId, project.id)).get();
|
|
57475
57852
|
const retainedBounds = [
|
|
57476
|
-
app.db.select({ min:
|
|
57477
|
-
app.db.select({ min:
|
|
57478
|
-
app.db.select({ min:
|
|
57853
|
+
app.db.select({ min: sql24`MIN(${gaTrafficSnapshots.date})`, max: sql24`MAX(${gaTrafficSnapshots.date})` }).from(gaTrafficSnapshots).where(eq60(gaTrafficSnapshots.projectId, project.id)).get(),
|
|
57854
|
+
app.db.select({ min: sql24`MIN(${gaAiReferrals.date})`, max: sql24`MAX(${gaAiReferrals.date})` }).from(gaAiReferrals).where(eq60(gaAiReferrals.projectId, project.id)).get(),
|
|
57855
|
+
app.db.select({ min: sql24`MIN(${gaSocialReferrals.date})`, max: sql24`MAX(${gaSocialReferrals.date})` }).from(gaSocialReferrals).where(eq60(gaSocialReferrals.projectId, project.id)).get()
|
|
57479
57856
|
];
|
|
57480
57857
|
const retainedDates = retainedBounds.flatMap((bound) => [bound?.min, bound?.max]).filter((date) => Boolean(date));
|
|
57481
57858
|
const projectSummaryCoversAll = Boolean(
|
|
@@ -57489,9 +57866,9 @@ async function ga4Routes(app, opts) {
|
|
|
57489
57866
|
const socialConditions = [eq60(gaSocialReferrals.projectId, project.id), ...dateRangeConditions(gaSocialReferrals.date, measuredRange)];
|
|
57490
57867
|
const snapshotTotalsRow = !denominatorRow ? (() => {
|
|
57491
57868
|
const summed = app.db.select({
|
|
57492
|
-
totalSessions:
|
|
57493
|
-
totalOrganicSessions:
|
|
57494
|
-
totalUsers:
|
|
57869
|
+
totalSessions: sql24`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)`,
|
|
57870
|
+
totalOrganicSessions: sql24`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)`,
|
|
57871
|
+
totalUsers: sql24`COALESCE(SUM(${gaTrafficSnapshots.users}), 0)`
|
|
57495
57872
|
}).from(gaTrafficSnapshots).where(and51(...snapshotConditions)).get();
|
|
57496
57873
|
return {
|
|
57497
57874
|
totalSessions: summed?.totalSessions ?? 0,
|
|
@@ -57504,35 +57881,35 @@ async function ga4Routes(app, opts) {
|
|
|
57504
57881
|
})() : null;
|
|
57505
57882
|
const summaryRow = denominatorRow ?? snapshotTotalsRow;
|
|
57506
57883
|
const directTotalRow = windowSummaryRow ? { totalDirectSessions: windowSummaryRow.totalDirectSessions } : app.db.select({
|
|
57507
|
-
totalDirectSessions:
|
|
57884
|
+
totalDirectSessions: sql24`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)`
|
|
57508
57885
|
}).from(gaTrafficSnapshots).where(and51(...snapshotConditions)).get();
|
|
57509
57886
|
const rows = app.db.select({
|
|
57510
|
-
landingPage:
|
|
57511
|
-
sessions:
|
|
57512
|
-
organicSessions:
|
|
57513
|
-
directSessions:
|
|
57514
|
-
users:
|
|
57515
|
-
}).from(gaTrafficSnapshots).where(and51(...snapshotConditions)).groupBy(
|
|
57887
|
+
landingPage: sql24`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`,
|
|
57888
|
+
sessions: sql24`SUM(${gaTrafficSnapshots.sessions})`,
|
|
57889
|
+
organicSessions: sql24`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
57890
|
+
directSessions: sql24`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)`,
|
|
57891
|
+
users: sql24`SUM(${gaTrafficSnapshots.users})`
|
|
57892
|
+
}).from(gaTrafficSnapshots).where(and51(...snapshotConditions)).groupBy(sql24`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`).orderBy(sql24`SUM(${gaTrafficSnapshots.sessions}) DESC`).limit(limit).all();
|
|
57516
57893
|
const aiReferralRows = app.db.select({
|
|
57517
57894
|
source: gaAiReferrals.source,
|
|
57518
57895
|
medium: gaAiReferrals.medium,
|
|
57519
57896
|
trafficClass: gaAiReferrals.trafficClass,
|
|
57520
57897
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
57521
|
-
sessions:
|
|
57898
|
+
sessions: sql24`SUM(${gaAiReferrals.sessions})`
|
|
57522
57899
|
}).from(gaAiReferrals).where(and51(...aiConditions)).groupBy(gaAiReferrals.source, gaAiReferrals.medium, gaAiReferrals.trafficClass, gaAiReferrals.sourceDimension).all();
|
|
57523
57900
|
const aiReferralLandingPageRows = app.db.select({
|
|
57524
57901
|
source: gaAiReferrals.source,
|
|
57525
57902
|
medium: gaAiReferrals.medium,
|
|
57526
57903
|
trafficClass: gaAiReferrals.trafficClass,
|
|
57527
57904
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
57528
|
-
landingPage:
|
|
57529
|
-
sessions:
|
|
57905
|
+
landingPage: sql24`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`,
|
|
57906
|
+
sessions: sql24`SUM(${gaAiReferrals.sessions})`
|
|
57530
57907
|
}).from(gaAiReferrals).where(and51(...aiConditions)).groupBy(
|
|
57531
57908
|
gaAiReferrals.source,
|
|
57532
57909
|
gaAiReferrals.medium,
|
|
57533
57910
|
gaAiReferrals.trafficClass,
|
|
57534
57911
|
gaAiReferrals.sourceDimension,
|
|
57535
|
-
|
|
57912
|
+
sql24`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`
|
|
57536
57913
|
).all();
|
|
57537
57914
|
const aiRowsForTotals = app.db.select({
|
|
57538
57915
|
date: gaAiReferrals.date,
|
|
@@ -57558,10 +57935,10 @@ async function ga4Routes(app, opts) {
|
|
|
57558
57935
|
// is GA's COUNT DISTINCT at that grain, so summing over the window's
|
|
57559
57936
|
// dates counts a returning visitor once per day they returned. Same
|
|
57560
57937
|
// conclusion the AI-referral surfaces reached in 4.135.0.
|
|
57561
|
-
sessions:
|
|
57562
|
-
}).from(gaSocialReferrals).where(and51(...socialConditions)).groupBy(gaSocialReferrals.source, gaSocialReferrals.medium, gaSocialReferrals.channelGroup).orderBy(
|
|
57938
|
+
sessions: sql24`SUM(${gaSocialReferrals.sessions})`
|
|
57939
|
+
}).from(gaSocialReferrals).where(and51(...socialConditions)).groupBy(gaSocialReferrals.source, gaSocialReferrals.medium, gaSocialReferrals.channelGroup).orderBy(sql24`SUM(${gaSocialReferrals.sessions}) DESC`).all();
|
|
57563
57940
|
const socialTotals = app.db.select({
|
|
57564
|
-
sessions:
|
|
57941
|
+
sessions: sql24`SUM(${gaSocialReferrals.sessions})`
|
|
57565
57942
|
}).from(gaSocialReferrals).where(and51(...socialConditions)).get();
|
|
57566
57943
|
const latestSync = app.db.select({ syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(eq60(gaTrafficSummaries.projectId, project.id)).orderBy(desc31(gaTrafficSummaries.syncedAt)).limit(1).get();
|
|
57567
57944
|
const total = summaryRow?.totalSessions ?? 0;
|
|
@@ -57682,20 +58059,20 @@ async function ga4Routes(app, opts) {
|
|
|
57682
58059
|
source: gaAiReferrals.source,
|
|
57683
58060
|
medium: gaAiReferrals.medium,
|
|
57684
58061
|
trafficClass: gaAiReferrals.trafficClass,
|
|
57685
|
-
landingPage:
|
|
58062
|
+
landingPage: sql24`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`,
|
|
57686
58063
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
57687
58064
|
// No `users`: ga-ai-referral-aggregation states the rule for this
|
|
57688
58065
|
// table, that there is no grain at which the stored users column may be
|
|
57689
58066
|
// added up. ga4AiReferralDtoSchema.users was deprecated for this in
|
|
57690
58067
|
// 4.135.0; the history endpoint kept emitting one.
|
|
57691
|
-
sessions:
|
|
58068
|
+
sessions: sql24`SUM(${gaAiReferrals.sessions})`
|
|
57692
58069
|
}).from(gaAiReferrals).where(and51(...conditions)).groupBy(
|
|
57693
58070
|
gaAiReferrals.date,
|
|
57694
58071
|
gaAiReferrals.source,
|
|
57695
58072
|
gaAiReferrals.medium,
|
|
57696
58073
|
gaAiReferrals.trafficClass,
|
|
57697
58074
|
gaAiReferrals.sourceDimension,
|
|
57698
|
-
|
|
58075
|
+
sql24`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`
|
|
57699
58076
|
).orderBy(gaAiReferrals.date).all();
|
|
57700
58077
|
return rows;
|
|
57701
58078
|
});
|
|
@@ -57740,10 +58117,10 @@ async function ga4Routes(app, opts) {
|
|
|
57740
58117
|
d.setDate(d.getDate() - n);
|
|
57741
58118
|
return fmt(d);
|
|
57742
58119
|
};
|
|
57743
|
-
const sumSocial = (from, to) => app.db.select({ sessions:
|
|
58120
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql24`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and51(
|
|
57744
58121
|
eq60(gaSocialReferrals.projectId, project.id),
|
|
57745
|
-
|
|
57746
|
-
|
|
58122
|
+
sql24`${gaSocialReferrals.date} >= ${from}`,
|
|
58123
|
+
sql24`${gaSocialReferrals.date} < ${to}`
|
|
57747
58124
|
)).get();
|
|
57748
58125
|
const current7d = sumSocial(daysAgo(7), fmt(today));
|
|
57749
58126
|
const prev7d = sumSocial(daysAgo(14), daysAgo(7));
|
|
@@ -57751,19 +58128,19 @@ async function ga4Routes(app, opts) {
|
|
|
57751
58128
|
const prev30d = sumSocial(daysAgo(60), daysAgo(30));
|
|
57752
58129
|
const sourceCurrent = app.db.select({
|
|
57753
58130
|
source: gaSocialReferrals.source,
|
|
57754
|
-
sessions:
|
|
58131
|
+
sessions: sql24`SUM(${gaSocialReferrals.sessions})`
|
|
57755
58132
|
}).from(gaSocialReferrals).where(and51(
|
|
57756
58133
|
eq60(gaSocialReferrals.projectId, project.id),
|
|
57757
|
-
|
|
57758
|
-
|
|
58134
|
+
sql24`${gaSocialReferrals.date} >= ${daysAgo(7)}`,
|
|
58135
|
+
sql24`${gaSocialReferrals.date} < ${fmt(today)}`
|
|
57759
58136
|
)).groupBy(gaSocialReferrals.source).all();
|
|
57760
58137
|
const sourcePrev = app.db.select({
|
|
57761
58138
|
source: gaSocialReferrals.source,
|
|
57762
|
-
sessions:
|
|
58139
|
+
sessions: sql24`SUM(${gaSocialReferrals.sessions})`
|
|
57763
58140
|
}).from(gaSocialReferrals).where(and51(
|
|
57764
58141
|
eq60(gaSocialReferrals.projectId, project.id),
|
|
57765
|
-
|
|
57766
|
-
|
|
58142
|
+
sql24`${gaSocialReferrals.date} >= ${daysAgo(14)}`,
|
|
58143
|
+
sql24`${gaSocialReferrals.date} < ${daysAgo(7)}`
|
|
57767
58144
|
)).groupBy(gaSocialReferrals.source).all();
|
|
57768
58145
|
const response = {
|
|
57769
58146
|
socialSessions7d: current7d?.sessions ?? 0,
|
|
@@ -57786,16 +58163,16 @@ async function ga4Routes(app, opts) {
|
|
|
57786
58163
|
d.setDate(d.getDate() - n);
|
|
57787
58164
|
return fmt(d);
|
|
57788
58165
|
};
|
|
57789
|
-
const sumTotal = (from, to) => app.db.select({ sessions:
|
|
57790
|
-
const sumOrganic = (from, to) => app.db.select({ sessions:
|
|
57791
|
-
const sumDirect = (from, to) => app.db.select({ sessions:
|
|
57792
|
-
const sumAi = (from, to) => app.db.select({ sessions:
|
|
58166
|
+
const sumTotal = (from, to) => app.db.select({ sessions: sql24`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)` }).from(gaTrafficSnapshots).where(and51(eq60(gaTrafficSnapshots.projectId, project.id), sql24`${gaTrafficSnapshots.date} >= ${from}`, sql24`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
58167
|
+
const sumOrganic = (from, to) => app.db.select({ sessions: sql24`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)` }).from(gaTrafficSnapshots).where(and51(eq60(gaTrafficSnapshots.projectId, project.id), sql24`${gaTrafficSnapshots.date} >= ${from}`, sql24`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
58168
|
+
const sumDirect = (from, to) => app.db.select({ sessions: sql24`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)` }).from(gaTrafficSnapshots).where(and51(eq60(gaTrafficSnapshots.projectId, project.id), sql24`${gaTrafficSnapshots.date} >= ${from}`, sql24`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
58169
|
+
const sumAi = (from, to) => app.db.select({ sessions: sql24`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and51(
|
|
57793
58170
|
eq60(gaAiReferrals.projectId, project.id),
|
|
57794
|
-
|
|
57795
|
-
|
|
58171
|
+
sql24`${gaAiReferrals.date} >= ${from}`,
|
|
58172
|
+
sql24`${gaAiReferrals.date} < ${to}`,
|
|
57796
58173
|
eq60(gaAiReferrals.sourceDimension, "session")
|
|
57797
58174
|
)).get();
|
|
57798
|
-
const sumSocial = (from, to) => app.db.select({ sessions:
|
|
58175
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql24`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and51(eq60(gaSocialReferrals.projectId, project.id), sql24`${gaSocialReferrals.date} >= ${from}`, sql24`${gaSocialReferrals.date} < ${to}`)).get();
|
|
57799
58176
|
const todayStr = fmt(today);
|
|
57800
58177
|
const buildTrend = (sum) => {
|
|
57801
58178
|
const c7 = sum(daysAgo(7), todayStr)?.sessions ?? 0;
|
|
@@ -57804,20 +58181,20 @@ async function ga4Routes(app, opts) {
|
|
|
57804
58181
|
const p30 = sum(daysAgo(60), daysAgo(30))?.sessions ?? 0;
|
|
57805
58182
|
return { sessions7d: c7, sessionsPrev7d: p7, trend7dPct: deltaPercent(c7, p7), sessions30d: c30, sessionsPrev30d: p30, trend30dPct: deltaPercent(c30, p30) };
|
|
57806
58183
|
};
|
|
57807
|
-
const aiSourceCurrent = app.db.select({ source: gaAiReferrals.source, sessions:
|
|
58184
|
+
const aiSourceCurrent = app.db.select({ source: gaAiReferrals.source, sessions: sql24`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and51(
|
|
57808
58185
|
eq60(gaAiReferrals.projectId, project.id),
|
|
57809
|
-
|
|
57810
|
-
|
|
58186
|
+
sql24`${gaAiReferrals.date} >= ${daysAgo(7)}`,
|
|
58187
|
+
sql24`${gaAiReferrals.date} < ${todayStr}`,
|
|
57811
58188
|
eq60(gaAiReferrals.sourceDimension, "session")
|
|
57812
58189
|
)).groupBy(gaAiReferrals.source).all();
|
|
57813
|
-
const aiSourcePrev = app.db.select({ source: gaAiReferrals.source, sessions:
|
|
58190
|
+
const aiSourcePrev = app.db.select({ source: gaAiReferrals.source, sessions: sql24`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and51(
|
|
57814
58191
|
eq60(gaAiReferrals.projectId, project.id),
|
|
57815
|
-
|
|
57816
|
-
|
|
58192
|
+
sql24`${gaAiReferrals.date} >= ${daysAgo(14)}`,
|
|
58193
|
+
sql24`${gaAiReferrals.date} < ${daysAgo(7)}`,
|
|
57817
58194
|
eq60(gaAiReferrals.sourceDimension, "session")
|
|
57818
58195
|
)).groupBy(gaAiReferrals.source).all();
|
|
57819
|
-
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions:
|
|
57820
|
-
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions:
|
|
58196
|
+
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions: sql24`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and51(eq60(gaSocialReferrals.projectId, project.id), sql24`${gaSocialReferrals.date} >= ${daysAgo(7)}`, sql24`${gaSocialReferrals.date} < ${todayStr}`)).groupBy(gaSocialReferrals.source).all();
|
|
58197
|
+
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions: sql24`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and51(eq60(gaSocialReferrals.projectId, project.id), sql24`${gaSocialReferrals.date} >= ${daysAgo(14)}`, sql24`${gaSocialReferrals.date} < ${daysAgo(7)}`)).groupBy(gaSocialReferrals.source).all();
|
|
57821
58198
|
const response = {
|
|
57822
58199
|
total: buildTrend(sumTotal),
|
|
57823
58200
|
organic: buildTrend(sumOrganic),
|
|
@@ -57836,9 +58213,9 @@ async function ga4Routes(app, opts) {
|
|
|
57836
58213
|
const conditions = [eq60(gaTrafficSnapshots.projectId, project.id), ...dateRangeConditions(gaTrafficSnapshots.date, range)];
|
|
57837
58214
|
const rows = app.db.select({
|
|
57838
58215
|
date: gaTrafficSnapshots.date,
|
|
57839
|
-
sessions:
|
|
57840
|
-
organicSessions:
|
|
57841
|
-
users:
|
|
58216
|
+
sessions: sql24`SUM(${gaTrafficSnapshots.sessions})`,
|
|
58217
|
+
organicSessions: sql24`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
58218
|
+
users: sql24`SUM(${gaTrafficSnapshots.users})`
|
|
57842
58219
|
}).from(gaTrafficSnapshots).where(and51(...conditions)).groupBy(gaTrafficSnapshots.date).orderBy(gaTrafficSnapshots.date).all();
|
|
57843
58220
|
const totalConditions = [eq60(gaDailyTotals.projectId, project.id), ...dateRangeConditions(gaDailyTotals.date, range)];
|
|
57844
58221
|
const totals = app.db.select({ date: gaDailyTotals.date, users: gaDailyTotals.users }).from(gaDailyTotals).where(and51(...totalConditions)).all();
|
|
@@ -57856,11 +58233,11 @@ async function ga4Routes(app, opts) {
|
|
|
57856
58233
|
const project = resolveProject(app.db, request.params.name);
|
|
57857
58234
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
57858
58235
|
const trafficPages = app.db.select({
|
|
57859
|
-
landingPage:
|
|
57860
|
-
sessions:
|
|
57861
|
-
organicSessions:
|
|
57862
|
-
users:
|
|
57863
|
-
}).from(gaTrafficSnapshots).where(eq60(gaTrafficSnapshots.projectId, project.id)).groupBy(
|
|
58236
|
+
landingPage: sql24`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`,
|
|
58237
|
+
sessions: sql24`SUM(${gaTrafficSnapshots.sessions})`,
|
|
58238
|
+
organicSessions: sql24`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
58239
|
+
users: sql24`SUM(${gaTrafficSnapshots.users})`
|
|
58240
|
+
}).from(gaTrafficSnapshots).where(eq60(gaTrafficSnapshots.projectId, project.id)).groupBy(sql24`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`).orderBy(sql24`SUM(${gaTrafficSnapshots.sessions}) DESC`).all();
|
|
57864
58241
|
return {
|
|
57865
58242
|
pages: trafficPages.map((r) => ({
|
|
57866
58243
|
landingPage: r.landingPage,
|
|
@@ -59515,7 +59892,7 @@ async function wordpressRoutes(app, opts) {
|
|
|
59515
59892
|
|
|
59516
59893
|
// ../api-routes/src/backlinks.ts
|
|
59517
59894
|
import crypto42 from "crypto";
|
|
59518
|
-
import { and as and53, asc as asc12, desc as desc32, eq as eq61, sql as
|
|
59895
|
+
import { and as and53, asc as asc12, desc as desc32, eq as eq61, sql as sql25 } from "drizzle-orm";
|
|
59519
59896
|
|
|
59520
59897
|
// ../integration-commoncrawl/src/constants.ts
|
|
59521
59898
|
import os2 from "os";
|
|
@@ -59612,7 +59989,7 @@ function parseContentLength(value) {
|
|
|
59612
59989
|
}
|
|
59613
59990
|
|
|
59614
59991
|
// ../integration-commoncrawl/src/downloader.ts
|
|
59615
|
-
import { createHash as
|
|
59992
|
+
import { createHash as createHash9 } from "crypto";
|
|
59616
59993
|
import { createWriteStream } from "fs";
|
|
59617
59994
|
import fs2 from "fs/promises";
|
|
59618
59995
|
import path3 from "path";
|
|
@@ -59638,7 +60015,7 @@ async function downloadFile(opts) {
|
|
|
59638
60015
|
throw new Error(`HTTP ${res.status} ${res.statusText} for ${opts.url}`);
|
|
59639
60016
|
}
|
|
59640
60017
|
const total = parseContentLength2(res.headers.get("content-length"));
|
|
59641
|
-
const hasher =
|
|
60018
|
+
const hasher = createHash9("sha256");
|
|
59642
60019
|
let bytes = 0;
|
|
59643
60020
|
const hashAndCount = new Transform({
|
|
59644
60021
|
transform(chunk, _enc, cb) {
|
|
@@ -59659,7 +60036,7 @@ async function downloadFile(opts) {
|
|
|
59659
60036
|
return { bytes, sha256: sha2564, cached: false, elapsedMs: Date.now() - start };
|
|
59660
60037
|
}
|
|
59661
60038
|
async function hashFile(filePath) {
|
|
59662
|
-
const hasher =
|
|
60039
|
+
const hasher = createHash9("sha256");
|
|
59663
60040
|
const handle = await fs2.open(filePath, "r");
|
|
59664
60041
|
try {
|
|
59665
60042
|
const stream = handle.createReadStream();
|
|
@@ -59805,7 +60182,7 @@ async function queryBacklinks(opts) {
|
|
|
59805
60182
|
const reversed = opts.targets.map(reverseDomain);
|
|
59806
60183
|
const targetList = reversed.map(quote).join(", ");
|
|
59807
60184
|
const limitClause = opts.limitPerTarget ? `QUALIFY row_number() OVER (PARTITION BY t.target_rev_domain ORDER BY v.num_hosts DESC) <= ${Math.floor(opts.limitPerTarget)}` : "";
|
|
59808
|
-
const
|
|
60185
|
+
const sql33 = `
|
|
59809
60186
|
WITH vertices AS (
|
|
59810
60187
|
SELECT * FROM read_csv(
|
|
59811
60188
|
${quote(opts.vertexPath)},
|
|
@@ -59841,7 +60218,7 @@ async function queryBacklinks(opts) {
|
|
|
59841
60218
|
const conn = await instance.connect();
|
|
59842
60219
|
let rows;
|
|
59843
60220
|
try {
|
|
59844
|
-
const reader = await conn.runAndReadAll(
|
|
60221
|
+
const reader = await conn.runAndReadAll(sql33);
|
|
59845
60222
|
rows = reader.getRowObjects();
|
|
59846
60223
|
} finally {
|
|
59847
60224
|
conn.disconnectSync?.();
|
|
@@ -60031,12 +60408,12 @@ function computeFilteredSummary(db, base) {
|
|
|
60031
60408
|
);
|
|
60032
60409
|
const filteredCondition = and53(baseDomainCondition, backlinkCrawlerExclusionClause());
|
|
60033
60410
|
const unfilteredAgg = db.select({
|
|
60034
|
-
count:
|
|
60035
|
-
total:
|
|
60411
|
+
count: sql25`count(*)`,
|
|
60412
|
+
total: sql25`coalesce(sum(${backlinkDomains.numHosts}), 0)`
|
|
60036
60413
|
}).from(backlinkDomains).where(baseDomainCondition).get();
|
|
60037
60414
|
const filteredAgg = db.select({
|
|
60038
|
-
count:
|
|
60039
|
-
total:
|
|
60415
|
+
count: sql25`count(*)`,
|
|
60416
|
+
total: sql25`coalesce(sum(${backlinkDomains.numHosts}), 0)`
|
|
60040
60417
|
}).from(backlinkDomains).where(filteredCondition).get();
|
|
60041
60418
|
const top10Rows = db.select({ numHosts: backlinkDomains.numHosts }).from(backlinkDomains).where(filteredCondition).orderBy(desc32(backlinkDomains.numHosts)).limit(10).all();
|
|
60042
60419
|
const totalLinkingDomains = Number(filteredAgg?.count ?? 0);
|
|
@@ -60062,7 +60439,7 @@ function buildSourceAvailability(db, projectId, source, connected, excludeCrawle
|
|
|
60062
60439
|
const summary3 = db.select().from(backlinkSummaries).where(and53(eq61(backlinkSummaries.projectId, projectId), eq61(backlinkSummaries.source, source))).orderBy(desc32(backlinkSummaries.queriedAt)).limit(1).get();
|
|
60063
60440
|
let totalLinkingDomains = summary3?.totalLinkingDomains ?? 0;
|
|
60064
60441
|
if (summary3 && excludeCrawlers) {
|
|
60065
|
-
const filtered = db.select({ count:
|
|
60442
|
+
const filtered = db.select({ count: sql25`count(*)` }).from(backlinkDomains).where(and53(
|
|
60066
60443
|
eq61(backlinkDomains.projectId, projectId),
|
|
60067
60444
|
eq61(backlinkDomains.source, source),
|
|
60068
60445
|
eq61(backlinkDomains.release, summary3.release),
|
|
@@ -60250,7 +60627,7 @@ async function backlinksRoutes(app, opts) {
|
|
|
60250
60627
|
eq61(backlinkDomains.release, targetRelease)
|
|
60251
60628
|
);
|
|
60252
60629
|
const domainCondition = excludeCrawlers ? and53(baseDomainCondition, backlinkCrawlerExclusionClause()) : baseDomainCondition;
|
|
60253
|
-
const totalRow = app.db.select({ count:
|
|
60630
|
+
const totalRow = app.db.select({ count: sql25`count(*)` }).from(backlinkDomains).where(domainCondition).get();
|
|
60254
60631
|
const rows = app.db.select({
|
|
60255
60632
|
linkingDomain: backlinkDomains.linkingDomain,
|
|
60256
60633
|
numHosts: backlinkDomains.numHosts,
|
|
@@ -60301,7 +60678,7 @@ import crypto45 from "crypto";
|
|
|
60301
60678
|
import { isIP } from "net";
|
|
60302
60679
|
import { isDeepStrictEqual as isDeepStrictEqual2 } from "util";
|
|
60303
60680
|
import { Agent as UndiciAgent } from "undici";
|
|
60304
|
-
import { and as and57, desc as desc33, eq as eq65, gte as gte16, inArray as inArray27, lte as lte17, sql as
|
|
60681
|
+
import { and as and57, desc as desc33, eq as eq65, gte as gte16, inArray as inArray27, lte as lte17, sql as sql28 } from "drizzle-orm";
|
|
60305
60682
|
|
|
60306
60683
|
// ../api-routes/src/cloudflare-worker-version.ts
|
|
60307
60684
|
var CURRENT_CLOUDFLARE_WORKER_VERSION = "1.0.1";
|
|
@@ -74319,7 +74696,7 @@ var perplexitybot_default = {
|
|
|
74319
74696
|
};
|
|
74320
74697
|
|
|
74321
74698
|
// ../integration-traffic/src/ip-range-manifest.ts
|
|
74322
|
-
import { createHash as
|
|
74699
|
+
import { createHash as createHash10 } from "crypto";
|
|
74323
74700
|
function isRecord2(value) {
|
|
74324
74701
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
74325
74702
|
}
|
|
@@ -74411,7 +74788,7 @@ function canonicalPrefixKey(entry) {
|
|
|
74411
74788
|
function ipRangeManifestContentHash(value) {
|
|
74412
74789
|
const prefixes = isRecord2(value) ? value.prefixes : void 0;
|
|
74413
74790
|
const canonical = Array.isArray(prefixes) ? prefixes.map(canonicalPrefixKey).sort() : [`invalid-prefixes:${stableJson2(prefixes)}`];
|
|
74414
|
-
return
|
|
74791
|
+
return createHash10("sha256").update(JSON.stringify(canonical)).digest("hex");
|
|
74415
74792
|
}
|
|
74416
74793
|
function validateIpRangeManifestPayload(value) {
|
|
74417
74794
|
if (!isRecord2(value)) return { ok: false, error: "response must be a JSON object" };
|
|
@@ -76971,10 +77348,10 @@ async function ackCloudflareQueueMessages(client, options) {
|
|
|
76971
77348
|
|
|
76972
77349
|
// ../api-routes/src/traffic-event-ingest.ts
|
|
76973
77350
|
import crypto44 from "crypto";
|
|
76974
|
-
import { and as and55, eq as eq63, gte as gte15, lt as lt12, lte as lte15, sql as
|
|
77351
|
+
import { and as and55, eq as eq63, gte as gte15, lt as lt12, lte as lte15, sql as sql27 } from "drizzle-orm";
|
|
76975
77352
|
|
|
76976
77353
|
// ../api-routes/src/raw-event-sample-retention.ts
|
|
76977
|
-
import { and as and54, eq as eq62, lt as lt11, sql as
|
|
77354
|
+
import { and as and54, eq as eq62, lt as lt11, sql as sql26 } from "drizzle-orm";
|
|
76978
77355
|
var RAW_EVENT_SAMPLE_RETENTION_MS = 30 * 24 * 60 * 6e4;
|
|
76979
77356
|
var RAW_EVENT_SAMPLE_RETENTION_SWEEP_INTERVAL_MS = 24 * 60 * 6e4;
|
|
76980
77357
|
var CANONICAL_UTC_TIMESTAMP_GLOB = "[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9][0-9][0-9]Z";
|
|
@@ -76985,16 +77362,16 @@ function rawEventSampleRetentionCutoff(referenceAt) {
|
|
|
76985
77362
|
}
|
|
76986
77363
|
function deleteExpiredRawEventSamples(db, cutoff, sourceId) {
|
|
76987
77364
|
const sourceScope = sourceId === void 0 ? void 0 : eq62(rawEventSamples.sourceId, sourceId);
|
|
76988
|
-
db.delete(rawEventSamples).where(and54(sourceScope,
|
|
77365
|
+
db.delete(rawEventSamples).where(and54(sourceScope, sql26`julianday(${rawEventSamples.ts}) IS NULL`)).run();
|
|
76989
77366
|
db.delete(rawEventSamples).where(and54(
|
|
76990
77367
|
sourceScope,
|
|
76991
|
-
|
|
77368
|
+
sql26`${rawEventSamples.ts} GLOB ${CANONICAL_UTC_TIMESTAMP_GLOB}`,
|
|
76992
77369
|
lt11(rawEventSamples.ts, cutoff)
|
|
76993
77370
|
)).run();
|
|
76994
77371
|
db.delete(rawEventSamples).where(and54(
|
|
76995
77372
|
sourceScope,
|
|
76996
|
-
|
|
76997
|
-
|
|
77373
|
+
sql26`${rawEventSamples.ts} NOT GLOB ${CANONICAL_UTC_TIMESTAMP_GLOB}`,
|
|
77374
|
+
sql26`julianday(${rawEventSamples.ts}) < julianday(${cutoff})`
|
|
76998
77375
|
)).run();
|
|
76999
77376
|
}
|
|
77000
77377
|
function enforceRawEventSampleRetention(db, sourceId, referenceAt) {
|
|
@@ -77040,7 +77417,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
77040
77417
|
let aiReferralHits = 0;
|
|
77041
77418
|
let unknownHits = 0;
|
|
77042
77419
|
opts.db.transaction((tx) => {
|
|
77043
|
-
tx.update(trafficSources).set({ updatedAt:
|
|
77420
|
+
tx.update(trafficSources).set({ updatedAt: sql27`${trafficSources.updatedAt}` }).where(eq63(trafficSources.id, opts.sourceId)).run();
|
|
77044
77421
|
const source = tx.select().from(trafficSources).where(eq63(trafficSources.id, opts.sourceId)).get();
|
|
77045
77422
|
opts.validateSource(source);
|
|
77046
77423
|
const rawSampleCutoff = enforceRawEventSampleRetention(
|
|
@@ -77097,7 +77474,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
77097
77474
|
crawlerEventsHourly.status
|
|
77098
77475
|
],
|
|
77099
77476
|
set: {
|
|
77100
|
-
hits:
|
|
77477
|
+
hits: sql27`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
77101
77478
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
77102
77479
|
updatedAt: opts.receivedAt
|
|
77103
77480
|
}
|
|
@@ -77128,7 +77505,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
77128
77505
|
crawlerVerificationManifestsHourly.manifestId
|
|
77129
77506
|
],
|
|
77130
77507
|
set: {
|
|
77131
|
-
hits:
|
|
77508
|
+
hits: sql27`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
77132
77509
|
manifestJson: bucket.verificationManifest,
|
|
77133
77510
|
updatedAt: opts.receivedAt
|
|
77134
77511
|
}
|
|
@@ -77162,7 +77539,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
77162
77539
|
aiUserFetchEventsHourly.status
|
|
77163
77540
|
],
|
|
77164
77541
|
set: {
|
|
77165
|
-
hits:
|
|
77542
|
+
hits: sql27`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
77166
77543
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
77167
77544
|
updatedAt: opts.receivedAt
|
|
77168
77545
|
}
|
|
@@ -77193,7 +77570,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
77193
77570
|
aiUserFetchVerificationManifestsHourly.manifestId
|
|
77194
77571
|
],
|
|
77195
77572
|
set: {
|
|
77196
|
-
hits:
|
|
77573
|
+
hits: sql27`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
77197
77574
|
manifestJson: bucket.verificationManifest,
|
|
77198
77575
|
updatedAt: opts.receivedAt
|
|
77199
77576
|
}
|
|
@@ -77231,9 +77608,9 @@ function writeTrafficEventBatch(opts) {
|
|
|
77231
77608
|
aiReferralEventsHourly.status
|
|
77232
77609
|
],
|
|
77233
77610
|
set: {
|
|
77234
|
-
sessionsOrHits:
|
|
77235
|
-
paidSessionsOrHits:
|
|
77236
|
-
organicSessionsOrHits:
|
|
77611
|
+
sessionsOrHits: sql27`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
77612
|
+
paidSessionsOrHits: sql27`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
|
|
77613
|
+
organicSessionsOrHits: sql27`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
|
|
77237
77614
|
updatedAt: opts.receivedAt
|
|
77238
77615
|
}
|
|
77239
77616
|
}).run();
|
|
@@ -77248,7 +77625,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
77248
77625
|
if (!hour || hourlySampleLimit === 0) continue;
|
|
77249
77626
|
let remaining = remainingSamplesByHour.get(hour.start);
|
|
77250
77627
|
if (remaining === void 0) {
|
|
77251
|
-
const existing = tx.select({ total:
|
|
77628
|
+
const existing = tx.select({ total: sql27`COUNT(*)` }).from(rawEventSamples).where(and55(
|
|
77252
77629
|
eq63(rawEventSamples.sourceId, opts.sourceId),
|
|
77253
77630
|
gte15(rawEventSamples.ts, hour.start),
|
|
77254
77631
|
lt12(rawEventSamples.ts, hour.end)
|
|
@@ -77574,7 +77951,7 @@ function crawlerVerificationProvenance(db, projectId, events) {
|
|
|
77574
77951
|
manifestId: crawlerVerificationManifestsHourly.manifestId,
|
|
77575
77952
|
manifest: crawlerVerificationManifestsHourly.manifestJson,
|
|
77576
77953
|
hits: crawlerVerificationManifestsHourly.hits
|
|
77577
|
-
}).from(crawlerVerificationManifestsHourly).where(
|
|
77954
|
+
}).from(crawlerVerificationManifestsHourly).where(sql28`(
|
|
77578
77955
|
${crawlerVerificationManifestsHourly.projectId},
|
|
77579
77956
|
${crawlerVerificationManifestsHourly.sourceId},
|
|
77580
77957
|
${crawlerVerificationManifestsHourly.tsHour},
|
|
@@ -77608,7 +77985,7 @@ function aiUserFetchVerificationProvenance(db, projectId, events) {
|
|
|
77608
77985
|
manifestId: aiUserFetchVerificationManifestsHourly.manifestId,
|
|
77609
77986
|
manifest: aiUserFetchVerificationManifestsHourly.manifestJson,
|
|
77610
77987
|
hits: aiUserFetchVerificationManifestsHourly.hits
|
|
77611
|
-
}).from(aiUserFetchVerificationManifestsHourly).where(
|
|
77988
|
+
}).from(aiUserFetchVerificationManifestsHourly).where(sql28`(
|
|
77612
77989
|
${aiUserFetchVerificationManifestsHourly.projectId},
|
|
77613
77990
|
${aiUserFetchVerificationManifestsHourly.sourceId},
|
|
77614
77991
|
${aiUserFetchVerificationManifestsHourly.tsHour},
|
|
@@ -77872,7 +78249,7 @@ async function runBackfillTask(options) {
|
|
|
77872
78249
|
crawlerEventsHourly.status
|
|
77873
78250
|
],
|
|
77874
78251
|
set: {
|
|
77875
|
-
hits:
|
|
78252
|
+
hits: sql28`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
77876
78253
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
77877
78254
|
updatedAt: finishedAt
|
|
77878
78255
|
}
|
|
@@ -77903,7 +78280,7 @@ async function runBackfillTask(options) {
|
|
|
77903
78280
|
crawlerVerificationManifestsHourly.manifestId
|
|
77904
78281
|
],
|
|
77905
78282
|
set: {
|
|
77906
|
-
hits:
|
|
78283
|
+
hits: sql28`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
77907
78284
|
manifestJson: bucket.verificationManifest,
|
|
77908
78285
|
updatedAt: finishedAt
|
|
77909
78286
|
}
|
|
@@ -77936,7 +78313,7 @@ async function runBackfillTask(options) {
|
|
|
77936
78313
|
aiUserFetchEventsHourly.status
|
|
77937
78314
|
],
|
|
77938
78315
|
set: {
|
|
77939
|
-
hits:
|
|
78316
|
+
hits: sql28`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
77940
78317
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
77941
78318
|
updatedAt: finishedAt
|
|
77942
78319
|
}
|
|
@@ -77967,7 +78344,7 @@ async function runBackfillTask(options) {
|
|
|
77967
78344
|
aiUserFetchVerificationManifestsHourly.manifestId
|
|
77968
78345
|
],
|
|
77969
78346
|
set: {
|
|
77970
|
-
hits:
|
|
78347
|
+
hits: sql28`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
77971
78348
|
manifestJson: bucket.verificationManifest,
|
|
77972
78349
|
updatedAt: finishedAt
|
|
77973
78350
|
}
|
|
@@ -79572,7 +79949,7 @@ async function trafficRoutes(app, opts) {
|
|
|
79572
79949
|
crawlerEventsHourly.status
|
|
79573
79950
|
],
|
|
79574
79951
|
set: {
|
|
79575
|
-
hits:
|
|
79952
|
+
hits: sql28`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
79576
79953
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
79577
79954
|
updatedAt: finishedAt
|
|
79578
79955
|
}
|
|
@@ -79603,7 +79980,7 @@ async function trafficRoutes(app, opts) {
|
|
|
79603
79980
|
crawlerVerificationManifestsHourly.manifestId
|
|
79604
79981
|
],
|
|
79605
79982
|
set: {
|
|
79606
|
-
hits:
|
|
79983
|
+
hits: sql28`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
79607
79984
|
manifestJson: bucket.verificationManifest,
|
|
79608
79985
|
updatedAt: finishedAt
|
|
79609
79986
|
}
|
|
@@ -79637,7 +80014,7 @@ async function trafficRoutes(app, opts) {
|
|
|
79637
80014
|
aiUserFetchEventsHourly.status
|
|
79638
80015
|
],
|
|
79639
80016
|
set: {
|
|
79640
|
-
hits:
|
|
80017
|
+
hits: sql28`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
79641
80018
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
79642
80019
|
updatedAt: finishedAt
|
|
79643
80020
|
}
|
|
@@ -79668,7 +80045,7 @@ async function trafficRoutes(app, opts) {
|
|
|
79668
80045
|
aiUserFetchVerificationManifestsHourly.manifestId
|
|
79669
80046
|
],
|
|
79670
80047
|
set: {
|
|
79671
|
-
hits:
|
|
80048
|
+
hits: sql28`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
79672
80049
|
manifestJson: bucket.verificationManifest,
|
|
79673
80050
|
updatedAt: finishedAt
|
|
79674
80051
|
}
|
|
@@ -79709,9 +80086,9 @@ async function trafficRoutes(app, opts) {
|
|
|
79709
80086
|
// colliding bucket adds its paid and organic arrivals to the ones
|
|
79710
80087
|
// already there rather than overwriting a single label.
|
|
79711
80088
|
set: {
|
|
79712
|
-
sessionsOrHits:
|
|
79713
|
-
paidSessionsOrHits:
|
|
79714
|
-
organicSessionsOrHits:
|
|
80089
|
+
sessionsOrHits: sql28`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
80090
|
+
paidSessionsOrHits: sql28`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
|
|
80091
|
+
organicSessionsOrHits: sql28`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
|
|
79715
80092
|
updatedAt: finishedAt
|
|
79716
80093
|
}
|
|
79717
80094
|
}).run();
|
|
@@ -80004,7 +80381,7 @@ async function trafficRoutes(app, opts) {
|
|
|
80004
80381
|
function buildSourceDetail(projectId, row, since) {
|
|
80005
80382
|
const crawlerPathRows = app.db.select({
|
|
80006
80383
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
80007
|
-
hits:
|
|
80384
|
+
hits: sql28`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
80008
80385
|
}).from(crawlerEventsHourly).where(
|
|
80009
80386
|
and57(
|
|
80010
80387
|
eq65(crawlerEventsHourly.sourceId, row.id),
|
|
@@ -80015,23 +80392,23 @@ async function trafficRoutes(app, opts) {
|
|
|
80015
80392
|
crawlerPathRows.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
|
|
80016
80393
|
);
|
|
80017
80394
|
const crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
80018
|
-
const aiUserFetchTotals = app.db.select({ total:
|
|
80395
|
+
const aiUserFetchTotals = app.db.select({ total: sql28`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
80019
80396
|
and57(
|
|
80020
80397
|
eq65(aiUserFetchEventsHourly.sourceId, row.id),
|
|
80021
80398
|
gte16(aiUserFetchEventsHourly.tsHour, since)
|
|
80022
80399
|
)
|
|
80023
80400
|
).get();
|
|
80024
80401
|
const aiTotals = app.db.select({
|
|
80025
|
-
total:
|
|
80026
|
-
nonhop:
|
|
80027
|
-
landed:
|
|
80402
|
+
total: sql28`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
80403
|
+
nonhop: sql28`COALESCE(SUM(CASE WHEN ${referralLandedCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`,
|
|
80404
|
+
landed: sql28`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`
|
|
80028
80405
|
}).from(aiReferralEventsHourly).where(
|
|
80029
80406
|
and57(
|
|
80030
80407
|
eq65(aiReferralEventsHourly.sourceId, row.id),
|
|
80031
80408
|
gte16(aiReferralEventsHourly.tsHour, since)
|
|
80032
80409
|
)
|
|
80033
80410
|
).get();
|
|
80034
|
-
const sampleTotals = app.db.select({ total:
|
|
80411
|
+
const sampleTotals = app.db.select({ total: sql28`COUNT(*)` }).from(rawEventSamples).where(
|
|
80035
80412
|
and57(
|
|
80036
80413
|
eq65(rawEventSamples.sourceId, row.id),
|
|
80037
80414
|
gte16(rawEventSamples.ts, since)
|
|
@@ -80220,15 +80597,15 @@ async function trafficRoutes(app, opts) {
|
|
|
80220
80597
|
const crawlerWhere = and57(...crawlerFilters);
|
|
80221
80598
|
const pathTotals = app.db.select({
|
|
80222
80599
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
80223
|
-
hits:
|
|
80224
|
-
rows:
|
|
80600
|
+
hits: sql28`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
80601
|
+
rows: sql28`COUNT(*)`
|
|
80225
80602
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerEventsHourly.pathNormalized).all();
|
|
80226
80603
|
crawlerSegments = segmentCrawlerHits(
|
|
80227
80604
|
pathTotals.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
|
|
80228
80605
|
);
|
|
80229
80606
|
crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
80230
80607
|
totalEventRows += pathTotals.reduce((sum, row) => sum + Number(row.rows), 0);
|
|
80231
|
-
const crawlerSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
80608
|
+
const crawlerSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql28`substr(${crawlerEventsHourly.tsHour}, 1, 10)` : crawlerEventsHourly.tsHour;
|
|
80232
80609
|
const distinctPaths = pathTotals.map((r) => ({ pathNormalized: r.pathNormalized }));
|
|
80233
80610
|
const contentPaths = [];
|
|
80234
80611
|
const infraPaths = [];
|
|
@@ -80240,12 +80617,12 @@ async function trafficRoutes(app, opts) {
|
|
|
80240
80617
|
const listed = useContentList ? contentPaths : infraPaths;
|
|
80241
80618
|
const IN_LIST_CAP = 2e4;
|
|
80242
80619
|
const listFitsInSql = listed.length <= IN_LIST_CAP;
|
|
80243
|
-
const contentExpr = listed.length === 0 ? useContentList ?
|
|
80620
|
+
const contentExpr = listed.length === 0 ? useContentList ? sql28`0` : sql28`${crawlerEventsHourly.hits}` : useContentList ? sql28`CASE WHEN ${inArray27(crawlerEventsHourly.pathNormalized, listed)} THEN ${crawlerEventsHourly.hits} ELSE 0 END` : sql28`CASE WHEN ${inArray27(crawlerEventsHourly.pathNormalized, listed)} THEN 0 ELSE ${crawlerEventsHourly.hits} END`;
|
|
80244
80621
|
if (listFitsInSql) {
|
|
80245
80622
|
const crawlerSeries = app.db.select({
|
|
80246
80623
|
bucket: crawlerSeriesBucket,
|
|
80247
|
-
hits:
|
|
80248
|
-
content:
|
|
80624
|
+
hits: sql28`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
80625
|
+
content: sql28`COALESCE(SUM(${contentExpr}), 0)`
|
|
80249
80626
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerSeriesBucket).all();
|
|
80250
80627
|
for (const row of crawlerSeries) {
|
|
80251
80628
|
const point = trafficSeriesPoint(seriesByBucket, String(row.bucket));
|
|
@@ -80257,7 +80634,7 @@ async function trafficRoutes(app, opts) {
|
|
|
80257
80634
|
const grouped = app.db.select({
|
|
80258
80635
|
bucket: crawlerSeriesBucket,
|
|
80259
80636
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
80260
|
-
hits:
|
|
80637
|
+
hits: sql28`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
80261
80638
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerSeriesBucket, crawlerEventsHourly.pathNormalized).all();
|
|
80262
80639
|
for (const row of grouped) {
|
|
80263
80640
|
const point = trafficSeriesPoint(seriesByBucket, String(row.bucket));
|
|
@@ -80294,15 +80671,15 @@ async function trafficRoutes(app, opts) {
|
|
|
80294
80671
|
if (sourceIdParam) userFetchFilters.push(eq65(aiUserFetchEventsHourly.sourceId, sourceIdParam));
|
|
80295
80672
|
const userFetchWhere = and57(...userFetchFilters);
|
|
80296
80673
|
const total = app.db.select({
|
|
80297
|
-
total:
|
|
80298
|
-
rows:
|
|
80674
|
+
total: sql28`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`,
|
|
80675
|
+
rows: sql28`COUNT(*)`
|
|
80299
80676
|
}).from(aiUserFetchEventsHourly).where(userFetchWhere).get();
|
|
80300
80677
|
aiUserFetchTotal = Number(total?.total ?? 0);
|
|
80301
80678
|
totalEventRows += Number(total?.rows ?? 0);
|
|
80302
|
-
const userFetchSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
80679
|
+
const userFetchSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql28`substr(${aiUserFetchEventsHourly.tsHour}, 1, 10)` : aiUserFetchEventsHourly.tsHour;
|
|
80303
80680
|
const userFetchSeries = app.db.select({
|
|
80304
80681
|
bucket: userFetchSeriesBucket,
|
|
80305
|
-
hits:
|
|
80682
|
+
hits: sql28`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`
|
|
80306
80683
|
}).from(aiUserFetchEventsHourly).where(userFetchWhere).groupBy(userFetchSeriesBucket).all();
|
|
80307
80684
|
for (const row of userFetchSeries) {
|
|
80308
80685
|
trafficSeriesPoint(seriesByBucket, row.bucket).aiUserFetchHits = Number(row.hits);
|
|
@@ -80333,12 +80710,12 @@ async function trafficRoutes(app, opts) {
|
|
|
80333
80710
|
if (sourceIdParam) aiFilters.push(eq65(aiReferralEventsHourly.sourceId, sourceIdParam));
|
|
80334
80711
|
const aiWhere = and57(...aiFilters);
|
|
80335
80712
|
const total = app.db.select({
|
|
80336
|
-
total:
|
|
80337
|
-
nonhop:
|
|
80338
|
-
landed:
|
|
80339
|
-
paid:
|
|
80340
|
-
organic:
|
|
80341
|
-
rows:
|
|
80713
|
+
total: sql28`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
80714
|
+
nonhop: sql28`COALESCE(SUM(CASE WHEN ${referralLandedCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`,
|
|
80715
|
+
landed: sql28`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`,
|
|
80716
|
+
paid: sql28`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.paidSessionsOrHits} ELSE 0 END), 0)`,
|
|
80717
|
+
organic: sql28`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.organicSessionsOrHits} ELSE 0 END), 0)`,
|
|
80718
|
+
rows: sql28`COUNT(*)`
|
|
80342
80719
|
}).from(aiReferralEventsHourly).where(aiWhere).get();
|
|
80343
80720
|
aiReferralTotalHits = Number(total?.total ?? 0);
|
|
80344
80721
|
aiReferralNonHopHits = Number(total?.nonhop ?? 0);
|
|
@@ -80349,11 +80726,11 @@ async function trafficRoutes(app, opts) {
|
|
|
80349
80726
|
Number(total?.organic ?? 0)
|
|
80350
80727
|
);
|
|
80351
80728
|
totalEventRows += Number(total?.rows ?? 0);
|
|
80352
|
-
const referralSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
80729
|
+
const referralSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql28`substr(${aiReferralEventsHourly.tsHour}, 1, 10)` : aiReferralEventsHourly.tsHour;
|
|
80353
80730
|
const referralSeries = app.db.select({
|
|
80354
80731
|
bucket: referralSeriesBucket,
|
|
80355
|
-
hits:
|
|
80356
|
-
landed:
|
|
80732
|
+
hits: sql28`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
80733
|
+
landed: sql28`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`
|
|
80357
80734
|
}).from(aiReferralEventsHourly).where(aiWhere).groupBy(referralSeriesBucket).all();
|
|
80358
80735
|
for (const row of referralSeries) {
|
|
80359
80736
|
const point = trafficSeriesPoint(seriesByBucket, row.bucket);
|
|
@@ -80408,9 +80785,9 @@ async function trafficRoutes(app, opts) {
|
|
|
80408
80785
|
const points = completeTrafficSeries(since, until, granularity, seriesByBucket);
|
|
80409
80786
|
const coverageScope = (col, srcCol) => sourceIdParam ? and57(eq65(col, project.id), eq65(srcCol, sourceIdParam)) : eq65(col, project.id);
|
|
80410
80787
|
const firsts = [
|
|
80411
|
-
app.db.select({ v:
|
|
80412
|
-
app.db.select({ v:
|
|
80413
|
-
app.db.select({ v:
|
|
80788
|
+
app.db.select({ v: sql28`MIN(${crawlerEventsHourly.tsHour})` }).from(crawlerEventsHourly).where(coverageScope(crawlerEventsHourly.projectId, crawlerEventsHourly.sourceId)).get()?.v,
|
|
80789
|
+
app.db.select({ v: sql28`MIN(${aiUserFetchEventsHourly.tsHour})` }).from(aiUserFetchEventsHourly).where(coverageScope(aiUserFetchEventsHourly.projectId, aiUserFetchEventsHourly.sourceId)).get()?.v,
|
|
80790
|
+
app.db.select({ v: sql28`MIN(${aiReferralEventsHourly.tsHour})` }).from(aiReferralEventsHourly).where(coverageScope(aiReferralEventsHourly.projectId, aiReferralEventsHourly.sourceId)).get()?.v
|
|
80414
80791
|
].filter((v) => typeof v === "string" && v.length > 0);
|
|
80415
80792
|
const coverageStart = firsts.length ? firsts.slice().sort()[0] : null;
|
|
80416
80793
|
const bucketStartMs = (bucket) => Date.parse(
|
|
@@ -83118,7 +83495,7 @@ var RUNTIME_STATE_CHECKS = [
|
|
|
83118
83495
|
];
|
|
83119
83496
|
|
|
83120
83497
|
// ../api-routes/src/doctor/checks/traffic-source.ts
|
|
83121
|
-
import { and as and65, eq as eq77, gte as gte19, inArray as inArray30, ne as ne14, sql as
|
|
83498
|
+
import { and as and65, eq as eq77, gte as gte19, inArray as inArray30, ne as ne14, sql as sql29 } from "drizzle-orm";
|
|
83122
83499
|
var RECENT_DATA_WARN_DAYS = 7;
|
|
83123
83500
|
var RECENT_DATA_FAIL_DAYS = 30;
|
|
83124
83501
|
function isCloudflareDirectPush(source) {
|
|
@@ -83263,7 +83640,7 @@ var recentDataCheck = {
|
|
|
83263
83640
|
const warnCutoff = new Date(now.getTime() - RECENT_DATA_WARN_DAYS * 24 * 60 * 6e4).toISOString();
|
|
83264
83641
|
const failCutoff = new Date(now.getTime() - RECENT_DATA_FAIL_DAYS * 24 * 60 * 6e4).toISOString();
|
|
83265
83642
|
const recentCrawlers = Number(
|
|
83266
|
-
ctx.db.select({ total:
|
|
83643
|
+
ctx.db.select({ total: sql29`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
83267
83644
|
and65(
|
|
83268
83645
|
eq77(crawlerEventsHourly.projectId, ctx.project.id),
|
|
83269
83646
|
inArray30(crawlerEventsHourly.sourceId, activeSourceIds),
|
|
@@ -83272,7 +83649,7 @@ var recentDataCheck = {
|
|
|
83272
83649
|
).get()?.total ?? 0
|
|
83273
83650
|
);
|
|
83274
83651
|
const recentReferrals = Number(
|
|
83275
|
-
ctx.db.select({ total:
|
|
83652
|
+
ctx.db.select({ total: sql29`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
83276
83653
|
and65(
|
|
83277
83654
|
eq77(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
83278
83655
|
inArray30(aiReferralEventsHourly.sourceId, activeSourceIds),
|
|
@@ -83281,7 +83658,7 @@ var recentDataCheck = {
|
|
|
83281
83658
|
).get()?.total ?? 0
|
|
83282
83659
|
);
|
|
83283
83660
|
const recentUserFetches = Number(
|
|
83284
|
-
ctx.db.select({ total:
|
|
83661
|
+
ctx.db.select({ total: sql29`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
83285
83662
|
and65(
|
|
83286
83663
|
eq77(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
83287
83664
|
inArray30(aiUserFetchEventsHourly.sourceId, activeSourceIds),
|
|
@@ -83307,7 +83684,7 @@ var recentDataCheck = {
|
|
|
83307
83684
|
};
|
|
83308
83685
|
}
|
|
83309
83686
|
const olderCrawlers = Number(
|
|
83310
|
-
ctx.db.select({ total:
|
|
83687
|
+
ctx.db.select({ total: sql29`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
83311
83688
|
and65(
|
|
83312
83689
|
eq77(crawlerEventsHourly.projectId, ctx.project.id),
|
|
83313
83690
|
inArray30(crawlerEventsHourly.sourceId, activeSourceIds),
|
|
@@ -83316,7 +83693,7 @@ var recentDataCheck = {
|
|
|
83316
83693
|
).get()?.total ?? 0
|
|
83317
83694
|
);
|
|
83318
83695
|
const olderReferrals = Number(
|
|
83319
|
-
ctx.db.select({ total:
|
|
83696
|
+
ctx.db.select({ total: sql29`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
83320
83697
|
and65(
|
|
83321
83698
|
eq77(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
83322
83699
|
inArray30(aiReferralEventsHourly.sourceId, activeSourceIds),
|
|
@@ -83325,7 +83702,7 @@ var recentDataCheck = {
|
|
|
83325
83702
|
).get()?.total ?? 0
|
|
83326
83703
|
);
|
|
83327
83704
|
const olderUserFetches = Number(
|
|
83328
|
-
ctx.db.select({ total:
|
|
83705
|
+
ctx.db.select({ total: sql29`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
83329
83706
|
and65(
|
|
83330
83707
|
eq77(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
83331
83708
|
inArray30(aiUserFetchEventsHourly.sourceId, activeSourceIds),
|
|
@@ -85089,7 +85466,7 @@ function dedupeStrings(input) {
|
|
|
85089
85466
|
|
|
85090
85467
|
// ../api-routes/src/technical-aeo.ts
|
|
85091
85468
|
import crypto49 from "crypto";
|
|
85092
|
-
import { and as and67, asc as asc14, count as count2, desc as desc36, eq as eq80, inArray as inArray32, isNotNull as isNotNull8, isNull as isNull15, lt as lt13, or as or18, sql as
|
|
85469
|
+
import { and as and67, asc as asc14, count as count2, desc as desc36, eq as eq80, gte as gte21, inArray as inArray32, isNotNull as isNotNull8, isNull as isNull15, lt as lt13, or as or18, sql as sql30 } from "drizzle-orm";
|
|
85093
85470
|
import { alias } from "drizzle-orm/sqlite-core";
|
|
85094
85471
|
var FETCHED_SITE_CRAWL_STATES = /* @__PURE__ */ new Set([
|
|
85095
85472
|
...SiteCrawlFetchedStates,
|
|
@@ -85171,6 +85548,26 @@ function decodeCursor(value) {
|
|
|
85171
85548
|
return 0;
|
|
85172
85549
|
}
|
|
85173
85550
|
}
|
|
85551
|
+
function parseCrawlDate(value, runId) {
|
|
85552
|
+
if (value === void 0) return void 0;
|
|
85553
|
+
const parsed = calendarDateSchema.safeParse(value);
|
|
85554
|
+
if (!parsed.success) throw validationError('"date" must be a calendar date as YYYY-MM-DD');
|
|
85555
|
+
if (runId !== void 0) throw validationError('"date" and "runId" are mutually exclusive');
|
|
85556
|
+
return parsed.data;
|
|
85557
|
+
}
|
|
85558
|
+
function decodeCrawlPageCursor(value, selectionKey) {
|
|
85559
|
+
if (typeof value !== "string" || value.length === 0 || value.length > 256) return 0;
|
|
85560
|
+
let parsed;
|
|
85561
|
+
try {
|
|
85562
|
+
parsed = JSON.parse(Buffer.from(value, "base64url").toString("utf8"));
|
|
85563
|
+
} catch {
|
|
85564
|
+
return 0;
|
|
85565
|
+
}
|
|
85566
|
+
if (parsed?.selectionKey !== void 0 && parsed.selectionKey !== selectionKey) {
|
|
85567
|
+
throw validationError("Crawl page cursor no longer matches the selected scan and filters; restart pagination");
|
|
85568
|
+
}
|
|
85569
|
+
return decodeCursor(value);
|
|
85570
|
+
}
|
|
85174
85571
|
function normalizeParentPath(value) {
|
|
85175
85572
|
if (typeof value !== "string" || value.trim() === "") return "/";
|
|
85176
85573
|
const bare = value.trim().split(/[?#]/, 1)[0] ?? "/";
|
|
@@ -85215,7 +85612,8 @@ function mapCrawlPage(row) {
|
|
|
85215
85612
|
outboundOccurrences: row.outboundOccurrences,
|
|
85216
85613
|
linkScoreRaw: row.linkScoreRaw,
|
|
85217
85614
|
linkScoreNormalized: row.linkScoreNormalized,
|
|
85218
|
-
healthState: deriveSiteHealthState(row)
|
|
85615
|
+
healthState: deriveSiteHealthState(row),
|
|
85616
|
+
healthReason: deriveSiteHealthReason(row)
|
|
85219
85617
|
};
|
|
85220
85618
|
}
|
|
85221
85619
|
function withRecordedShare(factor) {
|
|
@@ -85311,7 +85709,7 @@ function linkKindFilter(linkKind) {
|
|
|
85311
85709
|
}
|
|
85312
85710
|
var graphSourceNode = alias(siteCrawlGraphNodes, "site_crawl_graph_source_node");
|
|
85313
85711
|
var graphTargetNode = alias(siteCrawlGraphNodes, "site_crawl_graph_target_node");
|
|
85314
|
-
var PAGE_CHANGE_PREDICATE =
|
|
85712
|
+
var PAGE_CHANGE_PREDICATE = sql30`(
|
|
85315
85713
|
current.url IS NOT previous.url
|
|
85316
85714
|
OR current.final_url IS NOT previous.final_url
|
|
85317
85715
|
OR current.path IS NOT previous.path
|
|
@@ -85333,7 +85731,7 @@ var PAGE_CHANGE_PREDICATE = sql29`(
|
|
|
85333
85731
|
OR current.link_score_raw IS NOT previous.link_score_raw
|
|
85334
85732
|
OR current.link_score_normalized IS NOT previous.link_score_normalized
|
|
85335
85733
|
)`;
|
|
85336
|
-
var LINK_CHANGE_PREDICATE =
|
|
85734
|
+
var LINK_CHANGE_PREDICATE = sql30`(
|
|
85337
85735
|
current.source_node_key IS NOT previous.source_node_key
|
|
85338
85736
|
OR current.source_url IS NOT previous.source_url
|
|
85339
85737
|
OR current.target_node_key IS NOT previous.target_node_key
|
|
@@ -85409,21 +85807,191 @@ function changedFields(before, after, fields) {
|
|
|
85409
85807
|
return fields.filter((field) => valueChanged(before[field], after[field])).map(String);
|
|
85410
85808
|
}
|
|
85411
85809
|
async function technicalAeoRoutes(app, opts) {
|
|
85412
|
-
const
|
|
85810
|
+
const visibleCrawlFilters = (projectId) => [
|
|
85811
|
+
eq80(siteCrawlSnapshots.projectId, projectId),
|
|
85812
|
+
eq80(runs.projectId, projectId),
|
|
85813
|
+
eq80(runs.kind, RunKinds["site-audit"]),
|
|
85814
|
+
inArray32(runs.status, SURFACEABLE_STATUSES),
|
|
85815
|
+
notProbeRun()
|
|
85816
|
+
];
|
|
85817
|
+
const availableScanDatesFor = (projectId, requestedDate) => {
|
|
85818
|
+
if (!requestedDate) return void 0;
|
|
85819
|
+
const date = sql30`substr(${siteCrawlSnapshots.createdAt}, 1, 10)`;
|
|
85820
|
+
const filters = visibleCrawlFilters(projectId);
|
|
85821
|
+
const allDates = and67(...filters);
|
|
85822
|
+
const matchingDates = and67(...filters, eq80(sql30`substr(${siteCrawlSnapshots.createdAt}, 6, 5)`, requestedDate.slice(5)));
|
|
85823
|
+
const readDates = (where) => app.db.select({ date }).from(siteCrawlSnapshots).innerJoin(runs, eq80(siteCrawlSnapshots.runId, runs.id)).where(where).groupBy(date).orderBy(desc36(date)).limit(10).all().map((row) => row.date);
|
|
85824
|
+
const countDates = (where) => app.db.select({ value: sql30`count(distinct ${date})` }).from(siteCrawlSnapshots).innerJoin(runs, eq80(siteCrawlSnapshots.runId, runs.id)).where(where).get()?.value ?? 0;
|
|
85825
|
+
return {
|
|
85826
|
+
recentDates: readDates(allDates),
|
|
85827
|
+
totalDates: countDates(allDates),
|
|
85828
|
+
matchingMonthDayDates: readDates(matchingDates),
|
|
85829
|
+
matchingMonthDayTotal: countDates(matchingDates)
|
|
85830
|
+
};
|
|
85831
|
+
};
|
|
85832
|
+
const resolveCrawl = (projectId, runId, requestedDate) => {
|
|
85833
|
+
const filters = visibleCrawlFilters(projectId);
|
|
85834
|
+
if (requestedDate) {
|
|
85835
|
+
const nextDate2 = /* @__PURE__ */ new Date(`${requestedDate}T00:00:00.000Z`);
|
|
85836
|
+
nextDate2.setUTCDate(nextDate2.getUTCDate() + 1);
|
|
85837
|
+
filters.push(gte21(siteCrawlSnapshots.createdAt, requestedDate), lt13(siteCrawlSnapshots.createdAt, nextDate2.toISOString()));
|
|
85838
|
+
}
|
|
85839
|
+
const selected = app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq80(siteCrawlSnapshots.runId, runs.id)).where(and67(...filters, runId ? eq80(siteCrawlSnapshots.runId, runId) : void 0)).orderBy(
|
|
85840
|
+
desc36(sql30`substr(${siteCrawlSnapshots.createdAt}, 1, 10)`),
|
|
85841
|
+
desc36(siteCrawlSnapshots.complete),
|
|
85842
|
+
desc36(siteCrawlSnapshots.pagesFetched),
|
|
85843
|
+
desc36(siteCrawlSnapshots.createdAt),
|
|
85844
|
+
desc36(siteCrawlSnapshots.runId)
|
|
85845
|
+
).limit(1).get();
|
|
85846
|
+
if (!selected) return void 0;
|
|
85847
|
+
const date = selected.snapshot.createdAt.slice(0, 10);
|
|
85848
|
+
const nextDate = /* @__PURE__ */ new Date(`${date}T00:00:00.000Z`);
|
|
85849
|
+
nextDate.setUTCDate(nextDate.getUTCDate() + 1);
|
|
85850
|
+
const sameDate = and67(
|
|
85851
|
+
...filters,
|
|
85852
|
+
gte21(siteCrawlSnapshots.createdAt, date),
|
|
85853
|
+
lt13(siteCrawlSnapshots.createdAt, nextDate.toISOString())
|
|
85854
|
+
);
|
|
85855
|
+
const sameDateRunCount = app.db.select({ value: count2() }).from(siteCrawlSnapshots).innerJoin(runs, eq80(siteCrawlSnapshots.runId, runs.id)).where(sameDate).get()?.value ?? 0;
|
|
85856
|
+
const candidates = app.db.select({
|
|
85857
|
+
runId: siteCrawlSnapshots.runId,
|
|
85858
|
+
createdAt: siteCrawlSnapshots.createdAt,
|
|
85859
|
+
complete: siteCrawlSnapshots.complete,
|
|
85860
|
+
pages: siteCrawlSnapshots.pagesFetched
|
|
85861
|
+
}).from(siteCrawlSnapshots).innerJoin(runs, eq80(siteCrawlSnapshots.runId, runs.id)).where(sameDate).orderBy(
|
|
85862
|
+
desc36(siteCrawlSnapshots.complete),
|
|
85863
|
+
desc36(siteCrawlSnapshots.pagesFetched),
|
|
85864
|
+
desc36(siteCrawlSnapshots.createdAt),
|
|
85865
|
+
desc36(siteCrawlSnapshots.runId)
|
|
85866
|
+
).limit(10).all();
|
|
85867
|
+
const runSelection = {
|
|
85868
|
+
reason: runId ? "explicit-run" : requestedDate ? selected.snapshot.complete ? "requested-date-complete" : "requested-date-most-pages" : selected.snapshot.complete ? "latest-date-complete" : "latest-date-most-pages",
|
|
85869
|
+
date,
|
|
85870
|
+
sameDateRunCount,
|
|
85871
|
+
ambiguousDate: sameDateRunCount > 1,
|
|
85872
|
+
candidates,
|
|
85873
|
+
candidatesTruncated: sameDateRunCount > candidates.length
|
|
85874
|
+
};
|
|
85875
|
+
return { ...selected, runSelection };
|
|
85876
|
+
};
|
|
85877
|
+
const resolveAudit = (projectId, runId) => {
|
|
85878
|
+
const crawl = resolveCrawl(projectId, runId);
|
|
85879
|
+
const selectedRunId = runId ?? crawl?.snapshot.runId;
|
|
85413
85880
|
const filters = [
|
|
85414
|
-
eq80(
|
|
85881
|
+
eq80(siteAuditSnapshots.projectId, projectId),
|
|
85415
85882
|
eq80(runs.projectId, projectId),
|
|
85416
85883
|
eq80(runs.kind, RunKinds["site-audit"]),
|
|
85417
85884
|
inArray32(runs.status, SURFACEABLE_STATUSES),
|
|
85418
85885
|
notProbeRun()
|
|
85419
85886
|
];
|
|
85420
|
-
|
|
85421
|
-
|
|
85887
|
+
const complete = sql30`case when ${runs.status} = ${RunStatuses.completed} then 1 else 0 end`;
|
|
85888
|
+
const selected = app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs, eq80(siteAuditSnapshots.runId, runs.id)).where(and67(...filters, selectedRunId ? eq80(siteAuditSnapshots.runId, selectedRunId) : void 0)).orderBy(
|
|
85889
|
+
desc36(sql30`substr(${siteAuditSnapshots.createdAt}, 1, 10)`),
|
|
85890
|
+
desc36(complete),
|
|
85891
|
+
desc36(siteAuditSnapshots.pagesAudited),
|
|
85892
|
+
desc36(siteAuditSnapshots.createdAt),
|
|
85893
|
+
desc36(siteAuditSnapshots.runId)
|
|
85894
|
+
).limit(1).get();
|
|
85895
|
+
if (!selected) return void 0;
|
|
85896
|
+
if (crawl) return { ...selected, runSelection: crawl.runSelection };
|
|
85897
|
+
const date = selected.snap.createdAt.slice(0, 10);
|
|
85898
|
+
const nextDate = /* @__PURE__ */ new Date(`${date}T00:00:00.000Z`);
|
|
85899
|
+
nextDate.setUTCDate(nextDate.getUTCDate() + 1);
|
|
85900
|
+
const sameDate = and67(
|
|
85901
|
+
...filters,
|
|
85902
|
+
gte21(siteAuditSnapshots.createdAt, date),
|
|
85903
|
+
lt13(siteAuditSnapshots.createdAt, nextDate.toISOString())
|
|
85904
|
+
);
|
|
85905
|
+
const sameDateRunCount = app.db.select({ value: count2() }).from(siteAuditSnapshots).innerJoin(runs, eq80(siteAuditSnapshots.runId, runs.id)).where(sameDate).get()?.value ?? 0;
|
|
85906
|
+
const candidates = app.db.select({
|
|
85907
|
+
runId: siteAuditSnapshots.runId,
|
|
85908
|
+
createdAt: siteAuditSnapshots.createdAt,
|
|
85909
|
+
complete,
|
|
85910
|
+
pages: siteAuditSnapshots.pagesAudited
|
|
85911
|
+
}).from(siteAuditSnapshots).innerJoin(runs, eq80(siteAuditSnapshots.runId, runs.id)).where(sameDate).orderBy(
|
|
85912
|
+
desc36(complete),
|
|
85913
|
+
desc36(siteAuditSnapshots.pagesAudited),
|
|
85914
|
+
desc36(siteAuditSnapshots.createdAt),
|
|
85915
|
+
desc36(siteAuditSnapshots.runId)
|
|
85916
|
+
).limit(10).all().map((candidate) => ({ ...candidate, complete: Boolean(candidate.complete) }));
|
|
85917
|
+
const runSelection = {
|
|
85918
|
+
reason: runId ? "explicit-run" : selected.runStatus === RunStatuses.completed ? "latest-date-complete" : "latest-date-most-pages",
|
|
85919
|
+
date,
|
|
85920
|
+
sameDateRunCount,
|
|
85921
|
+
ambiguousDate: sameDateRunCount > 1,
|
|
85922
|
+
candidates,
|
|
85923
|
+
candidatesTruncated: sameDateRunCount > candidates.length
|
|
85924
|
+
};
|
|
85925
|
+
return { ...selected, runSelection };
|
|
85422
85926
|
};
|
|
85423
85927
|
const detailScopeFor = (projectId, snapshot) => snapshot.detailsAvailable && snapshot.attemptId ? { projectId, runId: snapshot.runId, attemptId: snapshot.attemptId } : null;
|
|
85424
|
-
const
|
|
85425
|
-
|
|
85426
|
-
|
|
85928
|
+
const crawlReasonGroups = (where) => {
|
|
85929
|
+
const canonicalOther = sql30`case when ${siteCrawlPages.canonicalNodeKey} is not null
|
|
85930
|
+
and ${siteCrawlPages.canonicalNodeKey} <> ${siteCrawlPages.nodeKey} then 1 else 0 end`;
|
|
85931
|
+
const canonicalReason = sql30`exists (select 1 from json_each(${siteCrawlPages.indexabilityReasons})
|
|
85932
|
+
where value = ${SiteCrawlIndexabilityReasons.canonicalToOther})`;
|
|
85933
|
+
const groups = app.db.select({
|
|
85934
|
+
fetchState: siteCrawlPages.fetchState,
|
|
85935
|
+
indexabilityState: siteCrawlPages.indexabilityState,
|
|
85936
|
+
inventoryEligible: siteCrawlPages.inventoryEligible,
|
|
85937
|
+
canonicalOther,
|
|
85938
|
+
canonicalReason,
|
|
85939
|
+
pages: count2(),
|
|
85940
|
+
exampleUrl: sql30`min(${siteCrawlPages.url})`
|
|
85941
|
+
}).from(siteCrawlPages).where(where).groupBy(
|
|
85942
|
+
siteCrawlPages.fetchState,
|
|
85943
|
+
siteCrawlPages.indexabilityState,
|
|
85944
|
+
siteCrawlPages.inventoryEligible,
|
|
85945
|
+
canonicalOther,
|
|
85946
|
+
canonicalReason
|
|
85947
|
+
).limit(257).all();
|
|
85948
|
+
if (groups.length > 256) throw validationError("Persisted crawl has too many distinct health-state variants to summarize");
|
|
85949
|
+
return groups.map((group) => ({
|
|
85950
|
+
inventoryEligible: group.inventoryEligible,
|
|
85951
|
+
healthReason: deriveSiteHealthReason({
|
|
85952
|
+
fetchState: group.fetchState,
|
|
85953
|
+
indexabilityState: group.indexabilityState,
|
|
85954
|
+
indexabilityReasons: group.canonicalReason ? [SiteCrawlIndexabilityReasons.canonicalToOther] : [],
|
|
85955
|
+
// Preserve only the identity inequality tested by the canonical helper.
|
|
85956
|
+
nodeKey: "source",
|
|
85957
|
+
canonicalNodeKey: group.canonicalOther ? "other" : null
|
|
85958
|
+
}),
|
|
85959
|
+
pages: group.pages,
|
|
85960
|
+
exampleUrl: group.exampleUrl
|
|
85961
|
+
}));
|
|
85962
|
+
};
|
|
85963
|
+
const reasonCounts = (groups) => {
|
|
85964
|
+
const counts2 = /* @__PURE__ */ new Map();
|
|
85965
|
+
for (const group of groups) {
|
|
85966
|
+
const existing = counts2.get(group.healthReason);
|
|
85967
|
+
if (existing) {
|
|
85968
|
+
existing.pages += group.pages;
|
|
85969
|
+
if (group.exampleUrl < existing.exampleUrl) existing.exampleUrl = group.exampleUrl;
|
|
85970
|
+
} else counts2.set(group.healthReason, { healthReason: group.healthReason, pages: group.pages, exampleUrl: group.exampleUrl });
|
|
85971
|
+
}
|
|
85972
|
+
return [...counts2.values()].sort((a, b) => a.healthReason.localeCompare(b.healthReason));
|
|
85973
|
+
};
|
|
85974
|
+
const inventorySummaryFor = (scope) => {
|
|
85975
|
+
if (!scope) return null;
|
|
85976
|
+
const groups = crawlReasonGroups(and67(
|
|
85977
|
+
eq80(siteCrawlPages.projectId, scope.projectId),
|
|
85978
|
+
eq80(siteCrawlPages.runId, scope.runId),
|
|
85979
|
+
eq80(siteCrawlPages.attemptId, scope.attemptId)
|
|
85980
|
+
));
|
|
85981
|
+
const total = groups.reduce((sum, group) => sum + group.pages, 0);
|
|
85982
|
+
const eligible = groups.reduce((sum, group) => sum + (group.inventoryEligible ? group.pages : 0), 0);
|
|
85983
|
+
return {
|
|
85984
|
+
scope: "selected-snapshot",
|
|
85985
|
+
total,
|
|
85986
|
+
eligible,
|
|
85987
|
+
excluded: total - eligible,
|
|
85988
|
+
excludedReasons: reasonCounts(groups.filter((group) => !group.inventoryEligible))
|
|
85989
|
+
};
|
|
85990
|
+
};
|
|
85991
|
+
const completenessOf = (target) => ({
|
|
85992
|
+
complete: target.snapshot.complete,
|
|
85993
|
+
termination: target.snapshot.termination,
|
|
85994
|
+
runSelection: target.runSelection
|
|
85427
85995
|
});
|
|
85428
85996
|
const NO_CRAWL_COMPLETENESS = { complete: false, termination: null };
|
|
85429
85997
|
const pageInScope = (scope, selector) => {
|
|
@@ -85463,6 +86031,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85463
86031
|
hasCrawlData: false,
|
|
85464
86032
|
legacyAuditAvailable,
|
|
85465
86033
|
runId: null,
|
|
86034
|
+
inventorySummary: null,
|
|
85466
86035
|
runStatus: null,
|
|
85467
86036
|
requestedRootUrl: null,
|
|
85468
86037
|
rootUrl: null,
|
|
@@ -85482,16 +86051,42 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85482
86051
|
inArray32(runs.status, SURFACEABLE_STATUSES),
|
|
85483
86052
|
notProbeRun()
|
|
85484
86053
|
];
|
|
85485
|
-
const
|
|
85486
|
-
const latest = app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs, eq80(siteAuditSnapshots.runId, runs.id)).where(and67(...targetFilters)).orderBy(desc36(siteAuditSnapshots.createdAt)).limit(1).get();
|
|
86054
|
+
const latest = resolveAudit(project.id, request.query.runId);
|
|
85487
86055
|
if (!latest) {
|
|
85488
|
-
|
|
85489
|
-
|
|
86056
|
+
const crawl = resolveCrawl(project.id, request.query.runId);
|
|
86057
|
+
if (!crawl && request.query.runId) throw notFound("Site audit run", request.query.runId);
|
|
86058
|
+
return crawl ? {
|
|
86059
|
+
...emptyScore(project.name),
|
|
86060
|
+
runId: crawl.snapshot.runId,
|
|
86061
|
+
runStatus: crawl.runStatus,
|
|
86062
|
+
runSelection: crawl.runSelection
|
|
86063
|
+
} : emptyScore(project.name);
|
|
85490
86064
|
}
|
|
85491
86065
|
const snap = latest.snap;
|
|
85492
86066
|
const previous = app.db.select({ snap: siteAuditSnapshots }).from(siteAuditSnapshots).innerJoin(runs, eq80(siteAuditSnapshots.runId, runs.id)).where(and67(...baseFilters, lt13(siteAuditSnapshots.createdAt, snap.createdAt))).orderBy(desc36(siteAuditSnapshots.createdAt)).limit(1).get()?.snap ?? null;
|
|
85493
86067
|
const deltaScore = previous ? snap.aggregateScore - previous.aggregateScore : null;
|
|
85494
86068
|
const trend = deltaScore == null ? null : deltaScore > 0 ? SiteAuditTrendDirections.up : deltaScore < 0 ? SiteAuditTrendDirections.down : SiteAuditTrendDirections.flat;
|
|
86069
|
+
const affectedPageRanking = {
|
|
86070
|
+
basis: "pages-below-pass",
|
|
86071
|
+
scope: "audited-pages",
|
|
86072
|
+
pagesAudited: snap.pagesAudited,
|
|
86073
|
+
items: [],
|
|
86074
|
+
unavailableFactorIds: []
|
|
86075
|
+
};
|
|
86076
|
+
for (const factor of snap.factorAverages) {
|
|
86077
|
+
if (!Number.isInteger(factor.pagesFailing) || factor.pagesFailing < 0 || !Number.isInteger(factor.pagesPartial) || factor.pagesPartial < 0) {
|
|
86078
|
+
affectedPageRanking.unavailableFactorIds.push(factor.id);
|
|
86079
|
+
continue;
|
|
86080
|
+
}
|
|
86081
|
+
affectedPageRanking.items.push({
|
|
86082
|
+
factorId: factor.id,
|
|
86083
|
+
pagesBelowPass: factor.pagesFailing + factor.pagesPartial,
|
|
86084
|
+
pagesFailing: factor.pagesFailing,
|
|
86085
|
+
pagesPartial: factor.pagesPartial
|
|
86086
|
+
});
|
|
86087
|
+
}
|
|
86088
|
+
affectedPageRanking.items.sort((a, b) => b.pagesBelowPass - a.pagesBelowPass || a.factorId.localeCompare(b.factorId));
|
|
86089
|
+
affectedPageRanking.unavailableFactorIds.sort();
|
|
85495
86090
|
return {
|
|
85496
86091
|
project: project.name,
|
|
85497
86092
|
hasData: true,
|
|
@@ -85508,30 +86103,33 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85508
86103
|
trend,
|
|
85509
86104
|
previousScore: previous?.aggregateScore ?? null,
|
|
85510
86105
|
previousAuditedAt: previous?.auditedAt ?? null,
|
|
85511
|
-
|
|
86106
|
+
runSelection: latest.runSelection,
|
|
86107
|
+
factors: snap.factorAverages.map((factor) => ({
|
|
86108
|
+
...withRecordedShare(factor),
|
|
86109
|
+
status: factorStatusFromDistribution(factor)
|
|
86110
|
+
})),
|
|
86111
|
+
affectedPageRanking,
|
|
85512
86112
|
crossCuttingIssues: snap.crossCuttingIssues,
|
|
85513
86113
|
prioritizedFixes: snap.prioritizedFixes
|
|
85514
86114
|
};
|
|
85515
86115
|
});
|
|
85516
86116
|
app.get("/projects/:name/technical-aeo/pages", async (request) => {
|
|
85517
86117
|
const project = resolveProject(app.db, request.params.name);
|
|
85518
|
-
const
|
|
85519
|
-
eq80(siteAuditSnapshots.projectId, project.id),
|
|
85520
|
-
eq80(runs.projectId, project.id),
|
|
85521
|
-
eq80(runs.kind, RunKinds["site-audit"]),
|
|
85522
|
-
inArray32(runs.status, SURFACEABLE_STATUSES),
|
|
85523
|
-
notProbeRun()
|
|
85524
|
-
];
|
|
85525
|
-
if (request.query.runId) targetFilters.push(eq80(siteAuditSnapshots.runId, request.query.runId));
|
|
85526
|
-
const latest = app.db.select({ runId: siteAuditSnapshots.runId, auditedAt: siteAuditSnapshots.auditedAt }).from(siteAuditSnapshots).innerJoin(runs, eq80(siteAuditSnapshots.runId, runs.id)).where(and67(...targetFilters)).orderBy(desc36(siteAuditSnapshots.createdAt)).limit(1).get();
|
|
85527
|
-
if (!latest && request.query.runId) {
|
|
85528
|
-
throw notFound("Site audit run", request.query.runId);
|
|
85529
|
-
}
|
|
86118
|
+
const latest = resolveAudit(project.id, request.query.runId);
|
|
85530
86119
|
if (!latest) {
|
|
85531
|
-
|
|
86120
|
+
const crawl = resolveCrawl(project.id, request.query.runId);
|
|
86121
|
+
if (!crawl && request.query.runId) throw notFound("Site audit run", request.query.runId);
|
|
86122
|
+
return {
|
|
86123
|
+
project: project.name,
|
|
86124
|
+
runId: crawl?.snapshot.runId ?? null,
|
|
86125
|
+
auditedAt: null,
|
|
86126
|
+
total: 0,
|
|
86127
|
+
pages: [],
|
|
86128
|
+
...crawl ? { runSelection: crawl.runSelection } : {}
|
|
86129
|
+
};
|
|
85532
86130
|
}
|
|
85533
86131
|
const statusFilter = request.query.status === "success" || request.query.status === "error" ? request.query.status : null;
|
|
85534
|
-
const conds = [eq80(siteAuditPages.projectId, project.id), eq80(siteAuditPages.runId, latest.runId)];
|
|
86132
|
+
const conds = [eq80(siteAuditPages.projectId, project.id), eq80(siteAuditPages.runId, latest.snap.runId)];
|
|
85535
86133
|
if (statusFilter) conds.push(eq80(siteAuditPages.status, statusFilter));
|
|
85536
86134
|
const where = and67(...conds);
|
|
85537
86135
|
const totalRow = app.db.select({ value: count2() }).from(siteAuditPages).where(where).get();
|
|
@@ -85547,7 +86145,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85547
86145
|
error: row.error,
|
|
85548
86146
|
factors: row.factors.map(withRecordedShare)
|
|
85549
86147
|
}));
|
|
85550
|
-
return { project: project.name, runId: latest.runId, auditedAt: latest.auditedAt, total, pages };
|
|
86148
|
+
return { project: project.name, runId: latest.snap.runId, auditedAt: latest.snap.auditedAt, runSelection: latest.runSelection, total, pages };
|
|
85551
86149
|
});
|
|
85552
86150
|
app.get("/projects/:name/technical-aeo/trend", async (request) => {
|
|
85553
86151
|
const project = resolveProject(app.db, request.params.name);
|
|
@@ -85568,11 +86166,16 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85568
86166
|
});
|
|
85569
86167
|
app.get("/projects/:name/technical-aeo/crawl", async (request) => {
|
|
85570
86168
|
const project = resolveProject(app.db, request.params.name);
|
|
85571
|
-
const
|
|
86169
|
+
const requestedDate = parseCrawlDate(request.query.date, request.query.runId);
|
|
86170
|
+
const target = resolveCrawl(project.id, request.query.runId, requestedDate);
|
|
85572
86171
|
const legacyAuditAvailable = hasLegacyAudit(project.id);
|
|
85573
86172
|
if (!target) {
|
|
85574
86173
|
assertKnownAuditRun(project.id, request.query.runId);
|
|
85575
|
-
return
|
|
86174
|
+
return {
|
|
86175
|
+
...emptyCrawlSummary(project.name, legacyAuditAvailable),
|
|
86176
|
+
requestedDate,
|
|
86177
|
+
availableScanDates: availableScanDatesFor(project.id, requestedDate)
|
|
86178
|
+
};
|
|
85576
86179
|
}
|
|
85577
86180
|
const snapshot = target.snapshot;
|
|
85578
86181
|
const deadLinks = !snapshot.checkDeadLinks || snapshot.deadLinkState === "disabled" ? { state: "disabled" } : snapshot.deadLinkState === "complete" ? { state: "complete", checked: snapshot.deadLinksChecked, found: snapshot.deadLinksFound, unverified: snapshot.deadLinksUnverified } : snapshot.deadLinkState === "partial" ? { state: "partial", checked: snapshot.deadLinksChecked, found: snapshot.deadLinksFound, unverified: snapshot.deadLinksUnverified } : { state: "unavailable" };
|
|
@@ -85581,6 +86184,8 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85581
86184
|
hasCrawlData: true,
|
|
85582
86185
|
legacyAuditAvailable,
|
|
85583
86186
|
runId: snapshot.runId,
|
|
86187
|
+
requestedDate,
|
|
86188
|
+
inventorySummary: inventorySummaryFor(detailScopeFor(project.id, snapshot)),
|
|
85584
86189
|
runStatus: target.runStatus,
|
|
85585
86190
|
requestedRootUrl: snapshot.requestedRootUrl,
|
|
85586
86191
|
rootUrl: snapshot.rootUrl,
|
|
@@ -85590,8 +86195,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85590
86195
|
indexabilityVersion: snapshot.indexabilityVersion,
|
|
85591
86196
|
linkScoreVersion: snapshot.linkScoreVersion,
|
|
85592
86197
|
effectiveOptions: snapshot.effectiveOptions,
|
|
85593
|
-
|
|
85594
|
-
termination: snapshot.termination,
|
|
86198
|
+
...completenessOf(target),
|
|
85595
86199
|
detailsAvailable: snapshot.detailsAvailable,
|
|
85596
86200
|
counts: {
|
|
85597
86201
|
pagesDiscovered: snapshot.pagesDiscovered,
|
|
@@ -85637,7 +86241,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85637
86241
|
project: project.name,
|
|
85638
86242
|
hasCrawlData: true,
|
|
85639
86243
|
runId: snapshot.runId,
|
|
85640
|
-
...completenessOf(
|
|
86244
|
+
...completenessOf(target),
|
|
85641
86245
|
rootNodeKey: null,
|
|
85642
86246
|
layout: { state: "unavailable", version: null, reason: "details-unavailable" },
|
|
85643
86247
|
templateDetection,
|
|
@@ -85667,7 +86271,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85667
86271
|
project: project.name,
|
|
85668
86272
|
hasCrawlData: true,
|
|
85669
86273
|
runId: snapshot.runId,
|
|
85670
|
-
...completenessOf(
|
|
86274
|
+
...completenessOf(target),
|
|
85671
86275
|
rootNodeKey,
|
|
85672
86276
|
layout: { state: "unavailable", version: null, reason: "legacy-snapshot" },
|
|
85673
86277
|
templateDetection,
|
|
@@ -85690,7 +86294,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85690
86294
|
project: project.name,
|
|
85691
86295
|
hasCrawlData: true,
|
|
85692
86296
|
runId: snapshot.runId,
|
|
85693
|
-
...completenessOf(
|
|
86297
|
+
...completenessOf(target),
|
|
85694
86298
|
rootNodeKey,
|
|
85695
86299
|
layout: {
|
|
85696
86300
|
state: "unavailable",
|
|
@@ -85742,7 +86346,8 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85742
86346
|
)).where(and67(...graphScope, lt13(siteCrawlGraphNodes.sampleRank, maxNodes))).orderBy(asc14(siteCrawlGraphNodes.sampleRank)).all();
|
|
85743
86347
|
const nodes = nodeRows.map(({ indexabilityReasons, canonicalNodeKey, ...row }) => ({
|
|
85744
86348
|
...row,
|
|
85745
|
-
healthState: deriveSiteHealthState({ ...row, indexabilityReasons, canonicalNodeKey })
|
|
86349
|
+
healthState: deriveSiteHealthState({ ...row, indexabilityReasons, canonicalNodeKey }),
|
|
86350
|
+
healthReason: deriveSiteHealthReason({ ...row, indexabilityReasons, canonicalNodeKey })
|
|
85746
86351
|
}));
|
|
85747
86352
|
const graphLinkKindFilter = linkKind === SiteHealthLinkKinds.all ? void 0 : eq80(siteCrawlGraphEdges.isTemplate, linkKind === SiteHealthLinkKinds.template);
|
|
85748
86353
|
const edges = app.db.select({
|
|
@@ -85780,7 +86385,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85780
86385
|
project: project.name,
|
|
85781
86386
|
hasCrawlData: true,
|
|
85782
86387
|
runId: snapshot.runId,
|
|
85783
|
-
...completenessOf(
|
|
86388
|
+
...completenessOf(target),
|
|
85784
86389
|
rootNodeKey,
|
|
85785
86390
|
layout: {
|
|
85786
86391
|
state: "ready",
|
|
@@ -85937,8 +86542,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85937
86542
|
project: project.name,
|
|
85938
86543
|
hasCrawlData: true,
|
|
85939
86544
|
runId: snapshot.runId,
|
|
85940
|
-
|
|
85941
|
-
termination: snapshot.termination,
|
|
86545
|
+
...completenessOf(target),
|
|
85942
86546
|
state: "ready",
|
|
85943
86547
|
focusNodeKey: focus.nodeKey,
|
|
85944
86548
|
focusUrl: focus.url,
|
|
@@ -85989,8 +86593,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
85989
86593
|
state: "details-unavailable",
|
|
85990
86594
|
from: null,
|
|
85991
86595
|
to: null,
|
|
85992
|
-
|
|
85993
|
-
termination: snapshot.termination,
|
|
86596
|
+
...completenessOf(target),
|
|
85994
86597
|
maxDepth,
|
|
85995
86598
|
visitedNodes: 0,
|
|
85996
86599
|
nodes: [],
|
|
@@ -86052,8 +86655,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86052
86655
|
return {
|
|
86053
86656
|
project: project.name,
|
|
86054
86657
|
runId: snapshot.runId,
|
|
86055
|
-
|
|
86056
|
-
termination: snapshot.termination,
|
|
86658
|
+
...completenessOf(target),
|
|
86057
86659
|
state: truncated || frontier.length > 0 ? "truncated" : "unreachable",
|
|
86058
86660
|
from: reference(fromPage),
|
|
86059
86661
|
to: reference(toPage),
|
|
@@ -86082,8 +86684,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86082
86684
|
return {
|
|
86083
86685
|
project: project.name,
|
|
86084
86686
|
runId: snapshot.runId,
|
|
86085
|
-
|
|
86086
|
-
termination: snapshot.termination,
|
|
86687
|
+
...completenessOf(target),
|
|
86087
86688
|
state: "found",
|
|
86088
86689
|
from: reference(fromPage),
|
|
86089
86690
|
to: reference(toPage),
|
|
@@ -86206,9 +86807,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86206
86807
|
const summary4 = { pages: zero2(), links: zero2() };
|
|
86207
86808
|
if (scopeFilter !== "links") {
|
|
86208
86809
|
if (includeAdded || includeChanged) {
|
|
86209
|
-
const addedExpression = includeAdded ?
|
|
86210
|
-
const changedExpression = includeChanged ?
|
|
86211
|
-
const [counts2 = { added: 0, changed: 0 }] = app.db.all(
|
|
86810
|
+
const addedExpression = includeAdded ? sql30`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql30`0`;
|
|
86811
|
+
const changedExpression = includeChanged ? sql30`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${PAGE_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql30`0`;
|
|
86812
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql30`
|
|
86212
86813
|
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
86213
86814
|
FROM site_crawl_pages AS current
|
|
86214
86815
|
LEFT JOIN site_crawl_pages AS previous
|
|
@@ -86224,7 +86825,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86224
86825
|
summary4.pages.changed = Number(counts2.changed);
|
|
86225
86826
|
}
|
|
86226
86827
|
if (includeRemoved) {
|
|
86227
|
-
const [counts2 = { removed: 0 }] = app.db.all(
|
|
86828
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql30`
|
|
86228
86829
|
SELECT COUNT(*) AS removed
|
|
86229
86830
|
FROM site_crawl_pages AS previous
|
|
86230
86831
|
LEFT JOIN site_crawl_pages AS current
|
|
@@ -86242,9 +86843,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86242
86843
|
}
|
|
86243
86844
|
if (scopeFilter !== "pages") {
|
|
86244
86845
|
if (includeAdded || includeChanged) {
|
|
86245
|
-
const addedExpression = includeAdded ?
|
|
86246
|
-
const changedExpression = includeChanged ?
|
|
86247
|
-
const [counts2 = { added: 0, changed: 0 }] = app.db.all(
|
|
86846
|
+
const addedExpression = includeAdded ? sql30`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql30`0`;
|
|
86847
|
+
const changedExpression = includeChanged ? sql30`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${LINK_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql30`0`;
|
|
86848
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql30`
|
|
86248
86849
|
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
86249
86850
|
FROM site_crawl_edges AS current
|
|
86250
86851
|
LEFT JOIN site_crawl_edges AS previous
|
|
@@ -86262,7 +86863,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86262
86863
|
summary4.links.changed = Number(counts2.changed);
|
|
86263
86864
|
}
|
|
86264
86865
|
if (includeRemoved) {
|
|
86265
|
-
const [counts2 = { removed: 0 }] = app.db.all(
|
|
86866
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql30`
|
|
86266
86867
|
SELECT COUNT(*) AS removed
|
|
86267
86868
|
FROM site_crawl_edges AS previous
|
|
86268
86869
|
LEFT JOIN site_crawl_edges AS current
|
|
@@ -86292,48 +86893,48 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86292
86893
|
const includeRemoved = changeFilter === "all" || changeFilter === "removed";
|
|
86293
86894
|
const table = entity === "page" ? "site_crawl_pages" : "site_crawl_edges";
|
|
86294
86895
|
const keyColumn = entity === "page" ? "node_key" : "edge_key";
|
|
86295
|
-
const internalCurrent = entity === "link" ?
|
|
86296
|
-
const internalPreviousJoin = entity === "link" ?
|
|
86297
|
-
const internalPrevious = entity === "link" ?
|
|
86298
|
-
const internalCurrentJoin = entity === "link" ?
|
|
86896
|
+
const internalCurrent = entity === "link" ? sql30`AND current.internal = 1` : sql30``;
|
|
86897
|
+
const internalPreviousJoin = entity === "link" ? sql30`AND previous.internal = 1` : sql30``;
|
|
86898
|
+
const internalPrevious = entity === "link" ? sql30`AND previous.internal = 1` : sql30``;
|
|
86899
|
+
const internalCurrentJoin = entity === "link" ? sql30`AND current.internal = 1` : sql30``;
|
|
86299
86900
|
const difference2 = entity === "page" ? PAGE_CHANGE_PREDICATE : LINK_CHANGE_PREDICATE;
|
|
86300
|
-
const afterKind = changeFilter === "added" ?
|
|
86301
|
-
const afterRows = includeAfter ? app.db.all(
|
|
86901
|
+
const afterKind = changeFilter === "added" ? sql30`previous.id IS NULL` : changeFilter === "changed" ? sql30`previous.id IS NOT NULL AND ${difference2}` : sql30`previous.id IS NULL OR (previous.id IS NOT NULL AND ${difference2})`;
|
|
86902
|
+
const afterRows = includeAfter ? app.db.all(sql30`
|
|
86302
86903
|
SELECT ${entity} AS entity,
|
|
86303
86904
|
CASE WHEN previous.id IS NULL THEN 'added' ELSE 'changed' END AS change,
|
|
86304
|
-
current.${
|
|
86305
|
-
FROM ${
|
|
86306
|
-
LEFT JOIN ${
|
|
86905
|
+
current.${sql30.raw(keyColumn)} AS key
|
|
86906
|
+
FROM ${sql30.raw(table)} AS current
|
|
86907
|
+
LEFT JOIN ${sql30.raw(table)} AS previous
|
|
86307
86908
|
ON previous.project_id = ${beforeScope.projectId}
|
|
86308
86909
|
AND previous.run_id = ${beforeScope.runId}
|
|
86309
86910
|
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
86310
86911
|
${internalPreviousJoin}
|
|
86311
|
-
AND previous.${
|
|
86912
|
+
AND previous.${sql30.raw(keyColumn)} = current.${sql30.raw(keyColumn)}
|
|
86312
86913
|
WHERE current.project_id = ${afterScope.projectId}
|
|
86313
86914
|
AND current.run_id = ${afterScope.runId}
|
|
86314
86915
|
AND current.attempt_id = ${afterScope.attemptId}
|
|
86315
86916
|
${internalCurrent}
|
|
86316
|
-
AND current.${
|
|
86917
|
+
AND current.${sql30.raw(keyColumn)} > ${afterKey}
|
|
86317
86918
|
AND (${afterKind})
|
|
86318
|
-
ORDER BY current.${
|
|
86919
|
+
ORDER BY current.${sql30.raw(keyColumn)} ASC
|
|
86319
86920
|
LIMIT ${wanted2}
|
|
86320
86921
|
`) : [];
|
|
86321
|
-
const removedRows = includeRemoved ? app.db.all(
|
|
86322
|
-
SELECT ${entity} AS entity, 'removed' AS change, previous.${
|
|
86323
|
-
FROM ${
|
|
86324
|
-
LEFT JOIN ${
|
|
86922
|
+
const removedRows = includeRemoved ? app.db.all(sql30`
|
|
86923
|
+
SELECT ${entity} AS entity, 'removed' AS change, previous.${sql30.raw(keyColumn)} AS key
|
|
86924
|
+
FROM ${sql30.raw(table)} AS previous
|
|
86925
|
+
LEFT JOIN ${sql30.raw(table)} AS current
|
|
86325
86926
|
ON current.project_id = ${afterScope.projectId}
|
|
86326
86927
|
AND current.run_id = ${afterScope.runId}
|
|
86327
86928
|
AND current.attempt_id = ${afterScope.attemptId}
|
|
86328
86929
|
${internalCurrentJoin}
|
|
86329
|
-
AND current.${
|
|
86930
|
+
AND current.${sql30.raw(keyColumn)} = previous.${sql30.raw(keyColumn)}
|
|
86330
86931
|
WHERE previous.project_id = ${beforeScope.projectId}
|
|
86331
86932
|
AND previous.run_id = ${beforeScope.runId}
|
|
86332
86933
|
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
86333
86934
|
${internalPrevious}
|
|
86334
|
-
AND previous.${
|
|
86935
|
+
AND previous.${sql30.raw(keyColumn)} > ${afterKey}
|
|
86335
86936
|
AND current.id IS NULL
|
|
86336
|
-
ORDER BY previous.${
|
|
86937
|
+
ORDER BY previous.${sql30.raw(keyColumn)} ASC
|
|
86337
86938
|
LIMIT ${wanted2}
|
|
86338
86939
|
`) : [];
|
|
86339
86940
|
const merged = [];
|
|
@@ -86457,8 +87058,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86457
87058
|
const provenance = {
|
|
86458
87059
|
project: project.name,
|
|
86459
87060
|
runId: snapshot.runId,
|
|
86460
|
-
|
|
86461
|
-
termination: snapshot.termination
|
|
87061
|
+
...completenessOf(target)
|
|
86462
87062
|
};
|
|
86463
87063
|
const scope = detailScopeFor(project.id, snapshot);
|
|
86464
87064
|
if (!scope) return { state: "details-unavailable", ...provenance };
|
|
@@ -86489,15 +87089,44 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86489
87089
|
});
|
|
86490
87090
|
app.get("/projects/:name/technical-aeo/crawl/pages", async (request) => {
|
|
86491
87091
|
const project = resolveProject(app.db, request.params.name);
|
|
86492
|
-
const
|
|
87092
|
+
const requestedDate = parseCrawlDate(request.query.date, request.query.runId);
|
|
87093
|
+
const target = resolveCrawl(project.id, request.query.runId, requestedDate);
|
|
86493
87094
|
if (!target) {
|
|
86494
87095
|
assertKnownAuditRun(project.id, request.query.runId);
|
|
86495
|
-
|
|
87096
|
+
decodeCrawlPageCursor(request.query.cursor, null);
|
|
87097
|
+
return {
|
|
87098
|
+
project: project.name,
|
|
87099
|
+
hasCrawlData: false,
|
|
87100
|
+
runId: null,
|
|
87101
|
+
requestedDate,
|
|
87102
|
+
inventorySummary: null,
|
|
87103
|
+
healthReasonCounts: null,
|
|
87104
|
+
availableScanDates: availableScanDatesFor(project.id, requestedDate),
|
|
87105
|
+
...NO_CRAWL_COMPLETENESS,
|
|
87106
|
+
total: 0,
|
|
87107
|
+
nextCursor: null,
|
|
87108
|
+
healthStateFilter: null,
|
|
87109
|
+
pages: []
|
|
87110
|
+
};
|
|
86496
87111
|
}
|
|
86497
87112
|
const snapshot = target.snapshot;
|
|
86498
87113
|
if (!snapshot.detailsAvailable || !snapshot.attemptId) {
|
|
86499
|
-
|
|
87114
|
+
decodeCrawlPageCursor(request.query.cursor, null);
|
|
87115
|
+
return {
|
|
87116
|
+
project: project.name,
|
|
87117
|
+
hasCrawlData: true,
|
|
87118
|
+
runId: snapshot.runId,
|
|
87119
|
+
requestedDate,
|
|
87120
|
+
inventorySummary: null,
|
|
87121
|
+
healthReasonCounts: null,
|
|
87122
|
+
...completenessOf(target),
|
|
87123
|
+
total: 0,
|
|
87124
|
+
nextCursor: null,
|
|
87125
|
+
healthStateFilter: null,
|
|
87126
|
+
pages: []
|
|
87127
|
+
};
|
|
86500
87128
|
}
|
|
87129
|
+
const inventorySummary = inventorySummaryFor(detailScopeFor(project.id, snapshot));
|
|
86501
87130
|
const filters = [
|
|
86502
87131
|
eq80(siteCrawlPages.projectId, project.id),
|
|
86503
87132
|
eq80(siteCrawlPages.runId, snapshot.runId),
|
|
@@ -86511,8 +87140,22 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86511
87140
|
if (request.query.auditState) filters.push(eq80(siteCrawlPages.auditState, request.query.auditState));
|
|
86512
87141
|
const healthState = parseSiteHealthState(request.query.healthState);
|
|
86513
87142
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
86514
|
-
const
|
|
86515
|
-
const
|
|
87143
|
+
const sort = ["score-desc", "score-asc", "path"].includes(request.query.sort ?? "") ? request.query.sort : "url";
|
|
87144
|
+
const selectionKey = crypto49.createHash("sha256").update(JSON.stringify({
|
|
87145
|
+
projectId: project.id,
|
|
87146
|
+
runId: snapshot.runId,
|
|
87147
|
+
attemptId: snapshot.attemptId,
|
|
87148
|
+
requestedDate: requestedDate ?? null,
|
|
87149
|
+
nodeKey: request.query.nodeKey || null,
|
|
87150
|
+
inventoryEligible,
|
|
87151
|
+
fetchState: request.query.fetchState || null,
|
|
87152
|
+
indexabilityState: request.query.indexabilityState || null,
|
|
87153
|
+
auditState: request.query.auditState || null,
|
|
87154
|
+
healthState,
|
|
87155
|
+
sort
|
|
87156
|
+
})).digest("base64url");
|
|
87157
|
+
const offset = decodeCrawlPageCursor(request.query.cursor, selectionKey);
|
|
87158
|
+
const orderBy = sort === "score-desc" ? [desc36(siteCrawlPages.auditScore), asc14(siteCrawlPages.nodeKey)] : sort === "score-asc" ? [asc14(siteCrawlPages.auditScore), asc14(siteCrawlPages.nodeKey)] : sort === "path" ? [asc14(siteCrawlPages.path), asc14(siteCrawlPages.nodeKey)] : [asc14(siteCrawlPages.url), asc14(siteCrawlPages.nodeKey)];
|
|
86516
87159
|
let healthStateFilter = null;
|
|
86517
87160
|
if (healthState) {
|
|
86518
87161
|
const legacyRow = app.db.select({ id: siteCrawlPages.id }).from(siteCrawlPages).where(and67(
|
|
@@ -86529,7 +87172,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86529
87172
|
project: project.name,
|
|
86530
87173
|
hasCrawlData: true,
|
|
86531
87174
|
runId: snapshot.runId,
|
|
86532
|
-
|
|
87175
|
+
requestedDate,
|
|
87176
|
+
inventorySummary,
|
|
87177
|
+
healthReasonCounts: null,
|
|
87178
|
+
...completenessOf(target),
|
|
86533
87179
|
total: 0,
|
|
86534
87180
|
nextCursor: null,
|
|
86535
87181
|
healthStateFilter,
|
|
@@ -86544,9 +87190,12 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86544
87190
|
project: project.name,
|
|
86545
87191
|
hasCrawlData: true,
|
|
86546
87192
|
runId: snapshot.runId,
|
|
86547
|
-
|
|
87193
|
+
requestedDate,
|
|
87194
|
+
inventorySummary,
|
|
87195
|
+
healthReasonCounts: reasonCounts(crawlReasonGroups(where)),
|
|
87196
|
+
...completenessOf(target),
|
|
86548
87197
|
total,
|
|
86549
|
-
nextCursor: nextOffset < total ?
|
|
87198
|
+
nextCursor: nextOffset < total ? Buffer.from(JSON.stringify({ offset: nextOffset, selectionKey })).toString("base64url") : null,
|
|
86550
87199
|
healthStateFilter,
|
|
86551
87200
|
pages: rows.map(mapCrawlPage)
|
|
86552
87201
|
};
|
|
@@ -86561,7 +87210,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86561
87210
|
}
|
|
86562
87211
|
const snapshot = target.snapshot;
|
|
86563
87212
|
if (!snapshot.detailsAvailable || !snapshot.attemptId) {
|
|
86564
|
-
return { project: project.name, hasCrawlData: true, runId: snapshot.runId, ...completenessOf(
|
|
87213
|
+
return { project: project.name, hasCrawlData: true, runId: snapshot.runId, ...completenessOf(target), parentPath, nextCursor: null, children: [] };
|
|
86565
87214
|
}
|
|
86566
87215
|
const sourceRows = app.db.select({
|
|
86567
87216
|
path: siteCrawlPages.path,
|
|
@@ -86606,7 +87255,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86606
87255
|
project: project.name,
|
|
86607
87256
|
hasCrawlData: true,
|
|
86608
87257
|
runId: snapshot.runId,
|
|
86609
|
-
...completenessOf(
|
|
87258
|
+
...completenessOf(target),
|
|
86610
87259
|
parentPath,
|
|
86611
87260
|
nextCursor: hasMore ? encodeCursor2(offset + visible.length) : null,
|
|
86612
87261
|
children: visible.map(([path8, child]) => ({
|
|
@@ -86644,7 +87293,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86644
87293
|
project: project.name,
|
|
86645
87294
|
hasCrawlData: true,
|
|
86646
87295
|
runId: snapshot.runId,
|
|
86647
|
-
...completenessOf(
|
|
87296
|
+
...completenessOf(target),
|
|
86648
87297
|
total: 0,
|
|
86649
87298
|
nextCursor: null,
|
|
86650
87299
|
templateDetection,
|
|
@@ -86673,7 +87322,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86673
87322
|
project: project.name,
|
|
86674
87323
|
hasCrawlData: true,
|
|
86675
87324
|
runId: snapshot.runId,
|
|
86676
|
-
...completenessOf(
|
|
87325
|
+
...completenessOf(target),
|
|
86677
87326
|
total,
|
|
86678
87327
|
nextCursor: nextOffset < total ? encodeCursor2(nextOffset) : null,
|
|
86679
87328
|
templateDetection,
|
|
@@ -86712,7 +87361,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86712
87361
|
project: project.name,
|
|
86713
87362
|
hasCrawlData: true,
|
|
86714
87363
|
runId: snapshot.runId,
|
|
86715
|
-
...completenessOf(
|
|
87364
|
+
...completenessOf(target),
|
|
86716
87365
|
nodeKey: request.query.nodeKey ?? null,
|
|
86717
87366
|
url: request.query.url ?? null,
|
|
86718
87367
|
templateDetection,
|
|
@@ -86739,7 +87388,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86739
87388
|
project: project.name,
|
|
86740
87389
|
hasCrawlData: true,
|
|
86741
87390
|
runId: snapshot.runId,
|
|
86742
|
-
...completenessOf(
|
|
87391
|
+
...completenessOf(target),
|
|
86743
87392
|
nodeKey: request.query.nodeKey ?? null,
|
|
86744
87393
|
url: request.query.url ?? null,
|
|
86745
87394
|
templateDetection,
|
|
@@ -86760,13 +87409,13 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86760
87409
|
}
|
|
86761
87410
|
const snapshot = target.snapshot;
|
|
86762
87411
|
if (!snapshot.checkDeadLinks || snapshot.deadLinkState === "disabled") {
|
|
86763
|
-
return { project: project.name, runId: snapshot.runId, state: "disabled", checkDeadLinks: false };
|
|
87412
|
+
return { project: project.name, runId: snapshot.runId, state: "disabled", checkDeadLinks: false, runSelection: target.runSelection };
|
|
86764
87413
|
}
|
|
86765
87414
|
if (snapshot.deadLinkState === "unavailable") {
|
|
86766
|
-
return { project: project.name, runId: snapshot.runId, state: "unavailable", legacyAuditAvailable };
|
|
87415
|
+
return { project: project.name, runId: snapshot.runId, state: "unavailable", legacyAuditAvailable, runSelection: target.runSelection };
|
|
86767
87416
|
}
|
|
86768
87417
|
if (!snapshot.attemptId) {
|
|
86769
|
-
return { project: project.name, runId: snapshot.runId, state: "partial", checkDeadLinks: true, checked: snapshot.deadLinksChecked, found: snapshot.deadLinksFound, unverified: snapshot.deadLinksUnverified, total: 0, nextCursor: null, deadLinks: [] };
|
|
87418
|
+
return { project: project.name, runId: snapshot.runId, state: "partial", checkDeadLinks: true, checked: snapshot.deadLinksChecked, found: snapshot.deadLinksFound, unverified: snapshot.deadLinksUnverified, total: 0, nextCursor: null, deadLinks: [], runSelection: target.runSelection };
|
|
86770
87419
|
}
|
|
86771
87420
|
const where = and67(
|
|
86772
87421
|
eq80(siteCrawlFindings.projectId, project.id),
|
|
@@ -86782,7 +87431,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86782
87431
|
// a fabricated broken link shown to a client is not worth the saved
|
|
86783
87432
|
// predicate. Applied to `total` as well as the rows, or the count would
|
|
86784
87433
|
// promise a page the list cannot fill.
|
|
86785
|
-
isNotNull8(
|
|
87434
|
+
isNotNull8(sql30`json_extract(${siteCrawlFindings.evidence}, '$.statusCode')`)
|
|
86786
87435
|
);
|
|
86787
87436
|
const total = app.db.select({ value: count2() }).from(siteCrawlFindings).where(where).get()?.value ?? 0;
|
|
86788
87437
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
@@ -86802,6 +87451,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86802
87451
|
return {
|
|
86803
87452
|
project: project.name,
|
|
86804
87453
|
runId: snapshot.runId,
|
|
87454
|
+
runSelection: target.runSelection,
|
|
86805
87455
|
state,
|
|
86806
87456
|
checkDeadLinks: true,
|
|
86807
87457
|
checked: snapshot.deadLinksChecked,
|
|
@@ -86837,6 +87487,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
86837
87487
|
)).all().map((row) => row.runId));
|
|
86838
87488
|
return {
|
|
86839
87489
|
project: project.name,
|
|
87490
|
+
preferredRunId: resolveCrawl(project.id)?.snapshot.runId ?? resolveAudit(project.id)?.snap.runId ?? null,
|
|
86840
87491
|
scans: rows.map((row) => ({
|
|
86841
87492
|
...row,
|
|
86842
87493
|
status: row.status,
|
|
@@ -87027,7 +87678,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87027
87678
|
// ../api-routes/src/research.ts
|
|
87028
87679
|
import crypto50 from "crypto";
|
|
87029
87680
|
import { z as z5 } from "zod";
|
|
87030
|
-
import { and as and68, count as count3, desc as desc37, eq as eq81, gte as
|
|
87681
|
+
import { and as and68, count as count3, desc as desc37, eq as eq81, gte as gte22, lt as lt14, or as or19, sql as sql31 } from "drizzle-orm";
|
|
87031
87682
|
var sameLocation = (a, b) => a.label === b.label && a.city === b.city && a.region === b.region && a.country === b.country && a.timezone === b.timezone;
|
|
87032
87683
|
var BATCH_RECEIPT_PREFIX = "__canonry_research_batch__:";
|
|
87033
87684
|
async function researchRoutes(app, opts) {
|
|
@@ -87087,9 +87738,9 @@ async function researchRoutes(app, opts) {
|
|
|
87087
87738
|
const { start, end, date } = utcDayBounds(now);
|
|
87088
87739
|
const used = tx.select({ value: count3() }).from(researchRuns).where(and68(
|
|
87089
87740
|
eq81(researchRuns.projectId, project.id),
|
|
87090
|
-
|
|
87741
|
+
gte22(researchRuns.createdAt, start),
|
|
87091
87742
|
lt14(researchRuns.createdAt, end),
|
|
87092
|
-
|
|
87743
|
+
sql31`(json_extract(${researchRuns.initiatedBy}, '$.role') = ${UserRoles.viewer} OR json_extract(${researchRuns.initiatedBy}, '$.limited') = 1)`
|
|
87093
87744
|
)).get()?.value ?? 0;
|
|
87094
87745
|
if (used >= viewerDailyRunLimit) {
|
|
87095
87746
|
throw researchDailyLimitExceeded(project.name, viewerDailyRunLimit, date);
|
|
@@ -87142,9 +87793,9 @@ async function researchRoutes(app, opts) {
|
|
|
87142
87793
|
const { start, end, date } = utcDayBounds(now);
|
|
87143
87794
|
const used = tx.select({ value: count3() }).from(researchRuns).where(and68(
|
|
87144
87795
|
eq81(researchRuns.projectId, project.id),
|
|
87145
|
-
|
|
87796
|
+
gte22(researchRuns.createdAt, start),
|
|
87146
87797
|
lt14(researchRuns.createdAt, end),
|
|
87147
|
-
|
|
87798
|
+
sql31`(json_extract(${researchRuns.initiatedBy}, '$.role') = ${UserRoles.viewer} OR json_extract(${researchRuns.initiatedBy}, '$.limited') = 1)`
|
|
87148
87799
|
)).get()?.value ?? 0;
|
|
87149
87800
|
if (used + prepared.length > viewerDailyRunLimit) throw researchDailyLimitExceeded(project.name, viewerDailyRunLimit, date);
|
|
87150
87801
|
}
|
|
@@ -88408,7 +89059,7 @@ function computeCitedCompetitorDomains(citedDomains, competitorDomains) {
|
|
|
88408
89059
|
var RECURRENCE_LOOKBACK_RUNS = 5;
|
|
88409
89060
|
var HISTORY_WINDOW_RUNS = Math.max(PERSISTENT_GAP_THRESHOLD, 5);
|
|
88410
89061
|
var log = createLogger("IntelligenceService");
|
|
88411
|
-
var runChronologyKey =
|
|
89062
|
+
var runChronologyKey = sql32`coalesce(${runs.finishedAt}, ${runs.createdAt})`;
|
|
88412
89063
|
function analyzableRunPredicates(projectId) {
|
|
88413
89064
|
return [
|
|
88414
89065
|
eq84(runs.projectId, projectId),
|
|
@@ -88476,7 +89127,7 @@ var IntelligenceService = class {
|
|
|
88476
89127
|
// MEASUREMENT: only runs that actually wrote snapshots — see above.
|
|
88477
89128
|
// Index seek on idx_snapshots_run, short-circuited on the first row.
|
|
88478
89129
|
exists2(
|
|
88479
|
-
this.db.select({ one:
|
|
89130
|
+
this.db.select({ one: sql32`1` }).from(querySnapshots).where(eq84(querySnapshots.runId, runs.id))
|
|
88480
89131
|
)
|
|
88481
89132
|
)
|
|
88482
89133
|
).orderBy(desc38(runs.finishedAt), desc38(runs.createdAt)).limit(HISTORY_WINDOW_RUNS).all();
|
|
@@ -88562,7 +89213,7 @@ var IntelligenceService = class {
|
|
|
88562
89213
|
const selected = this.db.select().from(gbpLocations).where(and71(
|
|
88563
89214
|
eq84(gbpLocations.projectId, projectId),
|
|
88564
89215
|
eq84(gbpLocations.selected, true),
|
|
88565
|
-
|
|
89216
|
+
gte23(gbpLocations.syncedAt, windowStart),
|
|
88566
89217
|
lte20(gbpLocations.syncedAt, windowEnd)
|
|
88567
89218
|
)).all();
|
|
88568
89219
|
if (selected.length === 0) {
|
|
@@ -88942,7 +89593,7 @@ var IntelligenceService = class {
|
|
|
88942
89593
|
position: gscSearchData.position
|
|
88943
89594
|
}).from(gscSearchData).where(and71(
|
|
88944
89595
|
eq84(gscSearchData.projectId, projectId),
|
|
88945
|
-
|
|
89596
|
+
gte23(gscSearchData.date, window.startDate),
|
|
88946
89597
|
lte20(gscSearchData.date, window.endDate)
|
|
88947
89598
|
)).all();
|
|
88948
89599
|
const byDay = /* @__PURE__ */ new Map();
|