@canonry/canonry 7.14.0 → 7.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +5 -2
- package/assets/assets/{AuditHistoryPanel-gAkqWz9z.js → AuditHistoryPanel-Cl9tUvSZ.js} +1 -1
- package/assets/assets/{BacklinksPage-C67DtHo8.js → BacklinksPage-CTeTJ9dh.js} +1 -1
- package/assets/assets/{HistoryPage-BQuW6Z6m.js → HistoryPage-DpOoXr2P.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-B1XS6DR0.js → MeasurementPropertyPage-DxXsgM25.js} +1 -1
- package/assets/assets/{ProjectPage-CV_7bOA7.js → ProjectPage-By9arjLJ.js} +1 -1
- package/assets/assets/{RunsPage-CJYihZYC.js → RunsPage-CfDi7I1z.js} +1 -1
- package/assets/assets/{SettingsPage-Dd3J2DQE.js → SettingsPage-BHDb1apK.js} +1 -1
- package/assets/assets/{SiteHealthSection-iktyuzt2.js → SiteHealthSection-DQZyy4P_.js} +3 -3
- package/assets/assets/{TrafficPage-CWjK7F3j.js → TrafficPage-DX_vN2RL.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-D4_AXQNE.js → TrafficSourceDetailPage-Cg92zrtZ.js} +1 -1
- package/assets/assets/{extract-error-message-Cxefvthz.js → extract-error-message-BKxVCbov.js} +1 -1
- package/assets/assets/index-DoQJETc6.js +86 -0
- package/assets/assets/{react-sigma_core.esm.min-DDEPa7AV.js → react-sigma_core.esm.min-BhqZcSgh.js} +1 -1
- package/assets/assets/{v2-overview-adapter-CLxlMdb2.js → v2-overview-adapter-DZ5YsE_G.js} +1 -1
- package/assets/index.html +1 -1
- package/dist/{chunk-WHVEDCHY.js → chunk-2S7U2YHI.js} +2 -2
- package/dist/{chunk-ZHXOR2LJ.js → chunk-3ZGBSHZ2.js} +2 -2
- package/dist/{chunk-A25FO3Z5.js → chunk-6LS2R477.js} +5671 -5399
- package/dist/{chunk-4LQ4HKVK.js → chunk-KDWTNI4Z.js} +2 -2
- package/dist/{chunk-LUQ5G5OO.js → chunk-O6XPXEN6.js} +3 -3
- package/dist/{chunk-EZRBFH77.js → chunk-SPZRSKCF.js} +859 -455
- package/dist/{chunk-KXHPQMBP.js → chunk-VWSUFTW5.js} +43 -20
- package/dist/cli.js +52 -9
- package/dist/{demo-server-SIATTXXS.js → demo-server-6OE3H2KL.js} +4 -4
- package/dist/index.js +6 -6
- package/dist/{intelligence-service-UEEVETII.js → intelligence-service-USSTKAKC.js} +2 -2
- package/dist/mcp.js +4 -4
- package/package.json +15 -15
- package/assets/assets/index-BbmiGIM_.js +0 -86
|
@@ -30,6 +30,7 @@ import {
|
|
|
30
30
|
AdsOperationStepStates,
|
|
31
31
|
AdsReconcileStrategies,
|
|
32
32
|
AdsReviewStatuses,
|
|
33
|
+
AiEngineLeadRateUnavailableReasons,
|
|
33
34
|
AiReferralTrafficClasses,
|
|
34
35
|
AppError,
|
|
35
36
|
BacklinkSources,
|
|
@@ -63,6 +64,7 @@ import {
|
|
|
63
64
|
DiscoveryCompetitorTypes,
|
|
64
65
|
DiscoveryPromoteCompetitorSkipReasons,
|
|
65
66
|
DiscoverySessionStatuses,
|
|
67
|
+
GBP_NO_SELECTED_LOCATIONS_ERROR,
|
|
66
68
|
GOOGLE_MARKETING_LIVE_READ_SCOPE,
|
|
67
69
|
GOOGLE_MARKETING_STORED_SNAPSHOT_PAGE_MAX,
|
|
68
70
|
GOOGLE_MARKETING_WRITE_SCOPE,
|
|
@@ -223,7 +225,9 @@ import {
|
|
|
223
225
|
agentProvidersResponseDtoSchema,
|
|
224
226
|
aggregateHarvestedQueries,
|
|
225
227
|
aggregateSentiment,
|
|
228
|
+
aiEngineForReferralSource,
|
|
226
229
|
aiReferralClassCounts,
|
|
230
|
+
aiReferralEngineLabel,
|
|
227
231
|
alreadyExists,
|
|
228
232
|
answerProseForMentions,
|
|
229
233
|
apiKeyDtoSchema,
|
|
@@ -347,6 +351,7 @@ import {
|
|
|
347
351
|
deriveWinnabilityClass,
|
|
348
352
|
describeCompetitorAliasRejection,
|
|
349
353
|
describeError,
|
|
354
|
+
describeFetchError,
|
|
350
355
|
describeMarketPinAliasClaim,
|
|
351
356
|
determineAnswerMentioned,
|
|
352
357
|
diagnosticIdentity,
|
|
@@ -386,6 +391,7 @@ import {
|
|
|
386
391
|
formatIsoDate,
|
|
387
392
|
formatIsoDateInTimeZone,
|
|
388
393
|
formatPercent,
|
|
394
|
+
formatRunErrorOneLine,
|
|
389
395
|
formatSignedPercent,
|
|
390
396
|
ga4AiReferralDailyDtoSchema,
|
|
391
397
|
ga4AiReferralHistoryEntrySchema,
|
|
@@ -465,6 +471,8 @@ import {
|
|
|
465
471
|
internalError,
|
|
466
472
|
intersectScopes,
|
|
467
473
|
isBrowserProvider,
|
|
474
|
+
isFetchTransportError,
|
|
475
|
+
isGa4AiAssistantChannel,
|
|
468
476
|
isLocationRedirectStatus,
|
|
469
477
|
isNegativeReviewRating,
|
|
470
478
|
isProviderAccountFailure,
|
|
@@ -792,6 +800,7 @@ import {
|
|
|
792
800
|
trafficSourceListResponseSchema,
|
|
793
801
|
trafficStatusResponseSchema,
|
|
794
802
|
trafficSyncResponseSchema,
|
|
803
|
+
truncateUtf16,
|
|
795
804
|
uiTelemetryEventSchema,
|
|
796
805
|
unsupportedKind,
|
|
797
806
|
upgradeCaveatFor,
|
|
@@ -821,10 +830,10 @@ import {
|
|
|
821
830
|
wordpressSchemaDeployResultDtoSchema,
|
|
822
831
|
wordpressSchemaStatusResultDtoSchema,
|
|
823
832
|
wordpressStatusDtoSchema
|
|
824
|
-
} from "./chunk-
|
|
833
|
+
} from "./chunk-6LS2R477.js";
|
|
825
834
|
|
|
826
835
|
// src/intelligence-service.ts
|
|
827
|
-
import { eq as
|
|
836
|
+
import { eq as eq88, desc as desc40, asc as asc17, and as and74, ne as ne16, or as or22, inArray as inArray36, gte as gte24, lte as lte20, isNull as isNull17, sql as sql32, exists as exists2 } from "drizzle-orm";
|
|
828
837
|
|
|
829
838
|
// ../db/src/client.ts
|
|
830
839
|
import { mkdirSync } from "fs";
|
|
@@ -13378,10 +13387,10 @@ function selectSentimentSources(db, projectId, filters) {
|
|
|
13378
13387
|
for (const key of ["expected", "completed"]) out.sourceSlotIds[key].push(...slots[key].map((slot) => pairKey(runId, slot)));
|
|
13379
13388
|
};
|
|
13380
13389
|
let currentRunId = "";
|
|
13381
|
-
const skip = (reason,
|
|
13382
|
-
out.skipped[reason] = (out.skipped[reason] ?? 0) +
|
|
13390
|
+
const skip = (reason, count5 = 1) => {
|
|
13391
|
+
out.skipped[reason] = (out.skipped[reason] ?? 0) + count5;
|
|
13383
13392
|
const byRun = out.skippedByRun[currentRunId] ??= {};
|
|
13384
|
-
byRun[reason] = (byRun[reason] ?? 0) +
|
|
13393
|
+
byRun[reason] = (byRun[reason] ?? 0) + count5;
|
|
13385
13394
|
};
|
|
13386
13395
|
const otherClass = (filters.queryClass ?? "branded") === "branded" ? "non-brand" : "branded";
|
|
13387
13396
|
const otherClassFilters = { ...filters, queryClass: otherClass };
|
|
@@ -13587,7 +13596,7 @@ function selectedSlotCoverage(db, projectId, run, filters, plans) {
|
|
|
13587
13596
|
const coverage = () => ({ expected: expected.size, completed: completed.size, slotIds: { expected: [...expected], completed: [...completed] } });
|
|
13588
13597
|
if (run.status === "failed") return coverage();
|
|
13589
13598
|
const snapshots = db.select({ queryId: querySnapshots.queryId, queryText: querySnapshots.queryText, provider: querySnapshots.provider, servedModel: querySnapshots.servedModel, measurementExecutionId: querySnapshots.measurementExecutionId }).from(querySnapshots).where(eq8(querySnapshots.runId, run.id)).all();
|
|
13590
|
-
const
|
|
13599
|
+
const count5 = (key, edge, recorded) => {
|
|
13591
13600
|
if (!matchesSentimentEdge(edge, filters)) return;
|
|
13592
13601
|
expected.add(key);
|
|
13593
13602
|
if (recorded) completed.add(key);
|
|
@@ -13607,7 +13616,7 @@ function selectedSlotCoverage(db, projectId, run, filters, plans) {
|
|
|
13607
13616
|
if (scopedTargets && !scopedTargets.has(target.stableKey)) continue;
|
|
13608
13617
|
for (const provider of node.context.providers) {
|
|
13609
13618
|
const snapshot = bySlot.get(pairKey(node.stableKey, provider));
|
|
13610
|
-
|
|
13619
|
+
count5(`${node.stableKey}:${provider}`, { queryKey: usage.queryId, queryText: node.queryText, executionNodeKey: node.stableKey, queryClass: assignment.queryClass, propertyKey: target.stableKey, groupKeys: [...indexed.groupKeys.get(target.stableKey) ?? []], marketKeys: [...marketKeys], provider, sourceModel: node.context.models[provider] ?? null, servedModel: snapshot?.servedModel ?? null, context: node.context.location }, Boolean(snapshot));
|
|
13611
13620
|
}
|
|
13612
13621
|
}
|
|
13613
13622
|
} else {
|
|
@@ -13624,7 +13633,7 @@ function selectedSlotCoverage(db, projectId, run, filters, plans) {
|
|
|
13624
13633
|
for (const query of stored.data.queries) for (const engine of stored.data.engines) {
|
|
13625
13634
|
if (!query.queryClass) continue;
|
|
13626
13635
|
const snapshot = bySlot.get(simpleSlotKey(query.queryId, engine.provider));
|
|
13627
|
-
|
|
13636
|
+
count5(`${query.queryId}:${engine.provider}`, { queryKey: query.queryId, queryText: query.queryText, executionNodeKey: null, queryClass: query.queryClass, propertyKey: run.projectId, groupKeys: [], marketKeys: [], provider: engine.provider, sourceModel: engine.requestedModel, servedModel: snapshot?.servedModel ?? null, context: stored.data.location }, Boolean(snapshot));
|
|
13628
13637
|
}
|
|
13629
13638
|
}
|
|
13630
13639
|
return coverage();
|
|
@@ -13811,9 +13820,9 @@ var SentimentService = class {
|
|
|
13811
13820
|
const estimatedInputTokens = this.options.estimate ? inputs.reduce((sum, input) => sum + this.options.estimate(input), 0) : 0;
|
|
13812
13821
|
const expiresAt = new Date(Date.parse(this.now()) + 15 * 6e4).toISOString();
|
|
13813
13822
|
const previewToken = definitionId && settings.enabled && settings.ready && source.assessments.length ? this.sign({ projectId, selection: frozenSelection, definitionId, fingerprint: sentimentHash(source.assessments), expiresAt }) : null;
|
|
13814
|
-
const
|
|
13823
|
+
const skipped4 = source.runIds.flatMap((runId) => Object.entries(source.skippedByRun?.[runId] ?? {}).map(([reason, count5]) => ({ runId, reason, count: count5 })));
|
|
13815
13824
|
const estimateMethod = this.options.estimate ? `Adapter estimate of the full request including all questions and evidence candidates; final billed usage is recorded per attempt.${definitionId ? "" : " Sentiment is not configured for this project, so the estimate uses the current evaluator definition."}` : "Token and cost estimates unavailable on this host.";
|
|
13816
|
-
return { previewToken, expiresAt: previewToken ? expiresAt : null, selection: frozenSelection, evaluationDefinitionId: definitionId, eligibleAssessments: source.assessments.length, alreadyClassified, skipped:
|
|
13825
|
+
return { previewToken, expiresAt: previewToken ? expiresAt : null, selection: frozenSelection, evaluationDefinitionId: definitionId, eligibleAssessments: source.assessments.length, alreadyClassified, skipped: skipped4, estimatedInputTokens, estimatedCostUsd: this.options.estimate ? estimatedInputTokens * 0.042 / 1e6 : null, estimateMethod };
|
|
13817
13826
|
}
|
|
13818
13827
|
/** The receipt carries the first page of attempts; a replay of a long-running job never returns all of them. */
|
|
13819
13828
|
submit(projectId, previewToken, idempotencyKey, actor) {
|
|
@@ -14050,7 +14059,7 @@ var SentimentService = class {
|
|
|
14050
14059
|
const runIds = selections.flatMap((selection2) => selection2.runId ? [selection2.runId] : []).sort();
|
|
14051
14060
|
const definitions = new Set(selections.flatMap((selection2) => selection2.evaluationDefinitionId ? [selection2.evaluationDefinitionId] : []));
|
|
14052
14061
|
const source = { assessments: sources.flatMap((source2) => source2.assessments), runIds, sourceCoverage: { expected: sources.reduce((sum, source2) => sum + source2.sourceCoverage.expected, 0), completed: sources.reduce((sum, source2) => sum + source2.sourceCoverage.completed, 0) }, sourceSlotIds: { expected: sources.flatMap((source2) => source2.sourceSlotIds.expected), completed: sources.flatMap((source2) => source2.sourceSlotIds.completed) }, skipped: sources.reduce((counts2, source2) => {
|
|
14053
|
-
for (const [reason2,
|
|
14062
|
+
for (const [reason2, count5] of Object.entries(source2.skipped)) counts2[reason2] = (counts2[reason2] ?? 0) + count5;
|
|
14054
14063
|
return counts2;
|
|
14055
14064
|
}, {}) };
|
|
14056
14065
|
const selection = { ...selections[0], runId: runIds.length === 1 ? runIds[0] : null, runIds: runIds.length > 1 ? runIds : void 0, evaluationDefinitionId: definitions.size === 1 ? [...definitions][0] : null };
|
|
@@ -17515,7 +17524,7 @@ function heldProviderAccounts(db, params) {
|
|
|
17515
17524
|
const finished = (run) => Date.parse(run.finishedAt ?? run.createdAt);
|
|
17516
17525
|
const now = Date.parse(params.now);
|
|
17517
17526
|
for (const provider of calling) {
|
|
17518
|
-
let
|
|
17527
|
+
let count5 = 0;
|
|
17519
17528
|
let newest = null;
|
|
17520
17529
|
let oldest = null;
|
|
17521
17530
|
let code = null;
|
|
@@ -17542,11 +17551,11 @@ function heldProviderAccounts(db, params) {
|
|
|
17542
17551
|
newest ??= run;
|
|
17543
17552
|
code ??= entry.code;
|
|
17544
17553
|
oldest = run;
|
|
17545
|
-
|
|
17546
|
-
if (
|
|
17554
|
+
count5 += 1;
|
|
17555
|
+
if (count5 === PROVIDER_ACCOUNT_FAILURE_STREAK) break;
|
|
17547
17556
|
}
|
|
17548
17557
|
if (ended) continue;
|
|
17549
|
-
if (
|
|
17558
|
+
if (count5 < PROVIDER_ACCOUNT_FAILURE_STREAK && !vouched) continue;
|
|
17550
17559
|
const retryAfterMs = newest ? finished(newest) + PROVIDER_ACCOUNT_RETRY_HOURS * 36e5 : Date.parse(vouched.retryAfter);
|
|
17551
17560
|
if (!(now < retryAfterMs)) continue;
|
|
17552
17561
|
held.set(provider, {
|
|
@@ -18047,7 +18056,7 @@ async function projectRoutes(app, opts) {
|
|
|
18047
18056
|
app.get("/projects/:name/delete-preview", async (request, reply) => {
|
|
18048
18057
|
const project = resolveProject(app.db, request.params.name);
|
|
18049
18058
|
const pid = project.id;
|
|
18050
|
-
const
|
|
18059
|
+
const count5 = (n) => n ?? 0;
|
|
18051
18060
|
const queryCount = app.db.select({ n: sql7`count(*)` }).from(queries).where(eq19(queries.projectId, pid)).get();
|
|
18052
18061
|
const competitorCount = app.db.select({ n: sql7`count(*)` }).from(competitors).where(eq19(competitors.projectId, pid)).get();
|
|
18053
18062
|
const runCount = app.db.select({ n: sql7`count(*)` }).from(runs).where(eq19(runs.projectId, pid)).get();
|
|
@@ -18057,14 +18066,14 @@ async function projectRoutes(app, opts) {
|
|
|
18057
18066
|
return reply.send({
|
|
18058
18067
|
project: { id: project.id, name: project.name },
|
|
18059
18068
|
cascadeRows: {
|
|
18060
|
-
queries:
|
|
18061
|
-
competitors:
|
|
18062
|
-
runs:
|
|
18063
|
-
snapshots:
|
|
18064
|
-
insights:
|
|
18069
|
+
queries: count5(queryCount?.n),
|
|
18070
|
+
competitors: count5(competitorCount?.n),
|
|
18071
|
+
runs: count5(runCount?.n),
|
|
18072
|
+
snapshots: count5(snapshotCount?.n),
|
|
18073
|
+
insights: count5(insightCount?.n)
|
|
18065
18074
|
},
|
|
18066
18075
|
detachedRows: {
|
|
18067
|
-
auditLog:
|
|
18076
|
+
auditLog: count5(auditLogCount?.n)
|
|
18068
18077
|
}
|
|
18069
18078
|
});
|
|
18070
18079
|
});
|
|
@@ -18669,14 +18678,14 @@ async function queryRoutes(app, opts) {
|
|
|
18669
18678
|
validProviders: validNames
|
|
18670
18679
|
});
|
|
18671
18680
|
}
|
|
18672
|
-
const
|
|
18681
|
+
const count5 = body.count ?? 5;
|
|
18673
18682
|
if (!opts.onGenerateQueries) {
|
|
18674
18683
|
throw notImplemented("Query generation is not supported in this deployment");
|
|
18675
18684
|
}
|
|
18676
18685
|
const existingRows = app.db.select().from(queries).where(eq21(queries.projectId, project.id)).all();
|
|
18677
18686
|
const existingQueries = existingRows.map((r) => r.query);
|
|
18678
18687
|
try {
|
|
18679
|
-
const generated = await opts.onGenerateQueries(provider,
|
|
18688
|
+
const generated = await opts.onGenerateQueries(provider, count5, {
|
|
18680
18689
|
domain: project.canonicalDomain,
|
|
18681
18690
|
displayName: project.displayName,
|
|
18682
18691
|
country: project.country,
|
|
@@ -18802,14 +18811,14 @@ async function queryRoutes(app, opts) {
|
|
|
18802
18811
|
validProviders: validNames
|
|
18803
18812
|
});
|
|
18804
18813
|
}
|
|
18805
|
-
const
|
|
18814
|
+
const count5 = body.count ?? 5;
|
|
18806
18815
|
if (!opts.onGenerateQueries) {
|
|
18807
18816
|
throw notImplemented("Keyword generation is not supported in this deployment");
|
|
18808
18817
|
}
|
|
18809
18818
|
const existingRows = app.db.select().from(queries).where(eq21(queries.projectId, project.id)).all();
|
|
18810
18819
|
const existingQueries = existingRows.map((r) => r.query);
|
|
18811
18820
|
try {
|
|
18812
|
-
const generated = await opts.onGenerateQueries(provider,
|
|
18821
|
+
const generated = await opts.onGenerateQueries(provider, count5, {
|
|
18813
18822
|
domain: project.canonicalDomain,
|
|
18814
18823
|
displayName: project.displayName,
|
|
18815
18824
|
country: project.country,
|
|
@@ -24379,19 +24388,19 @@ function groupChangesFromRows(rows, identities) {
|
|
|
24379
24388
|
});
|
|
24380
24389
|
}
|
|
24381
24390
|
function countsFromRows(rows, groupChanges) {
|
|
24382
|
-
const
|
|
24383
|
-
const ambiguousRows =
|
|
24384
|
-
const unmatchedRows =
|
|
24385
|
-
const invalidRows =
|
|
24386
|
-
const proposedRows =
|
|
24387
|
-
const excludedRows =
|
|
24391
|
+
const count5 = (status) => rows.filter((row) => row.status === status).length;
|
|
24392
|
+
const ambiguousRows = count5("ambiguous");
|
|
24393
|
+
const unmatchedRows = count5("unmatched");
|
|
24394
|
+
const invalidRows = count5("invalid");
|
|
24395
|
+
const proposedRows = count5("proposed");
|
|
24396
|
+
const excludedRows = count5("excluded");
|
|
24388
24397
|
return {
|
|
24389
24398
|
dataRows: rows.length,
|
|
24390
|
-
matchedRows:
|
|
24399
|
+
matchedRows: count5("matched"),
|
|
24391
24400
|
ambiguousRows,
|
|
24392
24401
|
unmatchedRows,
|
|
24393
24402
|
invalidRows,
|
|
24394
|
-
duplicateRows:
|
|
24403
|
+
duplicateRows: count5("duplicate"),
|
|
24395
24404
|
proposedRows,
|
|
24396
24405
|
excludedRows,
|
|
24397
24406
|
needsAttention: ambiguousRows + unmatchedRows + invalidRows + proposedRows + excludedRows,
|
|
@@ -27170,7 +27179,7 @@ function domainRows(answers, limit, readHost = createCitedHostReader()) {
|
|
|
27170
27179
|
const domains = new Set(answerDomains(answer, readHost).values());
|
|
27171
27180
|
for (const domain of domains) counts2.set(domain, (counts2.get(domain) ?? 0) + 1);
|
|
27172
27181
|
}
|
|
27173
|
-
const rows = [...counts2].map(([domain,
|
|
27182
|
+
const rows = [...counts2].map(([domain, count5]) => ({ domain, answers: count5 })).sort((left, right) => right.answers - left.answers || compareText13(left.domain, right.domain));
|
|
27174
27183
|
return { rows: rows.slice(0, limit), total: rows.length };
|
|
27175
27184
|
}
|
|
27176
27185
|
function targetAliasKeys(target) {
|
|
@@ -27458,9 +27467,9 @@ function compactPortfolioPage(response, query, versionId, evidence, cursor) {
|
|
|
27458
27467
|
if (arrays.some((rows, index2) => offsets[index2] > rows.length)) throw validationError("Invalid measurement portfolio cursor offset.");
|
|
27459
27468
|
const requested = Math.min(query.limit ?? 50, arrays[selected].length - offsets[selected]);
|
|
27460
27469
|
const initialSummaryLimit = query.list === void 0 && cursor === void 0 ? MEASUREMENT_PORTFOLIO_DEFAULT_LIMIT : 0;
|
|
27461
|
-
const build = (
|
|
27462
|
-
const pages = arrays.map((rows, index2) => index2 === selected ? rows.slice(offsets[index2], offsets[index2] +
|
|
27463
|
-
const nextOffsets = offsets.map((offset, index2) => index2 === selected ? offset +
|
|
27470
|
+
const build = (count5, summaryLimit2) => {
|
|
27471
|
+
const pages = arrays.map((rows, index2) => index2 === selected ? rows.slice(offsets[index2], offsets[index2] + count5) : rows.slice(0, summaryLimit2));
|
|
27472
|
+
const nextOffsets = offsets.map((offset, index2) => index2 === selected ? offset + count5 : offset);
|
|
27464
27473
|
const hasMore = nextOffsets[selected] < arrays[selected].length;
|
|
27465
27474
|
const omitted = paths.filter((_, index2) => index2 !== selected && pages[index2].length < arrays[index2].length);
|
|
27466
27475
|
return {
|
|
@@ -27518,12 +27527,12 @@ function compactPortfolioPage(response, query, versionId, evidence, cursor) {
|
|
|
27518
27527
|
let low = minimum + 1;
|
|
27519
27528
|
let high = requested;
|
|
27520
27529
|
while (low <= high) {
|
|
27521
|
-
const
|
|
27522
|
-
const candidate = build(
|
|
27530
|
+
const count5 = Math.floor((low + high) / 2);
|
|
27531
|
+
const candidate = build(count5, summaryLimit);
|
|
27523
27532
|
if (fits(candidate)) {
|
|
27524
27533
|
page3 = candidate;
|
|
27525
|
-
low =
|
|
27526
|
-
} else high =
|
|
27534
|
+
low = count5 + 1;
|
|
27535
|
+
} else high = count5 - 1;
|
|
27527
27536
|
}
|
|
27528
27537
|
return measurementPortfolioSummaryResponseSchema.parse(page3);
|
|
27529
27538
|
}
|
|
@@ -27739,7 +27748,7 @@ function tieByMetro(rows) {
|
|
|
27739
27748
|
add(row.metro);
|
|
27740
27749
|
for (const other of row.otherMetros ?? []) add(other);
|
|
27741
27750
|
}
|
|
27742
|
-
return [...counts2.values()].sort((left, right) => right.count - left.count || (left.metro === null ? 1 : 0) - (right.metro === null ? 1 : 0) || compareText13(left.metro?.label ?? "", right.metro?.label ?? "") || compareText13(left.metro?.groupKey ?? "", right.metro?.groupKey ?? "")).map(({ metro, count:
|
|
27751
|
+
return [...counts2.values()].sort((left, right) => right.count - left.count || (left.metro === null ? 1 : 0) - (right.metro === null ? 1 : 0) || compareText13(left.metro?.label ?? "", right.metro?.label ?? "") || compareText13(left.metro?.groupKey ?? "", right.metro?.groupKey ?? "")).map(({ metro, count: count5 }) => ({ metro: metro?.label ?? null, count: count5 }));
|
|
27743
27752
|
}
|
|
27744
27753
|
function tieNamedInstead(rows) {
|
|
27745
27754
|
const grouped = /* @__PURE__ */ new Map();
|
|
@@ -30778,9 +30787,9 @@ function workloadDiff(before, after) {
|
|
|
30778
30787
|
nextSweepNodes: after.size,
|
|
30779
30788
|
nextSweepProviderCalls: sum(after.values()),
|
|
30780
30789
|
addedNodes: added.length,
|
|
30781
|
-
addedProviderCalls: sum(added.map(([,
|
|
30790
|
+
addedProviderCalls: sum(added.map(([, count5]) => count5)),
|
|
30782
30791
|
removedNodes: removed.length,
|
|
30783
|
-
removedProviderCalls: sum(removed.map(([,
|
|
30792
|
+
removedProviderCalls: sum(removed.map(([, count5]) => count5))
|
|
30784
30793
|
};
|
|
30785
30794
|
}
|
|
30786
30795
|
function candidateDiff(state, nextRows, plan, reusedQueryIds, mutatedQueryIds, scopedRemovalQueryIds) {
|
|
@@ -33228,10 +33237,10 @@ function computeQueryChanges(projectQueries, cutoff) {
|
|
|
33228
33237
|
}
|
|
33229
33238
|
const days = [...byDay.entries()].sort((a, b) => a[0].localeCompare(b[0]));
|
|
33230
33239
|
if (days.length <= 1) return [];
|
|
33231
|
-
return days.slice(1).map(([date,
|
|
33240
|
+
return days.slice(1).map(([date, count5]) => ({
|
|
33232
33241
|
date: (/* @__PURE__ */ new Date(date + "T00:00:00.000Z")).toISOString(),
|
|
33233
|
-
delta:
|
|
33234
|
-
label: `+${
|
|
33242
|
+
delta: count5,
|
|
33243
|
+
label: `+${count5} kp`
|
|
33235
33244
|
}));
|
|
33236
33245
|
}
|
|
33237
33246
|
function pooledRate(buckets, rateKey) {
|
|
@@ -33323,15 +33332,15 @@ function buildRankedList(domains, limit, answerTotal, answersWithSources) {
|
|
|
33323
33332
|
function buildCategoryCounts(counts2) {
|
|
33324
33333
|
let grandTotal = 0;
|
|
33325
33334
|
for (const domains of counts2.values()) {
|
|
33326
|
-
for (const
|
|
33335
|
+
for (const count5 of domains.values()) grandTotal += count5;
|
|
33327
33336
|
}
|
|
33328
33337
|
const result = [];
|
|
33329
33338
|
for (const [category, domains] of counts2) {
|
|
33330
33339
|
let categoryTotal = 0;
|
|
33331
33340
|
const domainEntries = [];
|
|
33332
|
-
for (const [domain,
|
|
33333
|
-
categoryTotal +=
|
|
33334
|
-
domainEntries.push({ domain, count:
|
|
33341
|
+
for (const [domain, count5] of domains) {
|
|
33342
|
+
categoryTotal += count5;
|
|
33343
|
+
domainEntries.push({ domain, count: count5 });
|
|
33335
33344
|
}
|
|
33336
33345
|
domainEntries.sort((a, b) => b.count - a.count);
|
|
33337
33346
|
result.push({
|
|
@@ -34233,6 +34242,128 @@ function clickPeriods(periods, clicks, impressions) {
|
|
|
34233
34242
|
impressions: impressions[index2] ?? 0
|
|
34234
34243
|
}));
|
|
34235
34244
|
}
|
|
34245
|
+
function compareCodeUnits(left, right) {
|
|
34246
|
+
if (left < right) return -1;
|
|
34247
|
+
return left > right ? 1 : 0;
|
|
34248
|
+
}
|
|
34249
|
+
function aiEngineLeadPeriods(periods, eventCounts, sessionCounts, leadRateAvailable) {
|
|
34250
|
+
return periods.map((period2, index2) => {
|
|
34251
|
+
const eventCount = eventCounts[index2] ?? 0;
|
|
34252
|
+
const sessions = sessionCounts[index2] ?? 0;
|
|
34253
|
+
return {
|
|
34254
|
+
...period2,
|
|
34255
|
+
eventCount,
|
|
34256
|
+
sessions,
|
|
34257
|
+
leadRate: leadRateAvailable && sessions > 0 ? roundRatio(eventCount / sessions, RatioUnits.fraction) : null
|
|
34258
|
+
};
|
|
34259
|
+
});
|
|
34260
|
+
}
|
|
34261
|
+
var UNATTRIBUTED_AI = "(unattributed)";
|
|
34262
|
+
function buildAiEngineLeadClass(input) {
|
|
34263
|
+
const { periods, leadRateAvailable } = input;
|
|
34264
|
+
const sourcesByKey = /* @__PURE__ */ new Map();
|
|
34265
|
+
for (const row of [...input.leads, ...input.sessions]) {
|
|
34266
|
+
const sources = sourcesByKey.get(row.key) ?? /* @__PURE__ */ new Set();
|
|
34267
|
+
sources.add(row.source);
|
|
34268
|
+
sourcesByKey.set(row.key, sources);
|
|
34269
|
+
}
|
|
34270
|
+
const leadsByKey = aggregateByKey(input.leads, periods, (row) => row.key, (row) => row.value);
|
|
34271
|
+
const sessionsByKey = aggregateByKey(input.sessions, periods, (row) => row.key, (row) => row.value);
|
|
34272
|
+
const zeros = Array(periods.length).fill(0);
|
|
34273
|
+
const sum = (values) => values.reduce((total, value) => total + value, 0);
|
|
34274
|
+
const countsFor = (key) => ({
|
|
34275
|
+
leads: leadsByKey.get(key) ?? zeros,
|
|
34276
|
+
sessions: sessionsByKey.get(key) ?? zeros
|
|
34277
|
+
});
|
|
34278
|
+
const sourcesFor = (key) => [...sourcesByKey.get(key) ?? []].sort(compareCodeUnits);
|
|
34279
|
+
const engineRows = [...sourcesByKey.keys()].filter((key) => key !== UNATTRIBUTED_AI).map((engine) => ({ engine, ...countsFor(engine) })).sort((left, right) => (right.leads.at(-1) ?? 0) - (left.leads.at(-1) ?? 0) || sum(right.leads) - sum(left.leads) || (right.sessions.at(-1) ?? 0) - (left.sessions.at(-1) ?? 0) || sum(right.sessions) - sum(left.sessions) || left.engine.localeCompare(right.engine));
|
|
34280
|
+
const unattributed = countsFor(UNATTRIBUTED_AI);
|
|
34281
|
+
const allRows = [...engineRows, unattributed];
|
|
34282
|
+
const totalLeads = periods.map((_, index2) => sum(allRows.map((row) => row.leads[index2] ?? 0)));
|
|
34283
|
+
const totalSessions = periods.map((_, index2) => sum(allRows.map((row) => row.sessions[index2] ?? 0)));
|
|
34284
|
+
return {
|
|
34285
|
+
periods: aiEngineLeadPeriods(periods, totalLeads, totalSessions, leadRateAvailable),
|
|
34286
|
+
engines: engineRows.map((row) => ({
|
|
34287
|
+
engine: row.engine,
|
|
34288
|
+
label: aiReferralEngineLabel(row.engine),
|
|
34289
|
+
sources: sourcesFor(row.engine),
|
|
34290
|
+
periods: aiEngineLeadPeriods(periods, row.leads, row.sessions, leadRateAvailable)
|
|
34291
|
+
})),
|
|
34292
|
+
unattributed: {
|
|
34293
|
+
sources: sourcesFor(UNATTRIBUTED_AI),
|
|
34294
|
+
periods: aiEngineLeadPeriods(periods, unattributed.leads, unattributed.sessions, leadRateAvailable)
|
|
34295
|
+
}
|
|
34296
|
+
};
|
|
34297
|
+
}
|
|
34298
|
+
function attributeAiEngineRows(input) {
|
|
34299
|
+
const engineBySource = /* @__PURE__ */ new Map();
|
|
34300
|
+
const engineOf = (source) => {
|
|
34301
|
+
let engine = engineBySource.get(source);
|
|
34302
|
+
if (engine === void 0) {
|
|
34303
|
+
engine = aiEngineForReferralSource(source);
|
|
34304
|
+
engineBySource.set(source, engine);
|
|
34305
|
+
}
|
|
34306
|
+
return engine;
|
|
34307
|
+
};
|
|
34308
|
+
let paidOnlyByLandingPage = false;
|
|
34309
|
+
const attribute = (rows, valueOf) => rows.flatMap((row) => {
|
|
34310
|
+
const engine = engineOf(row.source);
|
|
34311
|
+
if (engine === null && !isGa4AiAssistantChannel(row.channelGroup)) return [];
|
|
34312
|
+
const evidence = { source: row.source, medium: row.medium, channelGroup: row.channelGroup };
|
|
34313
|
+
const withLandingPage = classifyAiReferralTrafficClass({ ...evidence, landingPage: row.landingPage });
|
|
34314
|
+
const trafficClass = input.landingPageEvidence ? withLandingPage : classifyAiReferralTrafficClass({ ...evidence, landingPage: null });
|
|
34315
|
+
if (trafficClass !== withLandingPage) paidOnlyByLandingPage = true;
|
|
34316
|
+
return [{
|
|
34317
|
+
date: row.date,
|
|
34318
|
+
source: row.source,
|
|
34319
|
+
key: engine ?? UNATTRIBUTED_AI,
|
|
34320
|
+
trafficClass,
|
|
34321
|
+
value: valueOf(row)
|
|
34322
|
+
}];
|
|
34323
|
+
});
|
|
34324
|
+
const leads = attribute(input.leads, (row) => row.eventCount);
|
|
34325
|
+
const sessions = attribute(input.sessions, (row) => row.sessions);
|
|
34326
|
+
return { leads, sessions, paidOnlyByLandingPage };
|
|
34327
|
+
}
|
|
34328
|
+
function buildAiEngineLeads(input) {
|
|
34329
|
+
const leadRateAvailable = input.leadRateUnavailableReason === null;
|
|
34330
|
+
const classBlock = (trafficClass) => buildAiEngineLeadClass({
|
|
34331
|
+
periods: input.periods,
|
|
34332
|
+
leads: input.leads.filter((row) => row.trafficClass === trafficClass),
|
|
34333
|
+
sessions: input.sessions.filter((row) => row.trafficClass === trafficClass),
|
|
34334
|
+
leadRateAvailable
|
|
34335
|
+
});
|
|
34336
|
+
return {
|
|
34337
|
+
leadRateAvailable,
|
|
34338
|
+
leadRateUnavailableReason: input.leadRateUnavailableReason,
|
|
34339
|
+
organic: classBlock(AiReferralTrafficClasses.organic),
|
|
34340
|
+
paid: classBlock(AiReferralTrafficClasses.paid)
|
|
34341
|
+
};
|
|
34342
|
+
}
|
|
34343
|
+
function latestDate(rows) {
|
|
34344
|
+
let latest = null;
|
|
34345
|
+
for (const row of rows) {
|
|
34346
|
+
if (latest === null || row.date > latest) latest = row.date;
|
|
34347
|
+
}
|
|
34348
|
+
return latest;
|
|
34349
|
+
}
|
|
34350
|
+
function hasLeadDayWithoutSessions(acquisitionRows, leadRows) {
|
|
34351
|
+
const acquisitionDates = new Set(acquisitionRows.map((row) => row.date));
|
|
34352
|
+
return leadRows.some((row) => !acquisitionDates.has(row.date));
|
|
34353
|
+
}
|
|
34354
|
+
function aiEngineLeadRateUnavailableReason(input) {
|
|
34355
|
+
if (!input.hasData) return AiEngineLeadRateUnavailableReasons["no-data"];
|
|
34356
|
+
if (input.acquisitionStatus !== "ready" || input.leadStatus !== "ready") {
|
|
34357
|
+
return AiEngineLeadRateUnavailableReasons["sync-not-ready"];
|
|
34358
|
+
}
|
|
34359
|
+
if (input.channelLeadsUnfiltered) return AiEngineLeadRateUnavailableReasons["channel-leads-unfiltered"];
|
|
34360
|
+
if (input.paidOnlyByLandingPage) return AiEngineLeadRateUnavailableReasons["paid-split-needs-landing-page"];
|
|
34361
|
+
if (input.latestLeadDate !== null && (input.latestAcquisitionDate === null || input.latestAcquisitionDate < input.latestLeadDate)) {
|
|
34362
|
+
return AiEngineLeadRateUnavailableReasons["sessions-behind-leads"];
|
|
34363
|
+
}
|
|
34364
|
+
if (input.leadDayWithoutSessions) return AiEngineLeadRateUnavailableReasons["sessions-missing-on-lead-days"];
|
|
34365
|
+
return null;
|
|
34366
|
+
}
|
|
34236
34367
|
function engagementPeriods(periods, rows) {
|
|
34237
34368
|
return periods.map((period2) => {
|
|
34238
34369
|
const inPeriod = rows.filter((row) => row.date >= period2.startDate && row.date <= period2.endDate);
|
|
@@ -34449,6 +34580,28 @@ function buildGaMeasurementAnalysis(db, projectName, options = {}) {
|
|
|
34449
34580
|
(row) => row.channelGroup,
|
|
34450
34581
|
(row) => row.eventCount
|
|
34451
34582
|
);
|
|
34583
|
+
const filtersNarrowSessions = parsedHostScope.data === "marketing" || pathPrefix !== null && pathPrefix !== "/";
|
|
34584
|
+
const hasAiEngineLeadData = gaAnchor !== null && hasLeadTimeline;
|
|
34585
|
+
const aiEngineRows = attributeAiEngineRows({
|
|
34586
|
+
leads: hasAiEngineLeadData ? leads : [],
|
|
34587
|
+
sessions: hasAiEngineLeadData ? acquisition : [],
|
|
34588
|
+
landingPageEvidence: attributionScope !== "channel"
|
|
34589
|
+
});
|
|
34590
|
+
const aiEngineLeads = buildAiEngineLeads({
|
|
34591
|
+
periods: hasAiEngineLeadData ? gaPeriods : [],
|
|
34592
|
+
leads: aiEngineRows.leads,
|
|
34593
|
+
sessions: aiEngineRows.sessions,
|
|
34594
|
+
leadRateUnavailableReason: aiEngineLeadRateUnavailableReason({
|
|
34595
|
+
hasData: hasAiEngineLeadData,
|
|
34596
|
+
acquisitionStatus: state?.acquisitionStatus,
|
|
34597
|
+
leadStatus: state?.leadStatus,
|
|
34598
|
+
channelLeadsUnfiltered: attributionScope === "channel" && filtersNarrowSessions,
|
|
34599
|
+
paidOnlyByLandingPage: aiEngineRows.paidOnlyByLandingPage,
|
|
34600
|
+
latestAcquisitionDate: latestDate(acquisitionRows),
|
|
34601
|
+
latestLeadDate: latestDate(leadRows),
|
|
34602
|
+
leadDayWithoutSessions: hasLeadDayWithoutSessions(acquisitionRows, leadRows)
|
|
34603
|
+
})
|
|
34604
|
+
});
|
|
34452
34605
|
const propertyClicks = aggregateByKey(
|
|
34453
34606
|
propertyRows,
|
|
34454
34607
|
gscPeriods,
|
|
@@ -34553,7 +34706,8 @@ function buildGaMeasurementAnalysis(db, projectName, options = {}) {
|
|
|
34553
34706
|
channels: rankEntries(leadChannels).map(([channelGroup, values]) => ({
|
|
34554
34707
|
channelGroup,
|
|
34555
34708
|
periods: eventPeriods(gaPeriods, values)
|
|
34556
|
-
}))
|
|
34709
|
+
})),
|
|
34710
|
+
aiEngines: aiEngineLeads
|
|
34557
34711
|
},
|
|
34558
34712
|
engagement: {
|
|
34559
34713
|
status: engagementAnchor === null ? "unavailable" : "ready",
|
|
@@ -34682,9 +34836,9 @@ function summarizeVerificationHits(rows) {
|
|
|
34682
34836
|
return { counts: counts2, unsupportedRows, unsupportedHits };
|
|
34683
34837
|
}
|
|
34684
34838
|
function summarizeServer(crawlers, fetches, referrals) {
|
|
34685
|
-
const total = referrals.reduce((
|
|
34686
|
-
const paid = referrals.reduce((
|
|
34687
|
-
const organic = referrals.reduce((
|
|
34839
|
+
const total = referrals.reduce((count5, row) => count5 + row.sessionsOrHits, 0);
|
|
34840
|
+
const paid = referrals.reduce((count5, row) => count5 + row.paidSessionsOrHits, 0);
|
|
34841
|
+
const organic = referrals.reduce((count5, row) => count5 + row.organicSessionsOrHits, 0);
|
|
34688
34842
|
const crawlerHits = summarizeVerificationHits(crawlers);
|
|
34689
34843
|
const userFetchHits = summarizeVerificationHits(fetches);
|
|
34690
34844
|
return {
|
|
@@ -34833,10 +34987,10 @@ function buildOrganicEvidence(db, projectName, periodDays) {
|
|
|
34833
34987
|
const gaWindow = gaStartDate && gaEndDate ? gaRows.filter((row) => inRange(row.date, gaStartDate, gaEndDate)) : [];
|
|
34834
34988
|
const legacyGaCohorts = gaCohorts.map((cohort) => ({
|
|
34835
34989
|
...cohort,
|
|
34836
|
-
organicSessions: gaRows.filter((row) => inRange(row.date, cohort.startDate, cohort.endDate)).reduce((
|
|
34990
|
+
organicSessions: gaRows.filter((row) => inRange(row.date, cohort.startDate, cohort.endDate)).reduce((count5, row) => count5 + row.organicSessions, 0)
|
|
34837
34991
|
}));
|
|
34838
34992
|
let ga4 = gaRows.length ? {
|
|
34839
|
-
organicSessions: gaWindow.reduce((
|
|
34993
|
+
organicSessions: gaWindow.reduce((count5, row) => count5 + row.organicSessions, 0),
|
|
34840
34994
|
cohorts: legacyGaCohorts
|
|
34841
34995
|
} : null;
|
|
34842
34996
|
if (nativeAcquisitionActive && nativeAcquisition.periods.length === 0) {
|
|
@@ -34859,8 +35013,8 @@ function buildOrganicEvidence(db, projectName, periodDays) {
|
|
|
34859
35013
|
const allGaAiRows = db.select().from(gaAiReferrals).where(and38(eq45(gaAiReferrals.projectId, project.id), gte11(gaAiReferrals.date, gaStartDate ?? startDate), lte10(gaAiReferrals.date, gaEndDate ?? endDate))).all().filter((row) => row.sourceDimension === "session");
|
|
34860
35014
|
const aiRows = gaStartDate && gaEndDate ? allGaAiRows.filter((row) => inRange(row.date, gaStartDate, gaEndDate)) : [];
|
|
34861
35015
|
const gaAiReferralSummary = allGaAiRows.length ? {
|
|
34862
|
-
paidSessions: aiRows.filter((row) => row.trafficClass === AiReferralTrafficClasses.paid).reduce((
|
|
34863
|
-
organicSessions: aiRows.filter((row) => row.trafficClass === AiReferralTrafficClasses.organic).reduce((
|
|
35016
|
+
paidSessions: aiRows.filter((row) => row.trafficClass === AiReferralTrafficClasses.paid).reduce((count5, row) => count5 + row.sessions, 0),
|
|
35017
|
+
organicSessions: aiRows.filter((row) => row.trafficClass === AiReferralTrafficClasses.organic).reduce((count5, row) => count5 + row.sessions, 0)
|
|
34864
35018
|
} : null;
|
|
34865
35019
|
const serverSources = db.select().from(trafficSources).where(eq45(trafficSources.projectId, project.id)).all();
|
|
34866
35020
|
const serverStart = `${startDate}T00:00:00`;
|
|
@@ -49003,12 +49157,12 @@ async function getLodging(accessToken, locationName, opts = {}) {
|
|
|
49003
49157
|
}
|
|
49004
49158
|
}
|
|
49005
49159
|
function countPopulatedGroups(lodging) {
|
|
49006
|
-
let
|
|
49160
|
+
let count5 = 0;
|
|
49007
49161
|
for (const [key, value] of Object.entries(lodging)) {
|
|
49008
49162
|
if (key === "name" || key === "metadata") continue;
|
|
49009
|
-
if (isPopulated(value))
|
|
49163
|
+
if (isPopulated(value)) count5++;
|
|
49010
49164
|
}
|
|
49011
|
-
return
|
|
49165
|
+
return count5;
|
|
49012
49166
|
}
|
|
49013
49167
|
function isPopulated(value) {
|
|
49014
49168
|
if (value === null || value === void 0) return false;
|
|
@@ -54251,7 +54405,7 @@ function createActivationStore(app, request) {
|
|
|
54251
54405
|
function countManifest(manifest) {
|
|
54252
54406
|
return {
|
|
54253
54407
|
adGroupCount: manifest.campaign.adGroups.length,
|
|
54254
|
-
adCount: manifest.campaign.adGroups.reduce((
|
|
54408
|
+
adCount: manifest.campaign.adGroups.reduce((count5, group) => count5 + group.ads.length, 0)
|
|
54255
54409
|
};
|
|
54256
54410
|
}
|
|
54257
54411
|
function resolveAdsActivationMaxEntities(env = process.env) {
|
|
@@ -61167,11 +61321,11 @@ async function wordpressRoutes(app, opts) {
|
|
|
61167
61321
|
const result = await bulkSetSeoMeta(connection, metaEntries);
|
|
61168
61322
|
const applied = result.results.filter((r) => r.status === "applied").length;
|
|
61169
61323
|
const manual = result.results.filter((r) => r.status === "manual").length;
|
|
61170
|
-
const
|
|
61324
|
+
const skipped4 = result.results.filter((r) => r.status === "skipped").length;
|
|
61171
61325
|
steps.push({
|
|
61172
61326
|
name: "set-meta",
|
|
61173
61327
|
status: "completed",
|
|
61174
|
-
summary: `${applied} applied, ${manual} manual-assist, ${
|
|
61328
|
+
summary: `${applied} applied, ${manual} manual-assist, ${skipped4} skipped`
|
|
61175
61329
|
});
|
|
61176
61330
|
}
|
|
61177
61331
|
} catch (err) {
|
|
@@ -61193,11 +61347,11 @@ async function wordpressRoutes(app, opts) {
|
|
|
61193
61347
|
const result = await deploySchemaFromProfile(connection, profile);
|
|
61194
61348
|
const deployed = result.results.filter((r) => r.status === "deployed").length;
|
|
61195
61349
|
const stripped = result.results.filter((r) => r.status === "stripped").length;
|
|
61196
|
-
const
|
|
61350
|
+
const skipped4 = result.results.filter((r) => r.status === "skipped").length;
|
|
61197
61351
|
steps.push({
|
|
61198
61352
|
name: "schema-deploy",
|
|
61199
61353
|
status: "completed",
|
|
61200
|
-
summary: `${deployed} deployed, ${stripped} stripped (manual-assist), ${
|
|
61354
|
+
summary: `${deployed} deployed, ${stripped} stripped (manual-assist), ${skipped4} skipped`
|
|
61201
61355
|
});
|
|
61202
61356
|
}
|
|
61203
61357
|
} catch (err) {
|
|
@@ -76328,7 +76482,6 @@ function verifyIpForRuleDecision(ip, ruleId) {
|
|
|
76328
76482
|
}
|
|
76329
76483
|
|
|
76330
76484
|
// ../integration-traffic/src/rules.ts
|
|
76331
|
-
var LEGACY_CHATGPT_DOMAIN = "chat.openai.com";
|
|
76332
76485
|
var SELF_TRAFFIC_USER_AGENT_PATTERNS = [
|
|
76333
76486
|
/AINYC-AEO-Audit\//i
|
|
76334
76487
|
];
|
|
@@ -76711,7 +76864,7 @@ var DEFAULT_AI_CRAWLER_USER_AGENT_SUBSTRINGS = [
|
|
|
76711
76864
|
];
|
|
76712
76865
|
var DEFAULT_AI_REFERRER_RULES = [
|
|
76713
76866
|
{ domain: AI_ENGINE_DOMAINS.chatgpt, operator: "OpenAI", product: "ChatGPT" },
|
|
76714
|
-
{ domain:
|
|
76867
|
+
{ domain: AI_ENGINE_DOMAINS.chatgptLegacy, operator: "OpenAI", product: "ChatGPT" },
|
|
76715
76868
|
{ domain: AI_ENGINE_DOMAINS.perplexity, operator: "Perplexity", product: "Perplexity" },
|
|
76716
76869
|
{ domain: AI_ENGINE_DOMAINS.claude, operator: "Anthropic", product: "Claude" },
|
|
76717
76870
|
{ domain: AI_ENGINE_DOMAINS.gemini, operator: "Google", product: "Gemini" },
|
|
@@ -82501,10 +82654,10 @@ function dailyCoverage(ctx, month, source) {
|
|
|
82501
82654
|
beforeConnectionDays++;
|
|
82502
82655
|
continue;
|
|
82503
82656
|
}
|
|
82504
|
-
const
|
|
82505
|
-
if (
|
|
82657
|
+
const count5 = byDate.get(date);
|
|
82658
|
+
if (count5 !== void 0) {
|
|
82506
82659
|
observedDays++;
|
|
82507
|
-
if (
|
|
82660
|
+
if (count5 === 0) observedZeroDays++;
|
|
82508
82661
|
} else if (coverage && zeroProvenThrough && date >= coverage.start && date <= zeroProvenThrough) queriedEmptyDays++;
|
|
82509
82662
|
else absent.push(date);
|
|
82510
82663
|
}
|
|
@@ -83341,6 +83494,16 @@ var adsRecentSyncCheck = {
|
|
|
83341
83494
|
var ADS_CHECKS = [adsConnectionCheck, adsRecentSyncCheck];
|
|
83342
83495
|
|
|
83343
83496
|
// ../api-routes/src/doctor/checks/ga-auth.ts
|
|
83497
|
+
function unreachable(step, err, details) {
|
|
83498
|
+
const error = describeFetchError(err, step.requestUrl);
|
|
83499
|
+
return {
|
|
83500
|
+
status: CheckStatuses.fail,
|
|
83501
|
+
code: step.code,
|
|
83502
|
+
summary: `Could not reach Google to ${step.action}: ${error}`,
|
|
83503
|
+
remediation: "Check that this host can resolve and connect to Google (DNS filtering, firewall, or proxy), then re-run. The credential was not tested.",
|
|
83504
|
+
details: { ...details, error }
|
|
83505
|
+
};
|
|
83506
|
+
}
|
|
83344
83507
|
async function checkServiceAccount(conn) {
|
|
83345
83508
|
if (!conn.propertyId) {
|
|
83346
83509
|
return {
|
|
@@ -83365,6 +83528,9 @@ async function checkServiceAccount(conn) {
|
|
|
83365
83528
|
try {
|
|
83366
83529
|
await verifyConnection(conn.clientEmail, conn.privateKey, conn.propertyId);
|
|
83367
83530
|
} catch (err) {
|
|
83531
|
+
if (isFetchTransportError(err)) {
|
|
83532
|
+
return unreachable({ code: "ga.auth.verify-unreachable", action: "verify the GA4 service account" }, err, { propertyId: conn.propertyId, authMethod: "service-account" });
|
|
83533
|
+
}
|
|
83368
83534
|
const message = describeError(err);
|
|
83369
83535
|
return {
|
|
83370
83536
|
status: CheckStatuses.fail,
|
|
@@ -83414,6 +83580,9 @@ async function checkOAuthConnection(ctx, projectName, conn) {
|
|
|
83414
83580
|
const tokens = await refreshAccessToken(auth.clientId, auth.clientSecret, conn.refreshToken);
|
|
83415
83581
|
accessToken = tokens.access_token;
|
|
83416
83582
|
} catch (err) {
|
|
83583
|
+
if (isFetchTransportError(err)) {
|
|
83584
|
+
return unreachable({ code: "ga.auth.refresh-unreachable", action: "refresh the GA4 token", requestUrl: GOOGLE_TOKEN_URL }, err, { propertyId: conn.propertyId, authMethod: "oauth" });
|
|
83585
|
+
}
|
|
83417
83586
|
const message = describeError(err);
|
|
83418
83587
|
return {
|
|
83419
83588
|
status: CheckStatuses.fail,
|
|
@@ -83426,6 +83595,9 @@ async function checkOAuthConnection(ctx, projectName, conn) {
|
|
|
83426
83595
|
try {
|
|
83427
83596
|
await verifyConnectionWithToken(accessToken, conn.propertyId);
|
|
83428
83597
|
} catch (err) {
|
|
83598
|
+
if (isFetchTransportError(err)) {
|
|
83599
|
+
return unreachable({ code: "ga.auth.verify-unreachable", action: "verify GA4 property access", requestUrl: GA4_DATA_API_BASE }, err, { propertyId: conn.propertyId, authMethod: "oauth" });
|
|
83600
|
+
}
|
|
83429
83601
|
const message = describeError(err);
|
|
83430
83602
|
return {
|
|
83431
83603
|
status: CheckStatuses.fail,
|
|
@@ -83954,6 +84126,19 @@ async function resolveGbpToken(ctx) {
|
|
|
83954
84126
|
const tokens = await refreshAccessToken(auth.clientId, auth.clientSecret, conn.refreshToken);
|
|
83955
84127
|
return { ok: true, token: { accessToken: tokens.access_token, conn } };
|
|
83956
84128
|
} catch (err) {
|
|
84129
|
+
if (isFetchTransportError(err)) {
|
|
84130
|
+
const error = describeFetchError(err, GOOGLE_TOKEN_URL);
|
|
84131
|
+
return {
|
|
84132
|
+
ok: false,
|
|
84133
|
+
output: {
|
|
84134
|
+
status: CheckStatuses.fail,
|
|
84135
|
+
code: "gbp.auth.refresh-unreachable",
|
|
84136
|
+
summary: `Could not reach Google to refresh the GBP token: ${error}`,
|
|
84137
|
+
remediation: "Check that this host can resolve and connect to Google (DNS filtering, firewall, or proxy), then re-run. The stored grant was not tested.",
|
|
84138
|
+
details: { domain: conn.domain, error }
|
|
84139
|
+
}
|
|
84140
|
+
};
|
|
84141
|
+
}
|
|
83957
84142
|
const message = describeError(err);
|
|
83958
84143
|
return {
|
|
83959
84144
|
ok: false,
|
|
@@ -84086,6 +84271,16 @@ var accountAccessCheck = {
|
|
|
84086
84271
|
details: { reason: err.reason ?? void 0, status: err.status }
|
|
84087
84272
|
};
|
|
84088
84273
|
}
|
|
84274
|
+
if (isFetchTransportError(err)) {
|
|
84275
|
+
const error = describeFetchError(err, GBP_ACCOUNT_MANAGEMENT_BASE);
|
|
84276
|
+
return {
|
|
84277
|
+
status: CheckStatuses.fail,
|
|
84278
|
+
code: "gbp.account.list-unreachable",
|
|
84279
|
+
summary: `Could not reach the Business Profile API to list GBP accounts: ${error}`,
|
|
84280
|
+
remediation: "Check that this host can resolve and connect to the Business Profile API (DNS filtering, firewall, or proxy), then re-run. Account access was not tested.",
|
|
84281
|
+
details: { error }
|
|
84282
|
+
};
|
|
84283
|
+
}
|
|
84089
84284
|
const message = describeError(err);
|
|
84090
84285
|
return {
|
|
84091
84286
|
status: CheckStatuses.fail,
|
|
@@ -84322,10 +84517,10 @@ var reviewsAccessCheck = {
|
|
|
84322
84517
|
reason: loc.reviewsAccessReason,
|
|
84323
84518
|
coverage: coverageOf(loc)
|
|
84324
84519
|
}));
|
|
84325
|
-
const
|
|
84326
|
-
const full =
|
|
84327
|
-
const partial =
|
|
84328
|
-
const none =
|
|
84520
|
+
const count5 = (coverage) => perLocation.filter((loc) => loc.coverage === coverage).length;
|
|
84521
|
+
const full = count5("full");
|
|
84522
|
+
const partial = count5("partial");
|
|
84523
|
+
const none = count5("none");
|
|
84329
84524
|
const details = { placesFallback, full, partial, none, locations: perLocation };
|
|
84330
84525
|
if (full === locations.length) {
|
|
84331
84526
|
return {
|
|
@@ -84426,6 +84621,19 @@ async function resolveAccessToken(ctx) {
|
|
|
84426
84621
|
const tokens = await refreshAccessToken(auth.clientId, auth.clientSecret, conn.refreshToken);
|
|
84427
84622
|
return { ok: true, token: { accessToken: tokens.access_token } };
|
|
84428
84623
|
} catch (err) {
|
|
84624
|
+
if (isFetchTransportError(err)) {
|
|
84625
|
+
const error = describeFetchError(err, GOOGLE_TOKEN_URL);
|
|
84626
|
+
return {
|
|
84627
|
+
ok: false,
|
|
84628
|
+
output: {
|
|
84629
|
+
status: CheckStatuses.fail,
|
|
84630
|
+
code: "google.auth.refresh-unreachable",
|
|
84631
|
+
summary: `Could not reach Google to refresh the GSC token: ${error}`,
|
|
84632
|
+
remediation: "Check that this host can resolve and connect to Google (DNS filtering, firewall, or proxy), then re-run. The stored grant was not tested.",
|
|
84633
|
+
details: { domain: conn.domain, error }
|
|
84634
|
+
}
|
|
84635
|
+
};
|
|
84636
|
+
}
|
|
84429
84637
|
const message = describeError(err);
|
|
84430
84638
|
return {
|
|
84431
84639
|
ok: false,
|
|
@@ -84518,6 +84726,16 @@ var propertyAccessCheck = {
|
|
|
84518
84726
|
details: { error: err.message }
|
|
84519
84727
|
};
|
|
84520
84728
|
}
|
|
84729
|
+
if (isFetchTransportError(err)) {
|
|
84730
|
+
const error = describeFetchError(err, GSC_API_BASE);
|
|
84731
|
+
return {
|
|
84732
|
+
status: CheckStatuses.fail,
|
|
84733
|
+
code: "google.auth.list-sites-unreachable",
|
|
84734
|
+
summary: `Could not reach Search Console to list the authorized account's sites: ${error}`,
|
|
84735
|
+
remediation: "Check that this host can resolve and connect to the Search Console API (DNS filtering, firewall, or proxy), then re-run. Property access was not tested.",
|
|
84736
|
+
details: { error }
|
|
84737
|
+
};
|
|
84738
|
+
}
|
|
84521
84739
|
const message = describeError(err);
|
|
84522
84740
|
return {
|
|
84523
84741
|
status: CheckStatuses.fail,
|
|
@@ -85154,7 +85372,7 @@ async function runValidator(source, validator, fallbackId, fallbackLabel) {
|
|
|
85154
85372
|
function summarizePerSourceResults(fallbackId, fallbackLabel, results) {
|
|
85155
85373
|
const failed = results.filter((r) => r.output.status === CheckStatuses.fail);
|
|
85156
85374
|
const warned = results.filter((r) => r.output.status === CheckStatuses.warn);
|
|
85157
|
-
const
|
|
85375
|
+
const skipped4 = results.filter((r) => r.output.status === CheckStatuses.skipped);
|
|
85158
85376
|
const ok = results.filter((r) => r.output.status === CheckStatuses.ok);
|
|
85159
85377
|
const detail = {
|
|
85160
85378
|
sources: results.map((r) => ({
|
|
@@ -85188,7 +85406,7 @@ function summarizePerSourceResults(fallbackId, fallbackLabel, results) {
|
|
|
85188
85406
|
return {
|
|
85189
85407
|
status: CheckStatuses.ok,
|
|
85190
85408
|
code: `traffic.${fallbackId}.ok`,
|
|
85191
|
-
summary: `${ok.length} source(s) passed ${fallbackLabel} validation${
|
|
85409
|
+
summary: `${ok.length} source(s) passed ${fallbackLabel} validation${skipped4.length > 0 ? ` (${skipped4.length} skipped)` : ""}.`,
|
|
85192
85410
|
details: detail
|
|
85193
85411
|
};
|
|
85194
85412
|
}
|
|
@@ -85196,7 +85414,7 @@ function summarizePerSourceResults(fallbackId, fallbackLabel, results) {
|
|
|
85196
85414
|
status: CheckStatuses.skipped,
|
|
85197
85415
|
code: `traffic.${fallbackId}.all-skipped`,
|
|
85198
85416
|
summary: `No source-type validator was available for any of the ${results.length} connected source(s).`,
|
|
85199
|
-
remediation:
|
|
85417
|
+
remediation: skipped4.find((result) => result.output.remediation)?.output.remediation,
|
|
85200
85418
|
details: detail
|
|
85201
85419
|
};
|
|
85202
85420
|
}
|
|
@@ -85858,6 +86076,166 @@ var GOOGLE_MARKETING_DOCTOR_CHECKS = createGoogleMarketingDoctorChecks(
|
|
|
85858
86076
|
(ctx) => ctx.getGoogleMarketingDoctorInput?.(ctx)
|
|
85859
86077
|
);
|
|
85860
86078
|
|
|
86079
|
+
// ../api-routes/src/doctor/checks/integration-sync.ts
|
|
86080
|
+
import { and as and68, count as count2, desc as desc36, eq as eq81, gt as gt7, gte as gte20, inArray as inArray32, isNull as isNull14, notInArray as notInArray2, or as or18 } from "drizzle-orm";
|
|
86081
|
+
var SYNC_FAILURE_WINDOW = 3;
|
|
86082
|
+
var SUMMARY_ERROR_LIMIT = 300;
|
|
86083
|
+
var FINISHED_STATUSES = [RunStatuses.completed, RunStatuses.partial, RunStatuses.failed];
|
|
86084
|
+
var SUCCESS_STATUSES = [RunStatuses.completed, RunStatuses.partial];
|
|
86085
|
+
var CONNECTED = { state: "connected", since: null };
|
|
86086
|
+
function googleMarketingConnection(provider) {
|
|
86087
|
+
return (ctx) => {
|
|
86088
|
+
const input = ctx.getGoogleMarketingDoctorInput?.(ctx);
|
|
86089
|
+
if (!input) return { state: "store-unavailable" };
|
|
86090
|
+
return input[provider] ? CONNECTED : { state: "not-connected" };
|
|
86091
|
+
};
|
|
86092
|
+
}
|
|
86093
|
+
function gscConnection(ctx, project) {
|
|
86094
|
+
if (!ctx.googleConnectionStore) return { state: "store-unavailable" };
|
|
86095
|
+
return ctx.googleConnectionStore.getConnection(project.canonicalDomain, "gsc") ? CONNECTED : { state: "not-connected" };
|
|
86096
|
+
}
|
|
86097
|
+
function gbpConnection(ctx, project) {
|
|
86098
|
+
if (!ctx.googleConnectionStore) return { state: "store-unavailable" };
|
|
86099
|
+
if (!ctx.googleConnectionStore.getConnection(project.canonicalDomain, "gbp")) return { state: "not-connected" };
|
|
86100
|
+
const selected = ctx.db.select({ locationName: gbpLocations.locationName }).from(gbpLocations).where(and68(eq81(gbpLocations.projectId, project.id), eq81(gbpLocations.selected, true))).limit(1).get();
|
|
86101
|
+
return selected ? CONNECTED : { state: "no-selected-locations" };
|
|
86102
|
+
}
|
|
86103
|
+
function gaConnection(ctx, project) {
|
|
86104
|
+
const state = gaConnected(ctx);
|
|
86105
|
+
if (state === "store-unavailable") return { state };
|
|
86106
|
+
if (state === "not-connected") return { state };
|
|
86107
|
+
const serviceAccount = ctx.ga4CredentialStore?.getConnection(project.name);
|
|
86108
|
+
const since = serviceAccount ? serviceAccount.createdAt : ctx.googleConnectionStore?.getConnection(project.canonicalDomain, "ga4")?.createdAt;
|
|
86109
|
+
return { state: "connected", since: since || null };
|
|
86110
|
+
}
|
|
86111
|
+
var SPECS = [
|
|
86112
|
+
{
|
|
86113
|
+
prefix: "google-ads",
|
|
86114
|
+
label: "Google Ads",
|
|
86115
|
+
kind: RunKinds["google-ads-sync"],
|
|
86116
|
+
connection: googleMarketingConnection("googleAds"),
|
|
86117
|
+
syncCommand: (name) => `canonry google-ads sync ${name}`,
|
|
86118
|
+
causeCheckIds: ["google-ads.auth.connection", "google-ads.auth.scopes", "google-ads.account.context"]
|
|
86119
|
+
},
|
|
86120
|
+
{
|
|
86121
|
+
prefix: "gtm",
|
|
86122
|
+
label: "Google Tag Manager",
|
|
86123
|
+
kind: RunKinds["gtm-sync"],
|
|
86124
|
+
connection: googleMarketingConnection("gtm"),
|
|
86125
|
+
syncCommand: (name) => `canonry gtm sync ${name}`,
|
|
86126
|
+
causeCheckIds: ["gtm.auth.connection", "gtm.auth.scopes", "gtm.container.context"]
|
|
86127
|
+
},
|
|
86128
|
+
{
|
|
86129
|
+
prefix: "gbp",
|
|
86130
|
+
label: "Google Business Profile",
|
|
86131
|
+
kind: RunKinds["gbp-sync"],
|
|
86132
|
+
connection: gbpConnection,
|
|
86133
|
+
syncCommand: (name) => `canonry gbp sync ${name}`,
|
|
86134
|
+
causeCheckIds: ["gbp.auth.connection", "gbp.auth.scopes", "gbp.account.access"],
|
|
86135
|
+
// `gbp_locations` does not record when a location was selected, so the
|
|
86136
|
+
// runs from before the selection are recognized by the error they stored.
|
|
86137
|
+
nothingToSyncErrors: [serializeRunError({ message: GBP_NO_SELECTED_LOCATIONS_ERROR })]
|
|
86138
|
+
},
|
|
86139
|
+
{
|
|
86140
|
+
prefix: "ga",
|
|
86141
|
+
label: "GA4",
|
|
86142
|
+
kind: RunKinds["ga-sync"],
|
|
86143
|
+
connection: gaConnection,
|
|
86144
|
+
syncCommand: (name) => `canonry ga sync ${name}`,
|
|
86145
|
+
causeCheckIds: ["ga.auth.connection"]
|
|
86146
|
+
},
|
|
86147
|
+
{
|
|
86148
|
+
prefix: "gsc",
|
|
86149
|
+
label: "Search Console",
|
|
86150
|
+
kind: RunKinds["gsc-sync"],
|
|
86151
|
+
connection: gscConnection,
|
|
86152
|
+
syncCommand: (name) => `canonry google sync ${name}`,
|
|
86153
|
+
causeCheckIds: ["google.auth.connection", "google.auth.property-access", "google.auth.scopes"]
|
|
86154
|
+
}
|
|
86155
|
+
];
|
|
86156
|
+
function skipped3(code, summary3) {
|
|
86157
|
+
return { status: CheckStatuses.skipped, code, summary: summary3, remediation: null };
|
|
86158
|
+
}
|
|
86159
|
+
function latestErrorText(raw) {
|
|
86160
|
+
const parsed = parseRunError(raw);
|
|
86161
|
+
return parsed ? formatRunErrorOneLine(parsed) : "no error was recorded";
|
|
86162
|
+
}
|
|
86163
|
+
function gradeSyncFailures(ctx, spec) {
|
|
86164
|
+
const code = (suffix) => `${spec.prefix}.sync.${suffix}`;
|
|
86165
|
+
const project = ctx.project;
|
|
86166
|
+
if (!project) return skipped3(code("no-project"), "Project context required.");
|
|
86167
|
+
const connection = spec.connection(ctx, project);
|
|
86168
|
+
if (connection.state === "store-unavailable") {
|
|
86169
|
+
return skipped3(code("store-unavailable"), `No ${spec.label} connection store configured for this deployment.`);
|
|
86170
|
+
}
|
|
86171
|
+
if (connection.state === "not-connected") {
|
|
86172
|
+
return skipped3(code("not-connected"), `${spec.label} is not connected for this project.`);
|
|
86173
|
+
}
|
|
86174
|
+
if (connection.state === "no-selected-locations") {
|
|
86175
|
+
return skipped3(code("no-selected-locations"), `No ${spec.label} locations are selected for this project.`);
|
|
86176
|
+
}
|
|
86177
|
+
const nothingToSync = spec.nothingToSyncErrors ?? [];
|
|
86178
|
+
const scope = and68(
|
|
86179
|
+
eq81(runs.projectId, project.id),
|
|
86180
|
+
eq81(runs.kind, spec.kind),
|
|
86181
|
+
...nothingToSync.length > 0 ? [or18(isNull14(runs.error), notInArray2(runs.error, [...nothingToSync]))] : []
|
|
86182
|
+
);
|
|
86183
|
+
const graded = connection.since ? and68(scope, gte20(runs.createdAt, connection.since)) : scope;
|
|
86184
|
+
const recent = ctx.db.select({ status: runs.status, error: runs.error, createdAt: runs.createdAt }).from(runs).where(and68(graded, inArray32(runs.status, FINISHED_STATUSES))).orderBy(desc36(runs.createdAt)).limit(SYNC_FAILURE_WINDOW).all();
|
|
86185
|
+
if (recent.length === 0) {
|
|
86186
|
+
return skipped3(code("no-runs"), `${spec.label} is connected but no sync has finished yet.`);
|
|
86187
|
+
}
|
|
86188
|
+
const lastSuccess = ctx.db.select({ createdAt: runs.createdAt, finishedAt: runs.finishedAt }).from(runs).where(and68(graded, inArray32(runs.status, SUCCESS_STATUSES))).orderBy(desc36(runs.createdAt)).limit(1).get();
|
|
86189
|
+
const failuresSinceSuccess = ctx.db.select({ value: count2() }).from(runs).where(and68(
|
|
86190
|
+
graded,
|
|
86191
|
+
eq81(runs.status, RunStatuses.failed),
|
|
86192
|
+
...lastSuccess ? [gt7(runs.createdAt, lastSuccess.createdAt)] : []
|
|
86193
|
+
)).get()?.value ?? 0;
|
|
86194
|
+
const lastSuccessAt = lastSuccess ? lastSuccess.finishedAt ?? lastSuccess.createdAt : null;
|
|
86195
|
+
const latest = recent[0];
|
|
86196
|
+
const latestFailed = latest.status === RunStatuses.failed;
|
|
86197
|
+
const latestError = latestFailed ? latestErrorText(latest.error) : null;
|
|
86198
|
+
const details = {
|
|
86199
|
+
runKind: spec.kind,
|
|
86200
|
+
window: SYNC_FAILURE_WINDOW,
|
|
86201
|
+
consecutiveFailures: failuresSinceSuccess,
|
|
86202
|
+
latestStatus: latest.status,
|
|
86203
|
+
latestRunAt: latest.createdAt,
|
|
86204
|
+
latestError,
|
|
86205
|
+
lastSuccessAt,
|
|
86206
|
+
// Where the graded runs start (the current connection); null grades every run.
|
|
86207
|
+
gradedSince: connection.since
|
|
86208
|
+
};
|
|
86209
|
+
const repeatedFailure = recent.length >= SYNC_FAILURE_WINDOW && recent.every((run) => run.status === RunStatuses.failed);
|
|
86210
|
+
if (repeatedFailure) {
|
|
86211
|
+
const errorText = latestErrorText(latest.error);
|
|
86212
|
+
const shownError = errorText.length > SUMMARY_ERROR_LIMIT ? `${truncateUtf16(errorText, SUMMARY_ERROR_LIMIT)}...` : errorText;
|
|
86213
|
+
const lastSuccessText = lastSuccessAt ?? (connection.since ? `none since connecting at ${connection.since}` : "never");
|
|
86214
|
+
return {
|
|
86215
|
+
status: CheckStatuses.fail,
|
|
86216
|
+
code: code("repeated-failures"),
|
|
86217
|
+
summary: `The last ${failuresSinceSuccess} ${spec.label} syncs failed (last success: ${lastSuccessText}). Latest error: ${shownError}`,
|
|
86218
|
+
remediation: `Fix the cause the latest error names (DNS or network filtering of the API host, credentials, or account access), then run \`${spec.syncCommand(project.name)}\`.`,
|
|
86219
|
+
details
|
|
86220
|
+
};
|
|
86221
|
+
}
|
|
86222
|
+
return {
|
|
86223
|
+
status: CheckStatuses.ok,
|
|
86224
|
+
code: code("ok"),
|
|
86225
|
+
summary: latestFailed ? `The latest ${spec.label} sync failed, but not ${SYNC_FAILURE_WINDOW} in a row.` : `The latest ${spec.label} sync ${latest.status === RunStatuses.partial ? "partly completed" : "completed"}.`,
|
|
86226
|
+
remediation: null,
|
|
86227
|
+
details
|
|
86228
|
+
};
|
|
86229
|
+
}
|
|
86230
|
+
var INTEGRATION_SYNC_CHECKS = SPECS.map((spec) => ({
|
|
86231
|
+
id: `${spec.prefix}.sync.recent-failures`,
|
|
86232
|
+
category: CheckCategories.integrations,
|
|
86233
|
+
scope: CheckScopes.project,
|
|
86234
|
+
title: `${spec.label} sync failures`,
|
|
86235
|
+
supersededBy: spec.causeCheckIds,
|
|
86236
|
+
run: (ctx) => gradeSyncFailures(ctx, spec)
|
|
86237
|
+
}));
|
|
86238
|
+
|
|
85861
86239
|
// ../api-routes/src/doctor/checks/version.ts
|
|
85862
86240
|
var versionCurrentCheck = {
|
|
85863
86241
|
id: "canonry.version.current",
|
|
@@ -85938,6 +86316,7 @@ var ALL_CHECKS = [
|
|
|
85938
86316
|
...REFERRAL_ASSESSMENT_CHECKS,
|
|
85939
86317
|
...ADS_CHECKS,
|
|
85940
86318
|
...GOOGLE_MARKETING_DOCTOR_CHECKS,
|
|
86319
|
+
...INTEGRATION_SYNC_CHECKS,
|
|
85941
86320
|
...PROVIDERS_CHECKS,
|
|
85942
86321
|
...TRAFFIC_SOURCE_CHECKS,
|
|
85943
86322
|
...BACKLINKS_CHECKS,
|
|
@@ -85950,6 +86329,9 @@ function scheduledHealthCheckIds() {
|
|
|
85950
86329
|
return ALL_CHECKS.filter((check2) => check2.scope === CheckScopes.project && !check2.optIn && check2.notificationPolicy !== CheckNotificationPolicies.silent).map((check2) => check2.id);
|
|
85951
86330
|
}
|
|
85952
86331
|
|
|
86332
|
+
// ../api-routes/src/doctor/types.ts
|
|
86333
|
+
var UNREACHABLE_CODE_SUFFIX = "-unreachable";
|
|
86334
|
+
|
|
85953
86335
|
// ../api-routes/src/doctor/runner.ts
|
|
85954
86336
|
function matchesCheckId(checkId, filters) {
|
|
85955
86337
|
if (filters.length === 0) return true;
|
|
@@ -86004,14 +86386,36 @@ async function runChecks(ctx, checks, options = {}) {
|
|
|
86004
86386
|
durationMs: Date.now() - checkStarted
|
|
86005
86387
|
});
|
|
86006
86388
|
}
|
|
86389
|
+
const failedCauses = new Set(results.filter((result, index2) => namesCause(selected[index2], result)).map((result) => result.id));
|
|
86390
|
+
const graded = results.map((result, index2) => supersede(selected[index2], result, failedCauses));
|
|
86007
86391
|
return {
|
|
86008
86392
|
scope: targetScope,
|
|
86009
86393
|
project: projectName,
|
|
86010
86394
|
...ctx.project ? { reportMonths: reportMonthsForDoctor(ctx.reportMonth, startedAt) } : {},
|
|
86011
86395
|
generatedAt: startedAt.toISOString(),
|
|
86012
86396
|
durationMs: Date.now() - startedAt.getTime(),
|
|
86013
|
-
summary: summarizeCheckResults(
|
|
86014
|
-
checks:
|
|
86397
|
+
summary: summarizeCheckResults(graded),
|
|
86398
|
+
checks: graded
|
|
86399
|
+
};
|
|
86400
|
+
}
|
|
86401
|
+
function threw(definition, result) {
|
|
86402
|
+
return result.code === `${definition.id}.runtime-error`;
|
|
86403
|
+
}
|
|
86404
|
+
function namesCause(definition, result) {
|
|
86405
|
+
return result.status === CheckStatuses.fail && !threw(definition, result) && !result.code.endsWith(UNREACHABLE_CODE_SUFFIX);
|
|
86406
|
+
}
|
|
86407
|
+
function supersede(definition, result, failedCauses) {
|
|
86408
|
+
if (result.status !== CheckStatuses.fail && result.status !== CheckStatuses.warn) return result;
|
|
86409
|
+
if (threw(definition, result)) return result;
|
|
86410
|
+
const causes = (definition.supersededBy ?? []).filter((id) => failedCauses.has(id));
|
|
86411
|
+
if (causes.length === 0) return result;
|
|
86412
|
+
return {
|
|
86413
|
+
...result,
|
|
86414
|
+
status: CheckStatuses.skipped,
|
|
86415
|
+
code: `${definition.id}.superseded`,
|
|
86416
|
+
summary: `Superseded by failing ${causes.join(", ")}, which names the cause. ${result.summary}`,
|
|
86417
|
+
remediation: null,
|
|
86418
|
+
details: { ...result.details, supersededBy: causes, supersededStatus: result.status, supersededCode: result.code }
|
|
86015
86419
|
};
|
|
86016
86420
|
}
|
|
86017
86421
|
|
|
@@ -86096,7 +86500,7 @@ async function doctorRoutes(app, opts) {
|
|
|
86096
86500
|
|
|
86097
86501
|
// ../api-routes/src/discovery/routes.ts
|
|
86098
86502
|
import crypto49 from "crypto";
|
|
86099
|
-
import { and as
|
|
86503
|
+
import { and as and69, desc as desc37, eq as eq82, gte as gte21, inArray as inArray33, isNull as isNull15, or as or19 } from "drizzle-orm";
|
|
86100
86504
|
var MAX_INFLIGHT_DISCOVERY_AGE_MS = 2 * 60 * 60 * 1e3;
|
|
86101
86505
|
async function discoveryRoutes(app, opts) {
|
|
86102
86506
|
app.post("/projects/:name/discover/run", async (request, reply) => {
|
|
@@ -86130,13 +86534,13 @@ async function discoveryRoutes(app, opts) {
|
|
|
86130
86534
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
86131
86535
|
const ageFloorIso = new Date(Date.now() - MAX_INFLIGHT_DISCOVERY_AGE_MS).toISOString();
|
|
86132
86536
|
const decision = app.db.transaction((tx) => {
|
|
86133
|
-
const existing = tx.select({ id: discoverySessions.id, runId: discoverySessions.runId }).from(discoverySessions).where(
|
|
86134
|
-
|
|
86135
|
-
|
|
86537
|
+
const existing = tx.select({ id: discoverySessions.id, runId: discoverySessions.runId }).from(discoverySessions).where(and69(
|
|
86538
|
+
eq82(discoverySessions.projectId, project.id),
|
|
86539
|
+
eq82(discoverySessions.icpDescription, icpDescription),
|
|
86136
86540
|
// Buyer is part of session identity: it changes the seed prompt's
|
|
86137
86541
|
// semantics, so a request with a different (or no) buyer must start
|
|
86138
86542
|
// its own session, never adopt another buyer's probes.
|
|
86139
|
-
parsed.data.buyerDescription == null ?
|
|
86543
|
+
parsed.data.buyerDescription == null ? isNull15(discoverySessions.buyerDescription) : eq82(discoverySessions.buyerDescription, parsed.data.buyerDescription),
|
|
86140
86544
|
// Locations are identity too: a different service-area subset seeds
|
|
86141
86545
|
// and probes a different geo, so it must never reuse another geo's
|
|
86142
86546
|
// session. resolveLocations is deterministic (project-config order),
|
|
@@ -86146,18 +86550,18 @@ async function discoveryRoutes(app, opts) {
|
|
|
86146
86550
|
// locations a NULL row's subset is unknowable, so it conservatively
|
|
86147
86551
|
// never reuses — a one-time, bounded (2h window) non-reuse after
|
|
86148
86552
|
// upgrade, never a wrong reuse.
|
|
86149
|
-
locations.length === 0 ?
|
|
86553
|
+
locations.length === 0 ? or19(isNull15(discoverySessions.locations), eq82(discoverySessions.locations, locations)) : eq82(discoverySessions.locations, locations),
|
|
86150
86554
|
// Seed provider set is identity: a different phrasing distribution
|
|
86151
86555
|
// must never reuse another set's session. Null (= Gemini-only
|
|
86152
86556
|
// default, including explicit ['gemini']) matches legacy rows.
|
|
86153
|
-
seedProviders == null ?
|
|
86154
|
-
|
|
86557
|
+
seedProviders == null ? isNull15(discoverySessions.seedProviders) : eq82(discoverySessions.seedProviders, seedProviders),
|
|
86558
|
+
inArray33(discoverySessions.status, [
|
|
86155
86559
|
DiscoverySessionStatuses.queued,
|
|
86156
86560
|
DiscoverySessionStatuses.seeding,
|
|
86157
86561
|
DiscoverySessionStatuses.probing
|
|
86158
86562
|
]),
|
|
86159
|
-
|
|
86160
|
-
)).orderBy(
|
|
86563
|
+
gte21(discoverySessions.createdAt, ageFloorIso)
|
|
86564
|
+
)).orderBy(desc37(discoverySessions.createdAt)).get();
|
|
86161
86565
|
if (existing && existing.runId) {
|
|
86162
86566
|
return { reused: true, sessionId: existing.id, runId: existing.runId };
|
|
86163
86567
|
}
|
|
@@ -86226,7 +86630,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
86226
86630
|
const project = resolveProject(app.db, request.params.name);
|
|
86227
86631
|
const parsedLimit = parseInt(request.query.limit ?? "", 10);
|
|
86228
86632
|
const limit = Number.isNaN(parsedLimit) || parsedLimit <= 0 ? 50 : parsedLimit;
|
|
86229
|
-
const rows = app.db.select().from(discoverySessions).where(
|
|
86633
|
+
const rows = app.db.select().from(discoverySessions).where(eq82(discoverySessions.projectId, project.id)).orderBy(desc37(discoverySessions.createdAt)).limit(limit).all();
|
|
86230
86634
|
return reply.send(rows.map(serializeSession));
|
|
86231
86635
|
}
|
|
86232
86636
|
);
|
|
@@ -86234,11 +86638,11 @@ async function discoveryRoutes(app, opts) {
|
|
|
86234
86638
|
"/projects/:name/discover/sessions/:id",
|
|
86235
86639
|
async (request, reply) => {
|
|
86236
86640
|
const project = resolveProject(app.db, request.params.name);
|
|
86237
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
86641
|
+
const session = app.db.select().from(discoverySessions).where(eq82(discoverySessions.id, request.params.id)).get();
|
|
86238
86642
|
if (!session || session.projectId !== project.id) {
|
|
86239
86643
|
throw notFound("Discovery session", request.params.id);
|
|
86240
86644
|
}
|
|
86241
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
86645
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq82(discoveryProbes.sessionId, session.id)).all();
|
|
86242
86646
|
const detail = {
|
|
86243
86647
|
...serializeSession(session),
|
|
86244
86648
|
probes: probeRows.map(serializeProbe)
|
|
@@ -86250,7 +86654,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
86250
86654
|
"/projects/:name/discover/sessions/:id/harvest",
|
|
86251
86655
|
async (request, reply) => {
|
|
86252
86656
|
const project = resolveProject(app.db, request.params.name);
|
|
86253
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
86657
|
+
const session = app.db.select().from(discoverySessions).where(eq82(discoverySessions.id, request.params.id)).get();
|
|
86254
86658
|
if (!session || session.projectId !== project.id) {
|
|
86255
86659
|
throw notFound("Discovery session", request.params.id);
|
|
86256
86660
|
}
|
|
@@ -86258,7 +86662,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
86258
86662
|
const minProbeHits = Number.isNaN(parsedFloor) || parsedFloor < 1 ? 1 : parsedFloor;
|
|
86259
86663
|
const applyAnchor = request.query.anchor !== "false";
|
|
86260
86664
|
const provider = session.seedProvider ?? "gemini";
|
|
86261
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
86665
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq82(discoveryProbes.sessionId, session.id)).all();
|
|
86262
86666
|
const extract = opts.harvestSearchQueries;
|
|
86263
86667
|
const probesWithQueries = probeRows.map((row) => {
|
|
86264
86668
|
if (!extract || !row.rawResponse) return { searchQueries: [] };
|
|
@@ -86269,7 +86673,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
86269
86673
|
return { searchQueries: [] };
|
|
86270
86674
|
}
|
|
86271
86675
|
});
|
|
86272
|
-
const trackedQueries = app.db.select({ query: queries.query }).from(queries).where(
|
|
86676
|
+
const trackedQueries = app.db.select({ query: queries.query }).from(queries).where(eq82(queries.projectId, project.id)).all().map((r) => r.query);
|
|
86273
86677
|
const anchorTerms = buildHarvestAnchorTerms(
|
|
86274
86678
|
[session.icpDescription ?? "", ...trackedQueries],
|
|
86275
86679
|
effectiveDomains(project)
|
|
@@ -86316,11 +86720,11 @@ async function discoveryRoutes(app, opts) {
|
|
|
86316
86720
|
"/projects/:name/discover/sessions/:id/promote",
|
|
86317
86721
|
async (request, reply) => {
|
|
86318
86722
|
const project = resolveProject(app.db, request.params.name);
|
|
86319
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
86723
|
+
const session = app.db.select().from(discoverySessions).where(eq82(discoverySessions.id, request.params.id)).get();
|
|
86320
86724
|
if (!session || session.projectId !== project.id) {
|
|
86321
86725
|
throw notFound("Discovery session", request.params.id);
|
|
86322
86726
|
}
|
|
86323
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
86727
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq82(discoveryProbes.sessionId, session.id)).all();
|
|
86324
86728
|
const cited = /* @__PURE__ */ new Set();
|
|
86325
86729
|
const aspirational = /* @__PURE__ */ new Set();
|
|
86326
86730
|
const wasted = /* @__PURE__ */ new Set();
|
|
@@ -86355,7 +86759,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
86355
86759
|
);
|
|
86356
86760
|
app.post("/projects/:name/discover/sessions/:id/promote", async (request, reply) => {
|
|
86357
86761
|
const project = resolveProject(app.db, request.params.name);
|
|
86358
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
86762
|
+
const session = app.db.select().from(discoverySessions).where(eq82(discoverySessions.id, request.params.id)).get();
|
|
86359
86763
|
if (!session || session.projectId !== project.id) {
|
|
86360
86764
|
throw notFound("Discovery session", request.params.id);
|
|
86361
86765
|
}
|
|
@@ -86378,7 +86782,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
86378
86782
|
const bucketSet = new Set(buckets);
|
|
86379
86783
|
const includeCompetitors = parsed.data.includeCompetitors ?? true;
|
|
86380
86784
|
const competitorTypes = parsed.data.competitorTypes ?? DEFAULT_DISCOVERY_PROMOTE_COMPETITOR_TYPES;
|
|
86381
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
86785
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq82(discoveryProbes.sessionId, session.id)).all();
|
|
86382
86786
|
const candidateQueries = /* @__PURE__ */ new Set();
|
|
86383
86787
|
for (const probe of probeRows) {
|
|
86384
86788
|
if (!probe.bucket) continue;
|
|
@@ -86386,7 +86790,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
86386
86790
|
if (bucket.success && bucketSet.has(bucket.data)) candidateQueries.add(probe.query);
|
|
86387
86791
|
}
|
|
86388
86792
|
const existingQueries = new Set(
|
|
86389
|
-
app.db.select({ query: queries.query }).from(queries).where(
|
|
86793
|
+
app.db.select({ query: queries.query }).from(queries).where(eq82(queries.projectId, project.id)).all().map((r) => r.query.toLowerCase())
|
|
86390
86794
|
);
|
|
86391
86795
|
const promotedQueries = [];
|
|
86392
86796
|
const skippedQueries = [];
|
|
@@ -86557,21 +86961,21 @@ function planCompetitorPromotion(input) {
|
|
|
86557
86961
|
const preferred = new Set(input.competitorTypes ?? DEFAULT_DISCOVERY_PROMOTE_COMPETITOR_TYPES);
|
|
86558
86962
|
const eligible = groupCompetitorMap(input.competitorMap, input.probes, preferred).filter((group) => group.hits >= DISCOVERY_PROMOTE_COMPETITOR_MIN_HITS).filter((group) => !typeFilter || group.sources.some((source) => typeFilter.has(source.competitorType))).sort((a, b) => b.hits - a.hits || a.domain.localeCompare(b.domain));
|
|
86559
86963
|
const tracked = new Set(input.tracked.map((row) => normalizeCompetitorDomain(row.domain)));
|
|
86560
|
-
const
|
|
86964
|
+
const skipped4 = [];
|
|
86561
86965
|
const candidates = [];
|
|
86562
86966
|
for (const group of eligible) {
|
|
86563
86967
|
const projectDomain = competitorDomainProjectClaim(group.domain, input.project.domains);
|
|
86564
86968
|
const subdomains = [...new Set(group.sources.map((source) => hostOf(source.domain) ?? source.domain))].filter((host) => host !== group.domain);
|
|
86565
86969
|
if (projectDomain) {
|
|
86566
|
-
|
|
86970
|
+
skipped4.push({
|
|
86567
86971
|
...group,
|
|
86568
86972
|
reason: DiscoveryPromoteCompetitorSkipReasons["project-domain"],
|
|
86569
86973
|
message: `${group.domain} overlaps the project domain ${projectDomain}, so every citation of the project would count for it`
|
|
86570
86974
|
});
|
|
86571
86975
|
} else if (tracked.has(group.domain)) {
|
|
86572
|
-
|
|
86976
|
+
skipped4.push({ ...group, reason: DiscoveryPromoteCompetitorSkipReasons["already-tracked"], message: `${group.domain} is already tracked` });
|
|
86573
86977
|
} else if (subdomains.length > 1) {
|
|
86574
|
-
|
|
86978
|
+
skipped4.push({
|
|
86575
86979
|
...group,
|
|
86576
86980
|
reason: DiscoveryPromoteCompetitorSkipReasons["shared-host"],
|
|
86577
86981
|
message: `${group.domain} was cited as ${subdomains.length} different subdomains (${subdomains.join(", ")}), which may be different sites on one shared host, so it is not merged into one competitor; if they are one competitor, add it with canonry competitor add <project> ${group.domain}`
|
|
@@ -86585,7 +86989,7 @@ function planCompetitorPromotion(input) {
|
|
|
86585
86989
|
for (const group of candidates) {
|
|
86586
86990
|
const rejections = claimed.get(group.domain);
|
|
86587
86991
|
if (rejections) {
|
|
86588
|
-
|
|
86992
|
+
skipped4.push({
|
|
86589
86993
|
...group,
|
|
86590
86994
|
reason: DiscoveryPromoteCompetitorSkipReasons["claimed-by-alias"],
|
|
86591
86995
|
message: rejections.map((rejection) => `${group.domain} ${describeCompetitorAliasRejection(rejection)}`).join("; ")
|
|
@@ -86594,12 +86998,12 @@ function planCompetitorPromotion(input) {
|
|
|
86594
86998
|
promote.push(group);
|
|
86595
86999
|
}
|
|
86596
87000
|
}
|
|
86597
|
-
return { promote, skipped:
|
|
87001
|
+
return { promote, skipped: skipped4 };
|
|
86598
87002
|
}
|
|
86599
87003
|
|
|
86600
87004
|
// ../api-routes/src/discovery/orchestrate.ts
|
|
86601
87005
|
import crypto50 from "crypto";
|
|
86602
|
-
import { eq as
|
|
87006
|
+
import { eq as eq83 } from "drizzle-orm";
|
|
86603
87007
|
var DEFAULT_MAX_PROBES = 100;
|
|
86604
87008
|
var ABSOLUTE_MAX_PROBES = 500;
|
|
86605
87009
|
function classifyProbeBucket(input) {
|
|
@@ -86684,7 +87088,7 @@ async function executeDiscovery(opts) {
|
|
|
86684
87088
|
status: DiscoverySessionStatuses.seeding,
|
|
86685
87089
|
dedupThreshold,
|
|
86686
87090
|
startedAt
|
|
86687
|
-
}).where(
|
|
87091
|
+
}).where(eq83(discoverySessions.id, opts.sessionId)).run();
|
|
86688
87092
|
const seedResult = await opts.deps.seed({
|
|
86689
87093
|
project: opts.project,
|
|
86690
87094
|
icpDescription: opts.icpDescription,
|
|
@@ -86745,7 +87149,7 @@ async function executeDiscovery(opts) {
|
|
|
86745
87149
|
dedupBandPairFraction: dedupStats.bandPairFraction,
|
|
86746
87150
|
dedupPairsTotal: dedupStats.pairsTotal,
|
|
86747
87151
|
warning
|
|
86748
|
-
}).where(
|
|
87152
|
+
}).where(eq83(discoverySessions.id, opts.sessionId)).run();
|
|
86749
87153
|
const probeLocation = opts.locations?.[0];
|
|
86750
87154
|
const probeResults = await mapWithConcurrency(
|
|
86751
87155
|
probedCanonicals,
|
|
@@ -86799,7 +87203,7 @@ async function executeDiscovery(opts) {
|
|
|
86799
87203
|
wastedCount: buckets["wasted-surface"],
|
|
86800
87204
|
competitorMap,
|
|
86801
87205
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
86802
|
-
}).where(
|
|
87206
|
+
}).where(eq83(discoverySessions.id, opts.sessionId)).run();
|
|
86803
87207
|
upsertDomainClassifications(opts.db, opts.project.id, opts.sessionId, competitorMap);
|
|
86804
87208
|
return {
|
|
86805
87209
|
buckets,
|
|
@@ -86839,7 +87243,7 @@ function markSessionFailed(db, sessionId, error) {
|
|
|
86839
87243
|
status: DiscoverySessionStatuses.failed,
|
|
86840
87244
|
error,
|
|
86841
87245
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
86842
|
-
}).where(
|
|
87246
|
+
}).where(eq83(discoverySessions.id, sessionId)).run();
|
|
86843
87247
|
}
|
|
86844
87248
|
function dedupeStrings(input) {
|
|
86845
87249
|
const seen2 = /* @__PURE__ */ new Set();
|
|
@@ -86857,7 +87261,7 @@ function dedupeStrings(input) {
|
|
|
86857
87261
|
|
|
86858
87262
|
// ../api-routes/src/technical-aeo.ts
|
|
86859
87263
|
import crypto51 from "crypto";
|
|
86860
|
-
import { and as
|
|
87264
|
+
import { and as and70, asc as asc15, count as count3, desc as desc38, eq as eq84, gte as gte22, inArray as inArray34, isNotNull as isNotNull8, isNull as isNull16, lt as lt13, or as or20, sql as sql30 } from "drizzle-orm";
|
|
86861
87265
|
import { alias } from "drizzle-orm/sqlite-core";
|
|
86862
87266
|
var FETCHED_SITE_CRAWL_STATES = /* @__PURE__ */ new Set([
|
|
86863
87267
|
...SiteCrawlFetchedStates,
|
|
@@ -87094,8 +87498,8 @@ function parseLinkKind(value) {
|
|
|
87094
87498
|
return allowed.data;
|
|
87095
87499
|
}
|
|
87096
87500
|
function linkKindFilter(linkKind) {
|
|
87097
|
-
if (linkKind === SiteHealthLinkKinds.content) return
|
|
87098
|
-
if (linkKind === SiteHealthLinkKinds.template) return
|
|
87501
|
+
if (linkKind === SiteHealthLinkKinds.content) return eq84(siteCrawlEdges.isTemplate, false);
|
|
87502
|
+
if (linkKind === SiteHealthLinkKinds.template) return eq84(siteCrawlEdges.isTemplate, true);
|
|
87099
87503
|
return void 0;
|
|
87100
87504
|
}
|
|
87101
87505
|
var graphSourceNode = alias(siteCrawlGraphNodes, "site_crawl_graph_source_node");
|
|
@@ -87199,20 +87603,20 @@ function changedFields(before, after, fields) {
|
|
|
87199
87603
|
}
|
|
87200
87604
|
async function technicalAeoRoutes(app, opts) {
|
|
87201
87605
|
const visibleCrawlFilters = (projectId) => [
|
|
87202
|
-
|
|
87203
|
-
|
|
87204
|
-
|
|
87205
|
-
|
|
87606
|
+
eq84(siteCrawlSnapshots.projectId, projectId),
|
|
87607
|
+
eq84(runs.projectId, projectId),
|
|
87608
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
87609
|
+
inArray34(runs.status, SURFACEABLE_STATUSES),
|
|
87206
87610
|
notProbeRun()
|
|
87207
87611
|
];
|
|
87208
87612
|
const availableScanDatesFor = (projectId, requestedDate) => {
|
|
87209
87613
|
if (!requestedDate) return void 0;
|
|
87210
87614
|
const date = sql30`substr(${siteCrawlSnapshots.createdAt}, 1, 10)`;
|
|
87211
87615
|
const filters = visibleCrawlFilters(projectId);
|
|
87212
|
-
const allDates =
|
|
87213
|
-
const matchingDates =
|
|
87214
|
-
const readDates = (where) => app.db.select({ date }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
87215
|
-
const countDates = (where) => app.db.select({ value: sql30`count(distinct ${date})` }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
87616
|
+
const allDates = and70(...filters);
|
|
87617
|
+
const matchingDates = and70(...filters, eq84(sql30`substr(${siteCrawlSnapshots.createdAt}, 6, 5)`, requestedDate.slice(5)));
|
|
87618
|
+
const readDates = (where) => app.db.select({ date }).from(siteCrawlSnapshots).innerJoin(runs, eq84(siteCrawlSnapshots.runId, runs.id)).where(where).groupBy(date).orderBy(desc38(date)).limit(10).all().map((row) => row.date);
|
|
87619
|
+
const countDates = (where) => app.db.select({ value: sql30`count(distinct ${date})` }).from(siteCrawlSnapshots).innerJoin(runs, eq84(siteCrawlSnapshots.runId, runs.id)).where(where).get()?.value ?? 0;
|
|
87216
87620
|
return {
|
|
87217
87621
|
recentDates: readDates(allDates),
|
|
87218
87622
|
totalDates: countDates(allDates),
|
|
@@ -87226,36 +87630,36 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87226
87630
|
if (requestedDate) {
|
|
87227
87631
|
const nextDate2 = /* @__PURE__ */ new Date(`${requestedDate}T00:00:00.000Z`);
|
|
87228
87632
|
nextDate2.setUTCDate(nextDate2.getUTCDate() + 1);
|
|
87229
|
-
filters.push(
|
|
87633
|
+
filters.push(gte22(siteCrawlSnapshots.createdAt, requestedDate), lt13(siteCrawlSnapshots.createdAt, nextDate2.toISOString()));
|
|
87230
87634
|
}
|
|
87231
87635
|
const selectionFilters = scope.scoredOnly ? [...filters, sql30`exists (select 1 from ${siteAuditSnapshots} where ${siteAuditSnapshots.projectId} = ${siteCrawlSnapshots.projectId} and ${siteAuditSnapshots.runId} = ${siteCrawlSnapshots.runId})`] : filters;
|
|
87232
|
-
const selected = app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
87233
|
-
|
|
87234
|
-
|
|
87235
|
-
|
|
87236
|
-
|
|
87237
|
-
|
|
87636
|
+
const selected = app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq84(siteCrawlSnapshots.runId, runs.id)).where(and70(...selectionFilters, runId ? eq84(siteCrawlSnapshots.runId, runId) : void 0)).orderBy(
|
|
87637
|
+
desc38(sql30`substr(${siteCrawlSnapshots.createdAt}, 1, 10)`),
|
|
87638
|
+
desc38(siteCrawlSnapshots.complete),
|
|
87639
|
+
desc38(siteCrawlSnapshots.pagesFetched),
|
|
87640
|
+
desc38(siteCrawlSnapshots.createdAt),
|
|
87641
|
+
desc38(siteCrawlSnapshots.runId)
|
|
87238
87642
|
).limit(1).get();
|
|
87239
87643
|
if (!selected) return void 0;
|
|
87240
87644
|
const date = selected.snapshot.createdAt.slice(0, 10);
|
|
87241
87645
|
const nextDate = /* @__PURE__ */ new Date(`${date}T00:00:00.000Z`);
|
|
87242
87646
|
nextDate.setUTCDate(nextDate.getUTCDate() + 1);
|
|
87243
|
-
const sameDate =
|
|
87647
|
+
const sameDate = and70(
|
|
87244
87648
|
...filters,
|
|
87245
|
-
|
|
87649
|
+
gte22(siteCrawlSnapshots.createdAt, date),
|
|
87246
87650
|
lt13(siteCrawlSnapshots.createdAt, nextDate.toISOString())
|
|
87247
87651
|
);
|
|
87248
|
-
const sameDateRunCount = app.db.select({ value:
|
|
87652
|
+
const sameDateRunCount = app.db.select({ value: count3() }).from(siteCrawlSnapshots).innerJoin(runs, eq84(siteCrawlSnapshots.runId, runs.id)).where(sameDate).get()?.value ?? 0;
|
|
87249
87653
|
const candidates = app.db.select({
|
|
87250
87654
|
runId: siteCrawlSnapshots.runId,
|
|
87251
87655
|
createdAt: siteCrawlSnapshots.createdAt,
|
|
87252
87656
|
complete: siteCrawlSnapshots.complete,
|
|
87253
87657
|
pages: siteCrawlSnapshots.pagesFetched
|
|
87254
|
-
}).from(siteCrawlSnapshots).innerJoin(runs,
|
|
87255
|
-
|
|
87256
|
-
|
|
87257
|
-
|
|
87258
|
-
|
|
87658
|
+
}).from(siteCrawlSnapshots).innerJoin(runs, eq84(siteCrawlSnapshots.runId, runs.id)).where(sameDate).orderBy(
|
|
87659
|
+
desc38(siteCrawlSnapshots.complete),
|
|
87660
|
+
desc38(siteCrawlSnapshots.pagesFetched),
|
|
87661
|
+
desc38(siteCrawlSnapshots.createdAt),
|
|
87662
|
+
desc38(siteCrawlSnapshots.runId)
|
|
87259
87663
|
).limit(10).all();
|
|
87260
87664
|
const runSelection = {
|
|
87261
87665
|
reason: runId ? "explicit-run" : scope.scoredOnly ? "latest-scored-scan" : requestedDate ? selected.snapshot.complete ? "requested-date-complete" : "requested-date-most-pages" : selected.snapshot.complete ? "latest-date-complete" : "latest-date-most-pages",
|
|
@@ -87269,20 +87673,20 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87269
87673
|
};
|
|
87270
87674
|
const resolveAudit = (projectId, runId, recordedBefore) => {
|
|
87271
87675
|
const filters = [
|
|
87272
|
-
|
|
87273
|
-
|
|
87274
|
-
|
|
87275
|
-
|
|
87676
|
+
eq84(siteAuditSnapshots.projectId, projectId),
|
|
87677
|
+
eq84(runs.projectId, projectId),
|
|
87678
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
87679
|
+
inArray34(runs.status, SURFACEABLE_STATUSES),
|
|
87276
87680
|
notProbeRun(),
|
|
87277
87681
|
...recordedBefore ? [lt13(siteAuditSnapshots.createdAt, recordedBefore)] : []
|
|
87278
87682
|
];
|
|
87279
87683
|
const complete = sql30`case when ${runs.status} = ${RunStatuses.completed} then 1 else 0 end`;
|
|
87280
|
-
const selectAudit = (pinnedRunId) => app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs,
|
|
87281
|
-
|
|
87282
|
-
|
|
87283
|
-
|
|
87284
|
-
|
|
87285
|
-
|
|
87684
|
+
const selectAudit = (pinnedRunId) => app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs, eq84(siteAuditSnapshots.runId, runs.id)).where(and70(...filters, pinnedRunId ? eq84(siteAuditSnapshots.runId, pinnedRunId) : void 0)).orderBy(
|
|
87685
|
+
desc38(sql30`substr(${siteAuditSnapshots.createdAt}, 1, 10)`),
|
|
87686
|
+
desc38(complete),
|
|
87687
|
+
desc38(siteAuditSnapshots.pagesAudited),
|
|
87688
|
+
desc38(siteAuditSnapshots.createdAt),
|
|
87689
|
+
desc38(siteAuditSnapshots.runId)
|
|
87286
87690
|
).limit(1).get();
|
|
87287
87691
|
const auditOfCrawl = (target) => {
|
|
87288
87692
|
const selected2 = target ? selectAudit(target.snapshot.runId) : void 0;
|
|
@@ -87298,22 +87702,22 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87298
87702
|
const date = selected.snap.createdAt.slice(0, 10);
|
|
87299
87703
|
const nextDate = /* @__PURE__ */ new Date(`${date}T00:00:00.000Z`);
|
|
87300
87704
|
nextDate.setUTCDate(nextDate.getUTCDate() + 1);
|
|
87301
|
-
const sameDate =
|
|
87705
|
+
const sameDate = and70(
|
|
87302
87706
|
...filters,
|
|
87303
|
-
|
|
87707
|
+
gte22(siteAuditSnapshots.createdAt, date),
|
|
87304
87708
|
lt13(siteAuditSnapshots.createdAt, nextDate.toISOString())
|
|
87305
87709
|
);
|
|
87306
|
-
const sameDateRunCount = app.db.select({ value:
|
|
87710
|
+
const sameDateRunCount = app.db.select({ value: count3() }).from(siteAuditSnapshots).innerJoin(runs, eq84(siteAuditSnapshots.runId, runs.id)).where(sameDate).get()?.value ?? 0;
|
|
87307
87711
|
const candidates = app.db.select({
|
|
87308
87712
|
runId: siteAuditSnapshots.runId,
|
|
87309
87713
|
createdAt: siteAuditSnapshots.createdAt,
|
|
87310
87714
|
complete,
|
|
87311
87715
|
pages: siteAuditSnapshots.pagesAudited
|
|
87312
|
-
}).from(siteAuditSnapshots).innerJoin(runs,
|
|
87313
|
-
|
|
87314
|
-
|
|
87315
|
-
|
|
87316
|
-
|
|
87716
|
+
}).from(siteAuditSnapshots).innerJoin(runs, eq84(siteAuditSnapshots.runId, runs.id)).where(sameDate).orderBy(
|
|
87717
|
+
desc38(complete),
|
|
87718
|
+
desc38(siteAuditSnapshots.pagesAudited),
|
|
87719
|
+
desc38(siteAuditSnapshots.createdAt),
|
|
87720
|
+
desc38(siteAuditSnapshots.runId)
|
|
87317
87721
|
).limit(10).all().map((candidate) => ({ ...candidate, complete: Boolean(candidate.complete) }));
|
|
87318
87722
|
const runSelection = {
|
|
87319
87723
|
// A crawl exists here only when none stored a scorecard: this score is a fallback.
|
|
@@ -87338,7 +87742,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87338
87742
|
inventoryEligible: siteCrawlPages.inventoryEligible,
|
|
87339
87743
|
canonicalOther,
|
|
87340
87744
|
canonicalReason,
|
|
87341
|
-
pages:
|
|
87745
|
+
pages: count3(),
|
|
87342
87746
|
exampleUrl: sql30`min(${siteCrawlPages.url})`
|
|
87343
87747
|
}).from(siteCrawlPages).where(where).groupBy(
|
|
87344
87748
|
siteCrawlPages.fetchState,
|
|
@@ -87375,10 +87779,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87375
87779
|
};
|
|
87376
87780
|
const inventorySummaryFor = (scope) => {
|
|
87377
87781
|
if (!scope) return null;
|
|
87378
|
-
const groups = crawlReasonGroups(
|
|
87379
|
-
|
|
87380
|
-
|
|
87381
|
-
|
|
87782
|
+
const groups = crawlReasonGroups(and70(
|
|
87783
|
+
eq84(siteCrawlPages.projectId, scope.projectId),
|
|
87784
|
+
eq84(siteCrawlPages.runId, scope.runId),
|
|
87785
|
+
eq84(siteCrawlPages.attemptId, scope.attemptId)
|
|
87382
87786
|
));
|
|
87383
87787
|
const total = groups.reduce((sum, group) => sum + group.pages, 0);
|
|
87384
87788
|
const eligible = groups.reduce((sum, group) => sum + (group.inventoryEligible ? group.pages : 0), 0);
|
|
@@ -87398,34 +87802,34 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87398
87802
|
const NO_CRAWL_COMPLETENESS = { complete: false, termination: null };
|
|
87399
87803
|
const pageInScope = (scope, selector) => {
|
|
87400
87804
|
if (!selector.nodeKey && !selector.url) return null;
|
|
87401
|
-
return app.db.select().from(siteCrawlPages).where(
|
|
87402
|
-
|
|
87403
|
-
|
|
87404
|
-
|
|
87405
|
-
selector.nodeKey ?
|
|
87805
|
+
return app.db.select().from(siteCrawlPages).where(and70(
|
|
87806
|
+
eq84(siteCrawlPages.projectId, scope.projectId),
|
|
87807
|
+
eq84(siteCrawlPages.runId, scope.runId),
|
|
87808
|
+
eq84(siteCrawlPages.attemptId, scope.attemptId),
|
|
87809
|
+
selector.nodeKey ? eq84(siteCrawlPages.nodeKey, selector.nodeKey) : eq84(siteCrawlPages.url, selector.url)
|
|
87406
87810
|
)).orderBy(asc15(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
87407
87811
|
};
|
|
87408
|
-
const rootPageInScope = (scope, rootUrl) => pageInScope(scope, { url: rootUrl }) ?? app.db.select().from(siteCrawlPages).where(
|
|
87409
|
-
|
|
87410
|
-
|
|
87411
|
-
|
|
87412
|
-
|
|
87812
|
+
const rootPageInScope = (scope, rootUrl) => pageInScope(scope, { url: rootUrl }) ?? app.db.select().from(siteCrawlPages).where(and70(
|
|
87813
|
+
eq84(siteCrawlPages.projectId, scope.projectId),
|
|
87814
|
+
eq84(siteCrawlPages.runId, scope.runId),
|
|
87815
|
+
eq84(siteCrawlPages.attemptId, scope.attemptId),
|
|
87816
|
+
eq84(siteCrawlPages.depth, 0)
|
|
87413
87817
|
)).orderBy(asc15(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
87414
|
-
const isSurfaceableAuditRun = (projectId, runId) => Boolean(app.db.select({ id: runs.id }).from(runs).where(
|
|
87415
|
-
|
|
87416
|
-
|
|
87417
|
-
|
|
87418
|
-
|
|
87818
|
+
const isSurfaceableAuditRun = (projectId, runId) => Boolean(app.db.select({ id: runs.id }).from(runs).where(and70(
|
|
87819
|
+
eq84(runs.id, runId),
|
|
87820
|
+
eq84(runs.projectId, projectId),
|
|
87821
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
87822
|
+
inArray34(runs.status, SURFACEABLE_STATUSES),
|
|
87419
87823
|
notProbeRun()
|
|
87420
87824
|
)).limit(1).get());
|
|
87421
87825
|
const assertKnownAuditRun = (projectId, runId) => {
|
|
87422
87826
|
if (runId && !isSurfaceableAuditRun(projectId, runId)) throw notFound("Site crawl run", runId);
|
|
87423
87827
|
};
|
|
87424
|
-
const hasLegacyAudit = (projectId) => Boolean(app.db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).innerJoin(runs,
|
|
87425
|
-
|
|
87426
|
-
|
|
87427
|
-
|
|
87428
|
-
|
|
87828
|
+
const hasLegacyAudit = (projectId) => Boolean(app.db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).innerJoin(runs, eq84(siteAuditSnapshots.runId, runs.id)).where(and70(
|
|
87829
|
+
eq84(siteAuditSnapshots.projectId, projectId),
|
|
87830
|
+
eq84(runs.projectId, projectId),
|
|
87831
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
87832
|
+
inArray34(runs.status, SURFACEABLE_STATUSES),
|
|
87429
87833
|
notProbeRun()
|
|
87430
87834
|
)).limit(1).get());
|
|
87431
87835
|
const emptyCrawlSummary = (projectName, legacyAuditAvailable) => ({
|
|
@@ -87526,14 +87930,14 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87526
87930
|
};
|
|
87527
87931
|
}
|
|
87528
87932
|
const statusFilter = request.query.status === "success" || request.query.status === "error" ? request.query.status : null;
|
|
87529
|
-
const conds = [
|
|
87530
|
-
if (statusFilter) conds.push(
|
|
87531
|
-
const where =
|
|
87532
|
-
const totalRow = app.db.select({ value:
|
|
87933
|
+
const conds = [eq84(siteAuditPages.projectId, project.id), eq84(siteAuditPages.runId, latest.snap.runId)];
|
|
87934
|
+
if (statusFilter) conds.push(eq84(siteAuditPages.status, statusFilter));
|
|
87935
|
+
const where = and70(...conds);
|
|
87936
|
+
const totalRow = app.db.select({ value: count3() }).from(siteAuditPages).where(where).get();
|
|
87533
87937
|
const total = totalRow?.value ?? 0;
|
|
87534
87938
|
const limit = parsePositiveInt(request.query.limit, 100, 500);
|
|
87535
87939
|
const offset = parsePositiveInt(request.query.offset, 0, Number.MAX_SAFE_INTEGER);
|
|
87536
|
-
const orderBy = request.query.sort === "score-desc" ?
|
|
87940
|
+
const orderBy = request.query.sort === "score-desc" ? desc38(siteAuditPages.overallScore) : request.query.sort === "url" ? asc15(siteAuditPages.url) : asc15(siteAuditPages.overallScore);
|
|
87537
87941
|
const rows = app.db.select().from(siteAuditPages).where(where).orderBy(orderBy).limit(limit).offset(offset).all();
|
|
87538
87942
|
const pages = rows.map((row) => ({
|
|
87539
87943
|
url: row.url,
|
|
@@ -87552,13 +87956,13 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87552
87956
|
auditedAt: siteAuditSnapshots.auditedAt,
|
|
87553
87957
|
aggregateScore: siteAuditSnapshots.aggregateScore,
|
|
87554
87958
|
pagesAudited: siteAuditSnapshots.pagesAudited
|
|
87555
|
-
}).from(siteAuditSnapshots).innerJoin(runs,
|
|
87556
|
-
|
|
87557
|
-
|
|
87558
|
-
|
|
87559
|
-
|
|
87959
|
+
}).from(siteAuditSnapshots).innerJoin(runs, eq84(siteAuditSnapshots.runId, runs.id)).where(and70(
|
|
87960
|
+
eq84(siteAuditSnapshots.projectId, project.id),
|
|
87961
|
+
eq84(runs.projectId, project.id),
|
|
87962
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
87963
|
+
inArray34(runs.status, SURFACEABLE_STATUSES),
|
|
87560
87964
|
notProbeRun()
|
|
87561
|
-
)).orderBy(
|
|
87965
|
+
)).orderBy(desc38(siteAuditSnapshots.createdAt)).limit(limit).all();
|
|
87562
87966
|
return { project: project.name, points: rows.reverse() };
|
|
87563
87967
|
});
|
|
87564
87968
|
app.get("/projects/:name/technical-aeo/crawl", async (request) => {
|
|
@@ -87658,10 +88062,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87658
88062
|
{ projectId: project.id, runId: snapshot.runId, attemptId: snapshot.attemptId },
|
|
87659
88063
|
snapshot.rootUrl
|
|
87660
88064
|
)?.nodeKey ?? null;
|
|
87661
|
-
const persistedLayout = app.db.select().from(siteCrawlGraphLayouts).where(
|
|
87662
|
-
|
|
87663
|
-
|
|
87664
|
-
|
|
88065
|
+
const persistedLayout = app.db.select().from(siteCrawlGraphLayouts).where(and70(
|
|
88066
|
+
eq84(siteCrawlGraphLayouts.projectId, project.id),
|
|
88067
|
+
eq84(siteCrawlGraphLayouts.runId, snapshot.runId),
|
|
88068
|
+
eq84(siteCrawlGraphLayouts.attemptId, snapshot.attemptId)
|
|
87665
88069
|
)).limit(1).get();
|
|
87666
88070
|
if (!persistedLayout) {
|
|
87667
88071
|
return {
|
|
@@ -87714,9 +88118,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87714
88118
|
const maxNodes = parseBoundedLimit(request.query.maxNodes, SITE_CRAWL_GRAPH_DEFAULT_MAX_NODES, SITE_CRAWL_GRAPH_MAX_NODES);
|
|
87715
88119
|
const maxEdges = parseBoundedLimit(request.query.maxEdges, SITE_CRAWL_GRAPH_DEFAULT_MAX_EDGES, SITE_CRAWL_GRAPH_MAX_EDGES);
|
|
87716
88120
|
const graphScope = [
|
|
87717
|
-
|
|
87718
|
-
|
|
87719
|
-
|
|
88121
|
+
eq84(siteCrawlGraphNodes.projectId, project.id),
|
|
88122
|
+
eq84(siteCrawlGraphNodes.runId, snapshot.runId),
|
|
88123
|
+
eq84(siteCrawlGraphNodes.attemptId, snapshot.attemptId)
|
|
87720
88124
|
];
|
|
87721
88125
|
const nodeRows = app.db.select({
|
|
87722
88126
|
nodeKey: siteCrawlPages.nodeKey,
|
|
@@ -87735,18 +88139,18 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87735
88139
|
linkScoreNormalized: siteCrawlPages.linkScoreNormalized,
|
|
87736
88140
|
x: siteCrawlGraphNodes.x,
|
|
87737
88141
|
y: siteCrawlGraphNodes.y
|
|
87738
|
-
}).from(siteCrawlGraphNodes).innerJoin(siteCrawlPages,
|
|
87739
|
-
|
|
87740
|
-
|
|
87741
|
-
|
|
87742
|
-
|
|
87743
|
-
)).where(
|
|
88142
|
+
}).from(siteCrawlGraphNodes).innerJoin(siteCrawlPages, and70(
|
|
88143
|
+
eq84(siteCrawlPages.projectId, siteCrawlGraphNodes.projectId),
|
|
88144
|
+
eq84(siteCrawlPages.runId, siteCrawlGraphNodes.runId),
|
|
88145
|
+
eq84(siteCrawlPages.attemptId, siteCrawlGraphNodes.attemptId),
|
|
88146
|
+
eq84(siteCrawlPages.nodeKey, siteCrawlGraphNodes.nodeKey)
|
|
88147
|
+
)).where(and70(...graphScope, lt13(siteCrawlGraphNodes.sampleRank, maxNodes))).orderBy(asc15(siteCrawlGraphNodes.sampleRank)).all();
|
|
87744
88148
|
const nodes = nodeRows.map(({ indexabilityReasons, canonicalNodeKey, ...row }) => ({
|
|
87745
88149
|
...row,
|
|
87746
88150
|
healthState: deriveSiteHealthState({ ...row, indexabilityReasons, canonicalNodeKey }),
|
|
87747
88151
|
healthReason: deriveSiteHealthReason({ ...row, indexabilityReasons, canonicalNodeKey })
|
|
87748
88152
|
}));
|
|
87749
|
-
const graphLinkKindFilter = linkKind === SiteHealthLinkKinds.all ? void 0 :
|
|
88153
|
+
const graphLinkKindFilter = linkKind === SiteHealthLinkKinds.all ? void 0 : eq84(siteCrawlGraphEdges.isTemplate, linkKind === SiteHealthLinkKinds.template);
|
|
87750
88154
|
const edges = app.db.select({
|
|
87751
88155
|
edgeKey: siteCrawlGraphEdges.edgeKey,
|
|
87752
88156
|
sourceNodeKey: siteCrawlGraphEdges.sourceNodeKey,
|
|
@@ -87754,22 +88158,22 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87754
88158
|
followable: siteCrawlGraphEdges.followable,
|
|
87755
88159
|
occurrences: siteCrawlGraphEdges.occurrences,
|
|
87756
88160
|
isTemplate: siteCrawlGraphEdges.isTemplate
|
|
87757
|
-
}).from(siteCrawlGraphEdges).innerJoin(graphSourceNode,
|
|
87758
|
-
|
|
87759
|
-
|
|
87760
|
-
|
|
87761
|
-
|
|
88161
|
+
}).from(siteCrawlGraphEdges).innerJoin(graphSourceNode, and70(
|
|
88162
|
+
eq84(graphSourceNode.projectId, siteCrawlGraphEdges.projectId),
|
|
88163
|
+
eq84(graphSourceNode.runId, siteCrawlGraphEdges.runId),
|
|
88164
|
+
eq84(graphSourceNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
88165
|
+
eq84(graphSourceNode.nodeKey, siteCrawlGraphEdges.sourceNodeKey),
|
|
87762
88166
|
lt13(graphSourceNode.sampleRank, maxNodes)
|
|
87763
|
-
)).innerJoin(graphTargetNode,
|
|
87764
|
-
|
|
87765
|
-
|
|
87766
|
-
|
|
87767
|
-
|
|
88167
|
+
)).innerJoin(graphTargetNode, and70(
|
|
88168
|
+
eq84(graphTargetNode.projectId, siteCrawlGraphEdges.projectId),
|
|
88169
|
+
eq84(graphTargetNode.runId, siteCrawlGraphEdges.runId),
|
|
88170
|
+
eq84(graphTargetNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
88171
|
+
eq84(graphTargetNode.nodeKey, siteCrawlGraphEdges.targetNodeKey),
|
|
87768
88172
|
lt13(graphTargetNode.sampleRank, maxNodes)
|
|
87769
|
-
)).where(
|
|
87770
|
-
|
|
87771
|
-
|
|
87772
|
-
|
|
88173
|
+
)).where(and70(
|
|
88174
|
+
eq84(siteCrawlGraphEdges.projectId, project.id),
|
|
88175
|
+
eq84(siteCrawlGraphEdges.runId, snapshot.runId),
|
|
88176
|
+
eq84(siteCrawlGraphEdges.attemptId, snapshot.attemptId),
|
|
87773
88177
|
lt13(siteCrawlGraphEdges.sampleRank, maxEdges),
|
|
87774
88178
|
graphLinkKindFilter
|
|
87775
88179
|
)).orderBy(asc15(siteCrawlGraphEdges.sampleRank)).all();
|
|
@@ -87864,15 +88268,15 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87864
88268
|
let frontier = [focus.nodeKey];
|
|
87865
88269
|
let queryTruncated = false;
|
|
87866
88270
|
for (let distance = 0; distance < hops && frontier.length > 0; distance += 1) {
|
|
87867
|
-
const candidates = app.db.select().from(siteCrawlEdges).where(
|
|
87868
|
-
|
|
87869
|
-
|
|
87870
|
-
|
|
87871
|
-
|
|
88271
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and70(
|
|
88272
|
+
eq84(siteCrawlEdges.projectId, scope.projectId),
|
|
88273
|
+
eq84(siteCrawlEdges.runId, scope.runId),
|
|
88274
|
+
eq84(siteCrawlEdges.attemptId, scope.attemptId),
|
|
88275
|
+
eq84(siteCrawlEdges.internal, true),
|
|
87872
88276
|
isNotNull8(siteCrawlEdges.targetNodeKey),
|
|
87873
|
-
|
|
87874
|
-
|
|
87875
|
-
|
|
88277
|
+
or20(
|
|
88278
|
+
inArray34(siteCrawlEdges.sourceNodeKey, frontier),
|
|
88279
|
+
inArray34(siteCrawlEdges.targetNodeKey, frontier)
|
|
87876
88280
|
)
|
|
87877
88281
|
)).orderBy(asc15(siteCrawlEdges.edgeKey)).limit(maxEdges + maxNodes + 1).all();
|
|
87878
88282
|
if (candidates.length > maxEdges + maxNodes) queryTruncated = true;
|
|
@@ -87905,22 +88309,22 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87905
88309
|
frontier = [...next].sort();
|
|
87906
88310
|
}
|
|
87907
88311
|
const nodeKeys = [...distances.keys()];
|
|
87908
|
-
const pageRows = nodeKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(
|
|
87909
|
-
|
|
87910
|
-
|
|
87911
|
-
|
|
87912
|
-
|
|
88312
|
+
const pageRows = nodeKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and70(
|
|
88313
|
+
eq84(siteCrawlPages.projectId, scope.projectId),
|
|
88314
|
+
eq84(siteCrawlPages.runId, scope.runId),
|
|
88315
|
+
eq84(siteCrawlPages.attemptId, scope.attemptId),
|
|
88316
|
+
inArray34(siteCrawlPages.nodeKey, nodeKeys)
|
|
87913
88317
|
)).all();
|
|
87914
88318
|
const persistedNodeKeys = new Set(pageRows.map((row) => row.nodeKey));
|
|
87915
88319
|
const outermostNodeKeys = pageRows.filter((row) => distances.get(row.nodeKey) === hops).map((row) => row.nodeKey);
|
|
87916
|
-
const omittedOutermostEdge = outermostNodeKeys.length === 0 ? void 0 : app.db.select({ edgeKey: siteCrawlEdges.edgeKey }).from(siteCrawlEdges).where(
|
|
87917
|
-
|
|
87918
|
-
|
|
87919
|
-
|
|
87920
|
-
|
|
88320
|
+
const omittedOutermostEdge = outermostNodeKeys.length === 0 ? void 0 : app.db.select({ edgeKey: siteCrawlEdges.edgeKey }).from(siteCrawlEdges).where(and70(
|
|
88321
|
+
eq84(siteCrawlEdges.projectId, scope.projectId),
|
|
88322
|
+
eq84(siteCrawlEdges.runId, scope.runId),
|
|
88323
|
+
eq84(siteCrawlEdges.attemptId, scope.attemptId),
|
|
88324
|
+
eq84(siteCrawlEdges.internal, true),
|
|
87921
88325
|
isNotNull8(siteCrawlEdges.targetNodeKey),
|
|
87922
|
-
|
|
87923
|
-
|
|
88326
|
+
inArray34(siteCrawlEdges.sourceNodeKey, outermostNodeKeys),
|
|
88327
|
+
inArray34(siteCrawlEdges.targetNodeKey, outermostNodeKeys)
|
|
87924
88328
|
)).limit(1).get();
|
|
87925
88329
|
if (omittedOutermostEdge && !seenEdgeKeys.has(omittedOutermostEdge.edgeKey)) {
|
|
87926
88330
|
omittedEdgeKeys.add(omittedOutermostEdge.edgeKey);
|
|
@@ -88013,23 +88417,23 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88013
88417
|
let found = fromPage.nodeKey === toPage.nodeKey;
|
|
88014
88418
|
let truncated = false;
|
|
88015
88419
|
for (let depth = 0; depth < maxDepth && frontier.length > 0 && !found; depth += 1) {
|
|
88016
|
-
const candidates = app.db.select().from(siteCrawlEdges).where(
|
|
88017
|
-
|
|
88018
|
-
|
|
88019
|
-
|
|
88020
|
-
|
|
88021
|
-
|
|
88022
|
-
|
|
88420
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and70(
|
|
88421
|
+
eq84(siteCrawlEdges.projectId, scope.projectId),
|
|
88422
|
+
eq84(siteCrawlEdges.runId, scope.runId),
|
|
88423
|
+
eq84(siteCrawlEdges.attemptId, scope.attemptId),
|
|
88424
|
+
eq84(siteCrawlEdges.internal, true),
|
|
88425
|
+
eq84(siteCrawlEdges.followable, true),
|
|
88426
|
+
eq84(siteCrawlEdges.relation, "anchor"),
|
|
88023
88427
|
isNotNull8(siteCrawlEdges.targetNodeKey),
|
|
88024
|
-
|
|
88428
|
+
inArray34(siteCrawlEdges.sourceNodeKey, frontier)
|
|
88025
88429
|
)).orderBy(asc15(siteCrawlEdges.edgeKey)).limit(SITE_HEALTH_PATH_MAX_VISITED_NODES + 1).all();
|
|
88026
88430
|
if (candidates.length > SITE_HEALTH_PATH_MAX_VISITED_NODES) truncated = true;
|
|
88027
88431
|
const candidateTargetKeys = [...new Set(candidates.map((edge) => edge.targetNodeKey).filter((nodeKey) => nodeKey != null))];
|
|
88028
|
-
const persistedTargetKeys = new Set(candidateTargetKeys.length === 0 ? [] : app.db.select({ nodeKey: siteCrawlPages.nodeKey }).from(siteCrawlPages).where(
|
|
88029
|
-
|
|
88030
|
-
|
|
88031
|
-
|
|
88032
|
-
|
|
88432
|
+
const persistedTargetKeys = new Set(candidateTargetKeys.length === 0 ? [] : app.db.select({ nodeKey: siteCrawlPages.nodeKey }).from(siteCrawlPages).where(and70(
|
|
88433
|
+
eq84(siteCrawlPages.projectId, scope.projectId),
|
|
88434
|
+
eq84(siteCrawlPages.runId, scope.runId),
|
|
88435
|
+
eq84(siteCrawlPages.attemptId, scope.attemptId),
|
|
88436
|
+
inArray34(siteCrawlPages.nodeKey, candidateTargetKeys)
|
|
88033
88437
|
)).all().map((row) => row.nodeKey));
|
|
88034
88438
|
const next = /* @__PURE__ */ new Set();
|
|
88035
88439
|
for (const edge of candidates.slice(0, SITE_HEALTH_PATH_MAX_VISITED_NODES)) {
|
|
@@ -88070,11 +88474,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88070
88474
|
pathEdges.unshift(step.edge);
|
|
88071
88475
|
pathKeys.unshift(step.nodeKey);
|
|
88072
88476
|
}
|
|
88073
|
-
const pathRows = app.db.select().from(siteCrawlPages).where(
|
|
88074
|
-
|
|
88075
|
-
|
|
88076
|
-
|
|
88077
|
-
|
|
88477
|
+
const pathRows = app.db.select().from(siteCrawlPages).where(and70(
|
|
88478
|
+
eq84(siteCrawlPages.projectId, scope.projectId),
|
|
88479
|
+
eq84(siteCrawlPages.runId, scope.runId),
|
|
88480
|
+
eq84(siteCrawlPages.attemptId, scope.attemptId),
|
|
88481
|
+
inArray34(siteCrawlPages.nodeKey, pathKeys)
|
|
88078
88482
|
)).all();
|
|
88079
88483
|
const pageByKey = new Map(pathRows.map((row) => [row.nodeKey, row]));
|
|
88080
88484
|
const pathDetection = templateDetectionOf(snapshot.templateDetection);
|
|
@@ -88102,25 +88506,25 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88102
88506
|
throw validationError("change must be all, added, removed, or changed");
|
|
88103
88507
|
}
|
|
88104
88508
|
const completeSnapshotFilters = [
|
|
88105
|
-
|
|
88106
|
-
|
|
88107
|
-
|
|
88108
|
-
|
|
88109
|
-
|
|
88509
|
+
eq84(siteCrawlSnapshots.projectId, project.id),
|
|
88510
|
+
eq84(runs.projectId, project.id),
|
|
88511
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
88512
|
+
eq84(runs.status, RunStatuses.completed),
|
|
88513
|
+
eq84(siteCrawlSnapshots.complete, true),
|
|
88110
88514
|
notProbeRun()
|
|
88111
88515
|
];
|
|
88112
|
-
const afterTarget = request.query.toRunId ? resolveCrawl(project.id, request.query.toRunId) : app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
88516
|
+
const afterTarget = request.query.toRunId ? resolveCrawl(project.id, request.query.toRunId) : app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq84(siteCrawlSnapshots.runId, runs.id)).where(and70(...completeSnapshotFilters)).orderBy(desc38(siteCrawlSnapshots.createdAt), desc38(siteCrawlSnapshots.runId)).limit(1).get();
|
|
88113
88517
|
if (request.query.toRunId && !afterTarget) throw notFound("Site crawl run", request.query.toRunId);
|
|
88114
|
-
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
88518
|
+
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq84(siteCrawlSnapshots.runId, runs.id)).where(and70(
|
|
88115
88519
|
...completeSnapshotFilters,
|
|
88116
|
-
|
|
88520
|
+
or20(
|
|
88117
88521
|
lt13(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
88118
|
-
|
|
88119
|
-
|
|
88522
|
+
and70(
|
|
88523
|
+
eq84(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
88120
88524
|
lt13(siteCrawlSnapshots.runId, afterTarget.snapshot.runId)
|
|
88121
88525
|
)
|
|
88122
88526
|
)
|
|
88123
|
-
)).orderBy(
|
|
88527
|
+
)).orderBy(desc38(siteCrawlSnapshots.createdAt), desc38(siteCrawlSnapshots.runId)).limit(1).get() : void 0;
|
|
88124
88528
|
if (request.query.fromRunId && !beforeTarget) throw notFound("Site crawl run", request.query.fromRunId);
|
|
88125
88529
|
if (!afterTarget) {
|
|
88126
88530
|
const latestCrawl = resolveCrawl(project.id);
|
|
@@ -88361,18 +88765,18 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88361
88765
|
const visibleKeys = keyRows.slice(0, limit);
|
|
88362
88766
|
const pageKeys = visibleKeys.filter((row) => row.entity === "page").map((row) => row.key);
|
|
88363
88767
|
const linkKeys = visibleKeys.filter((row) => row.entity === "link").map((row) => row.key);
|
|
88364
|
-
const pageRowsFor = (scope) => pageKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(
|
|
88365
|
-
|
|
88366
|
-
|
|
88367
|
-
|
|
88368
|
-
|
|
88768
|
+
const pageRowsFor = (scope) => pageKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and70(
|
|
88769
|
+
eq84(siteCrawlPages.projectId, scope.projectId),
|
|
88770
|
+
eq84(siteCrawlPages.runId, scope.runId),
|
|
88771
|
+
eq84(siteCrawlPages.attemptId, scope.attemptId),
|
|
88772
|
+
inArray34(siteCrawlPages.nodeKey, pageKeys)
|
|
88369
88773
|
)).all();
|
|
88370
|
-
const edgeRowsFor = (scope) => linkKeys.length === 0 ? [] : app.db.select().from(siteCrawlEdges).where(
|
|
88371
|
-
|
|
88372
|
-
|
|
88373
|
-
|
|
88374
|
-
|
|
88375
|
-
|
|
88774
|
+
const edgeRowsFor = (scope) => linkKeys.length === 0 ? [] : app.db.select().from(siteCrawlEdges).where(and70(
|
|
88775
|
+
eq84(siteCrawlEdges.projectId, scope.projectId),
|
|
88776
|
+
eq84(siteCrawlEdges.runId, scope.runId),
|
|
88777
|
+
eq84(siteCrawlEdges.attemptId, scope.attemptId),
|
|
88778
|
+
eq84(siteCrawlEdges.internal, true),
|
|
88779
|
+
inArray34(siteCrawlEdges.edgeKey, linkKeys)
|
|
88376
88780
|
)).all();
|
|
88377
88781
|
const beforePages = new Map(pageRowsFor(beforeScope).map((row) => [row.nodeKey, mapCrawlPage(row)]));
|
|
88378
88782
|
const afterPages = new Map(pageRowsFor(afterScope).map((row) => [row.nodeKey, mapCrawlPage(row)]));
|
|
@@ -88525,16 +88929,16 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88525
88929
|
}
|
|
88526
88930
|
const inventorySummary = inventorySummaryFor(detailScopeFor(project.id, snapshot));
|
|
88527
88931
|
const filters = [
|
|
88528
|
-
|
|
88529
|
-
|
|
88530
|
-
|
|
88932
|
+
eq84(siteCrawlPages.projectId, project.id),
|
|
88933
|
+
eq84(siteCrawlPages.runId, snapshot.runId),
|
|
88934
|
+
eq84(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
88531
88935
|
];
|
|
88532
|
-
if (request.query.nodeKey) filters.push(
|
|
88936
|
+
if (request.query.nodeKey) filters.push(eq84(siteCrawlPages.nodeKey, request.query.nodeKey));
|
|
88533
88937
|
const inventoryEligible = parseBoolean2(request.query.inventoryEligible);
|
|
88534
|
-
if (inventoryEligible != null) filters.push(
|
|
88535
|
-
if (request.query.fetchState) filters.push(
|
|
88536
|
-
if (request.query.indexabilityState) filters.push(
|
|
88537
|
-
if (request.query.auditState) filters.push(
|
|
88938
|
+
if (inventoryEligible != null) filters.push(eq84(siteCrawlPages.inventoryEligible, inventoryEligible));
|
|
88939
|
+
if (request.query.fetchState) filters.push(eq84(siteCrawlPages.fetchState, request.query.fetchState));
|
|
88940
|
+
if (request.query.indexabilityState) filters.push(eq84(siteCrawlPages.indexabilityState, request.query.indexabilityState));
|
|
88941
|
+
if (request.query.auditState) filters.push(eq84(siteCrawlPages.auditState, request.query.auditState));
|
|
88538
88942
|
const healthState = parseSiteHealthState(request.query.healthState);
|
|
88539
88943
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
88540
88944
|
const sort = ["score-desc", "score-asc", "path"].includes(request.query.sort ?? "") ? request.query.sort : "url";
|
|
@@ -88552,17 +88956,17 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88552
88956
|
sort
|
|
88553
88957
|
})).digest("base64url");
|
|
88554
88958
|
const offset = decodeCrawlPageCursor(request.query.cursor, selectionKey);
|
|
88555
|
-
const orderBy = sort === "score-desc" ? [
|
|
88959
|
+
const orderBy = sort === "score-desc" ? [desc38(siteCrawlPages.auditScore), asc15(siteCrawlPages.nodeKey)] : sort === "score-asc" ? [asc15(siteCrawlPages.auditScore), asc15(siteCrawlPages.nodeKey)] : sort === "path" ? [asc15(siteCrawlPages.path), asc15(siteCrawlPages.nodeKey)] : [asc15(siteCrawlPages.url), asc15(siteCrawlPages.nodeKey)];
|
|
88556
88960
|
let healthStateFilter = null;
|
|
88557
88961
|
if (healthState) {
|
|
88558
|
-
const legacyRow = app.db.select({ id: siteCrawlPages.id }).from(siteCrawlPages).where(
|
|
88559
|
-
|
|
88560
|
-
|
|
88561
|
-
|
|
88562
|
-
|
|
88962
|
+
const legacyRow = app.db.select({ id: siteCrawlPages.id }).from(siteCrawlPages).where(and70(
|
|
88963
|
+
eq84(siteCrawlPages.projectId, project.id),
|
|
88964
|
+
eq84(siteCrawlPages.runId, snapshot.runId),
|
|
88965
|
+
eq84(siteCrawlPages.attemptId, snapshot.attemptId),
|
|
88966
|
+
isNull16(siteCrawlPages.healthState)
|
|
88563
88967
|
)).limit(1).get();
|
|
88564
88968
|
healthStateFilter = legacyRow ? "unavailable-legacy-scan" : "applied";
|
|
88565
|
-
if (healthStateFilter === "applied") filters.push(
|
|
88969
|
+
if (healthStateFilter === "applied") filters.push(eq84(siteCrawlPages.healthState, healthState));
|
|
88566
88970
|
}
|
|
88567
88971
|
if (healthStateFilter === "unavailable-legacy-scan") {
|
|
88568
88972
|
return {
|
|
@@ -88579,8 +88983,8 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88579
88983
|
pages: []
|
|
88580
88984
|
};
|
|
88581
88985
|
}
|
|
88582
|
-
const where =
|
|
88583
|
-
const total = app.db.select({ value:
|
|
88986
|
+
const where = and70(...filters);
|
|
88987
|
+
const total = app.db.select({ value: count3() }).from(siteCrawlPages).where(where).get()?.value ?? 0;
|
|
88584
88988
|
const rows = app.db.select().from(siteCrawlPages).where(where).orderBy(...orderBy).limit(limit).offset(offset).all();
|
|
88585
88989
|
const nextOffset = offset + rows.length;
|
|
88586
88990
|
return {
|
|
@@ -88614,10 +89018,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88614
89018
|
url: siteCrawlPages.url,
|
|
88615
89019
|
inventoryEligible: siteCrawlPages.inventoryEligible,
|
|
88616
89020
|
fetchState: siteCrawlPages.fetchState
|
|
88617
|
-
}).from(siteCrawlPages).where(
|
|
88618
|
-
|
|
88619
|
-
|
|
88620
|
-
|
|
89021
|
+
}).from(siteCrawlPages).where(and70(
|
|
89022
|
+
eq84(siteCrawlPages.projectId, project.id),
|
|
89023
|
+
eq84(siteCrawlPages.runId, snapshot.runId),
|
|
89024
|
+
eq84(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
88621
89025
|
)).limit(MAX_STRUCTURE_SOURCE_ROWS + 1).all();
|
|
88622
89026
|
if (sourceRows.length > MAX_STRUCTURE_SOURCE_ROWS) {
|
|
88623
89027
|
throw validationError(`Persisted crawl exceeds the ${MAX_STRUCTURE_SOURCE_ROWS}-page structure limit`);
|
|
@@ -88699,18 +89103,18 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88699
89103
|
};
|
|
88700
89104
|
}
|
|
88701
89105
|
const filters = [
|
|
88702
|
-
|
|
88703
|
-
|
|
88704
|
-
|
|
88705
|
-
|
|
89106
|
+
eq84(siteCrawlEdges.projectId, project.id),
|
|
89107
|
+
eq84(siteCrawlEdges.runId, snapshot.runId),
|
|
89108
|
+
eq84(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
89109
|
+
eq84(siteCrawlEdges.internal, true),
|
|
88706
89110
|
linkKindFilter(linkKind)
|
|
88707
89111
|
];
|
|
88708
|
-
if (request.query.sourceUrl) filters.push(
|
|
88709
|
-
if (request.query.targetUrl) filters.push(
|
|
89112
|
+
if (request.query.sourceUrl) filters.push(eq84(siteCrawlEdges.sourceUrl, request.query.sourceUrl));
|
|
89113
|
+
if (request.query.targetUrl) filters.push(eq84(siteCrawlEdges.targetUrl, request.query.targetUrl));
|
|
88710
89114
|
const followable = parseBoolean2(request.query.followable);
|
|
88711
|
-
if (followable != null) filters.push(
|
|
88712
|
-
const where =
|
|
88713
|
-
const total = app.db.select({ value:
|
|
89115
|
+
if (followable != null) filters.push(eq84(siteCrawlEdges.followable, followable));
|
|
89116
|
+
const where = and70(...filters);
|
|
89117
|
+
const total = app.db.select({ value: count3() }).from(siteCrawlEdges).where(where).get()?.value ?? 0;
|
|
88714
89118
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
88715
89119
|
const offset = decodeCursor(request.query.cursor);
|
|
88716
89120
|
const rows = app.db.select().from(siteCrawlEdges).where(where).orderBy(asc15(siteCrawlEdges.edgeKey)).limit(limit).offset(offset).all();
|
|
@@ -88770,17 +89174,17 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88770
89174
|
};
|
|
88771
89175
|
}
|
|
88772
89176
|
const scope = [
|
|
88773
|
-
|
|
88774
|
-
|
|
88775
|
-
|
|
88776
|
-
|
|
89177
|
+
eq84(siteCrawlEdges.projectId, project.id),
|
|
89178
|
+
eq84(siteCrawlEdges.runId, snapshot.runId),
|
|
89179
|
+
eq84(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
89180
|
+
eq84(siteCrawlEdges.internal, true),
|
|
88777
89181
|
linkKindFilter(linkKind)
|
|
88778
89182
|
];
|
|
88779
|
-
const inboundMatch = request.query.nodeKey && request.query.url ?
|
|
88780
|
-
const outboundMatch = request.query.nodeKey && request.query.url ?
|
|
89183
|
+
const inboundMatch = request.query.nodeKey && request.query.url ? or20(eq84(siteCrawlEdges.targetNodeKey, request.query.nodeKey), eq84(siteCrawlEdges.targetUrl, request.query.url)) : request.query.nodeKey ? eq84(siteCrawlEdges.targetNodeKey, request.query.nodeKey) : eq84(siteCrawlEdges.targetUrl, request.query.url);
|
|
89184
|
+
const outboundMatch = request.query.nodeKey && request.query.url ? or20(eq84(siteCrawlEdges.sourceNodeKey, request.query.nodeKey), eq84(siteCrawlEdges.sourceUrl, request.query.url)) : request.query.nodeKey ? eq84(siteCrawlEdges.sourceNodeKey, request.query.nodeKey) : eq84(siteCrawlEdges.sourceUrl, request.query.url);
|
|
88781
89185
|
const limit = parseBoundedLimit(request.query.limit, 50, 100);
|
|
88782
|
-
const inboundRows = app.db.select().from(siteCrawlEdges).where(
|
|
88783
|
-
const outboundRows = app.db.select().from(siteCrawlEdges).where(
|
|
89186
|
+
const inboundRows = app.db.select().from(siteCrawlEdges).where(and70(...scope, inboundMatch)).orderBy(asc15(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
89187
|
+
const outboundRows = app.db.select().from(siteCrawlEdges).where(and70(...scope, outboundMatch)).orderBy(asc15(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
88784
89188
|
return {
|
|
88785
89189
|
project: project.name,
|
|
88786
89190
|
hasCrawlData: true,
|
|
@@ -88814,11 +89218,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88814
89218
|
if (!snapshot.attemptId) {
|
|
88815
89219
|
return { project: project.name, runId: snapshot.runId, state: "partial", checkDeadLinks: true, checked: snapshot.deadLinksChecked, found: snapshot.deadLinksFound, unverified: snapshot.deadLinksUnverified, total: 0, nextCursor: null, deadLinks: [], runSelection: target.runSelection };
|
|
88816
89220
|
}
|
|
88817
|
-
const where =
|
|
88818
|
-
|
|
88819
|
-
|
|
88820
|
-
|
|
88821
|
-
|
|
89221
|
+
const where = and70(
|
|
89222
|
+
eq84(siteCrawlFindings.projectId, project.id),
|
|
89223
|
+
eq84(siteCrawlFindings.runId, snapshot.runId),
|
|
89224
|
+
eq84(siteCrawlFindings.attemptId, snapshot.attemptId),
|
|
89225
|
+
eq84(siteCrawlFindings.findingType, "dead-link"),
|
|
88822
89226
|
// Belt and braces on the client-facing surface: a row without a status
|
|
88823
89227
|
// code is not evidence of a broken link, and this response is where such
|
|
88824
89228
|
// a row would be read as one. Migration 140 removed every stored row that
|
|
@@ -88830,7 +89234,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88830
89234
|
// promise a page the list cannot fill.
|
|
88831
89235
|
isNotNull8(sql30`json_extract(${siteCrawlFindings.evidence}, '$.statusCode')`)
|
|
88832
89236
|
);
|
|
88833
|
-
const total = app.db.select({ value:
|
|
89237
|
+
const total = app.db.select({ value: count3() }).from(siteCrawlFindings).where(where).get()?.value ?? 0;
|
|
88834
89238
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
88835
89239
|
const offset = decodeCursor(request.query.cursor);
|
|
88836
89240
|
const rows = app.db.select().from(siteCrawlFindings).where(where).orderBy(asc15(siteCrawlFindings.findingKey)).limit(limit).offset(offset).all();
|
|
@@ -88868,19 +89272,19 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88868
89272
|
createdAt: runs.createdAt,
|
|
88869
89273
|
startedAt: runs.startedAt,
|
|
88870
89274
|
finishedAt: runs.finishedAt
|
|
88871
|
-
}).from(runs).where(
|
|
88872
|
-
|
|
88873
|
-
|
|
89275
|
+
}).from(runs).where(and70(
|
|
89276
|
+
eq84(runs.projectId, project.id),
|
|
89277
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
88874
89278
|
notProbeRun()
|
|
88875
|
-
)).orderBy(
|
|
89279
|
+
)).orderBy(desc38(runs.createdAt), desc38(runs.id)).limit(limit).all();
|
|
88876
89280
|
const runIds = rows.map((row) => row.runId);
|
|
88877
|
-
const crawlRunIds = new Set(runIds.length === 0 ? [] : app.db.select({ runId: siteCrawlSnapshots.runId }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
88878
|
-
|
|
88879
|
-
|
|
88880
|
-
|
|
88881
|
-
|
|
89281
|
+
const crawlRunIds = new Set(runIds.length === 0 ? [] : app.db.select({ runId: siteCrawlSnapshots.runId }).from(siteCrawlSnapshots).innerJoin(runs, eq84(siteCrawlSnapshots.runId, runs.id)).where(and70(
|
|
89282
|
+
eq84(siteCrawlSnapshots.projectId, project.id),
|
|
89283
|
+
eq84(runs.projectId, project.id),
|
|
89284
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
89285
|
+
inArray34(runs.status, SURFACEABLE_STATUSES),
|
|
88882
89286
|
notProbeRun(),
|
|
88883
|
-
|
|
89287
|
+
inArray34(siteCrawlSnapshots.runId, runIds)
|
|
88884
89288
|
)).all().map((row) => row.runId));
|
|
88885
89289
|
return {
|
|
88886
89290
|
project: project.name,
|
|
@@ -88894,21 +89298,21 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88894
89298
|
});
|
|
88895
89299
|
app.get("/projects/:name/technical-aeo/runs/:runId/progress", async (request) => {
|
|
88896
89300
|
const project = resolveProject(app.db, request.params.name);
|
|
88897
|
-
const run = app.db.select().from(runs).where(
|
|
88898
|
-
|
|
88899
|
-
|
|
88900
|
-
|
|
89301
|
+
const run = app.db.select().from(runs).where(and70(
|
|
89302
|
+
eq84(runs.id, request.params.runId),
|
|
89303
|
+
eq84(runs.projectId, project.id),
|
|
89304
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
88901
89305
|
notProbeRun()
|
|
88902
89306
|
)).get();
|
|
88903
89307
|
if (!run) throw notFound("Site audit run", request.params.runId);
|
|
88904
|
-
const attempt = app.db.select().from(siteCrawlAttempts).where(
|
|
88905
|
-
|
|
88906
|
-
|
|
88907
|
-
)).orderBy(
|
|
88908
|
-
const persistedLayout = attempt ? app.db.select().from(siteCrawlGraphLayouts).where(
|
|
88909
|
-
|
|
88910
|
-
|
|
88911
|
-
|
|
89308
|
+
const attempt = app.db.select().from(siteCrawlAttempts).where(and70(
|
|
89309
|
+
eq84(siteCrawlAttempts.projectId, project.id),
|
|
89310
|
+
eq84(siteCrawlAttempts.runId, run.id)
|
|
89311
|
+
)).orderBy(desc38(siteCrawlAttempts.attemptNumber), desc38(siteCrawlAttempts.updatedAt)).limit(1).get();
|
|
89312
|
+
const persistedLayout = attempt ? app.db.select().from(siteCrawlGraphLayouts).where(and70(
|
|
89313
|
+
eq84(siteCrawlGraphLayouts.projectId, project.id),
|
|
89314
|
+
eq84(siteCrawlGraphLayouts.runId, run.id),
|
|
89315
|
+
eq84(siteCrawlGraphLayouts.attemptId, attempt.id)
|
|
88912
89316
|
)).get() : void 0;
|
|
88913
89317
|
const layoutState = persistedLayout?.state === "ready" || persistedLayout?.state === "unavailable" ? persistedLayout.state : run.status === RunStatuses.completed || run.status === RunStatuses.partial ? "unavailable" : "pending";
|
|
88914
89318
|
return {
|
|
@@ -88939,19 +89343,19 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88939
89343
|
};
|
|
88940
89344
|
});
|
|
88941
89345
|
app.get("/projects/:name/technical-aeo/runs/:runId/page-health-preview", async (request) => app.db.transaction((tx) => {
|
|
88942
|
-
const project = tx.select().from(projects).where(
|
|
89346
|
+
const project = tx.select().from(projects).where(eq84(projects.name, request.params.name)).get();
|
|
88943
89347
|
if (!project) throw notFound("Project", request.params.name);
|
|
88944
|
-
const run = tx.select().from(runs).where(
|
|
88945
|
-
|
|
88946
|
-
|
|
88947
|
-
|
|
89348
|
+
const run = tx.select().from(runs).where(and70(
|
|
89349
|
+
eq84(runs.id, request.params.runId),
|
|
89350
|
+
eq84(runs.projectId, project.id),
|
|
89351
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
88948
89352
|
notProbeRun()
|
|
88949
89353
|
)).get();
|
|
88950
89354
|
if (!run) throw notFound("Site audit run", request.params.runId);
|
|
88951
|
-
const attempt = tx.select().from(siteCrawlAttempts).where(
|
|
88952
|
-
|
|
88953
|
-
|
|
88954
|
-
)).orderBy(
|
|
89355
|
+
const attempt = tx.select().from(siteCrawlAttempts).where(and70(
|
|
89356
|
+
eq84(siteCrawlAttempts.projectId, project.id),
|
|
89357
|
+
eq84(siteCrawlAttempts.runId, run.id)
|
|
89358
|
+
)).orderBy(desc38(siteCrawlAttempts.attemptNumber), desc38(siteCrawlAttempts.updatedAt)).limit(1).get();
|
|
88955
89359
|
const state = run.status === RunStatuses.queued ? "waiting" : run.status === RunStatuses.running ? "collecting" : "terminal";
|
|
88956
89360
|
const base = {
|
|
88957
89361
|
project: project.name,
|
|
@@ -88964,13 +89368,13 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88964
89368
|
if (!attempt) {
|
|
88965
89369
|
return { ...base, pagesAudited: 0, examples: [] };
|
|
88966
89370
|
}
|
|
88967
|
-
const auditedWhere =
|
|
88968
|
-
|
|
88969
|
-
|
|
88970
|
-
|
|
88971
|
-
|
|
89371
|
+
const auditedWhere = and70(
|
|
89372
|
+
eq84(siteCrawlPages.projectId, project.id),
|
|
89373
|
+
eq84(siteCrawlPages.runId, run.id),
|
|
89374
|
+
eq84(siteCrawlPages.attemptId, attempt.id),
|
|
89375
|
+
eq84(siteCrawlPages.auditState, "success")
|
|
88972
89376
|
);
|
|
88973
|
-
const pagesAudited = tx.select({ value:
|
|
89377
|
+
const pagesAudited = tx.select({ value: count3() }).from(siteCrawlPages).where(auditedWhere).get()?.value ?? 0;
|
|
88974
89378
|
if (state !== "collecting") {
|
|
88975
89379
|
return { ...base, pagesAudited, examples: [] };
|
|
88976
89380
|
}
|
|
@@ -88979,7 +89383,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88979
89383
|
url: siteCrawlPages.url,
|
|
88980
89384
|
auditScore: siteCrawlPages.auditScore,
|
|
88981
89385
|
auditFields: siteCrawlPages.auditFields
|
|
88982
|
-
}).from(siteCrawlPages).where(
|
|
89386
|
+
}).from(siteCrawlPages).where(and70(
|
|
88983
89387
|
auditedWhere,
|
|
88984
89388
|
isNotNull8(siteCrawlPages.auditScore),
|
|
88985
89389
|
lt13(siteCrawlPages.auditScore, 70)
|
|
@@ -89015,13 +89419,13 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
89015
89419
|
status: runs.status,
|
|
89016
89420
|
identityKey: siteCrawlRunRequests.identityKey,
|
|
89017
89421
|
effectiveOptions: siteCrawlRunRequests.effectiveOptions
|
|
89018
|
-
}).from(runs).leftJoin(siteCrawlRunRequests,
|
|
89019
|
-
|
|
89020
|
-
|
|
89021
|
-
)).where(
|
|
89022
|
-
|
|
89023
|
-
|
|
89024
|
-
|
|
89422
|
+
}).from(runs).leftJoin(siteCrawlRunRequests, and70(
|
|
89423
|
+
eq84(siteCrawlRunRequests.projectId, runs.projectId),
|
|
89424
|
+
eq84(siteCrawlRunRequests.runId, runs.id)
|
|
89425
|
+
)).where(and70(
|
|
89426
|
+
eq84(runs.projectId, project.id),
|
|
89427
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
89428
|
+
inArray34(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
89025
89429
|
)).get();
|
|
89026
89430
|
if (existing) {
|
|
89027
89431
|
if (existing.identityKey === identityKey) {
|
|
@@ -89075,7 +89479,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
89075
89479
|
// ../api-routes/src/research.ts
|
|
89076
89480
|
import crypto52 from "crypto";
|
|
89077
89481
|
import { z as z5 } from "zod";
|
|
89078
|
-
import { and as
|
|
89482
|
+
import { and as and71, count as count4, desc as desc39, eq as eq85, gte as gte23, lt as lt14, or as or21, sql as sql31 } from "drizzle-orm";
|
|
89079
89483
|
var sameLocation = (a, b) => a.label === b.label && a.city === b.city && a.region === b.region && a.country === b.country && a.timezone === b.timezone;
|
|
89080
89484
|
var BATCH_RECEIPT_PREFIX = "__canonry_research_batch__:";
|
|
89081
89485
|
async function researchRoutes(app, opts) {
|
|
@@ -89093,7 +89497,7 @@ async function researchRoutes(app, opts) {
|
|
|
89093
89497
|
const input = parsed.data;
|
|
89094
89498
|
if (input.idempotencyKey?.startsWith(BATCH_RECEIPT_PREFIX)) throw validationError("This idempotency key prefix is reserved for internal research batch receipts.");
|
|
89095
89499
|
const requestHash2 = directResearchRequestHash(input);
|
|
89096
|
-
const existingReceipt = input.idempotencyKey ? app.db.select().from(researchRuns).where(
|
|
89500
|
+
const existingReceipt = input.idempotencyKey ? app.db.select().from(researchRuns).where(and71(eq85(researchRuns.projectId, project.id), eq85(researchRuns.idempotencyKey, input.idempotencyKey))).get() : void 0;
|
|
89097
89501
|
if (existingReceipt) {
|
|
89098
89502
|
assertSameDirectResearchRequest(existingReceipt, input, requestHash2);
|
|
89099
89503
|
const result2 = getDetail(app, project.id, existingReceipt.id);
|
|
@@ -89125,7 +89529,7 @@ async function researchRoutes(app, opts) {
|
|
|
89125
89529
|
const initiatedBy = researchPrincipal(request);
|
|
89126
89530
|
const decision = app.db.transaction((tx) => {
|
|
89127
89531
|
if (input.idempotencyKey) {
|
|
89128
|
-
const existing = tx.select().from(researchRuns).where(
|
|
89532
|
+
const existing = tx.select().from(researchRuns).where(and71(eq85(researchRuns.projectId, project.id), eq85(researchRuns.idempotencyKey, input.idempotencyKey))).get();
|
|
89129
89533
|
if (existing) {
|
|
89130
89534
|
assertSameDirectResearchRequest(existing, input, requestHash2);
|
|
89131
89535
|
return { reused: true, id: existing.id, shouldDispatch: existing.status === ResearchRunStatuses.queued };
|
|
@@ -89133,9 +89537,9 @@ async function researchRoutes(app, opts) {
|
|
|
89133
89537
|
}
|
|
89134
89538
|
if (initiatedBy?.limited) {
|
|
89135
89539
|
const { start, end, date } = utcDayBounds(now);
|
|
89136
|
-
const used = tx.select({ value:
|
|
89137
|
-
|
|
89138
|
-
|
|
89540
|
+
const used = tx.select({ value: count4() }).from(researchRuns).where(and71(
|
|
89541
|
+
eq85(researchRuns.projectId, project.id),
|
|
89542
|
+
gte23(researchRuns.createdAt, start),
|
|
89139
89543
|
lt14(researchRuns.createdAt, end),
|
|
89140
89544
|
sql31`(json_extract(${researchRuns.initiatedBy}, '$.role') = ${UserRoles.viewer} OR json_extract(${researchRuns.initiatedBy}, '$.limited') = 1)`
|
|
89141
89545
|
)).get()?.value ?? 0;
|
|
@@ -89165,15 +89569,15 @@ async function researchRoutes(app, opts) {
|
|
|
89165
89569
|
if (input.idempotencyKey.startsWith(BATCH_RECEIPT_PREFIX)) throw validationError("This idempotency key prefix is reserved for internal research batch receipts.");
|
|
89166
89570
|
const requestHash2 = researchBatchRequestHash(input);
|
|
89167
89571
|
const receiptKeys = input.runs.map((_run, index2) => batchChildReceiptKey(input.idempotencyKey, index2));
|
|
89168
|
-
const firstReceipt = app.db.select().from(researchRuns).where(
|
|
89169
|
-
|
|
89170
|
-
|
|
89572
|
+
const firstReceipt = app.db.select().from(researchRuns).where(and71(
|
|
89573
|
+
eq85(researchRuns.projectId, project.id),
|
|
89574
|
+
eq85(researchRuns.idempotencyKey, receiptKeys[0])
|
|
89171
89575
|
)).get();
|
|
89172
89576
|
if (firstReceipt) {
|
|
89173
89577
|
if (firstReceipt.requestHash !== requestHash2) throw alreadyExists("Research batch idempotency key", input.idempotencyKey);
|
|
89174
|
-
const saved = receiptKeys.map((key) => app.db.select().from(researchRuns).where(
|
|
89175
|
-
|
|
89176
|
-
|
|
89578
|
+
const saved = receiptKeys.map((key) => app.db.select().from(researchRuns).where(and71(
|
|
89579
|
+
eq85(researchRuns.projectId, project.id),
|
|
89580
|
+
eq85(researchRuns.idempotencyKey, key)
|
|
89177
89581
|
)).get());
|
|
89178
89582
|
if (saved.some((row) => !row || row.requestHash !== requestHash2)) throw alreadyExists("Research batch idempotency key", input.idempotencyKey);
|
|
89179
89583
|
const runs3 = saved.map((row) => getDetail(app, project.id, row.id));
|
|
@@ -89188,9 +89592,9 @@ async function researchRoutes(app, opts) {
|
|
|
89188
89592
|
const decision = app.db.transaction((tx) => {
|
|
89189
89593
|
if (initiatedBy?.limited) {
|
|
89190
89594
|
const { start, end, date } = utcDayBounds(now);
|
|
89191
|
-
const used = tx.select({ value:
|
|
89192
|
-
|
|
89193
|
-
|
|
89595
|
+
const used = tx.select({ value: count4() }).from(researchRuns).where(and71(
|
|
89596
|
+
eq85(researchRuns.projectId, project.id),
|
|
89597
|
+
gte23(researchRuns.createdAt, start),
|
|
89194
89598
|
lt14(researchRuns.createdAt, end),
|
|
89195
89599
|
sql31`(json_extract(${researchRuns.initiatedBy}, '$.role') = ${UserRoles.viewer} OR json_extract(${researchRuns.initiatedBy}, '$.limited') = 1)`
|
|
89196
89600
|
)).get()?.value ?? 0;
|
|
@@ -89243,10 +89647,10 @@ async function researchRoutes(app, opts) {
|
|
|
89243
89647
|
const requested = Number.parseInt(request.query.limit ?? "", 10);
|
|
89244
89648
|
const limit = Number.isInteger(requested) && requested > 0 ? Math.min(requested, 100) : 20;
|
|
89245
89649
|
const cursor = request.query.cursor ? parseResearchCursor(request.query.cursor, project.id) : null;
|
|
89246
|
-
const rows = app.db.select().from(researchRuns).where(
|
|
89247
|
-
|
|
89248
|
-
cursor ?
|
|
89249
|
-
)).orderBy(
|
|
89650
|
+
const rows = app.db.select().from(researchRuns).where(and71(
|
|
89651
|
+
eq85(researchRuns.projectId, project.id),
|
|
89652
|
+
cursor ? or21(lt14(researchRuns.createdAt, cursor.createdAt), and71(eq85(researchRuns.createdAt, cursor.createdAt), lt14(researchRuns.id, cursor.id))) : void 0
|
|
89653
|
+
)).orderBy(desc39(researchRuns.createdAt), desc39(researchRuns.id)).limit(limit + 1).all();
|
|
89250
89654
|
const runs2 = rows.slice(0, limit).map(serializeRun);
|
|
89251
89655
|
const last = runs2.at(-1);
|
|
89252
89656
|
const nextCursor = rows.length > limit && last ? Buffer.from(JSON.stringify({ projectId: project.id, createdAt: last.createdAt, id: last.id })).toString("base64url") : null;
|
|
@@ -89372,9 +89776,9 @@ function resolveStoredResearchTemplate(stored, selection, idempotencyKey) {
|
|
|
89372
89776
|
return stored;
|
|
89373
89777
|
}
|
|
89374
89778
|
function resolveResearchTemplate(app, projectId, selection, scope, location) {
|
|
89375
|
-
const savedTemplate = app.db.select().from(measurementQueryTemplates).where(
|
|
89376
|
-
|
|
89377
|
-
|
|
89779
|
+
const savedTemplate = app.db.select().from(measurementQueryTemplates).where(and71(
|
|
89780
|
+
eq85(measurementQueryTemplates.projectId, projectId),
|
|
89781
|
+
eq85(measurementQueryTemplates.id, selection.templateId)
|
|
89378
89782
|
)).get();
|
|
89379
89783
|
if (!savedTemplate) throw validationError("Unknown or stale research template. Reload the selected template and try again.");
|
|
89380
89784
|
const templateVersion = savedTemplate.updatedAt;
|
|
@@ -89383,7 +89787,7 @@ function resolveResearchTemplate(app, projectId, selection, scope, location) {
|
|
|
89383
89787
|
}
|
|
89384
89788
|
const bindingLocation = selection.bindingLocation === void 0 ? location : selection.bindingLocation;
|
|
89385
89789
|
if (bindingLocation) {
|
|
89386
|
-
const project = app.db.select({ locations: projects.locations }).from(projects).where(
|
|
89790
|
+
const project = app.db.select({ locations: projects.locations }).from(projects).where(eq85(projects.id, projectId)).get();
|
|
89387
89791
|
if (!project.locations.some((item2) => sameLocation(item2, bindingLocation))) throw validationError("Research template binding location must match a configured project location.");
|
|
89388
89792
|
}
|
|
89389
89793
|
const { bindings, output } = expandResearchTemplate(savedTemplate, scope, bindingLocation);
|
|
@@ -89396,9 +89800,9 @@ function classifyResearchQueries(project, plan, queries2) {
|
|
|
89396
89800
|
return queries2.map((query) => classifier?.classify(query) ?? null);
|
|
89397
89801
|
}
|
|
89398
89802
|
function getDetail(app, projectId, id) {
|
|
89399
|
-
const row = app.db.select().from(researchRuns).where(
|
|
89803
|
+
const row = app.db.select().from(researchRuns).where(and71(eq85(researchRuns.id, id), eq85(researchRuns.projectId, projectId))).get();
|
|
89400
89804
|
if (!row) throw notFound("Research run", id);
|
|
89401
|
-
const queries2 = app.db.select().from(researchRunQueries).where(
|
|
89805
|
+
const queries2 = app.db.select().from(researchRunQueries).where(eq85(researchRunQueries.researchRunId, id)).orderBy(researchRunQueries.position).all().map(serializeQuery);
|
|
89402
89806
|
return { ...serializeRun(row), queries: queries2 };
|
|
89403
89807
|
}
|
|
89404
89808
|
function serializeRun(row) {
|
|
@@ -89428,7 +89832,7 @@ function utcDayBounds(now) {
|
|
|
89428
89832
|
|
|
89429
89833
|
// ../api-routes/src/snapshot-competitor-refresh.ts
|
|
89430
89834
|
import { setImmediate as yieldToEventLoop2 } from "timers/promises";
|
|
89431
|
-
import { and as
|
|
89835
|
+
import { and as and72, asc as asc16, eq as eq86, inArray as inArray35 } from "drizzle-orm";
|
|
89432
89836
|
var ANSWER_MENTIONS_CHUNK_SIZE = 50;
|
|
89433
89837
|
var ANSWER_MENTIONS_MAX_RESTARTS = 3;
|
|
89434
89838
|
function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
@@ -89478,14 +89882,14 @@ function* projectAnswerMentionsSteps(db, projectId, opts, driver) {
|
|
|
89478
89882
|
examined = 0;
|
|
89479
89883
|
wouldUpdate = 0;
|
|
89480
89884
|
mentioned = 0;
|
|
89481
|
-
const runRows = db.select({ id: runs.id, planVersionId: runs.measurementPlanVersionId }).from(runs).where(
|
|
89482
|
-
|
|
89483
|
-
|
|
89484
|
-
opts?.runId === void 0 ? void 0 :
|
|
89885
|
+
const runRows = db.select({ id: runs.id, planVersionId: runs.measurementPlanVersionId }).from(runs).where(and72(
|
|
89886
|
+
eq86(runs.kind, RunKinds["answer-visibility"]),
|
|
89887
|
+
eq86(runs.projectId, projectId),
|
|
89888
|
+
opts?.runId === void 0 ? void 0 : eq86(runs.id, opts.runId)
|
|
89485
89889
|
)).all();
|
|
89486
89890
|
const planVersionByRun = new Map(runRows.map((run) => [run.id, run.planVersionId]));
|
|
89487
89891
|
for (const run of runRows) {
|
|
89488
|
-
const snapshotIds = db.select({ id: querySnapshots.id }).from(querySnapshots).where(
|
|
89892
|
+
const snapshotIds = db.select({ id: querySnapshots.id }).from(querySnapshots).where(eq86(querySnapshots.runId, run.id)).orderBy(asc16(querySnapshots.id)).all().map((row) => row.id);
|
|
89489
89893
|
for (let offset = 0; offset < snapshotIds.length; offset += ANSWER_MENTIONS_CHUNK_SIZE) {
|
|
89490
89894
|
if (chunks++ > 0 && driver.recheckIdentity) {
|
|
89491
89895
|
yield;
|
|
@@ -89511,7 +89915,7 @@ function* projectAnswerMentionsSteps(db, projectId, opts, driver) {
|
|
|
89511
89915
|
competitorOverlap: querySnapshots.competitorOverlap,
|
|
89512
89916
|
recommendedCompetitors: querySnapshots.recommendedCompetitors,
|
|
89513
89917
|
rawResponse: querySnapshots.rawResponse
|
|
89514
|
-
}).from(querySnapshots).where(
|
|
89918
|
+
}).from(querySnapshots).where(inArray35(querySnapshots.id, snapshotIds.slice(offset, offset + ANSWER_MENTIONS_CHUNK_SIZE))).all();
|
|
89515
89919
|
const pendingUpdates = [];
|
|
89516
89920
|
for (const snapshot of snapshotRows) {
|
|
89517
89921
|
if (competitorFieldsOnly && snapshot.answerText == null) continue;
|
|
@@ -89561,7 +89965,7 @@ function* projectAnswerMentionsSteps(db, projectId, opts, driver) {
|
|
|
89561
89965
|
} else {
|
|
89562
89966
|
db.transaction((tx) => {
|
|
89563
89967
|
for (const update of pendingUpdates) {
|
|
89564
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
89968
|
+
tx.update(querySnapshots).set(update.patch).where(eq86(querySnapshots.id, update.id)).run();
|
|
89565
89969
|
}
|
|
89566
89970
|
});
|
|
89567
89971
|
updated += pendingUpdates.length;
|
|
@@ -89591,7 +89995,7 @@ function readStoredGroundingSources(rawResponse) {
|
|
|
89591
89995
|
|
|
89592
89996
|
// ../api-routes/src/simple-measurement-definitions.ts
|
|
89593
89997
|
import crypto53 from "crypto";
|
|
89594
|
-
import { and as
|
|
89998
|
+
import { and as and73, eq as eq87 } from "drizzle-orm";
|
|
89595
89999
|
function captureSimpleMeasurementDefinition(db, input) {
|
|
89596
90000
|
return db.transaction((tx) => {
|
|
89597
90001
|
const run = tx.select({
|
|
@@ -89599,7 +90003,7 @@ function captureSimpleMeasurementDefinition(db, input) {
|
|
|
89599
90003
|
trigger: runs.trigger,
|
|
89600
90004
|
status: runs.status,
|
|
89601
90005
|
measurementPlanVersionId: runs.measurementPlanVersionId
|
|
89602
|
-
}).from(runs).where(
|
|
90006
|
+
}).from(runs).where(and73(eq87(runs.projectId, input.projectId), eq87(runs.id, input.runId))).get();
|
|
89603
90007
|
if (!run) throw notFound("Run", input.runId);
|
|
89604
90008
|
if (run.kind !== RunKinds["answer-visibility"] || run.trigger === RunTriggers.probe || run.measurementPlanVersionId !== null) {
|
|
89605
90009
|
return null;
|
|
@@ -89620,7 +90024,7 @@ function captureSimpleMeasurementDefinition(db, input) {
|
|
|
89620
90024
|
throw validationError("Qualified aliases must be a sorted subset of the captured aliases.");
|
|
89621
90025
|
}
|
|
89622
90026
|
}
|
|
89623
|
-
const existing = tx.select().from(simpleMeasurementDefinitions).where(
|
|
90027
|
+
const existing = tx.select().from(simpleMeasurementDefinitions).where(and73(eq87(simpleMeasurementDefinitions.projectId, input.projectId), eq87(simpleMeasurementDefinitions.runId, input.runId))).get();
|
|
89624
90028
|
if (existing) {
|
|
89625
90029
|
const frozen = simpleMeasurementDefinitionSchema.parse(existing.definition);
|
|
89626
90030
|
const { qualifiedAliases: _liveQualifiedAliases, ...liveIdentity } = definition.identity;
|
|
@@ -89646,15 +90050,15 @@ function captureSimpleMeasurementDefinition(db, input) {
|
|
|
89646
90050
|
if (run.status !== RunStatuses.running) {
|
|
89647
90051
|
throw validationError("Only a running run can capture a new measurement definition.");
|
|
89648
90052
|
}
|
|
89649
|
-
if (tx.select({ id: querySnapshots.id }).from(querySnapshots).where(
|
|
90053
|
+
if (tx.select({ id: querySnapshots.id }).from(querySnapshots).where(eq87(querySnapshots.runId, input.runId)).limit(1).get()) {
|
|
89650
90054
|
throw validationError("This run already has stored answers. Its measurement definition cannot be inferred afterward.");
|
|
89651
90055
|
}
|
|
89652
|
-
const projectQueryIds = new Set(tx.select({ id: queries.id }).from(queries).where(
|
|
90056
|
+
const projectQueryIds = new Set(tx.select({ id: queries.id }).from(queries).where(eq87(queries.projectId, input.projectId)).all().map((query) => query.id));
|
|
89653
90057
|
if (definition.queries.some((query) => !projectQueryIds.has(query.queryId))) {
|
|
89654
90058
|
throw validationError("Every captured query must belong to the run project.");
|
|
89655
90059
|
}
|
|
89656
90060
|
if (definition.competitors !== void 0) {
|
|
89657
|
-
const liveCompetitorDomains = new Set(tx.select({ domain: competitors.domain }).from(competitors).where(
|
|
90061
|
+
const liveCompetitorDomains = new Set(tx.select({ domain: competitors.domain }).from(competitors).where(eq87(competitors.projectId, input.projectId)).orderBy(competitors.domain).all().map((competitor) => competitor.domain));
|
|
89658
90062
|
const capturedDomains = new Set(definition.competitors.map((competitor) => competitor.domain));
|
|
89659
90063
|
if (liveCompetitorDomains.size !== capturedDomains.size || [...liveCompetitorDomains].some((domain) => !capturedDomains.has(domain))) {
|
|
89660
90064
|
throw validationError("Captured competitors must exactly match the project competitors dispatched for this run.");
|
|
@@ -90510,10 +90914,10 @@ var log = createLogger("IntelligenceService");
|
|
|
90510
90914
|
var runChronologyKey = sql32`coalesce(${runs.finishedAt}, ${runs.createdAt})`;
|
|
90511
90915
|
function analyzableRunPredicates(projectId) {
|
|
90512
90916
|
return [
|
|
90513
|
-
|
|
90917
|
+
eq88(runs.projectId, projectId),
|
|
90514
90918
|
// Only this kind writes query_snapshots.
|
|
90515
|
-
|
|
90516
|
-
|
|
90919
|
+
eq88(runs.kind, RunKinds["answer-visibility"]),
|
|
90920
|
+
or22(eq88(runs.status, RunStatuses.completed), eq88(runs.status, RunStatuses.partial)),
|
|
90517
90921
|
// Defensive: RunCoordinator already skips probes before analyzeAndPersist
|
|
90518
90922
|
// is called, but a future call site invoking it directly for a probe must
|
|
90519
90923
|
// still not pollute the intelligence window.
|
|
@@ -90558,16 +90962,16 @@ var IntelligenceService = class {
|
|
|
90558
90962
|
* Returns the analysis result for the coordinator to inspect (e.g. for webhook dispatch).
|
|
90559
90963
|
*/
|
|
90560
90964
|
analyzeAndPersist(runId, projectId) {
|
|
90561
|
-
const currentRunRecord = this.db.select().from(runs).where(
|
|
90965
|
+
const currentRunRecord = this.db.select().from(runs).where(and74(eq88(runs.id, runId), ...analyzableRunPredicates(projectId))).get();
|
|
90562
90966
|
if (!currentRunRecord) {
|
|
90563
90967
|
log.info("intelligence.skip", { runId, reason: "run not eligible for analysis" });
|
|
90564
90968
|
return null;
|
|
90565
90969
|
}
|
|
90566
90970
|
const currentLocation = currentRunRecord.location ?? null;
|
|
90567
90971
|
const recentRuns = this.db.select().from(runs).where(
|
|
90568
|
-
|
|
90972
|
+
and74(
|
|
90569
90973
|
...analyzableRunPredicates(projectId),
|
|
90570
|
-
currentLocation === null ?
|
|
90974
|
+
currentLocation === null ? isNull17(runs.location) : eq88(runs.location, currentLocation),
|
|
90571
90975
|
// Never look forward. The window is this run's own past, so
|
|
90572
90976
|
// re-analyzing a historical run compares it against its true
|
|
90573
90977
|
// predecessor rather than against sweeps that came after it.
|
|
@@ -90575,10 +90979,10 @@ var IntelligenceService = class {
|
|
|
90575
90979
|
// MEASUREMENT: only runs that actually wrote snapshots — see above.
|
|
90576
90980
|
// Index seek on idx_snapshots_run, short-circuited on the first row.
|
|
90577
90981
|
exists2(
|
|
90578
|
-
this.db.select({ one: sql32`1` }).from(querySnapshots).where(
|
|
90982
|
+
this.db.select({ one: sql32`1` }).from(querySnapshots).where(eq88(querySnapshots.runId, runs.id))
|
|
90579
90983
|
)
|
|
90580
90984
|
)
|
|
90581
|
-
).orderBy(
|
|
90985
|
+
).orderBy(desc40(runs.finishedAt), desc40(runs.createdAt)).limit(HISTORY_WINDOW_RUNS).all();
|
|
90582
90986
|
const trackedCompetitors = this.loadTrackedCompetitors(projectId);
|
|
90583
90987
|
const trackedCompetitorDomains = trackedCompetitors.map((competitor) => competitor.domain);
|
|
90584
90988
|
const currentRun = this.buildRunData(
|
|
@@ -90651,7 +91055,7 @@ var IntelligenceService = class {
|
|
|
90651
91055
|
* Returns the persisted insights so the coordinator can count critical/high.
|
|
90652
91056
|
*/
|
|
90653
91057
|
analyzeAndPersistGbp(runId, projectId) {
|
|
90654
|
-
const runRow = this.db.select({ createdAt: runs.createdAt, startedAt: runs.startedAt, finishedAt: runs.finishedAt }).from(runs).where(
|
|
91058
|
+
const runRow = this.db.select({ createdAt: runs.createdAt, startedAt: runs.startedAt, finishedAt: runs.finishedAt }).from(runs).where(eq88(runs.id, runId)).get();
|
|
90655
91059
|
if (!runRow) {
|
|
90656
91060
|
log.info("gbp-intelligence.skip", { runId, reason: "run not found" });
|
|
90657
91061
|
this.persistGbpInsights(runId, projectId, [], []);
|
|
@@ -90659,10 +91063,10 @@ var IntelligenceService = class {
|
|
|
90659
91063
|
}
|
|
90660
91064
|
const windowStart = runRow.startedAt ?? runRow.createdAt;
|
|
90661
91065
|
const windowEnd = runRow.finishedAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
90662
|
-
const selected = this.db.select().from(gbpLocations).where(
|
|
90663
|
-
|
|
90664
|
-
|
|
90665
|
-
|
|
91066
|
+
const selected = this.db.select().from(gbpLocations).where(and74(
|
|
91067
|
+
eq88(gbpLocations.projectId, projectId),
|
|
91068
|
+
eq88(gbpLocations.selected, true),
|
|
91069
|
+
gte24(gbpLocations.syncedAt, windowStart),
|
|
90666
91070
|
lte20(gbpLocations.syncedAt, windowEnd)
|
|
90667
91071
|
)).all();
|
|
90668
91072
|
if (selected.length === 0) {
|
|
@@ -90696,12 +91100,12 @@ var IntelligenceService = class {
|
|
|
90696
91100
|
}
|
|
90697
91101
|
/** Build the per-location signal bundle the GBP analyzer consumes. */
|
|
90698
91102
|
buildGbpLocationSignals(projectId, locationName, displayName, fallbackDate) {
|
|
90699
|
-
const metricRows = this.db.select({ metric: gbpDailyMetrics.metric, date: gbpDailyMetrics.date, value: gbpDailyMetrics.value }).from(gbpDailyMetrics).where(
|
|
90700
|
-
const placeActionRows = this.db.select({ placeActionType: gbpPlaceActions.placeActionType, providerType: gbpPlaceActions.providerType }).from(gbpPlaceActions).where(
|
|
90701
|
-
const lodgingRow = this.db.select({ populatedGroupCount: gbpLodgingSnapshots.populatedGroupCount }).from(gbpLodgingSnapshots).where(
|
|
90702
|
-
const ownerRow = this.db.select({ description: gbpLocations.description }).from(gbpLocations).where(
|
|
91103
|
+
const metricRows = this.db.select({ metric: gbpDailyMetrics.metric, date: gbpDailyMetrics.date, value: gbpDailyMetrics.value }).from(gbpDailyMetrics).where(and74(eq88(gbpDailyMetrics.projectId, projectId), eq88(gbpDailyMetrics.locationName, locationName))).all();
|
|
91104
|
+
const placeActionRows = this.db.select({ placeActionType: gbpPlaceActions.placeActionType, providerType: gbpPlaceActions.providerType }).from(gbpPlaceActions).where(and74(eq88(gbpPlaceActions.projectId, projectId), eq88(gbpPlaceActions.locationName, locationName))).all();
|
|
91105
|
+
const lodgingRow = this.db.select({ populatedGroupCount: gbpLodgingSnapshots.populatedGroupCount }).from(gbpLodgingSnapshots).where(and74(eq88(gbpLodgingSnapshots.projectId, projectId), eq88(gbpLodgingSnapshots.locationName, locationName))).orderBy(desc40(gbpLodgingSnapshots.syncedAt)).limit(1).get();
|
|
91106
|
+
const ownerRow = this.db.select({ description: gbpLocations.description }).from(gbpLocations).where(and74(eq88(gbpLocations.projectId, projectId), eq88(gbpLocations.locationName, locationName))).get();
|
|
90703
91107
|
const descriptionMissing = !(ownerRow?.description ?? "").trim();
|
|
90704
|
-
const placeRow = this.db.select({ attributes: gbpPlaceDetails.attributes }).from(gbpPlaceDetails).where(
|
|
91108
|
+
const placeRow = this.db.select({ attributes: gbpPlaceDetails.attributes }).from(gbpPlaceDetails).where(and74(eq88(gbpPlaceDetails.projectId, projectId), eq88(gbpPlaceDetails.locationName, locationName))).orderBy(desc40(gbpPlaceDetails.syncedAt)).limit(1).get();
|
|
90705
91109
|
const placesAmenities = placeRow ? extractPlaceAmenities(placeRow.attributes) : [];
|
|
90706
91110
|
const summary3 = buildGbpSummary({
|
|
90707
91111
|
locationName,
|
|
@@ -90737,7 +91141,7 @@ var IntelligenceService = class {
|
|
|
90737
91141
|
/** Build the month-over-month keyword series for a location from the
|
|
90738
91142
|
* accumulating gbp_keyword_monthly table (latest complete month vs prior). */
|
|
90739
91143
|
buildGbpKeywordTrend(projectId, locationName) {
|
|
90740
|
-
const rows = this.db.select({ month: gbpKeywordMonthly.month, keyword: gbpKeywordMonthly.keyword, valueCount: gbpKeywordMonthly.valueCount }).from(gbpKeywordMonthly).where(
|
|
91144
|
+
const rows = this.db.select({ month: gbpKeywordMonthly.month, keyword: gbpKeywordMonthly.keyword, valueCount: gbpKeywordMonthly.valueCount }).from(gbpKeywordMonthly).where(and74(eq88(gbpKeywordMonthly.projectId, projectId), eq88(gbpKeywordMonthly.locationName, locationName))).all();
|
|
90741
91145
|
if (rows.length === 0) return { recentMonth: null, priorMonth: null, points: [] };
|
|
90742
91146
|
const months = [...new Set(rows.map((r) => r.month))].sort().reverse();
|
|
90743
91147
|
const recentMonth = months[0] ?? null;
|
|
@@ -90768,7 +91172,7 @@ var IntelligenceService = class {
|
|
|
90768
91172
|
*/
|
|
90769
91173
|
persistGbpInsights(runId, projectId, gbpInsights, coveredLocationNames) {
|
|
90770
91174
|
const covered = new Set(coveredLocationNames);
|
|
90771
|
-
const existing = this.db.select({ id: insights.id, dismissed: insights.dismissed }).from(insights).where(
|
|
91175
|
+
const existing = this.db.select({ id: insights.id, dismissed: insights.dismissed }).from(insights).where(and74(eq88(insights.projectId, projectId), eq88(insights.provider, GBP_INSIGHT_PROVIDER))).all();
|
|
90772
91176
|
const staleIds = [];
|
|
90773
91177
|
const dismissedSlots = /* @__PURE__ */ new Set();
|
|
90774
91178
|
for (const row of existing) {
|
|
@@ -90779,7 +91183,7 @@ var IntelligenceService = class {
|
|
|
90779
91183
|
}
|
|
90780
91184
|
this.db.transaction((tx) => {
|
|
90781
91185
|
for (const id of staleIds) {
|
|
90782
|
-
tx.delete(insights).where(
|
|
91186
|
+
tx.delete(insights).where(eq88(insights.id, id)).run();
|
|
90783
91187
|
}
|
|
90784
91188
|
for (const insight of gbpInsights) {
|
|
90785
91189
|
const parsed = parseGbpInsightId(insight.id);
|
|
@@ -90866,7 +91270,7 @@ var IntelligenceService = class {
|
|
|
90866
91270
|
* create per run + aggregate). DB is left untouched.
|
|
90867
91271
|
*/
|
|
90868
91272
|
backfill(projectName, opts, onProgress) {
|
|
90869
|
-
const project = this.db.select().from(projects).where(
|
|
91273
|
+
const project = this.db.select().from(projects).where(eq88(projects.name, projectName)).get();
|
|
90870
91274
|
if (!project) {
|
|
90871
91275
|
throw new Error(`Project "${projectName}" not found`);
|
|
90872
91276
|
}
|
|
@@ -90878,7 +91282,7 @@ var IntelligenceService = class {
|
|
|
90878
91282
|
}
|
|
90879
91283
|
sinceTimestamp = parsed;
|
|
90880
91284
|
}
|
|
90881
|
-
const allRuns = this.db.select().from(runs).where(
|
|
91285
|
+
const allRuns = this.db.select().from(runs).where(and74(...analyzableRunPredicates(project.id))).orderBy(asc17(runs.finishedAt)).all();
|
|
90882
91286
|
const measuredRunIds = this.runIdsWithSnapshots(allRuns.map((r) => r.id));
|
|
90883
91287
|
const measuredRuns = allRuns.filter((r) => measuredRunIds.has(r.id));
|
|
90884
91288
|
let startIdx = 0;
|
|
@@ -90902,14 +91306,14 @@ var IntelligenceService = class {
|
|
|
90902
91306
|
});
|
|
90903
91307
|
}
|
|
90904
91308
|
let processed = 0;
|
|
90905
|
-
let
|
|
91309
|
+
let skipped4 = 0;
|
|
90906
91310
|
let totalInsights = 0;
|
|
90907
91311
|
const isDryRun = opts?.dryRun === true;
|
|
90908
91312
|
const perRunDelta = [];
|
|
90909
91313
|
let wouldDeleteTotal = 0;
|
|
90910
91314
|
const existingByRunId = /* @__PURE__ */ new Map();
|
|
90911
91315
|
if (isDryRun && targetRuns.length > 0) {
|
|
90912
|
-
const rows = this.db.select({ runId: insights.runId }).from(insights).where(
|
|
91316
|
+
const rows = this.db.select({ runId: insights.runId }).from(insights).where(inArray36(insights.runId, targetRuns.map((r) => r.id))).all();
|
|
90913
91317
|
for (const r of rows) {
|
|
90914
91318
|
if (r.runId == null) continue;
|
|
90915
91319
|
existingByRunId.set(r.runId, (existingByRunId.get(r.runId) ?? 0) + 1);
|
|
@@ -90934,14 +91338,14 @@ var IntelligenceService = class {
|
|
|
90934
91338
|
}
|
|
90935
91339
|
onProgress?.({ runId: run.id, index: i + 1, total: targetRuns.length, insights: result.insights.length });
|
|
90936
91340
|
} else {
|
|
90937
|
-
|
|
91341
|
+
skipped4++;
|
|
90938
91342
|
onProgress?.({ runId: run.id, index: i + 1, total: targetRuns.length, insights: 0 });
|
|
90939
91343
|
}
|
|
90940
91344
|
}
|
|
90941
91345
|
if (isDryRun) {
|
|
90942
91346
|
return {
|
|
90943
91347
|
processed,
|
|
90944
|
-
skipped:
|
|
91348
|
+
skipped: skipped4,
|
|
90945
91349
|
totalInsights,
|
|
90946
91350
|
dryRun: true,
|
|
90947
91351
|
delta: {
|
|
@@ -90952,7 +91356,7 @@ var IntelligenceService = class {
|
|
|
90952
91356
|
}
|
|
90953
91357
|
};
|
|
90954
91358
|
}
|
|
90955
|
-
return { processed, skipped:
|
|
91359
|
+
return { processed, skipped: skipped4, totalInsights };
|
|
90956
91360
|
}
|
|
90957
91361
|
/**
|
|
90958
91362
|
* The project's tracked competitors with every name they answer to
|
|
@@ -90960,7 +91364,7 @@ var IntelligenceService = class {
|
|
|
90960
91364
|
* signals match the same identities the dashboard and reports do.
|
|
90961
91365
|
*/
|
|
90962
91366
|
loadTrackedCompetitors(projectId) {
|
|
90963
|
-
return this.db.select(competitorIdentityColumns).from(competitors).where(
|
|
91367
|
+
return this.db.select(competitorIdentityColumns).from(competitors).where(eq88(competitors.projectId, projectId)).orderBy(competitors.domain).all();
|
|
90964
91368
|
}
|
|
90965
91369
|
/**
|
|
90966
91370
|
* Which of `runIds` actually wrote query snapshots. One grouped read, so
|
|
@@ -90969,7 +91373,7 @@ var IntelligenceService = class {
|
|
|
90969
91373
|
*/
|
|
90970
91374
|
runIdsWithSnapshots(runIds) {
|
|
90971
91375
|
if (runIds.length === 0) return /* @__PURE__ */ new Set();
|
|
90972
|
-
const rows = this.db.select({ runId: querySnapshots.runId }).from(querySnapshots).where(
|
|
91376
|
+
const rows = this.db.select({ runId: querySnapshots.runId }).from(querySnapshots).where(inArray36(querySnapshots.runId, [...runIds])).groupBy(querySnapshots.runId).all();
|
|
90973
91377
|
return new Set(rows.map((r) => r.runId));
|
|
90974
91378
|
}
|
|
90975
91379
|
/**
|
|
@@ -90991,15 +91395,15 @@ var IntelligenceService = class {
|
|
|
90991
91395
|
}
|
|
90992
91396
|
persistResult(result, runId, projectId) {
|
|
90993
91397
|
const previouslyDismissed = /* @__PURE__ */ new Set();
|
|
90994
|
-
const existingInsights = this.db.select({ query: insights.query, provider: insights.provider, type: insights.type, dismissed: insights.dismissed }).from(insights).where(
|
|
91398
|
+
const existingInsights = this.db.select({ query: insights.query, provider: insights.provider, type: insights.type, dismissed: insights.dismissed }).from(insights).where(eq88(insights.runId, runId)).all();
|
|
90995
91399
|
for (const row of existingInsights) {
|
|
90996
91400
|
if (row.dismissed) {
|
|
90997
91401
|
previouslyDismissed.add(`${row.query}:${row.provider}:${row.type}`);
|
|
90998
91402
|
}
|
|
90999
91403
|
}
|
|
91000
91404
|
this.db.transaction((tx) => {
|
|
91001
|
-
tx.delete(insights).where(
|
|
91002
|
-
tx.delete(healthSnapshots).where(
|
|
91405
|
+
tx.delete(insights).where(eq88(insights.runId, runId)).run();
|
|
91406
|
+
tx.delete(healthSnapshots).where(eq88(healthSnapshots.runId, runId)).run();
|
|
91003
91407
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
91004
91408
|
for (const insight of result.insights) {
|
|
91005
91409
|
const wasDismissed = previouslyDismissed.has(`${insight.query}:${insight.provider}:${insight.type}`);
|
|
@@ -91046,9 +91450,9 @@ var IntelligenceService = class {
|
|
|
91046
91450
|
clicks: gscSearchData.clicks,
|
|
91047
91451
|
impressions: gscSearchData.impressions,
|
|
91048
91452
|
position: gscSearchData.position
|
|
91049
|
-
}).from(gscSearchData).where(
|
|
91050
|
-
|
|
91051
|
-
|
|
91453
|
+
}).from(gscSearchData).where(and74(
|
|
91454
|
+
eq88(gscSearchData.projectId, projectId),
|
|
91455
|
+
gte24(gscSearchData.date, window.startDate),
|
|
91052
91456
|
lte20(gscSearchData.date, window.endDate)
|
|
91053
91457
|
)).all();
|
|
91054
91458
|
const byDay = /* @__PURE__ */ new Map();
|
|
@@ -91100,21 +91504,21 @@ var IntelligenceService = class {
|
|
|
91100
91504
|
const key = row.query.toLowerCase();
|
|
91101
91505
|
gscImpressionsByQuery.set(key, (gscImpressionsByQuery.get(key) ?? 0) + row.impressions);
|
|
91102
91506
|
}
|
|
91103
|
-
const projectRow = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
91507
|
+
const projectRow = this.db.select({ locations: projects.locations }).from(projects).where(eq88(projects.id, projectId)).get();
|
|
91104
91508
|
const locationCount = Math.max(
|
|
91105
91509
|
1,
|
|
91106
91510
|
(projectRow?.locations ?? []).length
|
|
91107
91511
|
);
|
|
91108
91512
|
const ROWS_PER_GROUP_BUDGET = Math.max(2, locationCount);
|
|
91109
91513
|
const recentRunRows = this.db.select({ id: runs.id, createdAt: runs.createdAt }).from(runs).where(
|
|
91110
|
-
|
|
91111
|
-
|
|
91112
|
-
|
|
91113
|
-
|
|
91514
|
+
and74(
|
|
91515
|
+
eq88(runs.projectId, projectId),
|
|
91516
|
+
eq88(runs.kind, RunKinds["answer-visibility"]),
|
|
91517
|
+
or22(eq88(runs.status, "completed"), eq88(runs.status, "partial")),
|
|
91114
91518
|
// Defensive — see top of file.
|
|
91115
91519
|
ne16(runs.trigger, RunTriggers.probe)
|
|
91116
91520
|
)
|
|
91117
|
-
).orderBy(
|
|
91521
|
+
).orderBy(desc40(runs.createdAt), desc40(runs.id)).limit((RECURRENCE_LOOKBACK_RUNS + 1) * ROWS_PER_GROUP_BUDGET).all();
|
|
91118
91522
|
const recentGroups = groupRunsByCreatedAt(recentRunRows);
|
|
91119
91523
|
const recentRunIds = [];
|
|
91120
91524
|
const recentRunIdToCreatedAt = /* @__PURE__ */ new Map();
|
|
@@ -91130,7 +91534,7 @@ var IntelligenceService = class {
|
|
|
91130
91534
|
const haveHistory = recentRunIds.length > 0;
|
|
91131
91535
|
const priorRegressionsByPair = /* @__PURE__ */ new Map();
|
|
91132
91536
|
if (haveHistory) {
|
|
91133
|
-
const priorRows = this.db.select({ query: insights.query, provider: insights.provider, runId: insights.runId }).from(insights).where(
|
|
91537
|
+
const priorRows = this.db.select({ query: insights.query, provider: insights.provider, runId: insights.runId }).from(insights).where(and74(eq88(insights.type, "regression"), inArray36(insights.runId, recentRunIds))).all();
|
|
91134
91538
|
const regressionGroups = /* @__PURE__ */ new Map();
|
|
91135
91539
|
for (const row of priorRows) {
|
|
91136
91540
|
if (!row.runId) continue;
|
|
@@ -91159,7 +91563,7 @@ var IntelligenceService = class {
|
|
|
91159
91563
|
});
|
|
91160
91564
|
}
|
|
91161
91565
|
buildRunData(runId, projectId, completedAt, location = null, trackedCompetitors = []) {
|
|
91162
|
-
const projectDomainRow = this.db.select({ canonicalDomain: projects.canonicalDomain, ownedDomains: projects.ownedDomains }).from(projects).where(
|
|
91566
|
+
const projectDomainRow = this.db.select({ canonicalDomain: projects.canonicalDomain, ownedDomains: projects.ownedDomains }).from(projects).where(eq88(projects.id, projectId)).get();
|
|
91163
91567
|
const projectDomains = projectDomainRow ? effectiveDomains({
|
|
91164
91568
|
canonicalDomain: projectDomainRow.canonicalDomain,
|
|
91165
91569
|
ownedDomains: projectDomainRow.ownedDomains
|
|
@@ -91177,7 +91581,7 @@ var IntelligenceService = class {
|
|
|
91177
91581
|
citedDomains: querySnapshots.citedDomains,
|
|
91178
91582
|
rawResponse: querySnapshots.rawResponse,
|
|
91179
91583
|
snapshotLocation: querySnapshots.location
|
|
91180
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
91584
|
+
}).from(querySnapshots).leftJoin(queries, eq88(querySnapshots.queryId, queries.id)).where(eq88(querySnapshots.runId, runId)).all();
|
|
91181
91585
|
const snapshots = [];
|
|
91182
91586
|
let orphanCount = 0;
|
|
91183
91587
|
for (const r of rows) {
|