@canonry/canonry 7.15.0 → 7.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-SG6T3DEP.js → chunk-2S7U2YHI.js} +2 -2
- package/dist/{chunk-SGQBPTWQ.js → chunk-3ZGBSHZ2.js} +2 -2
- package/dist/{chunk-KOBK4I5N.js → chunk-6LS2R477.js} +58 -0
- package/dist/{chunk-LC7GMHOL.js → chunk-KDWTNI4Z.js} +1 -1
- package/dist/{chunk-YFCKHEKM.js → chunk-O6XPXEN6.js} +3 -3
- package/dist/{chunk-7HJYGDRW.js → chunk-SPZRSKCF.js} +708 -452
- package/dist/{chunk-N2MRW2DH.js → chunk-VWSUFTW5.js} +43 -20
- package/dist/cli.js +9 -9
- package/dist/{demo-server-YWGUGZMI.js → demo-server-6OE3H2KL.js} +4 -4
- package/dist/index.js +6 -6
- package/dist/{intelligence-service-7RKUUR7O.js → intelligence-service-USSTKAKC.js} +2 -2
- package/dist/mcp.js +4 -4
- package/package.json +14 -14
|
@@ -64,6 +64,7 @@ import {
|
|
|
64
64
|
DiscoveryCompetitorTypes,
|
|
65
65
|
DiscoveryPromoteCompetitorSkipReasons,
|
|
66
66
|
DiscoverySessionStatuses,
|
|
67
|
+
GBP_NO_SELECTED_LOCATIONS_ERROR,
|
|
67
68
|
GOOGLE_MARKETING_LIVE_READ_SCOPE,
|
|
68
69
|
GOOGLE_MARKETING_STORED_SNAPSHOT_PAGE_MAX,
|
|
69
70
|
GOOGLE_MARKETING_WRITE_SCOPE,
|
|
@@ -350,6 +351,7 @@ import {
|
|
|
350
351
|
deriveWinnabilityClass,
|
|
351
352
|
describeCompetitorAliasRejection,
|
|
352
353
|
describeError,
|
|
354
|
+
describeFetchError,
|
|
353
355
|
describeMarketPinAliasClaim,
|
|
354
356
|
determineAnswerMentioned,
|
|
355
357
|
diagnosticIdentity,
|
|
@@ -389,6 +391,7 @@ import {
|
|
|
389
391
|
formatIsoDate,
|
|
390
392
|
formatIsoDateInTimeZone,
|
|
391
393
|
formatPercent,
|
|
394
|
+
formatRunErrorOneLine,
|
|
392
395
|
formatSignedPercent,
|
|
393
396
|
ga4AiReferralDailyDtoSchema,
|
|
394
397
|
ga4AiReferralHistoryEntrySchema,
|
|
@@ -468,6 +471,7 @@ import {
|
|
|
468
471
|
internalError,
|
|
469
472
|
intersectScopes,
|
|
470
473
|
isBrowserProvider,
|
|
474
|
+
isFetchTransportError,
|
|
471
475
|
isGa4AiAssistantChannel,
|
|
472
476
|
isLocationRedirectStatus,
|
|
473
477
|
isNegativeReviewRating,
|
|
@@ -796,6 +800,7 @@ import {
|
|
|
796
800
|
trafficSourceListResponseSchema,
|
|
797
801
|
trafficStatusResponseSchema,
|
|
798
802
|
trafficSyncResponseSchema,
|
|
803
|
+
truncateUtf16,
|
|
799
804
|
uiTelemetryEventSchema,
|
|
800
805
|
unsupportedKind,
|
|
801
806
|
upgradeCaveatFor,
|
|
@@ -825,10 +830,10 @@ import {
|
|
|
825
830
|
wordpressSchemaDeployResultDtoSchema,
|
|
826
831
|
wordpressSchemaStatusResultDtoSchema,
|
|
827
832
|
wordpressStatusDtoSchema
|
|
828
|
-
} from "./chunk-
|
|
833
|
+
} from "./chunk-6LS2R477.js";
|
|
829
834
|
|
|
830
835
|
// src/intelligence-service.ts
|
|
831
|
-
import { eq as
|
|
836
|
+
import { eq as eq88, desc as desc40, asc as asc17, and as and74, ne as ne16, or as or22, inArray as inArray36, gte as gte24, lte as lte20, isNull as isNull17, sql as sql32, exists as exists2 } from "drizzle-orm";
|
|
832
837
|
|
|
833
838
|
// ../db/src/client.ts
|
|
834
839
|
import { mkdirSync } from "fs";
|
|
@@ -13382,10 +13387,10 @@ function selectSentimentSources(db, projectId, filters) {
|
|
|
13382
13387
|
for (const key of ["expected", "completed"]) out.sourceSlotIds[key].push(...slots[key].map((slot) => pairKey(runId, slot)));
|
|
13383
13388
|
};
|
|
13384
13389
|
let currentRunId = "";
|
|
13385
|
-
const skip = (reason,
|
|
13386
|
-
out.skipped[reason] = (out.skipped[reason] ?? 0) +
|
|
13390
|
+
const skip = (reason, count5 = 1) => {
|
|
13391
|
+
out.skipped[reason] = (out.skipped[reason] ?? 0) + count5;
|
|
13387
13392
|
const byRun = out.skippedByRun[currentRunId] ??= {};
|
|
13388
|
-
byRun[reason] = (byRun[reason] ?? 0) +
|
|
13393
|
+
byRun[reason] = (byRun[reason] ?? 0) + count5;
|
|
13389
13394
|
};
|
|
13390
13395
|
const otherClass = (filters.queryClass ?? "branded") === "branded" ? "non-brand" : "branded";
|
|
13391
13396
|
const otherClassFilters = { ...filters, queryClass: otherClass };
|
|
@@ -13591,7 +13596,7 @@ function selectedSlotCoverage(db, projectId, run, filters, plans) {
|
|
|
13591
13596
|
const coverage = () => ({ expected: expected.size, completed: completed.size, slotIds: { expected: [...expected], completed: [...completed] } });
|
|
13592
13597
|
if (run.status === "failed") return coverage();
|
|
13593
13598
|
const snapshots = db.select({ queryId: querySnapshots.queryId, queryText: querySnapshots.queryText, provider: querySnapshots.provider, servedModel: querySnapshots.servedModel, measurementExecutionId: querySnapshots.measurementExecutionId }).from(querySnapshots).where(eq8(querySnapshots.runId, run.id)).all();
|
|
13594
|
-
const
|
|
13599
|
+
const count5 = (key, edge, recorded) => {
|
|
13595
13600
|
if (!matchesSentimentEdge(edge, filters)) return;
|
|
13596
13601
|
expected.add(key);
|
|
13597
13602
|
if (recorded) completed.add(key);
|
|
@@ -13611,7 +13616,7 @@ function selectedSlotCoverage(db, projectId, run, filters, plans) {
|
|
|
13611
13616
|
if (scopedTargets && !scopedTargets.has(target.stableKey)) continue;
|
|
13612
13617
|
for (const provider of node.context.providers) {
|
|
13613
13618
|
const snapshot = bySlot.get(pairKey(node.stableKey, provider));
|
|
13614
|
-
|
|
13619
|
+
count5(`${node.stableKey}:${provider}`, { queryKey: usage.queryId, queryText: node.queryText, executionNodeKey: node.stableKey, queryClass: assignment.queryClass, propertyKey: target.stableKey, groupKeys: [...indexed.groupKeys.get(target.stableKey) ?? []], marketKeys: [...marketKeys], provider, sourceModel: node.context.models[provider] ?? null, servedModel: snapshot?.servedModel ?? null, context: node.context.location }, Boolean(snapshot));
|
|
13615
13620
|
}
|
|
13616
13621
|
}
|
|
13617
13622
|
} else {
|
|
@@ -13628,7 +13633,7 @@ function selectedSlotCoverage(db, projectId, run, filters, plans) {
|
|
|
13628
13633
|
for (const query of stored.data.queries) for (const engine of stored.data.engines) {
|
|
13629
13634
|
if (!query.queryClass) continue;
|
|
13630
13635
|
const snapshot = bySlot.get(simpleSlotKey(query.queryId, engine.provider));
|
|
13631
|
-
|
|
13636
|
+
count5(`${query.queryId}:${engine.provider}`, { queryKey: query.queryId, queryText: query.queryText, executionNodeKey: null, queryClass: query.queryClass, propertyKey: run.projectId, groupKeys: [], marketKeys: [], provider: engine.provider, sourceModel: engine.requestedModel, servedModel: snapshot?.servedModel ?? null, context: stored.data.location }, Boolean(snapshot));
|
|
13632
13637
|
}
|
|
13633
13638
|
}
|
|
13634
13639
|
return coverage();
|
|
@@ -13815,9 +13820,9 @@ var SentimentService = class {
|
|
|
13815
13820
|
const estimatedInputTokens = this.options.estimate ? inputs.reduce((sum, input) => sum + this.options.estimate(input), 0) : 0;
|
|
13816
13821
|
const expiresAt = new Date(Date.parse(this.now()) + 15 * 6e4).toISOString();
|
|
13817
13822
|
const previewToken = definitionId && settings.enabled && settings.ready && source.assessments.length ? this.sign({ projectId, selection: frozenSelection, definitionId, fingerprint: sentimentHash(source.assessments), expiresAt }) : null;
|
|
13818
|
-
const
|
|
13823
|
+
const skipped4 = source.runIds.flatMap((runId) => Object.entries(source.skippedByRun?.[runId] ?? {}).map(([reason, count5]) => ({ runId, reason, count: count5 })));
|
|
13819
13824
|
const estimateMethod = this.options.estimate ? `Adapter estimate of the full request including all questions and evidence candidates; final billed usage is recorded per attempt.${definitionId ? "" : " Sentiment is not configured for this project, so the estimate uses the current evaluator definition."}` : "Token and cost estimates unavailable on this host.";
|
|
13820
|
-
return { previewToken, expiresAt: previewToken ? expiresAt : null, selection: frozenSelection, evaluationDefinitionId: definitionId, eligibleAssessments: source.assessments.length, alreadyClassified, skipped:
|
|
13825
|
+
return { previewToken, expiresAt: previewToken ? expiresAt : null, selection: frozenSelection, evaluationDefinitionId: definitionId, eligibleAssessments: source.assessments.length, alreadyClassified, skipped: skipped4, estimatedInputTokens, estimatedCostUsd: this.options.estimate ? estimatedInputTokens * 0.042 / 1e6 : null, estimateMethod };
|
|
13821
13826
|
}
|
|
13822
13827
|
/** The receipt carries the first page of attempts; a replay of a long-running job never returns all of them. */
|
|
13823
13828
|
submit(projectId, previewToken, idempotencyKey, actor) {
|
|
@@ -14054,7 +14059,7 @@ var SentimentService = class {
|
|
|
14054
14059
|
const runIds = selections.flatMap((selection2) => selection2.runId ? [selection2.runId] : []).sort();
|
|
14055
14060
|
const definitions = new Set(selections.flatMap((selection2) => selection2.evaluationDefinitionId ? [selection2.evaluationDefinitionId] : []));
|
|
14056
14061
|
const source = { assessments: sources.flatMap((source2) => source2.assessments), runIds, sourceCoverage: { expected: sources.reduce((sum, source2) => sum + source2.sourceCoverage.expected, 0), completed: sources.reduce((sum, source2) => sum + source2.sourceCoverage.completed, 0) }, sourceSlotIds: { expected: sources.flatMap((source2) => source2.sourceSlotIds.expected), completed: sources.flatMap((source2) => source2.sourceSlotIds.completed) }, skipped: sources.reduce((counts2, source2) => {
|
|
14057
|
-
for (const [reason2,
|
|
14062
|
+
for (const [reason2, count5] of Object.entries(source2.skipped)) counts2[reason2] = (counts2[reason2] ?? 0) + count5;
|
|
14058
14063
|
return counts2;
|
|
14059
14064
|
}, {}) };
|
|
14060
14065
|
const selection = { ...selections[0], runId: runIds.length === 1 ? runIds[0] : null, runIds: runIds.length > 1 ? runIds : void 0, evaluationDefinitionId: definitions.size === 1 ? [...definitions][0] : null };
|
|
@@ -17519,7 +17524,7 @@ function heldProviderAccounts(db, params) {
|
|
|
17519
17524
|
const finished = (run) => Date.parse(run.finishedAt ?? run.createdAt);
|
|
17520
17525
|
const now = Date.parse(params.now);
|
|
17521
17526
|
for (const provider of calling) {
|
|
17522
|
-
let
|
|
17527
|
+
let count5 = 0;
|
|
17523
17528
|
let newest = null;
|
|
17524
17529
|
let oldest = null;
|
|
17525
17530
|
let code = null;
|
|
@@ -17546,11 +17551,11 @@ function heldProviderAccounts(db, params) {
|
|
|
17546
17551
|
newest ??= run;
|
|
17547
17552
|
code ??= entry.code;
|
|
17548
17553
|
oldest = run;
|
|
17549
|
-
|
|
17550
|
-
if (
|
|
17554
|
+
count5 += 1;
|
|
17555
|
+
if (count5 === PROVIDER_ACCOUNT_FAILURE_STREAK) break;
|
|
17551
17556
|
}
|
|
17552
17557
|
if (ended) continue;
|
|
17553
|
-
if (
|
|
17558
|
+
if (count5 < PROVIDER_ACCOUNT_FAILURE_STREAK && !vouched) continue;
|
|
17554
17559
|
const retryAfterMs = newest ? finished(newest) + PROVIDER_ACCOUNT_RETRY_HOURS * 36e5 : Date.parse(vouched.retryAfter);
|
|
17555
17560
|
if (!(now < retryAfterMs)) continue;
|
|
17556
17561
|
held.set(provider, {
|
|
@@ -18051,7 +18056,7 @@ async function projectRoutes(app, opts) {
|
|
|
18051
18056
|
app.get("/projects/:name/delete-preview", async (request, reply) => {
|
|
18052
18057
|
const project = resolveProject(app.db, request.params.name);
|
|
18053
18058
|
const pid = project.id;
|
|
18054
|
-
const
|
|
18059
|
+
const count5 = (n) => n ?? 0;
|
|
18055
18060
|
const queryCount = app.db.select({ n: sql7`count(*)` }).from(queries).where(eq19(queries.projectId, pid)).get();
|
|
18056
18061
|
const competitorCount = app.db.select({ n: sql7`count(*)` }).from(competitors).where(eq19(competitors.projectId, pid)).get();
|
|
18057
18062
|
const runCount = app.db.select({ n: sql7`count(*)` }).from(runs).where(eq19(runs.projectId, pid)).get();
|
|
@@ -18061,14 +18066,14 @@ async function projectRoutes(app, opts) {
|
|
|
18061
18066
|
return reply.send({
|
|
18062
18067
|
project: { id: project.id, name: project.name },
|
|
18063
18068
|
cascadeRows: {
|
|
18064
|
-
queries:
|
|
18065
|
-
competitors:
|
|
18066
|
-
runs:
|
|
18067
|
-
snapshots:
|
|
18068
|
-
insights:
|
|
18069
|
+
queries: count5(queryCount?.n),
|
|
18070
|
+
competitors: count5(competitorCount?.n),
|
|
18071
|
+
runs: count5(runCount?.n),
|
|
18072
|
+
snapshots: count5(snapshotCount?.n),
|
|
18073
|
+
insights: count5(insightCount?.n)
|
|
18069
18074
|
},
|
|
18070
18075
|
detachedRows: {
|
|
18071
|
-
auditLog:
|
|
18076
|
+
auditLog: count5(auditLogCount?.n)
|
|
18072
18077
|
}
|
|
18073
18078
|
});
|
|
18074
18079
|
});
|
|
@@ -18673,14 +18678,14 @@ async function queryRoutes(app, opts) {
|
|
|
18673
18678
|
validProviders: validNames
|
|
18674
18679
|
});
|
|
18675
18680
|
}
|
|
18676
|
-
const
|
|
18681
|
+
const count5 = body.count ?? 5;
|
|
18677
18682
|
if (!opts.onGenerateQueries) {
|
|
18678
18683
|
throw notImplemented("Query generation is not supported in this deployment");
|
|
18679
18684
|
}
|
|
18680
18685
|
const existingRows = app.db.select().from(queries).where(eq21(queries.projectId, project.id)).all();
|
|
18681
18686
|
const existingQueries = existingRows.map((r) => r.query);
|
|
18682
18687
|
try {
|
|
18683
|
-
const generated = await opts.onGenerateQueries(provider,
|
|
18688
|
+
const generated = await opts.onGenerateQueries(provider, count5, {
|
|
18684
18689
|
domain: project.canonicalDomain,
|
|
18685
18690
|
displayName: project.displayName,
|
|
18686
18691
|
country: project.country,
|
|
@@ -18806,14 +18811,14 @@ async function queryRoutes(app, opts) {
|
|
|
18806
18811
|
validProviders: validNames
|
|
18807
18812
|
});
|
|
18808
18813
|
}
|
|
18809
|
-
const
|
|
18814
|
+
const count5 = body.count ?? 5;
|
|
18810
18815
|
if (!opts.onGenerateQueries) {
|
|
18811
18816
|
throw notImplemented("Keyword generation is not supported in this deployment");
|
|
18812
18817
|
}
|
|
18813
18818
|
const existingRows = app.db.select().from(queries).where(eq21(queries.projectId, project.id)).all();
|
|
18814
18819
|
const existingQueries = existingRows.map((r) => r.query);
|
|
18815
18820
|
try {
|
|
18816
|
-
const generated = await opts.onGenerateQueries(provider,
|
|
18821
|
+
const generated = await opts.onGenerateQueries(provider, count5, {
|
|
18817
18822
|
domain: project.canonicalDomain,
|
|
18818
18823
|
displayName: project.displayName,
|
|
18819
18824
|
country: project.country,
|
|
@@ -24383,19 +24388,19 @@ function groupChangesFromRows(rows, identities) {
|
|
|
24383
24388
|
});
|
|
24384
24389
|
}
|
|
24385
24390
|
function countsFromRows(rows, groupChanges) {
|
|
24386
|
-
const
|
|
24387
|
-
const ambiguousRows =
|
|
24388
|
-
const unmatchedRows =
|
|
24389
|
-
const invalidRows =
|
|
24390
|
-
const proposedRows =
|
|
24391
|
-
const excludedRows =
|
|
24391
|
+
const count5 = (status) => rows.filter((row) => row.status === status).length;
|
|
24392
|
+
const ambiguousRows = count5("ambiguous");
|
|
24393
|
+
const unmatchedRows = count5("unmatched");
|
|
24394
|
+
const invalidRows = count5("invalid");
|
|
24395
|
+
const proposedRows = count5("proposed");
|
|
24396
|
+
const excludedRows = count5("excluded");
|
|
24392
24397
|
return {
|
|
24393
24398
|
dataRows: rows.length,
|
|
24394
|
-
matchedRows:
|
|
24399
|
+
matchedRows: count5("matched"),
|
|
24395
24400
|
ambiguousRows,
|
|
24396
24401
|
unmatchedRows,
|
|
24397
24402
|
invalidRows,
|
|
24398
|
-
duplicateRows:
|
|
24403
|
+
duplicateRows: count5("duplicate"),
|
|
24399
24404
|
proposedRows,
|
|
24400
24405
|
excludedRows,
|
|
24401
24406
|
needsAttention: ambiguousRows + unmatchedRows + invalidRows + proposedRows + excludedRows,
|
|
@@ -27174,7 +27179,7 @@ function domainRows(answers, limit, readHost = createCitedHostReader()) {
|
|
|
27174
27179
|
const domains = new Set(answerDomains(answer, readHost).values());
|
|
27175
27180
|
for (const domain of domains) counts2.set(domain, (counts2.get(domain) ?? 0) + 1);
|
|
27176
27181
|
}
|
|
27177
|
-
const rows = [...counts2].map(([domain,
|
|
27182
|
+
const rows = [...counts2].map(([domain, count5]) => ({ domain, answers: count5 })).sort((left, right) => right.answers - left.answers || compareText13(left.domain, right.domain));
|
|
27178
27183
|
return { rows: rows.slice(0, limit), total: rows.length };
|
|
27179
27184
|
}
|
|
27180
27185
|
function targetAliasKeys(target) {
|
|
@@ -27462,9 +27467,9 @@ function compactPortfolioPage(response, query, versionId, evidence, cursor) {
|
|
|
27462
27467
|
if (arrays.some((rows, index2) => offsets[index2] > rows.length)) throw validationError("Invalid measurement portfolio cursor offset.");
|
|
27463
27468
|
const requested = Math.min(query.limit ?? 50, arrays[selected].length - offsets[selected]);
|
|
27464
27469
|
const initialSummaryLimit = query.list === void 0 && cursor === void 0 ? MEASUREMENT_PORTFOLIO_DEFAULT_LIMIT : 0;
|
|
27465
|
-
const build = (
|
|
27466
|
-
const pages = arrays.map((rows, index2) => index2 === selected ? rows.slice(offsets[index2], offsets[index2] +
|
|
27467
|
-
const nextOffsets = offsets.map((offset, index2) => index2 === selected ? offset +
|
|
27470
|
+
const build = (count5, summaryLimit2) => {
|
|
27471
|
+
const pages = arrays.map((rows, index2) => index2 === selected ? rows.slice(offsets[index2], offsets[index2] + count5) : rows.slice(0, summaryLimit2));
|
|
27472
|
+
const nextOffsets = offsets.map((offset, index2) => index2 === selected ? offset + count5 : offset);
|
|
27468
27473
|
const hasMore = nextOffsets[selected] < arrays[selected].length;
|
|
27469
27474
|
const omitted = paths.filter((_, index2) => index2 !== selected && pages[index2].length < arrays[index2].length);
|
|
27470
27475
|
return {
|
|
@@ -27522,12 +27527,12 @@ function compactPortfolioPage(response, query, versionId, evidence, cursor) {
|
|
|
27522
27527
|
let low = minimum + 1;
|
|
27523
27528
|
let high = requested;
|
|
27524
27529
|
while (low <= high) {
|
|
27525
|
-
const
|
|
27526
|
-
const candidate = build(
|
|
27530
|
+
const count5 = Math.floor((low + high) / 2);
|
|
27531
|
+
const candidate = build(count5, summaryLimit);
|
|
27527
27532
|
if (fits(candidate)) {
|
|
27528
27533
|
page3 = candidate;
|
|
27529
|
-
low =
|
|
27530
|
-
} else high =
|
|
27534
|
+
low = count5 + 1;
|
|
27535
|
+
} else high = count5 - 1;
|
|
27531
27536
|
}
|
|
27532
27537
|
return measurementPortfolioSummaryResponseSchema.parse(page3);
|
|
27533
27538
|
}
|
|
@@ -27743,7 +27748,7 @@ function tieByMetro(rows) {
|
|
|
27743
27748
|
add(row.metro);
|
|
27744
27749
|
for (const other of row.otherMetros ?? []) add(other);
|
|
27745
27750
|
}
|
|
27746
|
-
return [...counts2.values()].sort((left, right) => right.count - left.count || (left.metro === null ? 1 : 0) - (right.metro === null ? 1 : 0) || compareText13(left.metro?.label ?? "", right.metro?.label ?? "") || compareText13(left.metro?.groupKey ?? "", right.metro?.groupKey ?? "")).map(({ metro, count:
|
|
27751
|
+
return [...counts2.values()].sort((left, right) => right.count - left.count || (left.metro === null ? 1 : 0) - (right.metro === null ? 1 : 0) || compareText13(left.metro?.label ?? "", right.metro?.label ?? "") || compareText13(left.metro?.groupKey ?? "", right.metro?.groupKey ?? "")).map(({ metro, count: count5 }) => ({ metro: metro?.label ?? null, count: count5 }));
|
|
27747
27752
|
}
|
|
27748
27753
|
function tieNamedInstead(rows) {
|
|
27749
27754
|
const grouped = /* @__PURE__ */ new Map();
|
|
@@ -30782,9 +30787,9 @@ function workloadDiff(before, after) {
|
|
|
30782
30787
|
nextSweepNodes: after.size,
|
|
30783
30788
|
nextSweepProviderCalls: sum(after.values()),
|
|
30784
30789
|
addedNodes: added.length,
|
|
30785
|
-
addedProviderCalls: sum(added.map(([,
|
|
30790
|
+
addedProviderCalls: sum(added.map(([, count5]) => count5)),
|
|
30786
30791
|
removedNodes: removed.length,
|
|
30787
|
-
removedProviderCalls: sum(removed.map(([,
|
|
30792
|
+
removedProviderCalls: sum(removed.map(([, count5]) => count5))
|
|
30788
30793
|
};
|
|
30789
30794
|
}
|
|
30790
30795
|
function candidateDiff(state, nextRows, plan, reusedQueryIds, mutatedQueryIds, scopedRemovalQueryIds) {
|
|
@@ -33232,10 +33237,10 @@ function computeQueryChanges(projectQueries, cutoff) {
|
|
|
33232
33237
|
}
|
|
33233
33238
|
const days = [...byDay.entries()].sort((a, b) => a[0].localeCompare(b[0]));
|
|
33234
33239
|
if (days.length <= 1) return [];
|
|
33235
|
-
return days.slice(1).map(([date,
|
|
33240
|
+
return days.slice(1).map(([date, count5]) => ({
|
|
33236
33241
|
date: (/* @__PURE__ */ new Date(date + "T00:00:00.000Z")).toISOString(),
|
|
33237
|
-
delta:
|
|
33238
|
-
label: `+${
|
|
33242
|
+
delta: count5,
|
|
33243
|
+
label: `+${count5} kp`
|
|
33239
33244
|
}));
|
|
33240
33245
|
}
|
|
33241
33246
|
function pooledRate(buckets, rateKey) {
|
|
@@ -33327,15 +33332,15 @@ function buildRankedList(domains, limit, answerTotal, answersWithSources) {
|
|
|
33327
33332
|
function buildCategoryCounts(counts2) {
|
|
33328
33333
|
let grandTotal = 0;
|
|
33329
33334
|
for (const domains of counts2.values()) {
|
|
33330
|
-
for (const
|
|
33335
|
+
for (const count5 of domains.values()) grandTotal += count5;
|
|
33331
33336
|
}
|
|
33332
33337
|
const result = [];
|
|
33333
33338
|
for (const [category, domains] of counts2) {
|
|
33334
33339
|
let categoryTotal = 0;
|
|
33335
33340
|
const domainEntries = [];
|
|
33336
|
-
for (const [domain,
|
|
33337
|
-
categoryTotal +=
|
|
33338
|
-
domainEntries.push({ domain, count:
|
|
33341
|
+
for (const [domain, count5] of domains) {
|
|
33342
|
+
categoryTotal += count5;
|
|
33343
|
+
domainEntries.push({ domain, count: count5 });
|
|
33339
33344
|
}
|
|
33340
33345
|
domainEntries.sort((a, b) => b.count - a.count);
|
|
33341
33346
|
result.push({
|
|
@@ -34831,9 +34836,9 @@ function summarizeVerificationHits(rows) {
|
|
|
34831
34836
|
return { counts: counts2, unsupportedRows, unsupportedHits };
|
|
34832
34837
|
}
|
|
34833
34838
|
function summarizeServer(crawlers, fetches, referrals) {
|
|
34834
|
-
const total = referrals.reduce((
|
|
34835
|
-
const paid = referrals.reduce((
|
|
34836
|
-
const organic = referrals.reduce((
|
|
34839
|
+
const total = referrals.reduce((count5, row) => count5 + row.sessionsOrHits, 0);
|
|
34840
|
+
const paid = referrals.reduce((count5, row) => count5 + row.paidSessionsOrHits, 0);
|
|
34841
|
+
const organic = referrals.reduce((count5, row) => count5 + row.organicSessionsOrHits, 0);
|
|
34837
34842
|
const crawlerHits = summarizeVerificationHits(crawlers);
|
|
34838
34843
|
const userFetchHits = summarizeVerificationHits(fetches);
|
|
34839
34844
|
return {
|
|
@@ -34982,10 +34987,10 @@ function buildOrganicEvidence(db, projectName, periodDays) {
|
|
|
34982
34987
|
const gaWindow = gaStartDate && gaEndDate ? gaRows.filter((row) => inRange(row.date, gaStartDate, gaEndDate)) : [];
|
|
34983
34988
|
const legacyGaCohorts = gaCohorts.map((cohort) => ({
|
|
34984
34989
|
...cohort,
|
|
34985
|
-
organicSessions: gaRows.filter((row) => inRange(row.date, cohort.startDate, cohort.endDate)).reduce((
|
|
34990
|
+
organicSessions: gaRows.filter((row) => inRange(row.date, cohort.startDate, cohort.endDate)).reduce((count5, row) => count5 + row.organicSessions, 0)
|
|
34986
34991
|
}));
|
|
34987
34992
|
let ga4 = gaRows.length ? {
|
|
34988
|
-
organicSessions: gaWindow.reduce((
|
|
34993
|
+
organicSessions: gaWindow.reduce((count5, row) => count5 + row.organicSessions, 0),
|
|
34989
34994
|
cohorts: legacyGaCohorts
|
|
34990
34995
|
} : null;
|
|
34991
34996
|
if (nativeAcquisitionActive && nativeAcquisition.periods.length === 0) {
|
|
@@ -35008,8 +35013,8 @@ function buildOrganicEvidence(db, projectName, periodDays) {
|
|
|
35008
35013
|
const allGaAiRows = db.select().from(gaAiReferrals).where(and38(eq45(gaAiReferrals.projectId, project.id), gte11(gaAiReferrals.date, gaStartDate ?? startDate), lte10(gaAiReferrals.date, gaEndDate ?? endDate))).all().filter((row) => row.sourceDimension === "session");
|
|
35009
35014
|
const aiRows = gaStartDate && gaEndDate ? allGaAiRows.filter((row) => inRange(row.date, gaStartDate, gaEndDate)) : [];
|
|
35010
35015
|
const gaAiReferralSummary = allGaAiRows.length ? {
|
|
35011
|
-
paidSessions: aiRows.filter((row) => row.trafficClass === AiReferralTrafficClasses.paid).reduce((
|
|
35012
|
-
organicSessions: aiRows.filter((row) => row.trafficClass === AiReferralTrafficClasses.organic).reduce((
|
|
35016
|
+
paidSessions: aiRows.filter((row) => row.trafficClass === AiReferralTrafficClasses.paid).reduce((count5, row) => count5 + row.sessions, 0),
|
|
35017
|
+
organicSessions: aiRows.filter((row) => row.trafficClass === AiReferralTrafficClasses.organic).reduce((count5, row) => count5 + row.sessions, 0)
|
|
35013
35018
|
} : null;
|
|
35014
35019
|
const serverSources = db.select().from(trafficSources).where(eq45(trafficSources.projectId, project.id)).all();
|
|
35015
35020
|
const serverStart = `${startDate}T00:00:00`;
|
|
@@ -49152,12 +49157,12 @@ async function getLodging(accessToken, locationName, opts = {}) {
|
|
|
49152
49157
|
}
|
|
49153
49158
|
}
|
|
49154
49159
|
function countPopulatedGroups(lodging) {
|
|
49155
|
-
let
|
|
49160
|
+
let count5 = 0;
|
|
49156
49161
|
for (const [key, value] of Object.entries(lodging)) {
|
|
49157
49162
|
if (key === "name" || key === "metadata") continue;
|
|
49158
|
-
if (isPopulated(value))
|
|
49163
|
+
if (isPopulated(value)) count5++;
|
|
49159
49164
|
}
|
|
49160
|
-
return
|
|
49165
|
+
return count5;
|
|
49161
49166
|
}
|
|
49162
49167
|
function isPopulated(value) {
|
|
49163
49168
|
if (value === null || value === void 0) return false;
|
|
@@ -54400,7 +54405,7 @@ function createActivationStore(app, request) {
|
|
|
54400
54405
|
function countManifest(manifest) {
|
|
54401
54406
|
return {
|
|
54402
54407
|
adGroupCount: manifest.campaign.adGroups.length,
|
|
54403
|
-
adCount: manifest.campaign.adGroups.reduce((
|
|
54408
|
+
adCount: manifest.campaign.adGroups.reduce((count5, group) => count5 + group.ads.length, 0)
|
|
54404
54409
|
};
|
|
54405
54410
|
}
|
|
54406
54411
|
function resolveAdsActivationMaxEntities(env = process.env) {
|
|
@@ -61316,11 +61321,11 @@ async function wordpressRoutes(app, opts) {
|
|
|
61316
61321
|
const result = await bulkSetSeoMeta(connection, metaEntries);
|
|
61317
61322
|
const applied = result.results.filter((r) => r.status === "applied").length;
|
|
61318
61323
|
const manual = result.results.filter((r) => r.status === "manual").length;
|
|
61319
|
-
const
|
|
61324
|
+
const skipped4 = result.results.filter((r) => r.status === "skipped").length;
|
|
61320
61325
|
steps.push({
|
|
61321
61326
|
name: "set-meta",
|
|
61322
61327
|
status: "completed",
|
|
61323
|
-
summary: `${applied} applied, ${manual} manual-assist, ${
|
|
61328
|
+
summary: `${applied} applied, ${manual} manual-assist, ${skipped4} skipped`
|
|
61324
61329
|
});
|
|
61325
61330
|
}
|
|
61326
61331
|
} catch (err) {
|
|
@@ -61342,11 +61347,11 @@ async function wordpressRoutes(app, opts) {
|
|
|
61342
61347
|
const result = await deploySchemaFromProfile(connection, profile);
|
|
61343
61348
|
const deployed = result.results.filter((r) => r.status === "deployed").length;
|
|
61344
61349
|
const stripped = result.results.filter((r) => r.status === "stripped").length;
|
|
61345
|
-
const
|
|
61350
|
+
const skipped4 = result.results.filter((r) => r.status === "skipped").length;
|
|
61346
61351
|
steps.push({
|
|
61347
61352
|
name: "schema-deploy",
|
|
61348
61353
|
status: "completed",
|
|
61349
|
-
summary: `${deployed} deployed, ${stripped} stripped (manual-assist), ${
|
|
61354
|
+
summary: `${deployed} deployed, ${stripped} stripped (manual-assist), ${skipped4} skipped`
|
|
61350
61355
|
});
|
|
61351
61356
|
}
|
|
61352
61357
|
} catch (err) {
|
|
@@ -82649,10 +82654,10 @@ function dailyCoverage(ctx, month, source) {
|
|
|
82649
82654
|
beforeConnectionDays++;
|
|
82650
82655
|
continue;
|
|
82651
82656
|
}
|
|
82652
|
-
const
|
|
82653
|
-
if (
|
|
82657
|
+
const count5 = byDate.get(date);
|
|
82658
|
+
if (count5 !== void 0) {
|
|
82654
82659
|
observedDays++;
|
|
82655
|
-
if (
|
|
82660
|
+
if (count5 === 0) observedZeroDays++;
|
|
82656
82661
|
} else if (coverage && zeroProvenThrough && date >= coverage.start && date <= zeroProvenThrough) queriedEmptyDays++;
|
|
82657
82662
|
else absent.push(date);
|
|
82658
82663
|
}
|
|
@@ -83489,6 +83494,16 @@ var adsRecentSyncCheck = {
|
|
|
83489
83494
|
var ADS_CHECKS = [adsConnectionCheck, adsRecentSyncCheck];
|
|
83490
83495
|
|
|
83491
83496
|
// ../api-routes/src/doctor/checks/ga-auth.ts
|
|
83497
|
+
function unreachable(step, err, details) {
|
|
83498
|
+
const error = describeFetchError(err, step.requestUrl);
|
|
83499
|
+
return {
|
|
83500
|
+
status: CheckStatuses.fail,
|
|
83501
|
+
code: step.code,
|
|
83502
|
+
summary: `Could not reach Google to ${step.action}: ${error}`,
|
|
83503
|
+
remediation: "Check that this host can resolve and connect to Google (DNS filtering, firewall, or proxy), then re-run. The credential was not tested.",
|
|
83504
|
+
details: { ...details, error }
|
|
83505
|
+
};
|
|
83506
|
+
}
|
|
83492
83507
|
async function checkServiceAccount(conn) {
|
|
83493
83508
|
if (!conn.propertyId) {
|
|
83494
83509
|
return {
|
|
@@ -83513,6 +83528,9 @@ async function checkServiceAccount(conn) {
|
|
|
83513
83528
|
try {
|
|
83514
83529
|
await verifyConnection(conn.clientEmail, conn.privateKey, conn.propertyId);
|
|
83515
83530
|
} catch (err) {
|
|
83531
|
+
if (isFetchTransportError(err)) {
|
|
83532
|
+
return unreachable({ code: "ga.auth.verify-unreachable", action: "verify the GA4 service account" }, err, { propertyId: conn.propertyId, authMethod: "service-account" });
|
|
83533
|
+
}
|
|
83516
83534
|
const message = describeError(err);
|
|
83517
83535
|
return {
|
|
83518
83536
|
status: CheckStatuses.fail,
|
|
@@ -83562,6 +83580,9 @@ async function checkOAuthConnection(ctx, projectName, conn) {
|
|
|
83562
83580
|
const tokens = await refreshAccessToken(auth.clientId, auth.clientSecret, conn.refreshToken);
|
|
83563
83581
|
accessToken = tokens.access_token;
|
|
83564
83582
|
} catch (err) {
|
|
83583
|
+
if (isFetchTransportError(err)) {
|
|
83584
|
+
return unreachable({ code: "ga.auth.refresh-unreachable", action: "refresh the GA4 token", requestUrl: GOOGLE_TOKEN_URL }, err, { propertyId: conn.propertyId, authMethod: "oauth" });
|
|
83585
|
+
}
|
|
83565
83586
|
const message = describeError(err);
|
|
83566
83587
|
return {
|
|
83567
83588
|
status: CheckStatuses.fail,
|
|
@@ -83574,6 +83595,9 @@ async function checkOAuthConnection(ctx, projectName, conn) {
|
|
|
83574
83595
|
try {
|
|
83575
83596
|
await verifyConnectionWithToken(accessToken, conn.propertyId);
|
|
83576
83597
|
} catch (err) {
|
|
83598
|
+
if (isFetchTransportError(err)) {
|
|
83599
|
+
return unreachable({ code: "ga.auth.verify-unreachable", action: "verify GA4 property access", requestUrl: GA4_DATA_API_BASE }, err, { propertyId: conn.propertyId, authMethod: "oauth" });
|
|
83600
|
+
}
|
|
83577
83601
|
const message = describeError(err);
|
|
83578
83602
|
return {
|
|
83579
83603
|
status: CheckStatuses.fail,
|
|
@@ -84102,6 +84126,19 @@ async function resolveGbpToken(ctx) {
|
|
|
84102
84126
|
const tokens = await refreshAccessToken(auth.clientId, auth.clientSecret, conn.refreshToken);
|
|
84103
84127
|
return { ok: true, token: { accessToken: tokens.access_token, conn } };
|
|
84104
84128
|
} catch (err) {
|
|
84129
|
+
if (isFetchTransportError(err)) {
|
|
84130
|
+
const error = describeFetchError(err, GOOGLE_TOKEN_URL);
|
|
84131
|
+
return {
|
|
84132
|
+
ok: false,
|
|
84133
|
+
output: {
|
|
84134
|
+
status: CheckStatuses.fail,
|
|
84135
|
+
code: "gbp.auth.refresh-unreachable",
|
|
84136
|
+
summary: `Could not reach Google to refresh the GBP token: ${error}`,
|
|
84137
|
+
remediation: "Check that this host can resolve and connect to Google (DNS filtering, firewall, or proxy), then re-run. The stored grant was not tested.",
|
|
84138
|
+
details: { domain: conn.domain, error }
|
|
84139
|
+
}
|
|
84140
|
+
};
|
|
84141
|
+
}
|
|
84105
84142
|
const message = describeError(err);
|
|
84106
84143
|
return {
|
|
84107
84144
|
ok: false,
|
|
@@ -84234,6 +84271,16 @@ var accountAccessCheck = {
|
|
|
84234
84271
|
details: { reason: err.reason ?? void 0, status: err.status }
|
|
84235
84272
|
};
|
|
84236
84273
|
}
|
|
84274
|
+
if (isFetchTransportError(err)) {
|
|
84275
|
+
const error = describeFetchError(err, GBP_ACCOUNT_MANAGEMENT_BASE);
|
|
84276
|
+
return {
|
|
84277
|
+
status: CheckStatuses.fail,
|
|
84278
|
+
code: "gbp.account.list-unreachable",
|
|
84279
|
+
summary: `Could not reach the Business Profile API to list GBP accounts: ${error}`,
|
|
84280
|
+
remediation: "Check that this host can resolve and connect to the Business Profile API (DNS filtering, firewall, or proxy), then re-run. Account access was not tested.",
|
|
84281
|
+
details: { error }
|
|
84282
|
+
};
|
|
84283
|
+
}
|
|
84237
84284
|
const message = describeError(err);
|
|
84238
84285
|
return {
|
|
84239
84286
|
status: CheckStatuses.fail,
|
|
@@ -84470,10 +84517,10 @@ var reviewsAccessCheck = {
|
|
|
84470
84517
|
reason: loc.reviewsAccessReason,
|
|
84471
84518
|
coverage: coverageOf(loc)
|
|
84472
84519
|
}));
|
|
84473
|
-
const
|
|
84474
|
-
const full =
|
|
84475
|
-
const partial =
|
|
84476
|
-
const none =
|
|
84520
|
+
const count5 = (coverage) => perLocation.filter((loc) => loc.coverage === coverage).length;
|
|
84521
|
+
const full = count5("full");
|
|
84522
|
+
const partial = count5("partial");
|
|
84523
|
+
const none = count5("none");
|
|
84477
84524
|
const details = { placesFallback, full, partial, none, locations: perLocation };
|
|
84478
84525
|
if (full === locations.length) {
|
|
84479
84526
|
return {
|
|
@@ -84574,6 +84621,19 @@ async function resolveAccessToken(ctx) {
|
|
|
84574
84621
|
const tokens = await refreshAccessToken(auth.clientId, auth.clientSecret, conn.refreshToken);
|
|
84575
84622
|
return { ok: true, token: { accessToken: tokens.access_token } };
|
|
84576
84623
|
} catch (err) {
|
|
84624
|
+
if (isFetchTransportError(err)) {
|
|
84625
|
+
const error = describeFetchError(err, GOOGLE_TOKEN_URL);
|
|
84626
|
+
return {
|
|
84627
|
+
ok: false,
|
|
84628
|
+
output: {
|
|
84629
|
+
status: CheckStatuses.fail,
|
|
84630
|
+
code: "google.auth.refresh-unreachable",
|
|
84631
|
+
summary: `Could not reach Google to refresh the GSC token: ${error}`,
|
|
84632
|
+
remediation: "Check that this host can resolve and connect to Google (DNS filtering, firewall, or proxy), then re-run. The stored grant was not tested.",
|
|
84633
|
+
details: { domain: conn.domain, error }
|
|
84634
|
+
}
|
|
84635
|
+
};
|
|
84636
|
+
}
|
|
84577
84637
|
const message = describeError(err);
|
|
84578
84638
|
return {
|
|
84579
84639
|
ok: false,
|
|
@@ -84666,6 +84726,16 @@ var propertyAccessCheck = {
|
|
|
84666
84726
|
details: { error: err.message }
|
|
84667
84727
|
};
|
|
84668
84728
|
}
|
|
84729
|
+
if (isFetchTransportError(err)) {
|
|
84730
|
+
const error = describeFetchError(err, GSC_API_BASE);
|
|
84731
|
+
return {
|
|
84732
|
+
status: CheckStatuses.fail,
|
|
84733
|
+
code: "google.auth.list-sites-unreachable",
|
|
84734
|
+
summary: `Could not reach Search Console to list the authorized account's sites: ${error}`,
|
|
84735
|
+
remediation: "Check that this host can resolve and connect to the Search Console API (DNS filtering, firewall, or proxy), then re-run. Property access was not tested.",
|
|
84736
|
+
details: { error }
|
|
84737
|
+
};
|
|
84738
|
+
}
|
|
84669
84739
|
const message = describeError(err);
|
|
84670
84740
|
return {
|
|
84671
84741
|
status: CheckStatuses.fail,
|
|
@@ -85302,7 +85372,7 @@ async function runValidator(source, validator, fallbackId, fallbackLabel) {
|
|
|
85302
85372
|
function summarizePerSourceResults(fallbackId, fallbackLabel, results) {
|
|
85303
85373
|
const failed = results.filter((r) => r.output.status === CheckStatuses.fail);
|
|
85304
85374
|
const warned = results.filter((r) => r.output.status === CheckStatuses.warn);
|
|
85305
|
-
const
|
|
85375
|
+
const skipped4 = results.filter((r) => r.output.status === CheckStatuses.skipped);
|
|
85306
85376
|
const ok = results.filter((r) => r.output.status === CheckStatuses.ok);
|
|
85307
85377
|
const detail = {
|
|
85308
85378
|
sources: results.map((r) => ({
|
|
@@ -85336,7 +85406,7 @@ function summarizePerSourceResults(fallbackId, fallbackLabel, results) {
|
|
|
85336
85406
|
return {
|
|
85337
85407
|
status: CheckStatuses.ok,
|
|
85338
85408
|
code: `traffic.${fallbackId}.ok`,
|
|
85339
|
-
summary: `${ok.length} source(s) passed ${fallbackLabel} validation${
|
|
85409
|
+
summary: `${ok.length} source(s) passed ${fallbackLabel} validation${skipped4.length > 0 ? ` (${skipped4.length} skipped)` : ""}.`,
|
|
85340
85410
|
details: detail
|
|
85341
85411
|
};
|
|
85342
85412
|
}
|
|
@@ -85344,7 +85414,7 @@ function summarizePerSourceResults(fallbackId, fallbackLabel, results) {
|
|
|
85344
85414
|
status: CheckStatuses.skipped,
|
|
85345
85415
|
code: `traffic.${fallbackId}.all-skipped`,
|
|
85346
85416
|
summary: `No source-type validator was available for any of the ${results.length} connected source(s).`,
|
|
85347
|
-
remediation:
|
|
85417
|
+
remediation: skipped4.find((result) => result.output.remediation)?.output.remediation,
|
|
85348
85418
|
details: detail
|
|
85349
85419
|
};
|
|
85350
85420
|
}
|
|
@@ -86006,6 +86076,166 @@ var GOOGLE_MARKETING_DOCTOR_CHECKS = createGoogleMarketingDoctorChecks(
|
|
|
86006
86076
|
(ctx) => ctx.getGoogleMarketingDoctorInput?.(ctx)
|
|
86007
86077
|
);
|
|
86008
86078
|
|
|
86079
|
+
// ../api-routes/src/doctor/checks/integration-sync.ts
|
|
86080
|
+
import { and as and68, count as count2, desc as desc36, eq as eq81, gt as gt7, gte as gte20, inArray as inArray32, isNull as isNull14, notInArray as notInArray2, or as or18 } from "drizzle-orm";
|
|
86081
|
+
var SYNC_FAILURE_WINDOW = 3;
|
|
86082
|
+
var SUMMARY_ERROR_LIMIT = 300;
|
|
86083
|
+
var FINISHED_STATUSES = [RunStatuses.completed, RunStatuses.partial, RunStatuses.failed];
|
|
86084
|
+
var SUCCESS_STATUSES = [RunStatuses.completed, RunStatuses.partial];
|
|
86085
|
+
var CONNECTED = { state: "connected", since: null };
|
|
86086
|
+
function googleMarketingConnection(provider) {
|
|
86087
|
+
return (ctx) => {
|
|
86088
|
+
const input = ctx.getGoogleMarketingDoctorInput?.(ctx);
|
|
86089
|
+
if (!input) return { state: "store-unavailable" };
|
|
86090
|
+
return input[provider] ? CONNECTED : { state: "not-connected" };
|
|
86091
|
+
};
|
|
86092
|
+
}
|
|
86093
|
+
function gscConnection(ctx, project) {
|
|
86094
|
+
if (!ctx.googleConnectionStore) return { state: "store-unavailable" };
|
|
86095
|
+
return ctx.googleConnectionStore.getConnection(project.canonicalDomain, "gsc") ? CONNECTED : { state: "not-connected" };
|
|
86096
|
+
}
|
|
86097
|
+
function gbpConnection(ctx, project) {
|
|
86098
|
+
if (!ctx.googleConnectionStore) return { state: "store-unavailable" };
|
|
86099
|
+
if (!ctx.googleConnectionStore.getConnection(project.canonicalDomain, "gbp")) return { state: "not-connected" };
|
|
86100
|
+
const selected = ctx.db.select({ locationName: gbpLocations.locationName }).from(gbpLocations).where(and68(eq81(gbpLocations.projectId, project.id), eq81(gbpLocations.selected, true))).limit(1).get();
|
|
86101
|
+
return selected ? CONNECTED : { state: "no-selected-locations" };
|
|
86102
|
+
}
|
|
86103
|
+
function gaConnection(ctx, project) {
|
|
86104
|
+
const state = gaConnected(ctx);
|
|
86105
|
+
if (state === "store-unavailable") return { state };
|
|
86106
|
+
if (state === "not-connected") return { state };
|
|
86107
|
+
const serviceAccount = ctx.ga4CredentialStore?.getConnection(project.name);
|
|
86108
|
+
const since = serviceAccount ? serviceAccount.createdAt : ctx.googleConnectionStore?.getConnection(project.canonicalDomain, "ga4")?.createdAt;
|
|
86109
|
+
return { state: "connected", since: since || null };
|
|
86110
|
+
}
|
|
86111
|
+
var SPECS = [
|
|
86112
|
+
{
|
|
86113
|
+
prefix: "google-ads",
|
|
86114
|
+
label: "Google Ads",
|
|
86115
|
+
kind: RunKinds["google-ads-sync"],
|
|
86116
|
+
connection: googleMarketingConnection("googleAds"),
|
|
86117
|
+
syncCommand: (name) => `canonry google-ads sync ${name}`,
|
|
86118
|
+
causeCheckIds: ["google-ads.auth.connection", "google-ads.auth.scopes", "google-ads.account.context"]
|
|
86119
|
+
},
|
|
86120
|
+
{
|
|
86121
|
+
prefix: "gtm",
|
|
86122
|
+
label: "Google Tag Manager",
|
|
86123
|
+
kind: RunKinds["gtm-sync"],
|
|
86124
|
+
connection: googleMarketingConnection("gtm"),
|
|
86125
|
+
syncCommand: (name) => `canonry gtm sync ${name}`,
|
|
86126
|
+
causeCheckIds: ["gtm.auth.connection", "gtm.auth.scopes", "gtm.container.context"]
|
|
86127
|
+
},
|
|
86128
|
+
{
|
|
86129
|
+
prefix: "gbp",
|
|
86130
|
+
label: "Google Business Profile",
|
|
86131
|
+
kind: RunKinds["gbp-sync"],
|
|
86132
|
+
connection: gbpConnection,
|
|
86133
|
+
syncCommand: (name) => `canonry gbp sync ${name}`,
|
|
86134
|
+
causeCheckIds: ["gbp.auth.connection", "gbp.auth.scopes", "gbp.account.access"],
|
|
86135
|
+
// `gbp_locations` does not record when a location was selected, so the
|
|
86136
|
+
// runs from before the selection are recognized by the error they stored.
|
|
86137
|
+
nothingToSyncErrors: [serializeRunError({ message: GBP_NO_SELECTED_LOCATIONS_ERROR })]
|
|
86138
|
+
},
|
|
86139
|
+
{
|
|
86140
|
+
prefix: "ga",
|
|
86141
|
+
label: "GA4",
|
|
86142
|
+
kind: RunKinds["ga-sync"],
|
|
86143
|
+
connection: gaConnection,
|
|
86144
|
+
syncCommand: (name) => `canonry ga sync ${name}`,
|
|
86145
|
+
causeCheckIds: ["ga.auth.connection"]
|
|
86146
|
+
},
|
|
86147
|
+
{
|
|
86148
|
+
prefix: "gsc",
|
|
86149
|
+
label: "Search Console",
|
|
86150
|
+
kind: RunKinds["gsc-sync"],
|
|
86151
|
+
connection: gscConnection,
|
|
86152
|
+
syncCommand: (name) => `canonry google sync ${name}`,
|
|
86153
|
+
causeCheckIds: ["google.auth.connection", "google.auth.property-access", "google.auth.scopes"]
|
|
86154
|
+
}
|
|
86155
|
+
];
|
|
86156
|
+
function skipped3(code, summary3) {
|
|
86157
|
+
return { status: CheckStatuses.skipped, code, summary: summary3, remediation: null };
|
|
86158
|
+
}
|
|
86159
|
+
function latestErrorText(raw) {
|
|
86160
|
+
const parsed = parseRunError(raw);
|
|
86161
|
+
return parsed ? formatRunErrorOneLine(parsed) : "no error was recorded";
|
|
86162
|
+
}
|
|
86163
|
+
function gradeSyncFailures(ctx, spec) {
|
|
86164
|
+
const code = (suffix) => `${spec.prefix}.sync.${suffix}`;
|
|
86165
|
+
const project = ctx.project;
|
|
86166
|
+
if (!project) return skipped3(code("no-project"), "Project context required.");
|
|
86167
|
+
const connection = spec.connection(ctx, project);
|
|
86168
|
+
if (connection.state === "store-unavailable") {
|
|
86169
|
+
return skipped3(code("store-unavailable"), `No ${spec.label} connection store configured for this deployment.`);
|
|
86170
|
+
}
|
|
86171
|
+
if (connection.state === "not-connected") {
|
|
86172
|
+
return skipped3(code("not-connected"), `${spec.label} is not connected for this project.`);
|
|
86173
|
+
}
|
|
86174
|
+
if (connection.state === "no-selected-locations") {
|
|
86175
|
+
return skipped3(code("no-selected-locations"), `No ${spec.label} locations are selected for this project.`);
|
|
86176
|
+
}
|
|
86177
|
+
const nothingToSync = spec.nothingToSyncErrors ?? [];
|
|
86178
|
+
const scope = and68(
|
|
86179
|
+
eq81(runs.projectId, project.id),
|
|
86180
|
+
eq81(runs.kind, spec.kind),
|
|
86181
|
+
...nothingToSync.length > 0 ? [or18(isNull14(runs.error), notInArray2(runs.error, [...nothingToSync]))] : []
|
|
86182
|
+
);
|
|
86183
|
+
const graded = connection.since ? and68(scope, gte20(runs.createdAt, connection.since)) : scope;
|
|
86184
|
+
const recent = ctx.db.select({ status: runs.status, error: runs.error, createdAt: runs.createdAt }).from(runs).where(and68(graded, inArray32(runs.status, FINISHED_STATUSES))).orderBy(desc36(runs.createdAt)).limit(SYNC_FAILURE_WINDOW).all();
|
|
86185
|
+
if (recent.length === 0) {
|
|
86186
|
+
return skipped3(code("no-runs"), `${spec.label} is connected but no sync has finished yet.`);
|
|
86187
|
+
}
|
|
86188
|
+
const lastSuccess = ctx.db.select({ createdAt: runs.createdAt, finishedAt: runs.finishedAt }).from(runs).where(and68(graded, inArray32(runs.status, SUCCESS_STATUSES))).orderBy(desc36(runs.createdAt)).limit(1).get();
|
|
86189
|
+
const failuresSinceSuccess = ctx.db.select({ value: count2() }).from(runs).where(and68(
|
|
86190
|
+
graded,
|
|
86191
|
+
eq81(runs.status, RunStatuses.failed),
|
|
86192
|
+
...lastSuccess ? [gt7(runs.createdAt, lastSuccess.createdAt)] : []
|
|
86193
|
+
)).get()?.value ?? 0;
|
|
86194
|
+
const lastSuccessAt = lastSuccess ? lastSuccess.finishedAt ?? lastSuccess.createdAt : null;
|
|
86195
|
+
const latest = recent[0];
|
|
86196
|
+
const latestFailed = latest.status === RunStatuses.failed;
|
|
86197
|
+
const latestError = latestFailed ? latestErrorText(latest.error) : null;
|
|
86198
|
+
const details = {
|
|
86199
|
+
runKind: spec.kind,
|
|
86200
|
+
window: SYNC_FAILURE_WINDOW,
|
|
86201
|
+
consecutiveFailures: failuresSinceSuccess,
|
|
86202
|
+
latestStatus: latest.status,
|
|
86203
|
+
latestRunAt: latest.createdAt,
|
|
86204
|
+
latestError,
|
|
86205
|
+
lastSuccessAt,
|
|
86206
|
+
// Where the graded runs start (the current connection); null grades every run.
|
|
86207
|
+
gradedSince: connection.since
|
|
86208
|
+
};
|
|
86209
|
+
const repeatedFailure = recent.length >= SYNC_FAILURE_WINDOW && recent.every((run) => run.status === RunStatuses.failed);
|
|
86210
|
+
if (repeatedFailure) {
|
|
86211
|
+
const errorText = latestErrorText(latest.error);
|
|
86212
|
+
const shownError = errorText.length > SUMMARY_ERROR_LIMIT ? `${truncateUtf16(errorText, SUMMARY_ERROR_LIMIT)}...` : errorText;
|
|
86213
|
+
const lastSuccessText = lastSuccessAt ?? (connection.since ? `none since connecting at ${connection.since}` : "never");
|
|
86214
|
+
return {
|
|
86215
|
+
status: CheckStatuses.fail,
|
|
86216
|
+
code: code("repeated-failures"),
|
|
86217
|
+
summary: `The last ${failuresSinceSuccess} ${spec.label} syncs failed (last success: ${lastSuccessText}). Latest error: ${shownError}`,
|
|
86218
|
+
remediation: `Fix the cause the latest error names (DNS or network filtering of the API host, credentials, or account access), then run \`${spec.syncCommand(project.name)}\`.`,
|
|
86219
|
+
details
|
|
86220
|
+
};
|
|
86221
|
+
}
|
|
86222
|
+
return {
|
|
86223
|
+
status: CheckStatuses.ok,
|
|
86224
|
+
code: code("ok"),
|
|
86225
|
+
summary: latestFailed ? `The latest ${spec.label} sync failed, but not ${SYNC_FAILURE_WINDOW} in a row.` : `The latest ${spec.label} sync ${latest.status === RunStatuses.partial ? "partly completed" : "completed"}.`,
|
|
86226
|
+
remediation: null,
|
|
86227
|
+
details
|
|
86228
|
+
};
|
|
86229
|
+
}
|
|
86230
|
+
var INTEGRATION_SYNC_CHECKS = SPECS.map((spec) => ({
|
|
86231
|
+
id: `${spec.prefix}.sync.recent-failures`,
|
|
86232
|
+
category: CheckCategories.integrations,
|
|
86233
|
+
scope: CheckScopes.project,
|
|
86234
|
+
title: `${spec.label} sync failures`,
|
|
86235
|
+
supersededBy: spec.causeCheckIds,
|
|
86236
|
+
run: (ctx) => gradeSyncFailures(ctx, spec)
|
|
86237
|
+
}));
|
|
86238
|
+
|
|
86009
86239
|
// ../api-routes/src/doctor/checks/version.ts
|
|
86010
86240
|
var versionCurrentCheck = {
|
|
86011
86241
|
id: "canonry.version.current",
|
|
@@ -86086,6 +86316,7 @@ var ALL_CHECKS = [
|
|
|
86086
86316
|
...REFERRAL_ASSESSMENT_CHECKS,
|
|
86087
86317
|
...ADS_CHECKS,
|
|
86088
86318
|
...GOOGLE_MARKETING_DOCTOR_CHECKS,
|
|
86319
|
+
...INTEGRATION_SYNC_CHECKS,
|
|
86089
86320
|
...PROVIDERS_CHECKS,
|
|
86090
86321
|
...TRAFFIC_SOURCE_CHECKS,
|
|
86091
86322
|
...BACKLINKS_CHECKS,
|
|
@@ -86098,6 +86329,9 @@ function scheduledHealthCheckIds() {
|
|
|
86098
86329
|
return ALL_CHECKS.filter((check2) => check2.scope === CheckScopes.project && !check2.optIn && check2.notificationPolicy !== CheckNotificationPolicies.silent).map((check2) => check2.id);
|
|
86099
86330
|
}
|
|
86100
86331
|
|
|
86332
|
+
// ../api-routes/src/doctor/types.ts
|
|
86333
|
+
var UNREACHABLE_CODE_SUFFIX = "-unreachable";
|
|
86334
|
+
|
|
86101
86335
|
// ../api-routes/src/doctor/runner.ts
|
|
86102
86336
|
function matchesCheckId(checkId, filters) {
|
|
86103
86337
|
if (filters.length === 0) return true;
|
|
@@ -86152,14 +86386,36 @@ async function runChecks(ctx, checks, options = {}) {
|
|
|
86152
86386
|
durationMs: Date.now() - checkStarted
|
|
86153
86387
|
});
|
|
86154
86388
|
}
|
|
86389
|
+
const failedCauses = new Set(results.filter((result, index2) => namesCause(selected[index2], result)).map((result) => result.id));
|
|
86390
|
+
const graded = results.map((result, index2) => supersede(selected[index2], result, failedCauses));
|
|
86155
86391
|
return {
|
|
86156
86392
|
scope: targetScope,
|
|
86157
86393
|
project: projectName,
|
|
86158
86394
|
...ctx.project ? { reportMonths: reportMonthsForDoctor(ctx.reportMonth, startedAt) } : {},
|
|
86159
86395
|
generatedAt: startedAt.toISOString(),
|
|
86160
86396
|
durationMs: Date.now() - startedAt.getTime(),
|
|
86161
|
-
summary: summarizeCheckResults(
|
|
86162
|
-
checks:
|
|
86397
|
+
summary: summarizeCheckResults(graded),
|
|
86398
|
+
checks: graded
|
|
86399
|
+
};
|
|
86400
|
+
}
|
|
86401
|
+
function threw(definition, result) {
|
|
86402
|
+
return result.code === `${definition.id}.runtime-error`;
|
|
86403
|
+
}
|
|
86404
|
+
function namesCause(definition, result) {
|
|
86405
|
+
return result.status === CheckStatuses.fail && !threw(definition, result) && !result.code.endsWith(UNREACHABLE_CODE_SUFFIX);
|
|
86406
|
+
}
|
|
86407
|
+
function supersede(definition, result, failedCauses) {
|
|
86408
|
+
if (result.status !== CheckStatuses.fail && result.status !== CheckStatuses.warn) return result;
|
|
86409
|
+
if (threw(definition, result)) return result;
|
|
86410
|
+
const causes = (definition.supersededBy ?? []).filter((id) => failedCauses.has(id));
|
|
86411
|
+
if (causes.length === 0) return result;
|
|
86412
|
+
return {
|
|
86413
|
+
...result,
|
|
86414
|
+
status: CheckStatuses.skipped,
|
|
86415
|
+
code: `${definition.id}.superseded`,
|
|
86416
|
+
summary: `Superseded by failing ${causes.join(", ")}, which names the cause. ${result.summary}`,
|
|
86417
|
+
remediation: null,
|
|
86418
|
+
details: { ...result.details, supersededBy: causes, supersededStatus: result.status, supersededCode: result.code }
|
|
86163
86419
|
};
|
|
86164
86420
|
}
|
|
86165
86421
|
|
|
@@ -86244,7 +86500,7 @@ async function doctorRoutes(app, opts) {
|
|
|
86244
86500
|
|
|
86245
86501
|
// ../api-routes/src/discovery/routes.ts
|
|
86246
86502
|
import crypto49 from "crypto";
|
|
86247
|
-
import { and as
|
|
86503
|
+
import { and as and69, desc as desc37, eq as eq82, gte as gte21, inArray as inArray33, isNull as isNull15, or as or19 } from "drizzle-orm";
|
|
86248
86504
|
var MAX_INFLIGHT_DISCOVERY_AGE_MS = 2 * 60 * 60 * 1e3;
|
|
86249
86505
|
async function discoveryRoutes(app, opts) {
|
|
86250
86506
|
app.post("/projects/:name/discover/run", async (request, reply) => {
|
|
@@ -86278,13 +86534,13 @@ async function discoveryRoutes(app, opts) {
|
|
|
86278
86534
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
86279
86535
|
const ageFloorIso = new Date(Date.now() - MAX_INFLIGHT_DISCOVERY_AGE_MS).toISOString();
|
|
86280
86536
|
const decision = app.db.transaction((tx) => {
|
|
86281
|
-
const existing = tx.select({ id: discoverySessions.id, runId: discoverySessions.runId }).from(discoverySessions).where(
|
|
86282
|
-
|
|
86283
|
-
|
|
86537
|
+
const existing = tx.select({ id: discoverySessions.id, runId: discoverySessions.runId }).from(discoverySessions).where(and69(
|
|
86538
|
+
eq82(discoverySessions.projectId, project.id),
|
|
86539
|
+
eq82(discoverySessions.icpDescription, icpDescription),
|
|
86284
86540
|
// Buyer is part of session identity: it changes the seed prompt's
|
|
86285
86541
|
// semantics, so a request with a different (or no) buyer must start
|
|
86286
86542
|
// its own session, never adopt another buyer's probes.
|
|
86287
|
-
parsed.data.buyerDescription == null ?
|
|
86543
|
+
parsed.data.buyerDescription == null ? isNull15(discoverySessions.buyerDescription) : eq82(discoverySessions.buyerDescription, parsed.data.buyerDescription),
|
|
86288
86544
|
// Locations are identity too: a different service-area subset seeds
|
|
86289
86545
|
// and probes a different geo, so it must never reuse another geo's
|
|
86290
86546
|
// session. resolveLocations is deterministic (project-config order),
|
|
@@ -86294,18 +86550,18 @@ async function discoveryRoutes(app, opts) {
|
|
|
86294
86550
|
// locations a NULL row's subset is unknowable, so it conservatively
|
|
86295
86551
|
// never reuses — a one-time, bounded (2h window) non-reuse after
|
|
86296
86552
|
// upgrade, never a wrong reuse.
|
|
86297
|
-
locations.length === 0 ?
|
|
86553
|
+
locations.length === 0 ? or19(isNull15(discoverySessions.locations), eq82(discoverySessions.locations, locations)) : eq82(discoverySessions.locations, locations),
|
|
86298
86554
|
// Seed provider set is identity: a different phrasing distribution
|
|
86299
86555
|
// must never reuse another set's session. Null (= Gemini-only
|
|
86300
86556
|
// default, including explicit ['gemini']) matches legacy rows.
|
|
86301
|
-
seedProviders == null ?
|
|
86302
|
-
|
|
86557
|
+
seedProviders == null ? isNull15(discoverySessions.seedProviders) : eq82(discoverySessions.seedProviders, seedProviders),
|
|
86558
|
+
inArray33(discoverySessions.status, [
|
|
86303
86559
|
DiscoverySessionStatuses.queued,
|
|
86304
86560
|
DiscoverySessionStatuses.seeding,
|
|
86305
86561
|
DiscoverySessionStatuses.probing
|
|
86306
86562
|
]),
|
|
86307
|
-
|
|
86308
|
-
)).orderBy(
|
|
86563
|
+
gte21(discoverySessions.createdAt, ageFloorIso)
|
|
86564
|
+
)).orderBy(desc37(discoverySessions.createdAt)).get();
|
|
86309
86565
|
if (existing && existing.runId) {
|
|
86310
86566
|
return { reused: true, sessionId: existing.id, runId: existing.runId };
|
|
86311
86567
|
}
|
|
@@ -86374,7 +86630,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
86374
86630
|
const project = resolveProject(app.db, request.params.name);
|
|
86375
86631
|
const parsedLimit = parseInt(request.query.limit ?? "", 10);
|
|
86376
86632
|
const limit = Number.isNaN(parsedLimit) || parsedLimit <= 0 ? 50 : parsedLimit;
|
|
86377
|
-
const rows = app.db.select().from(discoverySessions).where(
|
|
86633
|
+
const rows = app.db.select().from(discoverySessions).where(eq82(discoverySessions.projectId, project.id)).orderBy(desc37(discoverySessions.createdAt)).limit(limit).all();
|
|
86378
86634
|
return reply.send(rows.map(serializeSession));
|
|
86379
86635
|
}
|
|
86380
86636
|
);
|
|
@@ -86382,11 +86638,11 @@ async function discoveryRoutes(app, opts) {
|
|
|
86382
86638
|
"/projects/:name/discover/sessions/:id",
|
|
86383
86639
|
async (request, reply) => {
|
|
86384
86640
|
const project = resolveProject(app.db, request.params.name);
|
|
86385
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
86641
|
+
const session = app.db.select().from(discoverySessions).where(eq82(discoverySessions.id, request.params.id)).get();
|
|
86386
86642
|
if (!session || session.projectId !== project.id) {
|
|
86387
86643
|
throw notFound("Discovery session", request.params.id);
|
|
86388
86644
|
}
|
|
86389
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
86645
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq82(discoveryProbes.sessionId, session.id)).all();
|
|
86390
86646
|
const detail = {
|
|
86391
86647
|
...serializeSession(session),
|
|
86392
86648
|
probes: probeRows.map(serializeProbe)
|
|
@@ -86398,7 +86654,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
86398
86654
|
"/projects/:name/discover/sessions/:id/harvest",
|
|
86399
86655
|
async (request, reply) => {
|
|
86400
86656
|
const project = resolveProject(app.db, request.params.name);
|
|
86401
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
86657
|
+
const session = app.db.select().from(discoverySessions).where(eq82(discoverySessions.id, request.params.id)).get();
|
|
86402
86658
|
if (!session || session.projectId !== project.id) {
|
|
86403
86659
|
throw notFound("Discovery session", request.params.id);
|
|
86404
86660
|
}
|
|
@@ -86406,7 +86662,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
86406
86662
|
const minProbeHits = Number.isNaN(parsedFloor) || parsedFloor < 1 ? 1 : parsedFloor;
|
|
86407
86663
|
const applyAnchor = request.query.anchor !== "false";
|
|
86408
86664
|
const provider = session.seedProvider ?? "gemini";
|
|
86409
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
86665
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq82(discoveryProbes.sessionId, session.id)).all();
|
|
86410
86666
|
const extract = opts.harvestSearchQueries;
|
|
86411
86667
|
const probesWithQueries = probeRows.map((row) => {
|
|
86412
86668
|
if (!extract || !row.rawResponse) return { searchQueries: [] };
|
|
@@ -86417,7 +86673,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
86417
86673
|
return { searchQueries: [] };
|
|
86418
86674
|
}
|
|
86419
86675
|
});
|
|
86420
|
-
const trackedQueries = app.db.select({ query: queries.query }).from(queries).where(
|
|
86676
|
+
const trackedQueries = app.db.select({ query: queries.query }).from(queries).where(eq82(queries.projectId, project.id)).all().map((r) => r.query);
|
|
86421
86677
|
const anchorTerms = buildHarvestAnchorTerms(
|
|
86422
86678
|
[session.icpDescription ?? "", ...trackedQueries],
|
|
86423
86679
|
effectiveDomains(project)
|
|
@@ -86464,11 +86720,11 @@ async function discoveryRoutes(app, opts) {
|
|
|
86464
86720
|
"/projects/:name/discover/sessions/:id/promote",
|
|
86465
86721
|
async (request, reply) => {
|
|
86466
86722
|
const project = resolveProject(app.db, request.params.name);
|
|
86467
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
86723
|
+
const session = app.db.select().from(discoverySessions).where(eq82(discoverySessions.id, request.params.id)).get();
|
|
86468
86724
|
if (!session || session.projectId !== project.id) {
|
|
86469
86725
|
throw notFound("Discovery session", request.params.id);
|
|
86470
86726
|
}
|
|
86471
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
86727
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq82(discoveryProbes.sessionId, session.id)).all();
|
|
86472
86728
|
const cited = /* @__PURE__ */ new Set();
|
|
86473
86729
|
const aspirational = /* @__PURE__ */ new Set();
|
|
86474
86730
|
const wasted = /* @__PURE__ */ new Set();
|
|
@@ -86503,7 +86759,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
86503
86759
|
);
|
|
86504
86760
|
app.post("/projects/:name/discover/sessions/:id/promote", async (request, reply) => {
|
|
86505
86761
|
const project = resolveProject(app.db, request.params.name);
|
|
86506
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
86762
|
+
const session = app.db.select().from(discoverySessions).where(eq82(discoverySessions.id, request.params.id)).get();
|
|
86507
86763
|
if (!session || session.projectId !== project.id) {
|
|
86508
86764
|
throw notFound("Discovery session", request.params.id);
|
|
86509
86765
|
}
|
|
@@ -86526,7 +86782,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
86526
86782
|
const bucketSet = new Set(buckets);
|
|
86527
86783
|
const includeCompetitors = parsed.data.includeCompetitors ?? true;
|
|
86528
86784
|
const competitorTypes = parsed.data.competitorTypes ?? DEFAULT_DISCOVERY_PROMOTE_COMPETITOR_TYPES;
|
|
86529
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
86785
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq82(discoveryProbes.sessionId, session.id)).all();
|
|
86530
86786
|
const candidateQueries = /* @__PURE__ */ new Set();
|
|
86531
86787
|
for (const probe of probeRows) {
|
|
86532
86788
|
if (!probe.bucket) continue;
|
|
@@ -86534,7 +86790,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
86534
86790
|
if (bucket.success && bucketSet.has(bucket.data)) candidateQueries.add(probe.query);
|
|
86535
86791
|
}
|
|
86536
86792
|
const existingQueries = new Set(
|
|
86537
|
-
app.db.select({ query: queries.query }).from(queries).where(
|
|
86793
|
+
app.db.select({ query: queries.query }).from(queries).where(eq82(queries.projectId, project.id)).all().map((r) => r.query.toLowerCase())
|
|
86538
86794
|
);
|
|
86539
86795
|
const promotedQueries = [];
|
|
86540
86796
|
const skippedQueries = [];
|
|
@@ -86705,21 +86961,21 @@ function planCompetitorPromotion(input) {
|
|
|
86705
86961
|
const preferred = new Set(input.competitorTypes ?? DEFAULT_DISCOVERY_PROMOTE_COMPETITOR_TYPES);
|
|
86706
86962
|
const eligible = groupCompetitorMap(input.competitorMap, input.probes, preferred).filter((group) => group.hits >= DISCOVERY_PROMOTE_COMPETITOR_MIN_HITS).filter((group) => !typeFilter || group.sources.some((source) => typeFilter.has(source.competitorType))).sort((a, b) => b.hits - a.hits || a.domain.localeCompare(b.domain));
|
|
86707
86963
|
const tracked = new Set(input.tracked.map((row) => normalizeCompetitorDomain(row.domain)));
|
|
86708
|
-
const
|
|
86964
|
+
const skipped4 = [];
|
|
86709
86965
|
const candidates = [];
|
|
86710
86966
|
for (const group of eligible) {
|
|
86711
86967
|
const projectDomain = competitorDomainProjectClaim(group.domain, input.project.domains);
|
|
86712
86968
|
const subdomains = [...new Set(group.sources.map((source) => hostOf(source.domain) ?? source.domain))].filter((host) => host !== group.domain);
|
|
86713
86969
|
if (projectDomain) {
|
|
86714
|
-
|
|
86970
|
+
skipped4.push({
|
|
86715
86971
|
...group,
|
|
86716
86972
|
reason: DiscoveryPromoteCompetitorSkipReasons["project-domain"],
|
|
86717
86973
|
message: `${group.domain} overlaps the project domain ${projectDomain}, so every citation of the project would count for it`
|
|
86718
86974
|
});
|
|
86719
86975
|
} else if (tracked.has(group.domain)) {
|
|
86720
|
-
|
|
86976
|
+
skipped4.push({ ...group, reason: DiscoveryPromoteCompetitorSkipReasons["already-tracked"], message: `${group.domain} is already tracked` });
|
|
86721
86977
|
} else if (subdomains.length > 1) {
|
|
86722
|
-
|
|
86978
|
+
skipped4.push({
|
|
86723
86979
|
...group,
|
|
86724
86980
|
reason: DiscoveryPromoteCompetitorSkipReasons["shared-host"],
|
|
86725
86981
|
message: `${group.domain} was cited as ${subdomains.length} different subdomains (${subdomains.join(", ")}), which may be different sites on one shared host, so it is not merged into one competitor; if they are one competitor, add it with canonry competitor add <project> ${group.domain}`
|
|
@@ -86733,7 +86989,7 @@ function planCompetitorPromotion(input) {
|
|
|
86733
86989
|
for (const group of candidates) {
|
|
86734
86990
|
const rejections = claimed.get(group.domain);
|
|
86735
86991
|
if (rejections) {
|
|
86736
|
-
|
|
86992
|
+
skipped4.push({
|
|
86737
86993
|
...group,
|
|
86738
86994
|
reason: DiscoveryPromoteCompetitorSkipReasons["claimed-by-alias"],
|
|
86739
86995
|
message: rejections.map((rejection) => `${group.domain} ${describeCompetitorAliasRejection(rejection)}`).join("; ")
|
|
@@ -86742,12 +86998,12 @@ function planCompetitorPromotion(input) {
|
|
|
86742
86998
|
promote.push(group);
|
|
86743
86999
|
}
|
|
86744
87000
|
}
|
|
86745
|
-
return { promote, skipped:
|
|
87001
|
+
return { promote, skipped: skipped4 };
|
|
86746
87002
|
}
|
|
86747
87003
|
|
|
86748
87004
|
// ../api-routes/src/discovery/orchestrate.ts
|
|
86749
87005
|
import crypto50 from "crypto";
|
|
86750
|
-
import { eq as
|
|
87006
|
+
import { eq as eq83 } from "drizzle-orm";
|
|
86751
87007
|
var DEFAULT_MAX_PROBES = 100;
|
|
86752
87008
|
var ABSOLUTE_MAX_PROBES = 500;
|
|
86753
87009
|
function classifyProbeBucket(input) {
|
|
@@ -86832,7 +87088,7 @@ async function executeDiscovery(opts) {
|
|
|
86832
87088
|
status: DiscoverySessionStatuses.seeding,
|
|
86833
87089
|
dedupThreshold,
|
|
86834
87090
|
startedAt
|
|
86835
|
-
}).where(
|
|
87091
|
+
}).where(eq83(discoverySessions.id, opts.sessionId)).run();
|
|
86836
87092
|
const seedResult = await opts.deps.seed({
|
|
86837
87093
|
project: opts.project,
|
|
86838
87094
|
icpDescription: opts.icpDescription,
|
|
@@ -86893,7 +87149,7 @@ async function executeDiscovery(opts) {
|
|
|
86893
87149
|
dedupBandPairFraction: dedupStats.bandPairFraction,
|
|
86894
87150
|
dedupPairsTotal: dedupStats.pairsTotal,
|
|
86895
87151
|
warning
|
|
86896
|
-
}).where(
|
|
87152
|
+
}).where(eq83(discoverySessions.id, opts.sessionId)).run();
|
|
86897
87153
|
const probeLocation = opts.locations?.[0];
|
|
86898
87154
|
const probeResults = await mapWithConcurrency(
|
|
86899
87155
|
probedCanonicals,
|
|
@@ -86947,7 +87203,7 @@ async function executeDiscovery(opts) {
|
|
|
86947
87203
|
wastedCount: buckets["wasted-surface"],
|
|
86948
87204
|
competitorMap,
|
|
86949
87205
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
86950
|
-
}).where(
|
|
87206
|
+
}).where(eq83(discoverySessions.id, opts.sessionId)).run();
|
|
86951
87207
|
upsertDomainClassifications(opts.db, opts.project.id, opts.sessionId, competitorMap);
|
|
86952
87208
|
return {
|
|
86953
87209
|
buckets,
|
|
@@ -86987,7 +87243,7 @@ function markSessionFailed(db, sessionId, error) {
|
|
|
86987
87243
|
status: DiscoverySessionStatuses.failed,
|
|
86988
87244
|
error,
|
|
86989
87245
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
86990
|
-
}).where(
|
|
87246
|
+
}).where(eq83(discoverySessions.id, sessionId)).run();
|
|
86991
87247
|
}
|
|
86992
87248
|
function dedupeStrings(input) {
|
|
86993
87249
|
const seen2 = /* @__PURE__ */ new Set();
|
|
@@ -87005,7 +87261,7 @@ function dedupeStrings(input) {
|
|
|
87005
87261
|
|
|
87006
87262
|
// ../api-routes/src/technical-aeo.ts
|
|
87007
87263
|
import crypto51 from "crypto";
|
|
87008
|
-
import { and as
|
|
87264
|
+
import { and as and70, asc as asc15, count as count3, desc as desc38, eq as eq84, gte as gte22, inArray as inArray34, isNotNull as isNotNull8, isNull as isNull16, lt as lt13, or as or20, sql as sql30 } from "drizzle-orm";
|
|
87009
87265
|
import { alias } from "drizzle-orm/sqlite-core";
|
|
87010
87266
|
var FETCHED_SITE_CRAWL_STATES = /* @__PURE__ */ new Set([
|
|
87011
87267
|
...SiteCrawlFetchedStates,
|
|
@@ -87242,8 +87498,8 @@ function parseLinkKind(value) {
|
|
|
87242
87498
|
return allowed.data;
|
|
87243
87499
|
}
|
|
87244
87500
|
function linkKindFilter(linkKind) {
|
|
87245
|
-
if (linkKind === SiteHealthLinkKinds.content) return
|
|
87246
|
-
if (linkKind === SiteHealthLinkKinds.template) return
|
|
87501
|
+
if (linkKind === SiteHealthLinkKinds.content) return eq84(siteCrawlEdges.isTemplate, false);
|
|
87502
|
+
if (linkKind === SiteHealthLinkKinds.template) return eq84(siteCrawlEdges.isTemplate, true);
|
|
87247
87503
|
return void 0;
|
|
87248
87504
|
}
|
|
87249
87505
|
var graphSourceNode = alias(siteCrawlGraphNodes, "site_crawl_graph_source_node");
|
|
@@ -87347,20 +87603,20 @@ function changedFields(before, after, fields) {
|
|
|
87347
87603
|
}
|
|
87348
87604
|
async function technicalAeoRoutes(app, opts) {
|
|
87349
87605
|
const visibleCrawlFilters = (projectId) => [
|
|
87350
|
-
|
|
87351
|
-
|
|
87352
|
-
|
|
87353
|
-
|
|
87606
|
+
eq84(siteCrawlSnapshots.projectId, projectId),
|
|
87607
|
+
eq84(runs.projectId, projectId),
|
|
87608
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
87609
|
+
inArray34(runs.status, SURFACEABLE_STATUSES),
|
|
87354
87610
|
notProbeRun()
|
|
87355
87611
|
];
|
|
87356
87612
|
const availableScanDatesFor = (projectId, requestedDate) => {
|
|
87357
87613
|
if (!requestedDate) return void 0;
|
|
87358
87614
|
const date = sql30`substr(${siteCrawlSnapshots.createdAt}, 1, 10)`;
|
|
87359
87615
|
const filters = visibleCrawlFilters(projectId);
|
|
87360
|
-
const allDates =
|
|
87361
|
-
const matchingDates =
|
|
87362
|
-
const readDates = (where) => app.db.select({ date }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
87363
|
-
const countDates = (where) => app.db.select({ value: sql30`count(distinct ${date})` }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
87616
|
+
const allDates = and70(...filters);
|
|
87617
|
+
const matchingDates = and70(...filters, eq84(sql30`substr(${siteCrawlSnapshots.createdAt}, 6, 5)`, requestedDate.slice(5)));
|
|
87618
|
+
const readDates = (where) => app.db.select({ date }).from(siteCrawlSnapshots).innerJoin(runs, eq84(siteCrawlSnapshots.runId, runs.id)).where(where).groupBy(date).orderBy(desc38(date)).limit(10).all().map((row) => row.date);
|
|
87619
|
+
const countDates = (where) => app.db.select({ value: sql30`count(distinct ${date})` }).from(siteCrawlSnapshots).innerJoin(runs, eq84(siteCrawlSnapshots.runId, runs.id)).where(where).get()?.value ?? 0;
|
|
87364
87620
|
return {
|
|
87365
87621
|
recentDates: readDates(allDates),
|
|
87366
87622
|
totalDates: countDates(allDates),
|
|
@@ -87374,36 +87630,36 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87374
87630
|
if (requestedDate) {
|
|
87375
87631
|
const nextDate2 = /* @__PURE__ */ new Date(`${requestedDate}T00:00:00.000Z`);
|
|
87376
87632
|
nextDate2.setUTCDate(nextDate2.getUTCDate() + 1);
|
|
87377
|
-
filters.push(
|
|
87633
|
+
filters.push(gte22(siteCrawlSnapshots.createdAt, requestedDate), lt13(siteCrawlSnapshots.createdAt, nextDate2.toISOString()));
|
|
87378
87634
|
}
|
|
87379
87635
|
const selectionFilters = scope.scoredOnly ? [...filters, sql30`exists (select 1 from ${siteAuditSnapshots} where ${siteAuditSnapshots.projectId} = ${siteCrawlSnapshots.projectId} and ${siteAuditSnapshots.runId} = ${siteCrawlSnapshots.runId})`] : filters;
|
|
87380
|
-
const selected = app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
87381
|
-
|
|
87382
|
-
|
|
87383
|
-
|
|
87384
|
-
|
|
87385
|
-
|
|
87636
|
+
const selected = app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq84(siteCrawlSnapshots.runId, runs.id)).where(and70(...selectionFilters, runId ? eq84(siteCrawlSnapshots.runId, runId) : void 0)).orderBy(
|
|
87637
|
+
desc38(sql30`substr(${siteCrawlSnapshots.createdAt}, 1, 10)`),
|
|
87638
|
+
desc38(siteCrawlSnapshots.complete),
|
|
87639
|
+
desc38(siteCrawlSnapshots.pagesFetched),
|
|
87640
|
+
desc38(siteCrawlSnapshots.createdAt),
|
|
87641
|
+
desc38(siteCrawlSnapshots.runId)
|
|
87386
87642
|
).limit(1).get();
|
|
87387
87643
|
if (!selected) return void 0;
|
|
87388
87644
|
const date = selected.snapshot.createdAt.slice(0, 10);
|
|
87389
87645
|
const nextDate = /* @__PURE__ */ new Date(`${date}T00:00:00.000Z`);
|
|
87390
87646
|
nextDate.setUTCDate(nextDate.getUTCDate() + 1);
|
|
87391
|
-
const sameDate =
|
|
87647
|
+
const sameDate = and70(
|
|
87392
87648
|
...filters,
|
|
87393
|
-
|
|
87649
|
+
gte22(siteCrawlSnapshots.createdAt, date),
|
|
87394
87650
|
lt13(siteCrawlSnapshots.createdAt, nextDate.toISOString())
|
|
87395
87651
|
);
|
|
87396
|
-
const sameDateRunCount = app.db.select({ value:
|
|
87652
|
+
const sameDateRunCount = app.db.select({ value: count3() }).from(siteCrawlSnapshots).innerJoin(runs, eq84(siteCrawlSnapshots.runId, runs.id)).where(sameDate).get()?.value ?? 0;
|
|
87397
87653
|
const candidates = app.db.select({
|
|
87398
87654
|
runId: siteCrawlSnapshots.runId,
|
|
87399
87655
|
createdAt: siteCrawlSnapshots.createdAt,
|
|
87400
87656
|
complete: siteCrawlSnapshots.complete,
|
|
87401
87657
|
pages: siteCrawlSnapshots.pagesFetched
|
|
87402
|
-
}).from(siteCrawlSnapshots).innerJoin(runs,
|
|
87403
|
-
|
|
87404
|
-
|
|
87405
|
-
|
|
87406
|
-
|
|
87658
|
+
}).from(siteCrawlSnapshots).innerJoin(runs, eq84(siteCrawlSnapshots.runId, runs.id)).where(sameDate).orderBy(
|
|
87659
|
+
desc38(siteCrawlSnapshots.complete),
|
|
87660
|
+
desc38(siteCrawlSnapshots.pagesFetched),
|
|
87661
|
+
desc38(siteCrawlSnapshots.createdAt),
|
|
87662
|
+
desc38(siteCrawlSnapshots.runId)
|
|
87407
87663
|
).limit(10).all();
|
|
87408
87664
|
const runSelection = {
|
|
87409
87665
|
reason: runId ? "explicit-run" : scope.scoredOnly ? "latest-scored-scan" : requestedDate ? selected.snapshot.complete ? "requested-date-complete" : "requested-date-most-pages" : selected.snapshot.complete ? "latest-date-complete" : "latest-date-most-pages",
|
|
@@ -87417,20 +87673,20 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87417
87673
|
};
|
|
87418
87674
|
const resolveAudit = (projectId, runId, recordedBefore) => {
|
|
87419
87675
|
const filters = [
|
|
87420
|
-
|
|
87421
|
-
|
|
87422
|
-
|
|
87423
|
-
|
|
87676
|
+
eq84(siteAuditSnapshots.projectId, projectId),
|
|
87677
|
+
eq84(runs.projectId, projectId),
|
|
87678
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
87679
|
+
inArray34(runs.status, SURFACEABLE_STATUSES),
|
|
87424
87680
|
notProbeRun(),
|
|
87425
87681
|
...recordedBefore ? [lt13(siteAuditSnapshots.createdAt, recordedBefore)] : []
|
|
87426
87682
|
];
|
|
87427
87683
|
const complete = sql30`case when ${runs.status} = ${RunStatuses.completed} then 1 else 0 end`;
|
|
87428
|
-
const selectAudit = (pinnedRunId) => app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs,
|
|
87429
|
-
|
|
87430
|
-
|
|
87431
|
-
|
|
87432
|
-
|
|
87433
|
-
|
|
87684
|
+
const selectAudit = (pinnedRunId) => app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs, eq84(siteAuditSnapshots.runId, runs.id)).where(and70(...filters, pinnedRunId ? eq84(siteAuditSnapshots.runId, pinnedRunId) : void 0)).orderBy(
|
|
87685
|
+
desc38(sql30`substr(${siteAuditSnapshots.createdAt}, 1, 10)`),
|
|
87686
|
+
desc38(complete),
|
|
87687
|
+
desc38(siteAuditSnapshots.pagesAudited),
|
|
87688
|
+
desc38(siteAuditSnapshots.createdAt),
|
|
87689
|
+
desc38(siteAuditSnapshots.runId)
|
|
87434
87690
|
).limit(1).get();
|
|
87435
87691
|
const auditOfCrawl = (target) => {
|
|
87436
87692
|
const selected2 = target ? selectAudit(target.snapshot.runId) : void 0;
|
|
@@ -87446,22 +87702,22 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87446
87702
|
const date = selected.snap.createdAt.slice(0, 10);
|
|
87447
87703
|
const nextDate = /* @__PURE__ */ new Date(`${date}T00:00:00.000Z`);
|
|
87448
87704
|
nextDate.setUTCDate(nextDate.getUTCDate() + 1);
|
|
87449
|
-
const sameDate =
|
|
87705
|
+
const sameDate = and70(
|
|
87450
87706
|
...filters,
|
|
87451
|
-
|
|
87707
|
+
gte22(siteAuditSnapshots.createdAt, date),
|
|
87452
87708
|
lt13(siteAuditSnapshots.createdAt, nextDate.toISOString())
|
|
87453
87709
|
);
|
|
87454
|
-
const sameDateRunCount = app.db.select({ value:
|
|
87710
|
+
const sameDateRunCount = app.db.select({ value: count3() }).from(siteAuditSnapshots).innerJoin(runs, eq84(siteAuditSnapshots.runId, runs.id)).where(sameDate).get()?.value ?? 0;
|
|
87455
87711
|
const candidates = app.db.select({
|
|
87456
87712
|
runId: siteAuditSnapshots.runId,
|
|
87457
87713
|
createdAt: siteAuditSnapshots.createdAt,
|
|
87458
87714
|
complete,
|
|
87459
87715
|
pages: siteAuditSnapshots.pagesAudited
|
|
87460
|
-
}).from(siteAuditSnapshots).innerJoin(runs,
|
|
87461
|
-
|
|
87462
|
-
|
|
87463
|
-
|
|
87464
|
-
|
|
87716
|
+
}).from(siteAuditSnapshots).innerJoin(runs, eq84(siteAuditSnapshots.runId, runs.id)).where(sameDate).orderBy(
|
|
87717
|
+
desc38(complete),
|
|
87718
|
+
desc38(siteAuditSnapshots.pagesAudited),
|
|
87719
|
+
desc38(siteAuditSnapshots.createdAt),
|
|
87720
|
+
desc38(siteAuditSnapshots.runId)
|
|
87465
87721
|
).limit(10).all().map((candidate) => ({ ...candidate, complete: Boolean(candidate.complete) }));
|
|
87466
87722
|
const runSelection = {
|
|
87467
87723
|
// A crawl exists here only when none stored a scorecard: this score is a fallback.
|
|
@@ -87486,7 +87742,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87486
87742
|
inventoryEligible: siteCrawlPages.inventoryEligible,
|
|
87487
87743
|
canonicalOther,
|
|
87488
87744
|
canonicalReason,
|
|
87489
|
-
pages:
|
|
87745
|
+
pages: count3(),
|
|
87490
87746
|
exampleUrl: sql30`min(${siteCrawlPages.url})`
|
|
87491
87747
|
}).from(siteCrawlPages).where(where).groupBy(
|
|
87492
87748
|
siteCrawlPages.fetchState,
|
|
@@ -87523,10 +87779,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87523
87779
|
};
|
|
87524
87780
|
const inventorySummaryFor = (scope) => {
|
|
87525
87781
|
if (!scope) return null;
|
|
87526
|
-
const groups = crawlReasonGroups(
|
|
87527
|
-
|
|
87528
|
-
|
|
87529
|
-
|
|
87782
|
+
const groups = crawlReasonGroups(and70(
|
|
87783
|
+
eq84(siteCrawlPages.projectId, scope.projectId),
|
|
87784
|
+
eq84(siteCrawlPages.runId, scope.runId),
|
|
87785
|
+
eq84(siteCrawlPages.attemptId, scope.attemptId)
|
|
87530
87786
|
));
|
|
87531
87787
|
const total = groups.reduce((sum, group) => sum + group.pages, 0);
|
|
87532
87788
|
const eligible = groups.reduce((sum, group) => sum + (group.inventoryEligible ? group.pages : 0), 0);
|
|
@@ -87546,34 +87802,34 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87546
87802
|
const NO_CRAWL_COMPLETENESS = { complete: false, termination: null };
|
|
87547
87803
|
const pageInScope = (scope, selector) => {
|
|
87548
87804
|
if (!selector.nodeKey && !selector.url) return null;
|
|
87549
|
-
return app.db.select().from(siteCrawlPages).where(
|
|
87550
|
-
|
|
87551
|
-
|
|
87552
|
-
|
|
87553
|
-
selector.nodeKey ?
|
|
87805
|
+
return app.db.select().from(siteCrawlPages).where(and70(
|
|
87806
|
+
eq84(siteCrawlPages.projectId, scope.projectId),
|
|
87807
|
+
eq84(siteCrawlPages.runId, scope.runId),
|
|
87808
|
+
eq84(siteCrawlPages.attemptId, scope.attemptId),
|
|
87809
|
+
selector.nodeKey ? eq84(siteCrawlPages.nodeKey, selector.nodeKey) : eq84(siteCrawlPages.url, selector.url)
|
|
87554
87810
|
)).orderBy(asc15(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
87555
87811
|
};
|
|
87556
|
-
const rootPageInScope = (scope, rootUrl) => pageInScope(scope, { url: rootUrl }) ?? app.db.select().from(siteCrawlPages).where(
|
|
87557
|
-
|
|
87558
|
-
|
|
87559
|
-
|
|
87560
|
-
|
|
87812
|
+
const rootPageInScope = (scope, rootUrl) => pageInScope(scope, { url: rootUrl }) ?? app.db.select().from(siteCrawlPages).where(and70(
|
|
87813
|
+
eq84(siteCrawlPages.projectId, scope.projectId),
|
|
87814
|
+
eq84(siteCrawlPages.runId, scope.runId),
|
|
87815
|
+
eq84(siteCrawlPages.attemptId, scope.attemptId),
|
|
87816
|
+
eq84(siteCrawlPages.depth, 0)
|
|
87561
87817
|
)).orderBy(asc15(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
87562
|
-
const isSurfaceableAuditRun = (projectId, runId) => Boolean(app.db.select({ id: runs.id }).from(runs).where(
|
|
87563
|
-
|
|
87564
|
-
|
|
87565
|
-
|
|
87566
|
-
|
|
87818
|
+
const isSurfaceableAuditRun = (projectId, runId) => Boolean(app.db.select({ id: runs.id }).from(runs).where(and70(
|
|
87819
|
+
eq84(runs.id, runId),
|
|
87820
|
+
eq84(runs.projectId, projectId),
|
|
87821
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
87822
|
+
inArray34(runs.status, SURFACEABLE_STATUSES),
|
|
87567
87823
|
notProbeRun()
|
|
87568
87824
|
)).limit(1).get());
|
|
87569
87825
|
const assertKnownAuditRun = (projectId, runId) => {
|
|
87570
87826
|
if (runId && !isSurfaceableAuditRun(projectId, runId)) throw notFound("Site crawl run", runId);
|
|
87571
87827
|
};
|
|
87572
|
-
const hasLegacyAudit = (projectId) => Boolean(app.db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).innerJoin(runs,
|
|
87573
|
-
|
|
87574
|
-
|
|
87575
|
-
|
|
87576
|
-
|
|
87828
|
+
const hasLegacyAudit = (projectId) => Boolean(app.db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).innerJoin(runs, eq84(siteAuditSnapshots.runId, runs.id)).where(and70(
|
|
87829
|
+
eq84(siteAuditSnapshots.projectId, projectId),
|
|
87830
|
+
eq84(runs.projectId, projectId),
|
|
87831
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
87832
|
+
inArray34(runs.status, SURFACEABLE_STATUSES),
|
|
87577
87833
|
notProbeRun()
|
|
87578
87834
|
)).limit(1).get());
|
|
87579
87835
|
const emptyCrawlSummary = (projectName, legacyAuditAvailable) => ({
|
|
@@ -87674,14 +87930,14 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87674
87930
|
};
|
|
87675
87931
|
}
|
|
87676
87932
|
const statusFilter = request.query.status === "success" || request.query.status === "error" ? request.query.status : null;
|
|
87677
|
-
const conds = [
|
|
87678
|
-
if (statusFilter) conds.push(
|
|
87679
|
-
const where =
|
|
87680
|
-
const totalRow = app.db.select({ value:
|
|
87933
|
+
const conds = [eq84(siteAuditPages.projectId, project.id), eq84(siteAuditPages.runId, latest.snap.runId)];
|
|
87934
|
+
if (statusFilter) conds.push(eq84(siteAuditPages.status, statusFilter));
|
|
87935
|
+
const where = and70(...conds);
|
|
87936
|
+
const totalRow = app.db.select({ value: count3() }).from(siteAuditPages).where(where).get();
|
|
87681
87937
|
const total = totalRow?.value ?? 0;
|
|
87682
87938
|
const limit = parsePositiveInt(request.query.limit, 100, 500);
|
|
87683
87939
|
const offset = parsePositiveInt(request.query.offset, 0, Number.MAX_SAFE_INTEGER);
|
|
87684
|
-
const orderBy = request.query.sort === "score-desc" ?
|
|
87940
|
+
const orderBy = request.query.sort === "score-desc" ? desc38(siteAuditPages.overallScore) : request.query.sort === "url" ? asc15(siteAuditPages.url) : asc15(siteAuditPages.overallScore);
|
|
87685
87941
|
const rows = app.db.select().from(siteAuditPages).where(where).orderBy(orderBy).limit(limit).offset(offset).all();
|
|
87686
87942
|
const pages = rows.map((row) => ({
|
|
87687
87943
|
url: row.url,
|
|
@@ -87700,13 +87956,13 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87700
87956
|
auditedAt: siteAuditSnapshots.auditedAt,
|
|
87701
87957
|
aggregateScore: siteAuditSnapshots.aggregateScore,
|
|
87702
87958
|
pagesAudited: siteAuditSnapshots.pagesAudited
|
|
87703
|
-
}).from(siteAuditSnapshots).innerJoin(runs,
|
|
87704
|
-
|
|
87705
|
-
|
|
87706
|
-
|
|
87707
|
-
|
|
87959
|
+
}).from(siteAuditSnapshots).innerJoin(runs, eq84(siteAuditSnapshots.runId, runs.id)).where(and70(
|
|
87960
|
+
eq84(siteAuditSnapshots.projectId, project.id),
|
|
87961
|
+
eq84(runs.projectId, project.id),
|
|
87962
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
87963
|
+
inArray34(runs.status, SURFACEABLE_STATUSES),
|
|
87708
87964
|
notProbeRun()
|
|
87709
|
-
)).orderBy(
|
|
87965
|
+
)).orderBy(desc38(siteAuditSnapshots.createdAt)).limit(limit).all();
|
|
87710
87966
|
return { project: project.name, points: rows.reverse() };
|
|
87711
87967
|
});
|
|
87712
87968
|
app.get("/projects/:name/technical-aeo/crawl", async (request) => {
|
|
@@ -87806,10 +88062,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87806
88062
|
{ projectId: project.id, runId: snapshot.runId, attemptId: snapshot.attemptId },
|
|
87807
88063
|
snapshot.rootUrl
|
|
87808
88064
|
)?.nodeKey ?? null;
|
|
87809
|
-
const persistedLayout = app.db.select().from(siteCrawlGraphLayouts).where(
|
|
87810
|
-
|
|
87811
|
-
|
|
87812
|
-
|
|
88065
|
+
const persistedLayout = app.db.select().from(siteCrawlGraphLayouts).where(and70(
|
|
88066
|
+
eq84(siteCrawlGraphLayouts.projectId, project.id),
|
|
88067
|
+
eq84(siteCrawlGraphLayouts.runId, snapshot.runId),
|
|
88068
|
+
eq84(siteCrawlGraphLayouts.attemptId, snapshot.attemptId)
|
|
87813
88069
|
)).limit(1).get();
|
|
87814
88070
|
if (!persistedLayout) {
|
|
87815
88071
|
return {
|
|
@@ -87862,9 +88118,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87862
88118
|
const maxNodes = parseBoundedLimit(request.query.maxNodes, SITE_CRAWL_GRAPH_DEFAULT_MAX_NODES, SITE_CRAWL_GRAPH_MAX_NODES);
|
|
87863
88119
|
const maxEdges = parseBoundedLimit(request.query.maxEdges, SITE_CRAWL_GRAPH_DEFAULT_MAX_EDGES, SITE_CRAWL_GRAPH_MAX_EDGES);
|
|
87864
88120
|
const graphScope = [
|
|
87865
|
-
|
|
87866
|
-
|
|
87867
|
-
|
|
88121
|
+
eq84(siteCrawlGraphNodes.projectId, project.id),
|
|
88122
|
+
eq84(siteCrawlGraphNodes.runId, snapshot.runId),
|
|
88123
|
+
eq84(siteCrawlGraphNodes.attemptId, snapshot.attemptId)
|
|
87868
88124
|
];
|
|
87869
88125
|
const nodeRows = app.db.select({
|
|
87870
88126
|
nodeKey: siteCrawlPages.nodeKey,
|
|
@@ -87883,18 +88139,18 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87883
88139
|
linkScoreNormalized: siteCrawlPages.linkScoreNormalized,
|
|
87884
88140
|
x: siteCrawlGraphNodes.x,
|
|
87885
88141
|
y: siteCrawlGraphNodes.y
|
|
87886
|
-
}).from(siteCrawlGraphNodes).innerJoin(siteCrawlPages,
|
|
87887
|
-
|
|
87888
|
-
|
|
87889
|
-
|
|
87890
|
-
|
|
87891
|
-
)).where(
|
|
88142
|
+
}).from(siteCrawlGraphNodes).innerJoin(siteCrawlPages, and70(
|
|
88143
|
+
eq84(siteCrawlPages.projectId, siteCrawlGraphNodes.projectId),
|
|
88144
|
+
eq84(siteCrawlPages.runId, siteCrawlGraphNodes.runId),
|
|
88145
|
+
eq84(siteCrawlPages.attemptId, siteCrawlGraphNodes.attemptId),
|
|
88146
|
+
eq84(siteCrawlPages.nodeKey, siteCrawlGraphNodes.nodeKey)
|
|
88147
|
+
)).where(and70(...graphScope, lt13(siteCrawlGraphNodes.sampleRank, maxNodes))).orderBy(asc15(siteCrawlGraphNodes.sampleRank)).all();
|
|
87892
88148
|
const nodes = nodeRows.map(({ indexabilityReasons, canonicalNodeKey, ...row }) => ({
|
|
87893
88149
|
...row,
|
|
87894
88150
|
healthState: deriveSiteHealthState({ ...row, indexabilityReasons, canonicalNodeKey }),
|
|
87895
88151
|
healthReason: deriveSiteHealthReason({ ...row, indexabilityReasons, canonicalNodeKey })
|
|
87896
88152
|
}));
|
|
87897
|
-
const graphLinkKindFilter = linkKind === SiteHealthLinkKinds.all ? void 0 :
|
|
88153
|
+
const graphLinkKindFilter = linkKind === SiteHealthLinkKinds.all ? void 0 : eq84(siteCrawlGraphEdges.isTemplate, linkKind === SiteHealthLinkKinds.template);
|
|
87898
88154
|
const edges = app.db.select({
|
|
87899
88155
|
edgeKey: siteCrawlGraphEdges.edgeKey,
|
|
87900
88156
|
sourceNodeKey: siteCrawlGraphEdges.sourceNodeKey,
|
|
@@ -87902,22 +88158,22 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
87902
88158
|
followable: siteCrawlGraphEdges.followable,
|
|
87903
88159
|
occurrences: siteCrawlGraphEdges.occurrences,
|
|
87904
88160
|
isTemplate: siteCrawlGraphEdges.isTemplate
|
|
87905
|
-
}).from(siteCrawlGraphEdges).innerJoin(graphSourceNode,
|
|
87906
|
-
|
|
87907
|
-
|
|
87908
|
-
|
|
87909
|
-
|
|
88161
|
+
}).from(siteCrawlGraphEdges).innerJoin(graphSourceNode, and70(
|
|
88162
|
+
eq84(graphSourceNode.projectId, siteCrawlGraphEdges.projectId),
|
|
88163
|
+
eq84(graphSourceNode.runId, siteCrawlGraphEdges.runId),
|
|
88164
|
+
eq84(graphSourceNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
88165
|
+
eq84(graphSourceNode.nodeKey, siteCrawlGraphEdges.sourceNodeKey),
|
|
87910
88166
|
lt13(graphSourceNode.sampleRank, maxNodes)
|
|
87911
|
-
)).innerJoin(graphTargetNode,
|
|
87912
|
-
|
|
87913
|
-
|
|
87914
|
-
|
|
87915
|
-
|
|
88167
|
+
)).innerJoin(graphTargetNode, and70(
|
|
88168
|
+
eq84(graphTargetNode.projectId, siteCrawlGraphEdges.projectId),
|
|
88169
|
+
eq84(graphTargetNode.runId, siteCrawlGraphEdges.runId),
|
|
88170
|
+
eq84(graphTargetNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
88171
|
+
eq84(graphTargetNode.nodeKey, siteCrawlGraphEdges.targetNodeKey),
|
|
87916
88172
|
lt13(graphTargetNode.sampleRank, maxNodes)
|
|
87917
|
-
)).where(
|
|
87918
|
-
|
|
87919
|
-
|
|
87920
|
-
|
|
88173
|
+
)).where(and70(
|
|
88174
|
+
eq84(siteCrawlGraphEdges.projectId, project.id),
|
|
88175
|
+
eq84(siteCrawlGraphEdges.runId, snapshot.runId),
|
|
88176
|
+
eq84(siteCrawlGraphEdges.attemptId, snapshot.attemptId),
|
|
87921
88177
|
lt13(siteCrawlGraphEdges.sampleRank, maxEdges),
|
|
87922
88178
|
graphLinkKindFilter
|
|
87923
88179
|
)).orderBy(asc15(siteCrawlGraphEdges.sampleRank)).all();
|
|
@@ -88012,15 +88268,15 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88012
88268
|
let frontier = [focus.nodeKey];
|
|
88013
88269
|
let queryTruncated = false;
|
|
88014
88270
|
for (let distance = 0; distance < hops && frontier.length > 0; distance += 1) {
|
|
88015
|
-
const candidates = app.db.select().from(siteCrawlEdges).where(
|
|
88016
|
-
|
|
88017
|
-
|
|
88018
|
-
|
|
88019
|
-
|
|
88271
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and70(
|
|
88272
|
+
eq84(siteCrawlEdges.projectId, scope.projectId),
|
|
88273
|
+
eq84(siteCrawlEdges.runId, scope.runId),
|
|
88274
|
+
eq84(siteCrawlEdges.attemptId, scope.attemptId),
|
|
88275
|
+
eq84(siteCrawlEdges.internal, true),
|
|
88020
88276
|
isNotNull8(siteCrawlEdges.targetNodeKey),
|
|
88021
|
-
|
|
88022
|
-
|
|
88023
|
-
|
|
88277
|
+
or20(
|
|
88278
|
+
inArray34(siteCrawlEdges.sourceNodeKey, frontier),
|
|
88279
|
+
inArray34(siteCrawlEdges.targetNodeKey, frontier)
|
|
88024
88280
|
)
|
|
88025
88281
|
)).orderBy(asc15(siteCrawlEdges.edgeKey)).limit(maxEdges + maxNodes + 1).all();
|
|
88026
88282
|
if (candidates.length > maxEdges + maxNodes) queryTruncated = true;
|
|
@@ -88053,22 +88309,22 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88053
88309
|
frontier = [...next].sort();
|
|
88054
88310
|
}
|
|
88055
88311
|
const nodeKeys = [...distances.keys()];
|
|
88056
|
-
const pageRows = nodeKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(
|
|
88057
|
-
|
|
88058
|
-
|
|
88059
|
-
|
|
88060
|
-
|
|
88312
|
+
const pageRows = nodeKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and70(
|
|
88313
|
+
eq84(siteCrawlPages.projectId, scope.projectId),
|
|
88314
|
+
eq84(siteCrawlPages.runId, scope.runId),
|
|
88315
|
+
eq84(siteCrawlPages.attemptId, scope.attemptId),
|
|
88316
|
+
inArray34(siteCrawlPages.nodeKey, nodeKeys)
|
|
88061
88317
|
)).all();
|
|
88062
88318
|
const persistedNodeKeys = new Set(pageRows.map((row) => row.nodeKey));
|
|
88063
88319
|
const outermostNodeKeys = pageRows.filter((row) => distances.get(row.nodeKey) === hops).map((row) => row.nodeKey);
|
|
88064
|
-
const omittedOutermostEdge = outermostNodeKeys.length === 0 ? void 0 : app.db.select({ edgeKey: siteCrawlEdges.edgeKey }).from(siteCrawlEdges).where(
|
|
88065
|
-
|
|
88066
|
-
|
|
88067
|
-
|
|
88068
|
-
|
|
88320
|
+
const omittedOutermostEdge = outermostNodeKeys.length === 0 ? void 0 : app.db.select({ edgeKey: siteCrawlEdges.edgeKey }).from(siteCrawlEdges).where(and70(
|
|
88321
|
+
eq84(siteCrawlEdges.projectId, scope.projectId),
|
|
88322
|
+
eq84(siteCrawlEdges.runId, scope.runId),
|
|
88323
|
+
eq84(siteCrawlEdges.attemptId, scope.attemptId),
|
|
88324
|
+
eq84(siteCrawlEdges.internal, true),
|
|
88069
88325
|
isNotNull8(siteCrawlEdges.targetNodeKey),
|
|
88070
|
-
|
|
88071
|
-
|
|
88326
|
+
inArray34(siteCrawlEdges.sourceNodeKey, outermostNodeKeys),
|
|
88327
|
+
inArray34(siteCrawlEdges.targetNodeKey, outermostNodeKeys)
|
|
88072
88328
|
)).limit(1).get();
|
|
88073
88329
|
if (omittedOutermostEdge && !seenEdgeKeys.has(omittedOutermostEdge.edgeKey)) {
|
|
88074
88330
|
omittedEdgeKeys.add(omittedOutermostEdge.edgeKey);
|
|
@@ -88161,23 +88417,23 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88161
88417
|
let found = fromPage.nodeKey === toPage.nodeKey;
|
|
88162
88418
|
let truncated = false;
|
|
88163
88419
|
for (let depth = 0; depth < maxDepth && frontier.length > 0 && !found; depth += 1) {
|
|
88164
|
-
const candidates = app.db.select().from(siteCrawlEdges).where(
|
|
88165
|
-
|
|
88166
|
-
|
|
88167
|
-
|
|
88168
|
-
|
|
88169
|
-
|
|
88170
|
-
|
|
88420
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and70(
|
|
88421
|
+
eq84(siteCrawlEdges.projectId, scope.projectId),
|
|
88422
|
+
eq84(siteCrawlEdges.runId, scope.runId),
|
|
88423
|
+
eq84(siteCrawlEdges.attemptId, scope.attemptId),
|
|
88424
|
+
eq84(siteCrawlEdges.internal, true),
|
|
88425
|
+
eq84(siteCrawlEdges.followable, true),
|
|
88426
|
+
eq84(siteCrawlEdges.relation, "anchor"),
|
|
88171
88427
|
isNotNull8(siteCrawlEdges.targetNodeKey),
|
|
88172
|
-
|
|
88428
|
+
inArray34(siteCrawlEdges.sourceNodeKey, frontier)
|
|
88173
88429
|
)).orderBy(asc15(siteCrawlEdges.edgeKey)).limit(SITE_HEALTH_PATH_MAX_VISITED_NODES + 1).all();
|
|
88174
88430
|
if (candidates.length > SITE_HEALTH_PATH_MAX_VISITED_NODES) truncated = true;
|
|
88175
88431
|
const candidateTargetKeys = [...new Set(candidates.map((edge) => edge.targetNodeKey).filter((nodeKey) => nodeKey != null))];
|
|
88176
|
-
const persistedTargetKeys = new Set(candidateTargetKeys.length === 0 ? [] : app.db.select({ nodeKey: siteCrawlPages.nodeKey }).from(siteCrawlPages).where(
|
|
88177
|
-
|
|
88178
|
-
|
|
88179
|
-
|
|
88180
|
-
|
|
88432
|
+
const persistedTargetKeys = new Set(candidateTargetKeys.length === 0 ? [] : app.db.select({ nodeKey: siteCrawlPages.nodeKey }).from(siteCrawlPages).where(and70(
|
|
88433
|
+
eq84(siteCrawlPages.projectId, scope.projectId),
|
|
88434
|
+
eq84(siteCrawlPages.runId, scope.runId),
|
|
88435
|
+
eq84(siteCrawlPages.attemptId, scope.attemptId),
|
|
88436
|
+
inArray34(siteCrawlPages.nodeKey, candidateTargetKeys)
|
|
88181
88437
|
)).all().map((row) => row.nodeKey));
|
|
88182
88438
|
const next = /* @__PURE__ */ new Set();
|
|
88183
88439
|
for (const edge of candidates.slice(0, SITE_HEALTH_PATH_MAX_VISITED_NODES)) {
|
|
@@ -88218,11 +88474,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88218
88474
|
pathEdges.unshift(step.edge);
|
|
88219
88475
|
pathKeys.unshift(step.nodeKey);
|
|
88220
88476
|
}
|
|
88221
|
-
const pathRows = app.db.select().from(siteCrawlPages).where(
|
|
88222
|
-
|
|
88223
|
-
|
|
88224
|
-
|
|
88225
|
-
|
|
88477
|
+
const pathRows = app.db.select().from(siteCrawlPages).where(and70(
|
|
88478
|
+
eq84(siteCrawlPages.projectId, scope.projectId),
|
|
88479
|
+
eq84(siteCrawlPages.runId, scope.runId),
|
|
88480
|
+
eq84(siteCrawlPages.attemptId, scope.attemptId),
|
|
88481
|
+
inArray34(siteCrawlPages.nodeKey, pathKeys)
|
|
88226
88482
|
)).all();
|
|
88227
88483
|
const pageByKey = new Map(pathRows.map((row) => [row.nodeKey, row]));
|
|
88228
88484
|
const pathDetection = templateDetectionOf(snapshot.templateDetection);
|
|
@@ -88250,25 +88506,25 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88250
88506
|
throw validationError("change must be all, added, removed, or changed");
|
|
88251
88507
|
}
|
|
88252
88508
|
const completeSnapshotFilters = [
|
|
88253
|
-
|
|
88254
|
-
|
|
88255
|
-
|
|
88256
|
-
|
|
88257
|
-
|
|
88509
|
+
eq84(siteCrawlSnapshots.projectId, project.id),
|
|
88510
|
+
eq84(runs.projectId, project.id),
|
|
88511
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
88512
|
+
eq84(runs.status, RunStatuses.completed),
|
|
88513
|
+
eq84(siteCrawlSnapshots.complete, true),
|
|
88258
88514
|
notProbeRun()
|
|
88259
88515
|
];
|
|
88260
|
-
const afterTarget = request.query.toRunId ? resolveCrawl(project.id, request.query.toRunId) : app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
88516
|
+
const afterTarget = request.query.toRunId ? resolveCrawl(project.id, request.query.toRunId) : app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq84(siteCrawlSnapshots.runId, runs.id)).where(and70(...completeSnapshotFilters)).orderBy(desc38(siteCrawlSnapshots.createdAt), desc38(siteCrawlSnapshots.runId)).limit(1).get();
|
|
88261
88517
|
if (request.query.toRunId && !afterTarget) throw notFound("Site crawl run", request.query.toRunId);
|
|
88262
|
-
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
88518
|
+
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq84(siteCrawlSnapshots.runId, runs.id)).where(and70(
|
|
88263
88519
|
...completeSnapshotFilters,
|
|
88264
|
-
|
|
88520
|
+
or20(
|
|
88265
88521
|
lt13(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
88266
|
-
|
|
88267
|
-
|
|
88522
|
+
and70(
|
|
88523
|
+
eq84(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
88268
88524
|
lt13(siteCrawlSnapshots.runId, afterTarget.snapshot.runId)
|
|
88269
88525
|
)
|
|
88270
88526
|
)
|
|
88271
|
-
)).orderBy(
|
|
88527
|
+
)).orderBy(desc38(siteCrawlSnapshots.createdAt), desc38(siteCrawlSnapshots.runId)).limit(1).get() : void 0;
|
|
88272
88528
|
if (request.query.fromRunId && !beforeTarget) throw notFound("Site crawl run", request.query.fromRunId);
|
|
88273
88529
|
if (!afterTarget) {
|
|
88274
88530
|
const latestCrawl = resolveCrawl(project.id);
|
|
@@ -88509,18 +88765,18 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88509
88765
|
const visibleKeys = keyRows.slice(0, limit);
|
|
88510
88766
|
const pageKeys = visibleKeys.filter((row) => row.entity === "page").map((row) => row.key);
|
|
88511
88767
|
const linkKeys = visibleKeys.filter((row) => row.entity === "link").map((row) => row.key);
|
|
88512
|
-
const pageRowsFor = (scope) => pageKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(
|
|
88513
|
-
|
|
88514
|
-
|
|
88515
|
-
|
|
88516
|
-
|
|
88768
|
+
const pageRowsFor = (scope) => pageKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and70(
|
|
88769
|
+
eq84(siteCrawlPages.projectId, scope.projectId),
|
|
88770
|
+
eq84(siteCrawlPages.runId, scope.runId),
|
|
88771
|
+
eq84(siteCrawlPages.attemptId, scope.attemptId),
|
|
88772
|
+
inArray34(siteCrawlPages.nodeKey, pageKeys)
|
|
88517
88773
|
)).all();
|
|
88518
|
-
const edgeRowsFor = (scope) => linkKeys.length === 0 ? [] : app.db.select().from(siteCrawlEdges).where(
|
|
88519
|
-
|
|
88520
|
-
|
|
88521
|
-
|
|
88522
|
-
|
|
88523
|
-
|
|
88774
|
+
const edgeRowsFor = (scope) => linkKeys.length === 0 ? [] : app.db.select().from(siteCrawlEdges).where(and70(
|
|
88775
|
+
eq84(siteCrawlEdges.projectId, scope.projectId),
|
|
88776
|
+
eq84(siteCrawlEdges.runId, scope.runId),
|
|
88777
|
+
eq84(siteCrawlEdges.attemptId, scope.attemptId),
|
|
88778
|
+
eq84(siteCrawlEdges.internal, true),
|
|
88779
|
+
inArray34(siteCrawlEdges.edgeKey, linkKeys)
|
|
88524
88780
|
)).all();
|
|
88525
88781
|
const beforePages = new Map(pageRowsFor(beforeScope).map((row) => [row.nodeKey, mapCrawlPage(row)]));
|
|
88526
88782
|
const afterPages = new Map(pageRowsFor(afterScope).map((row) => [row.nodeKey, mapCrawlPage(row)]));
|
|
@@ -88673,16 +88929,16 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88673
88929
|
}
|
|
88674
88930
|
const inventorySummary = inventorySummaryFor(detailScopeFor(project.id, snapshot));
|
|
88675
88931
|
const filters = [
|
|
88676
|
-
|
|
88677
|
-
|
|
88678
|
-
|
|
88932
|
+
eq84(siteCrawlPages.projectId, project.id),
|
|
88933
|
+
eq84(siteCrawlPages.runId, snapshot.runId),
|
|
88934
|
+
eq84(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
88679
88935
|
];
|
|
88680
|
-
if (request.query.nodeKey) filters.push(
|
|
88936
|
+
if (request.query.nodeKey) filters.push(eq84(siteCrawlPages.nodeKey, request.query.nodeKey));
|
|
88681
88937
|
const inventoryEligible = parseBoolean2(request.query.inventoryEligible);
|
|
88682
|
-
if (inventoryEligible != null) filters.push(
|
|
88683
|
-
if (request.query.fetchState) filters.push(
|
|
88684
|
-
if (request.query.indexabilityState) filters.push(
|
|
88685
|
-
if (request.query.auditState) filters.push(
|
|
88938
|
+
if (inventoryEligible != null) filters.push(eq84(siteCrawlPages.inventoryEligible, inventoryEligible));
|
|
88939
|
+
if (request.query.fetchState) filters.push(eq84(siteCrawlPages.fetchState, request.query.fetchState));
|
|
88940
|
+
if (request.query.indexabilityState) filters.push(eq84(siteCrawlPages.indexabilityState, request.query.indexabilityState));
|
|
88941
|
+
if (request.query.auditState) filters.push(eq84(siteCrawlPages.auditState, request.query.auditState));
|
|
88686
88942
|
const healthState = parseSiteHealthState(request.query.healthState);
|
|
88687
88943
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
88688
88944
|
const sort = ["score-desc", "score-asc", "path"].includes(request.query.sort ?? "") ? request.query.sort : "url";
|
|
@@ -88700,17 +88956,17 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88700
88956
|
sort
|
|
88701
88957
|
})).digest("base64url");
|
|
88702
88958
|
const offset = decodeCrawlPageCursor(request.query.cursor, selectionKey);
|
|
88703
|
-
const orderBy = sort === "score-desc" ? [
|
|
88959
|
+
const orderBy = sort === "score-desc" ? [desc38(siteCrawlPages.auditScore), asc15(siteCrawlPages.nodeKey)] : sort === "score-asc" ? [asc15(siteCrawlPages.auditScore), asc15(siteCrawlPages.nodeKey)] : sort === "path" ? [asc15(siteCrawlPages.path), asc15(siteCrawlPages.nodeKey)] : [asc15(siteCrawlPages.url), asc15(siteCrawlPages.nodeKey)];
|
|
88704
88960
|
let healthStateFilter = null;
|
|
88705
88961
|
if (healthState) {
|
|
88706
|
-
const legacyRow = app.db.select({ id: siteCrawlPages.id }).from(siteCrawlPages).where(
|
|
88707
|
-
|
|
88708
|
-
|
|
88709
|
-
|
|
88710
|
-
|
|
88962
|
+
const legacyRow = app.db.select({ id: siteCrawlPages.id }).from(siteCrawlPages).where(and70(
|
|
88963
|
+
eq84(siteCrawlPages.projectId, project.id),
|
|
88964
|
+
eq84(siteCrawlPages.runId, snapshot.runId),
|
|
88965
|
+
eq84(siteCrawlPages.attemptId, snapshot.attemptId),
|
|
88966
|
+
isNull16(siteCrawlPages.healthState)
|
|
88711
88967
|
)).limit(1).get();
|
|
88712
88968
|
healthStateFilter = legacyRow ? "unavailable-legacy-scan" : "applied";
|
|
88713
|
-
if (healthStateFilter === "applied") filters.push(
|
|
88969
|
+
if (healthStateFilter === "applied") filters.push(eq84(siteCrawlPages.healthState, healthState));
|
|
88714
88970
|
}
|
|
88715
88971
|
if (healthStateFilter === "unavailable-legacy-scan") {
|
|
88716
88972
|
return {
|
|
@@ -88727,8 +88983,8 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88727
88983
|
pages: []
|
|
88728
88984
|
};
|
|
88729
88985
|
}
|
|
88730
|
-
const where =
|
|
88731
|
-
const total = app.db.select({ value:
|
|
88986
|
+
const where = and70(...filters);
|
|
88987
|
+
const total = app.db.select({ value: count3() }).from(siteCrawlPages).where(where).get()?.value ?? 0;
|
|
88732
88988
|
const rows = app.db.select().from(siteCrawlPages).where(where).orderBy(...orderBy).limit(limit).offset(offset).all();
|
|
88733
88989
|
const nextOffset = offset + rows.length;
|
|
88734
88990
|
return {
|
|
@@ -88762,10 +89018,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88762
89018
|
url: siteCrawlPages.url,
|
|
88763
89019
|
inventoryEligible: siteCrawlPages.inventoryEligible,
|
|
88764
89020
|
fetchState: siteCrawlPages.fetchState
|
|
88765
|
-
}).from(siteCrawlPages).where(
|
|
88766
|
-
|
|
88767
|
-
|
|
88768
|
-
|
|
89021
|
+
}).from(siteCrawlPages).where(and70(
|
|
89022
|
+
eq84(siteCrawlPages.projectId, project.id),
|
|
89023
|
+
eq84(siteCrawlPages.runId, snapshot.runId),
|
|
89024
|
+
eq84(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
88769
89025
|
)).limit(MAX_STRUCTURE_SOURCE_ROWS + 1).all();
|
|
88770
89026
|
if (sourceRows.length > MAX_STRUCTURE_SOURCE_ROWS) {
|
|
88771
89027
|
throw validationError(`Persisted crawl exceeds the ${MAX_STRUCTURE_SOURCE_ROWS}-page structure limit`);
|
|
@@ -88847,18 +89103,18 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88847
89103
|
};
|
|
88848
89104
|
}
|
|
88849
89105
|
const filters = [
|
|
88850
|
-
|
|
88851
|
-
|
|
88852
|
-
|
|
88853
|
-
|
|
89106
|
+
eq84(siteCrawlEdges.projectId, project.id),
|
|
89107
|
+
eq84(siteCrawlEdges.runId, snapshot.runId),
|
|
89108
|
+
eq84(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
89109
|
+
eq84(siteCrawlEdges.internal, true),
|
|
88854
89110
|
linkKindFilter(linkKind)
|
|
88855
89111
|
];
|
|
88856
|
-
if (request.query.sourceUrl) filters.push(
|
|
88857
|
-
if (request.query.targetUrl) filters.push(
|
|
89112
|
+
if (request.query.sourceUrl) filters.push(eq84(siteCrawlEdges.sourceUrl, request.query.sourceUrl));
|
|
89113
|
+
if (request.query.targetUrl) filters.push(eq84(siteCrawlEdges.targetUrl, request.query.targetUrl));
|
|
88858
89114
|
const followable = parseBoolean2(request.query.followable);
|
|
88859
|
-
if (followable != null) filters.push(
|
|
88860
|
-
const where =
|
|
88861
|
-
const total = app.db.select({ value:
|
|
89115
|
+
if (followable != null) filters.push(eq84(siteCrawlEdges.followable, followable));
|
|
89116
|
+
const where = and70(...filters);
|
|
89117
|
+
const total = app.db.select({ value: count3() }).from(siteCrawlEdges).where(where).get()?.value ?? 0;
|
|
88862
89118
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
88863
89119
|
const offset = decodeCursor(request.query.cursor);
|
|
88864
89120
|
const rows = app.db.select().from(siteCrawlEdges).where(where).orderBy(asc15(siteCrawlEdges.edgeKey)).limit(limit).offset(offset).all();
|
|
@@ -88918,17 +89174,17 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88918
89174
|
};
|
|
88919
89175
|
}
|
|
88920
89176
|
const scope = [
|
|
88921
|
-
|
|
88922
|
-
|
|
88923
|
-
|
|
88924
|
-
|
|
89177
|
+
eq84(siteCrawlEdges.projectId, project.id),
|
|
89178
|
+
eq84(siteCrawlEdges.runId, snapshot.runId),
|
|
89179
|
+
eq84(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
89180
|
+
eq84(siteCrawlEdges.internal, true),
|
|
88925
89181
|
linkKindFilter(linkKind)
|
|
88926
89182
|
];
|
|
88927
|
-
const inboundMatch = request.query.nodeKey && request.query.url ?
|
|
88928
|
-
const outboundMatch = request.query.nodeKey && request.query.url ?
|
|
89183
|
+
const inboundMatch = request.query.nodeKey && request.query.url ? or20(eq84(siteCrawlEdges.targetNodeKey, request.query.nodeKey), eq84(siteCrawlEdges.targetUrl, request.query.url)) : request.query.nodeKey ? eq84(siteCrawlEdges.targetNodeKey, request.query.nodeKey) : eq84(siteCrawlEdges.targetUrl, request.query.url);
|
|
89184
|
+
const outboundMatch = request.query.nodeKey && request.query.url ? or20(eq84(siteCrawlEdges.sourceNodeKey, request.query.nodeKey), eq84(siteCrawlEdges.sourceUrl, request.query.url)) : request.query.nodeKey ? eq84(siteCrawlEdges.sourceNodeKey, request.query.nodeKey) : eq84(siteCrawlEdges.sourceUrl, request.query.url);
|
|
88929
89185
|
const limit = parseBoundedLimit(request.query.limit, 50, 100);
|
|
88930
|
-
const inboundRows = app.db.select().from(siteCrawlEdges).where(
|
|
88931
|
-
const outboundRows = app.db.select().from(siteCrawlEdges).where(
|
|
89186
|
+
const inboundRows = app.db.select().from(siteCrawlEdges).where(and70(...scope, inboundMatch)).orderBy(asc15(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
89187
|
+
const outboundRows = app.db.select().from(siteCrawlEdges).where(and70(...scope, outboundMatch)).orderBy(asc15(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
88932
89188
|
return {
|
|
88933
89189
|
project: project.name,
|
|
88934
89190
|
hasCrawlData: true,
|
|
@@ -88962,11 +89218,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88962
89218
|
if (!snapshot.attemptId) {
|
|
88963
89219
|
return { project: project.name, runId: snapshot.runId, state: "partial", checkDeadLinks: true, checked: snapshot.deadLinksChecked, found: snapshot.deadLinksFound, unverified: snapshot.deadLinksUnverified, total: 0, nextCursor: null, deadLinks: [], runSelection: target.runSelection };
|
|
88964
89220
|
}
|
|
88965
|
-
const where =
|
|
88966
|
-
|
|
88967
|
-
|
|
88968
|
-
|
|
88969
|
-
|
|
89221
|
+
const where = and70(
|
|
89222
|
+
eq84(siteCrawlFindings.projectId, project.id),
|
|
89223
|
+
eq84(siteCrawlFindings.runId, snapshot.runId),
|
|
89224
|
+
eq84(siteCrawlFindings.attemptId, snapshot.attemptId),
|
|
89225
|
+
eq84(siteCrawlFindings.findingType, "dead-link"),
|
|
88970
89226
|
// Belt and braces on the client-facing surface: a row without a status
|
|
88971
89227
|
// code is not evidence of a broken link, and this response is where such
|
|
88972
89228
|
// a row would be read as one. Migration 140 removed every stored row that
|
|
@@ -88978,7 +89234,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
88978
89234
|
// promise a page the list cannot fill.
|
|
88979
89235
|
isNotNull8(sql30`json_extract(${siteCrawlFindings.evidence}, '$.statusCode')`)
|
|
88980
89236
|
);
|
|
88981
|
-
const total = app.db.select({ value:
|
|
89237
|
+
const total = app.db.select({ value: count3() }).from(siteCrawlFindings).where(where).get()?.value ?? 0;
|
|
88982
89238
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
88983
89239
|
const offset = decodeCursor(request.query.cursor);
|
|
88984
89240
|
const rows = app.db.select().from(siteCrawlFindings).where(where).orderBy(asc15(siteCrawlFindings.findingKey)).limit(limit).offset(offset).all();
|
|
@@ -89016,19 +89272,19 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
89016
89272
|
createdAt: runs.createdAt,
|
|
89017
89273
|
startedAt: runs.startedAt,
|
|
89018
89274
|
finishedAt: runs.finishedAt
|
|
89019
|
-
}).from(runs).where(
|
|
89020
|
-
|
|
89021
|
-
|
|
89275
|
+
}).from(runs).where(and70(
|
|
89276
|
+
eq84(runs.projectId, project.id),
|
|
89277
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
89022
89278
|
notProbeRun()
|
|
89023
|
-
)).orderBy(
|
|
89279
|
+
)).orderBy(desc38(runs.createdAt), desc38(runs.id)).limit(limit).all();
|
|
89024
89280
|
const runIds = rows.map((row) => row.runId);
|
|
89025
|
-
const crawlRunIds = new Set(runIds.length === 0 ? [] : app.db.select({ runId: siteCrawlSnapshots.runId }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
89026
|
-
|
|
89027
|
-
|
|
89028
|
-
|
|
89029
|
-
|
|
89281
|
+
const crawlRunIds = new Set(runIds.length === 0 ? [] : app.db.select({ runId: siteCrawlSnapshots.runId }).from(siteCrawlSnapshots).innerJoin(runs, eq84(siteCrawlSnapshots.runId, runs.id)).where(and70(
|
|
89282
|
+
eq84(siteCrawlSnapshots.projectId, project.id),
|
|
89283
|
+
eq84(runs.projectId, project.id),
|
|
89284
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
89285
|
+
inArray34(runs.status, SURFACEABLE_STATUSES),
|
|
89030
89286
|
notProbeRun(),
|
|
89031
|
-
|
|
89287
|
+
inArray34(siteCrawlSnapshots.runId, runIds)
|
|
89032
89288
|
)).all().map((row) => row.runId));
|
|
89033
89289
|
return {
|
|
89034
89290
|
project: project.name,
|
|
@@ -89042,21 +89298,21 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
89042
89298
|
});
|
|
89043
89299
|
app.get("/projects/:name/technical-aeo/runs/:runId/progress", async (request) => {
|
|
89044
89300
|
const project = resolveProject(app.db, request.params.name);
|
|
89045
|
-
const run = app.db.select().from(runs).where(
|
|
89046
|
-
|
|
89047
|
-
|
|
89048
|
-
|
|
89301
|
+
const run = app.db.select().from(runs).where(and70(
|
|
89302
|
+
eq84(runs.id, request.params.runId),
|
|
89303
|
+
eq84(runs.projectId, project.id),
|
|
89304
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
89049
89305
|
notProbeRun()
|
|
89050
89306
|
)).get();
|
|
89051
89307
|
if (!run) throw notFound("Site audit run", request.params.runId);
|
|
89052
|
-
const attempt = app.db.select().from(siteCrawlAttempts).where(
|
|
89053
|
-
|
|
89054
|
-
|
|
89055
|
-
)).orderBy(
|
|
89056
|
-
const persistedLayout = attempt ? app.db.select().from(siteCrawlGraphLayouts).where(
|
|
89057
|
-
|
|
89058
|
-
|
|
89059
|
-
|
|
89308
|
+
const attempt = app.db.select().from(siteCrawlAttempts).where(and70(
|
|
89309
|
+
eq84(siteCrawlAttempts.projectId, project.id),
|
|
89310
|
+
eq84(siteCrawlAttempts.runId, run.id)
|
|
89311
|
+
)).orderBy(desc38(siteCrawlAttempts.attemptNumber), desc38(siteCrawlAttempts.updatedAt)).limit(1).get();
|
|
89312
|
+
const persistedLayout = attempt ? app.db.select().from(siteCrawlGraphLayouts).where(and70(
|
|
89313
|
+
eq84(siteCrawlGraphLayouts.projectId, project.id),
|
|
89314
|
+
eq84(siteCrawlGraphLayouts.runId, run.id),
|
|
89315
|
+
eq84(siteCrawlGraphLayouts.attemptId, attempt.id)
|
|
89060
89316
|
)).get() : void 0;
|
|
89061
89317
|
const layoutState = persistedLayout?.state === "ready" || persistedLayout?.state === "unavailable" ? persistedLayout.state : run.status === RunStatuses.completed || run.status === RunStatuses.partial ? "unavailable" : "pending";
|
|
89062
89318
|
return {
|
|
@@ -89087,19 +89343,19 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
89087
89343
|
};
|
|
89088
89344
|
});
|
|
89089
89345
|
app.get("/projects/:name/technical-aeo/runs/:runId/page-health-preview", async (request) => app.db.transaction((tx) => {
|
|
89090
|
-
const project = tx.select().from(projects).where(
|
|
89346
|
+
const project = tx.select().from(projects).where(eq84(projects.name, request.params.name)).get();
|
|
89091
89347
|
if (!project) throw notFound("Project", request.params.name);
|
|
89092
|
-
const run = tx.select().from(runs).where(
|
|
89093
|
-
|
|
89094
|
-
|
|
89095
|
-
|
|
89348
|
+
const run = tx.select().from(runs).where(and70(
|
|
89349
|
+
eq84(runs.id, request.params.runId),
|
|
89350
|
+
eq84(runs.projectId, project.id),
|
|
89351
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
89096
89352
|
notProbeRun()
|
|
89097
89353
|
)).get();
|
|
89098
89354
|
if (!run) throw notFound("Site audit run", request.params.runId);
|
|
89099
|
-
const attempt = tx.select().from(siteCrawlAttempts).where(
|
|
89100
|
-
|
|
89101
|
-
|
|
89102
|
-
)).orderBy(
|
|
89355
|
+
const attempt = tx.select().from(siteCrawlAttempts).where(and70(
|
|
89356
|
+
eq84(siteCrawlAttempts.projectId, project.id),
|
|
89357
|
+
eq84(siteCrawlAttempts.runId, run.id)
|
|
89358
|
+
)).orderBy(desc38(siteCrawlAttempts.attemptNumber), desc38(siteCrawlAttempts.updatedAt)).limit(1).get();
|
|
89103
89359
|
const state = run.status === RunStatuses.queued ? "waiting" : run.status === RunStatuses.running ? "collecting" : "terminal";
|
|
89104
89360
|
const base = {
|
|
89105
89361
|
project: project.name,
|
|
@@ -89112,13 +89368,13 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
89112
89368
|
if (!attempt) {
|
|
89113
89369
|
return { ...base, pagesAudited: 0, examples: [] };
|
|
89114
89370
|
}
|
|
89115
|
-
const auditedWhere =
|
|
89116
|
-
|
|
89117
|
-
|
|
89118
|
-
|
|
89119
|
-
|
|
89371
|
+
const auditedWhere = and70(
|
|
89372
|
+
eq84(siteCrawlPages.projectId, project.id),
|
|
89373
|
+
eq84(siteCrawlPages.runId, run.id),
|
|
89374
|
+
eq84(siteCrawlPages.attemptId, attempt.id),
|
|
89375
|
+
eq84(siteCrawlPages.auditState, "success")
|
|
89120
89376
|
);
|
|
89121
|
-
const pagesAudited = tx.select({ value:
|
|
89377
|
+
const pagesAudited = tx.select({ value: count3() }).from(siteCrawlPages).where(auditedWhere).get()?.value ?? 0;
|
|
89122
89378
|
if (state !== "collecting") {
|
|
89123
89379
|
return { ...base, pagesAudited, examples: [] };
|
|
89124
89380
|
}
|
|
@@ -89127,7 +89383,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
89127
89383
|
url: siteCrawlPages.url,
|
|
89128
89384
|
auditScore: siteCrawlPages.auditScore,
|
|
89129
89385
|
auditFields: siteCrawlPages.auditFields
|
|
89130
|
-
}).from(siteCrawlPages).where(
|
|
89386
|
+
}).from(siteCrawlPages).where(and70(
|
|
89131
89387
|
auditedWhere,
|
|
89132
89388
|
isNotNull8(siteCrawlPages.auditScore),
|
|
89133
89389
|
lt13(siteCrawlPages.auditScore, 70)
|
|
@@ -89163,13 +89419,13 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
89163
89419
|
status: runs.status,
|
|
89164
89420
|
identityKey: siteCrawlRunRequests.identityKey,
|
|
89165
89421
|
effectiveOptions: siteCrawlRunRequests.effectiveOptions
|
|
89166
|
-
}).from(runs).leftJoin(siteCrawlRunRequests,
|
|
89167
|
-
|
|
89168
|
-
|
|
89169
|
-
)).where(
|
|
89170
|
-
|
|
89171
|
-
|
|
89172
|
-
|
|
89422
|
+
}).from(runs).leftJoin(siteCrawlRunRequests, and70(
|
|
89423
|
+
eq84(siteCrawlRunRequests.projectId, runs.projectId),
|
|
89424
|
+
eq84(siteCrawlRunRequests.runId, runs.id)
|
|
89425
|
+
)).where(and70(
|
|
89426
|
+
eq84(runs.projectId, project.id),
|
|
89427
|
+
eq84(runs.kind, RunKinds["site-audit"]),
|
|
89428
|
+
inArray34(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
89173
89429
|
)).get();
|
|
89174
89430
|
if (existing) {
|
|
89175
89431
|
if (existing.identityKey === identityKey) {
|
|
@@ -89223,7 +89479,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
89223
89479
|
// ../api-routes/src/research.ts
|
|
89224
89480
|
import crypto52 from "crypto";
|
|
89225
89481
|
import { z as z5 } from "zod";
|
|
89226
|
-
import { and as
|
|
89482
|
+
import { and as and71, count as count4, desc as desc39, eq as eq85, gte as gte23, lt as lt14, or as or21, sql as sql31 } from "drizzle-orm";
|
|
89227
89483
|
var sameLocation = (a, b) => a.label === b.label && a.city === b.city && a.region === b.region && a.country === b.country && a.timezone === b.timezone;
|
|
89228
89484
|
var BATCH_RECEIPT_PREFIX = "__canonry_research_batch__:";
|
|
89229
89485
|
async function researchRoutes(app, opts) {
|
|
@@ -89241,7 +89497,7 @@ async function researchRoutes(app, opts) {
|
|
|
89241
89497
|
const input = parsed.data;
|
|
89242
89498
|
if (input.idempotencyKey?.startsWith(BATCH_RECEIPT_PREFIX)) throw validationError("This idempotency key prefix is reserved for internal research batch receipts.");
|
|
89243
89499
|
const requestHash2 = directResearchRequestHash(input);
|
|
89244
|
-
const existingReceipt = input.idempotencyKey ? app.db.select().from(researchRuns).where(
|
|
89500
|
+
const existingReceipt = input.idempotencyKey ? app.db.select().from(researchRuns).where(and71(eq85(researchRuns.projectId, project.id), eq85(researchRuns.idempotencyKey, input.idempotencyKey))).get() : void 0;
|
|
89245
89501
|
if (existingReceipt) {
|
|
89246
89502
|
assertSameDirectResearchRequest(existingReceipt, input, requestHash2);
|
|
89247
89503
|
const result2 = getDetail(app, project.id, existingReceipt.id);
|
|
@@ -89273,7 +89529,7 @@ async function researchRoutes(app, opts) {
|
|
|
89273
89529
|
const initiatedBy = researchPrincipal(request);
|
|
89274
89530
|
const decision = app.db.transaction((tx) => {
|
|
89275
89531
|
if (input.idempotencyKey) {
|
|
89276
|
-
const existing = tx.select().from(researchRuns).where(
|
|
89532
|
+
const existing = tx.select().from(researchRuns).where(and71(eq85(researchRuns.projectId, project.id), eq85(researchRuns.idempotencyKey, input.idempotencyKey))).get();
|
|
89277
89533
|
if (existing) {
|
|
89278
89534
|
assertSameDirectResearchRequest(existing, input, requestHash2);
|
|
89279
89535
|
return { reused: true, id: existing.id, shouldDispatch: existing.status === ResearchRunStatuses.queued };
|
|
@@ -89281,9 +89537,9 @@ async function researchRoutes(app, opts) {
|
|
|
89281
89537
|
}
|
|
89282
89538
|
if (initiatedBy?.limited) {
|
|
89283
89539
|
const { start, end, date } = utcDayBounds(now);
|
|
89284
|
-
const used = tx.select({ value:
|
|
89285
|
-
|
|
89286
|
-
|
|
89540
|
+
const used = tx.select({ value: count4() }).from(researchRuns).where(and71(
|
|
89541
|
+
eq85(researchRuns.projectId, project.id),
|
|
89542
|
+
gte23(researchRuns.createdAt, start),
|
|
89287
89543
|
lt14(researchRuns.createdAt, end),
|
|
89288
89544
|
sql31`(json_extract(${researchRuns.initiatedBy}, '$.role') = ${UserRoles.viewer} OR json_extract(${researchRuns.initiatedBy}, '$.limited') = 1)`
|
|
89289
89545
|
)).get()?.value ?? 0;
|
|
@@ -89313,15 +89569,15 @@ async function researchRoutes(app, opts) {
|
|
|
89313
89569
|
if (input.idempotencyKey.startsWith(BATCH_RECEIPT_PREFIX)) throw validationError("This idempotency key prefix is reserved for internal research batch receipts.");
|
|
89314
89570
|
const requestHash2 = researchBatchRequestHash(input);
|
|
89315
89571
|
const receiptKeys = input.runs.map((_run, index2) => batchChildReceiptKey(input.idempotencyKey, index2));
|
|
89316
|
-
const firstReceipt = app.db.select().from(researchRuns).where(
|
|
89317
|
-
|
|
89318
|
-
|
|
89572
|
+
const firstReceipt = app.db.select().from(researchRuns).where(and71(
|
|
89573
|
+
eq85(researchRuns.projectId, project.id),
|
|
89574
|
+
eq85(researchRuns.idempotencyKey, receiptKeys[0])
|
|
89319
89575
|
)).get();
|
|
89320
89576
|
if (firstReceipt) {
|
|
89321
89577
|
if (firstReceipt.requestHash !== requestHash2) throw alreadyExists("Research batch idempotency key", input.idempotencyKey);
|
|
89322
|
-
const saved = receiptKeys.map((key) => app.db.select().from(researchRuns).where(
|
|
89323
|
-
|
|
89324
|
-
|
|
89578
|
+
const saved = receiptKeys.map((key) => app.db.select().from(researchRuns).where(and71(
|
|
89579
|
+
eq85(researchRuns.projectId, project.id),
|
|
89580
|
+
eq85(researchRuns.idempotencyKey, key)
|
|
89325
89581
|
)).get());
|
|
89326
89582
|
if (saved.some((row) => !row || row.requestHash !== requestHash2)) throw alreadyExists("Research batch idempotency key", input.idempotencyKey);
|
|
89327
89583
|
const runs3 = saved.map((row) => getDetail(app, project.id, row.id));
|
|
@@ -89336,9 +89592,9 @@ async function researchRoutes(app, opts) {
|
|
|
89336
89592
|
const decision = app.db.transaction((tx) => {
|
|
89337
89593
|
if (initiatedBy?.limited) {
|
|
89338
89594
|
const { start, end, date } = utcDayBounds(now);
|
|
89339
|
-
const used = tx.select({ value:
|
|
89340
|
-
|
|
89341
|
-
|
|
89595
|
+
const used = tx.select({ value: count4() }).from(researchRuns).where(and71(
|
|
89596
|
+
eq85(researchRuns.projectId, project.id),
|
|
89597
|
+
gte23(researchRuns.createdAt, start),
|
|
89342
89598
|
lt14(researchRuns.createdAt, end),
|
|
89343
89599
|
sql31`(json_extract(${researchRuns.initiatedBy}, '$.role') = ${UserRoles.viewer} OR json_extract(${researchRuns.initiatedBy}, '$.limited') = 1)`
|
|
89344
89600
|
)).get()?.value ?? 0;
|
|
@@ -89391,10 +89647,10 @@ async function researchRoutes(app, opts) {
|
|
|
89391
89647
|
const requested = Number.parseInt(request.query.limit ?? "", 10);
|
|
89392
89648
|
const limit = Number.isInteger(requested) && requested > 0 ? Math.min(requested, 100) : 20;
|
|
89393
89649
|
const cursor = request.query.cursor ? parseResearchCursor(request.query.cursor, project.id) : null;
|
|
89394
|
-
const rows = app.db.select().from(researchRuns).where(
|
|
89395
|
-
|
|
89396
|
-
cursor ?
|
|
89397
|
-
)).orderBy(
|
|
89650
|
+
const rows = app.db.select().from(researchRuns).where(and71(
|
|
89651
|
+
eq85(researchRuns.projectId, project.id),
|
|
89652
|
+
cursor ? or21(lt14(researchRuns.createdAt, cursor.createdAt), and71(eq85(researchRuns.createdAt, cursor.createdAt), lt14(researchRuns.id, cursor.id))) : void 0
|
|
89653
|
+
)).orderBy(desc39(researchRuns.createdAt), desc39(researchRuns.id)).limit(limit + 1).all();
|
|
89398
89654
|
const runs2 = rows.slice(0, limit).map(serializeRun);
|
|
89399
89655
|
const last = runs2.at(-1);
|
|
89400
89656
|
const nextCursor = rows.length > limit && last ? Buffer.from(JSON.stringify({ projectId: project.id, createdAt: last.createdAt, id: last.id })).toString("base64url") : null;
|
|
@@ -89520,9 +89776,9 @@ function resolveStoredResearchTemplate(stored, selection, idempotencyKey) {
|
|
|
89520
89776
|
return stored;
|
|
89521
89777
|
}
|
|
89522
89778
|
function resolveResearchTemplate(app, projectId, selection, scope, location) {
|
|
89523
|
-
const savedTemplate = app.db.select().from(measurementQueryTemplates).where(
|
|
89524
|
-
|
|
89525
|
-
|
|
89779
|
+
const savedTemplate = app.db.select().from(measurementQueryTemplates).where(and71(
|
|
89780
|
+
eq85(measurementQueryTemplates.projectId, projectId),
|
|
89781
|
+
eq85(measurementQueryTemplates.id, selection.templateId)
|
|
89526
89782
|
)).get();
|
|
89527
89783
|
if (!savedTemplate) throw validationError("Unknown or stale research template. Reload the selected template and try again.");
|
|
89528
89784
|
const templateVersion = savedTemplate.updatedAt;
|
|
@@ -89531,7 +89787,7 @@ function resolveResearchTemplate(app, projectId, selection, scope, location) {
|
|
|
89531
89787
|
}
|
|
89532
89788
|
const bindingLocation = selection.bindingLocation === void 0 ? location : selection.bindingLocation;
|
|
89533
89789
|
if (bindingLocation) {
|
|
89534
|
-
const project = app.db.select({ locations: projects.locations }).from(projects).where(
|
|
89790
|
+
const project = app.db.select({ locations: projects.locations }).from(projects).where(eq85(projects.id, projectId)).get();
|
|
89535
89791
|
if (!project.locations.some((item2) => sameLocation(item2, bindingLocation))) throw validationError("Research template binding location must match a configured project location.");
|
|
89536
89792
|
}
|
|
89537
89793
|
const { bindings, output } = expandResearchTemplate(savedTemplate, scope, bindingLocation);
|
|
@@ -89544,9 +89800,9 @@ function classifyResearchQueries(project, plan, queries2) {
|
|
|
89544
89800
|
return queries2.map((query) => classifier?.classify(query) ?? null);
|
|
89545
89801
|
}
|
|
89546
89802
|
function getDetail(app, projectId, id) {
|
|
89547
|
-
const row = app.db.select().from(researchRuns).where(
|
|
89803
|
+
const row = app.db.select().from(researchRuns).where(and71(eq85(researchRuns.id, id), eq85(researchRuns.projectId, projectId))).get();
|
|
89548
89804
|
if (!row) throw notFound("Research run", id);
|
|
89549
|
-
const queries2 = app.db.select().from(researchRunQueries).where(
|
|
89805
|
+
const queries2 = app.db.select().from(researchRunQueries).where(eq85(researchRunQueries.researchRunId, id)).orderBy(researchRunQueries.position).all().map(serializeQuery);
|
|
89550
89806
|
return { ...serializeRun(row), queries: queries2 };
|
|
89551
89807
|
}
|
|
89552
89808
|
function serializeRun(row) {
|
|
@@ -89576,7 +89832,7 @@ function utcDayBounds(now) {
|
|
|
89576
89832
|
|
|
89577
89833
|
// ../api-routes/src/snapshot-competitor-refresh.ts
|
|
89578
89834
|
import { setImmediate as yieldToEventLoop2 } from "timers/promises";
|
|
89579
|
-
import { and as
|
|
89835
|
+
import { and as and72, asc as asc16, eq as eq86, inArray as inArray35 } from "drizzle-orm";
|
|
89580
89836
|
var ANSWER_MENTIONS_CHUNK_SIZE = 50;
|
|
89581
89837
|
var ANSWER_MENTIONS_MAX_RESTARTS = 3;
|
|
89582
89838
|
function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
@@ -89626,14 +89882,14 @@ function* projectAnswerMentionsSteps(db, projectId, opts, driver) {
|
|
|
89626
89882
|
examined = 0;
|
|
89627
89883
|
wouldUpdate = 0;
|
|
89628
89884
|
mentioned = 0;
|
|
89629
|
-
const runRows = db.select({ id: runs.id, planVersionId: runs.measurementPlanVersionId }).from(runs).where(
|
|
89630
|
-
|
|
89631
|
-
|
|
89632
|
-
opts?.runId === void 0 ? void 0 :
|
|
89885
|
+
const runRows = db.select({ id: runs.id, planVersionId: runs.measurementPlanVersionId }).from(runs).where(and72(
|
|
89886
|
+
eq86(runs.kind, RunKinds["answer-visibility"]),
|
|
89887
|
+
eq86(runs.projectId, projectId),
|
|
89888
|
+
opts?.runId === void 0 ? void 0 : eq86(runs.id, opts.runId)
|
|
89633
89889
|
)).all();
|
|
89634
89890
|
const planVersionByRun = new Map(runRows.map((run) => [run.id, run.planVersionId]));
|
|
89635
89891
|
for (const run of runRows) {
|
|
89636
|
-
const snapshotIds = db.select({ id: querySnapshots.id }).from(querySnapshots).where(
|
|
89892
|
+
const snapshotIds = db.select({ id: querySnapshots.id }).from(querySnapshots).where(eq86(querySnapshots.runId, run.id)).orderBy(asc16(querySnapshots.id)).all().map((row) => row.id);
|
|
89637
89893
|
for (let offset = 0; offset < snapshotIds.length; offset += ANSWER_MENTIONS_CHUNK_SIZE) {
|
|
89638
89894
|
if (chunks++ > 0 && driver.recheckIdentity) {
|
|
89639
89895
|
yield;
|
|
@@ -89659,7 +89915,7 @@ function* projectAnswerMentionsSteps(db, projectId, opts, driver) {
|
|
|
89659
89915
|
competitorOverlap: querySnapshots.competitorOverlap,
|
|
89660
89916
|
recommendedCompetitors: querySnapshots.recommendedCompetitors,
|
|
89661
89917
|
rawResponse: querySnapshots.rawResponse
|
|
89662
|
-
}).from(querySnapshots).where(
|
|
89918
|
+
}).from(querySnapshots).where(inArray35(querySnapshots.id, snapshotIds.slice(offset, offset + ANSWER_MENTIONS_CHUNK_SIZE))).all();
|
|
89663
89919
|
const pendingUpdates = [];
|
|
89664
89920
|
for (const snapshot of snapshotRows) {
|
|
89665
89921
|
if (competitorFieldsOnly && snapshot.answerText == null) continue;
|
|
@@ -89709,7 +89965,7 @@ function* projectAnswerMentionsSteps(db, projectId, opts, driver) {
|
|
|
89709
89965
|
} else {
|
|
89710
89966
|
db.transaction((tx) => {
|
|
89711
89967
|
for (const update of pendingUpdates) {
|
|
89712
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
89968
|
+
tx.update(querySnapshots).set(update.patch).where(eq86(querySnapshots.id, update.id)).run();
|
|
89713
89969
|
}
|
|
89714
89970
|
});
|
|
89715
89971
|
updated += pendingUpdates.length;
|
|
@@ -89739,7 +89995,7 @@ function readStoredGroundingSources(rawResponse) {
|
|
|
89739
89995
|
|
|
89740
89996
|
// ../api-routes/src/simple-measurement-definitions.ts
|
|
89741
89997
|
import crypto53 from "crypto";
|
|
89742
|
-
import { and as
|
|
89998
|
+
import { and as and73, eq as eq87 } from "drizzle-orm";
|
|
89743
89999
|
function captureSimpleMeasurementDefinition(db, input) {
|
|
89744
90000
|
return db.transaction((tx) => {
|
|
89745
90001
|
const run = tx.select({
|
|
@@ -89747,7 +90003,7 @@ function captureSimpleMeasurementDefinition(db, input) {
|
|
|
89747
90003
|
trigger: runs.trigger,
|
|
89748
90004
|
status: runs.status,
|
|
89749
90005
|
measurementPlanVersionId: runs.measurementPlanVersionId
|
|
89750
|
-
}).from(runs).where(
|
|
90006
|
+
}).from(runs).where(and73(eq87(runs.projectId, input.projectId), eq87(runs.id, input.runId))).get();
|
|
89751
90007
|
if (!run) throw notFound("Run", input.runId);
|
|
89752
90008
|
if (run.kind !== RunKinds["answer-visibility"] || run.trigger === RunTriggers.probe || run.measurementPlanVersionId !== null) {
|
|
89753
90009
|
return null;
|
|
@@ -89768,7 +90024,7 @@ function captureSimpleMeasurementDefinition(db, input) {
|
|
|
89768
90024
|
throw validationError("Qualified aliases must be a sorted subset of the captured aliases.");
|
|
89769
90025
|
}
|
|
89770
90026
|
}
|
|
89771
|
-
const existing = tx.select().from(simpleMeasurementDefinitions).where(
|
|
90027
|
+
const existing = tx.select().from(simpleMeasurementDefinitions).where(and73(eq87(simpleMeasurementDefinitions.projectId, input.projectId), eq87(simpleMeasurementDefinitions.runId, input.runId))).get();
|
|
89772
90028
|
if (existing) {
|
|
89773
90029
|
const frozen = simpleMeasurementDefinitionSchema.parse(existing.definition);
|
|
89774
90030
|
const { qualifiedAliases: _liveQualifiedAliases, ...liveIdentity } = definition.identity;
|
|
@@ -89794,15 +90050,15 @@ function captureSimpleMeasurementDefinition(db, input) {
|
|
|
89794
90050
|
if (run.status !== RunStatuses.running) {
|
|
89795
90051
|
throw validationError("Only a running run can capture a new measurement definition.");
|
|
89796
90052
|
}
|
|
89797
|
-
if (tx.select({ id: querySnapshots.id }).from(querySnapshots).where(
|
|
90053
|
+
if (tx.select({ id: querySnapshots.id }).from(querySnapshots).where(eq87(querySnapshots.runId, input.runId)).limit(1).get()) {
|
|
89798
90054
|
throw validationError("This run already has stored answers. Its measurement definition cannot be inferred afterward.");
|
|
89799
90055
|
}
|
|
89800
|
-
const projectQueryIds = new Set(tx.select({ id: queries.id }).from(queries).where(
|
|
90056
|
+
const projectQueryIds = new Set(tx.select({ id: queries.id }).from(queries).where(eq87(queries.projectId, input.projectId)).all().map((query) => query.id));
|
|
89801
90057
|
if (definition.queries.some((query) => !projectQueryIds.has(query.queryId))) {
|
|
89802
90058
|
throw validationError("Every captured query must belong to the run project.");
|
|
89803
90059
|
}
|
|
89804
90060
|
if (definition.competitors !== void 0) {
|
|
89805
|
-
const liveCompetitorDomains = new Set(tx.select({ domain: competitors.domain }).from(competitors).where(
|
|
90061
|
+
const liveCompetitorDomains = new Set(tx.select({ domain: competitors.domain }).from(competitors).where(eq87(competitors.projectId, input.projectId)).orderBy(competitors.domain).all().map((competitor) => competitor.domain));
|
|
89806
90062
|
const capturedDomains = new Set(definition.competitors.map((competitor) => competitor.domain));
|
|
89807
90063
|
if (liveCompetitorDomains.size !== capturedDomains.size || [...liveCompetitorDomains].some((domain) => !capturedDomains.has(domain))) {
|
|
89808
90064
|
throw validationError("Captured competitors must exactly match the project competitors dispatched for this run.");
|
|
@@ -90658,10 +90914,10 @@ var log = createLogger("IntelligenceService");
|
|
|
90658
90914
|
var runChronologyKey = sql32`coalesce(${runs.finishedAt}, ${runs.createdAt})`;
|
|
90659
90915
|
function analyzableRunPredicates(projectId) {
|
|
90660
90916
|
return [
|
|
90661
|
-
|
|
90917
|
+
eq88(runs.projectId, projectId),
|
|
90662
90918
|
// Only this kind writes query_snapshots.
|
|
90663
|
-
|
|
90664
|
-
|
|
90919
|
+
eq88(runs.kind, RunKinds["answer-visibility"]),
|
|
90920
|
+
or22(eq88(runs.status, RunStatuses.completed), eq88(runs.status, RunStatuses.partial)),
|
|
90665
90921
|
// Defensive: RunCoordinator already skips probes before analyzeAndPersist
|
|
90666
90922
|
// is called, but a future call site invoking it directly for a probe must
|
|
90667
90923
|
// still not pollute the intelligence window.
|
|
@@ -90706,16 +90962,16 @@ var IntelligenceService = class {
|
|
|
90706
90962
|
* Returns the analysis result for the coordinator to inspect (e.g. for webhook dispatch).
|
|
90707
90963
|
*/
|
|
90708
90964
|
analyzeAndPersist(runId, projectId) {
|
|
90709
|
-
const currentRunRecord = this.db.select().from(runs).where(
|
|
90965
|
+
const currentRunRecord = this.db.select().from(runs).where(and74(eq88(runs.id, runId), ...analyzableRunPredicates(projectId))).get();
|
|
90710
90966
|
if (!currentRunRecord) {
|
|
90711
90967
|
log.info("intelligence.skip", { runId, reason: "run not eligible for analysis" });
|
|
90712
90968
|
return null;
|
|
90713
90969
|
}
|
|
90714
90970
|
const currentLocation = currentRunRecord.location ?? null;
|
|
90715
90971
|
const recentRuns = this.db.select().from(runs).where(
|
|
90716
|
-
|
|
90972
|
+
and74(
|
|
90717
90973
|
...analyzableRunPredicates(projectId),
|
|
90718
|
-
currentLocation === null ?
|
|
90974
|
+
currentLocation === null ? isNull17(runs.location) : eq88(runs.location, currentLocation),
|
|
90719
90975
|
// Never look forward. The window is this run's own past, so
|
|
90720
90976
|
// re-analyzing a historical run compares it against its true
|
|
90721
90977
|
// predecessor rather than against sweeps that came after it.
|
|
@@ -90723,10 +90979,10 @@ var IntelligenceService = class {
|
|
|
90723
90979
|
// MEASUREMENT: only runs that actually wrote snapshots — see above.
|
|
90724
90980
|
// Index seek on idx_snapshots_run, short-circuited on the first row.
|
|
90725
90981
|
exists2(
|
|
90726
|
-
this.db.select({ one: sql32`1` }).from(querySnapshots).where(
|
|
90982
|
+
this.db.select({ one: sql32`1` }).from(querySnapshots).where(eq88(querySnapshots.runId, runs.id))
|
|
90727
90983
|
)
|
|
90728
90984
|
)
|
|
90729
|
-
).orderBy(
|
|
90985
|
+
).orderBy(desc40(runs.finishedAt), desc40(runs.createdAt)).limit(HISTORY_WINDOW_RUNS).all();
|
|
90730
90986
|
const trackedCompetitors = this.loadTrackedCompetitors(projectId);
|
|
90731
90987
|
const trackedCompetitorDomains = trackedCompetitors.map((competitor) => competitor.domain);
|
|
90732
90988
|
const currentRun = this.buildRunData(
|
|
@@ -90799,7 +91055,7 @@ var IntelligenceService = class {
|
|
|
90799
91055
|
* Returns the persisted insights so the coordinator can count critical/high.
|
|
90800
91056
|
*/
|
|
90801
91057
|
analyzeAndPersistGbp(runId, projectId) {
|
|
90802
|
-
const runRow = this.db.select({ createdAt: runs.createdAt, startedAt: runs.startedAt, finishedAt: runs.finishedAt }).from(runs).where(
|
|
91058
|
+
const runRow = this.db.select({ createdAt: runs.createdAt, startedAt: runs.startedAt, finishedAt: runs.finishedAt }).from(runs).where(eq88(runs.id, runId)).get();
|
|
90803
91059
|
if (!runRow) {
|
|
90804
91060
|
log.info("gbp-intelligence.skip", { runId, reason: "run not found" });
|
|
90805
91061
|
this.persistGbpInsights(runId, projectId, [], []);
|
|
@@ -90807,10 +91063,10 @@ var IntelligenceService = class {
|
|
|
90807
91063
|
}
|
|
90808
91064
|
const windowStart = runRow.startedAt ?? runRow.createdAt;
|
|
90809
91065
|
const windowEnd = runRow.finishedAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
90810
|
-
const selected = this.db.select().from(gbpLocations).where(
|
|
90811
|
-
|
|
90812
|
-
|
|
90813
|
-
|
|
91066
|
+
const selected = this.db.select().from(gbpLocations).where(and74(
|
|
91067
|
+
eq88(gbpLocations.projectId, projectId),
|
|
91068
|
+
eq88(gbpLocations.selected, true),
|
|
91069
|
+
gte24(gbpLocations.syncedAt, windowStart),
|
|
90814
91070
|
lte20(gbpLocations.syncedAt, windowEnd)
|
|
90815
91071
|
)).all();
|
|
90816
91072
|
if (selected.length === 0) {
|
|
@@ -90844,12 +91100,12 @@ var IntelligenceService = class {
|
|
|
90844
91100
|
}
|
|
90845
91101
|
/** Build the per-location signal bundle the GBP analyzer consumes. */
|
|
90846
91102
|
buildGbpLocationSignals(projectId, locationName, displayName, fallbackDate) {
|
|
90847
|
-
const metricRows = this.db.select({ metric: gbpDailyMetrics.metric, date: gbpDailyMetrics.date, value: gbpDailyMetrics.value }).from(gbpDailyMetrics).where(
|
|
90848
|
-
const placeActionRows = this.db.select({ placeActionType: gbpPlaceActions.placeActionType, providerType: gbpPlaceActions.providerType }).from(gbpPlaceActions).where(
|
|
90849
|
-
const lodgingRow = this.db.select({ populatedGroupCount: gbpLodgingSnapshots.populatedGroupCount }).from(gbpLodgingSnapshots).where(
|
|
90850
|
-
const ownerRow = this.db.select({ description: gbpLocations.description }).from(gbpLocations).where(
|
|
91103
|
+
const metricRows = this.db.select({ metric: gbpDailyMetrics.metric, date: gbpDailyMetrics.date, value: gbpDailyMetrics.value }).from(gbpDailyMetrics).where(and74(eq88(gbpDailyMetrics.projectId, projectId), eq88(gbpDailyMetrics.locationName, locationName))).all();
|
|
91104
|
+
const placeActionRows = this.db.select({ placeActionType: gbpPlaceActions.placeActionType, providerType: gbpPlaceActions.providerType }).from(gbpPlaceActions).where(and74(eq88(gbpPlaceActions.projectId, projectId), eq88(gbpPlaceActions.locationName, locationName))).all();
|
|
91105
|
+
const lodgingRow = this.db.select({ populatedGroupCount: gbpLodgingSnapshots.populatedGroupCount }).from(gbpLodgingSnapshots).where(and74(eq88(gbpLodgingSnapshots.projectId, projectId), eq88(gbpLodgingSnapshots.locationName, locationName))).orderBy(desc40(gbpLodgingSnapshots.syncedAt)).limit(1).get();
|
|
91106
|
+
const ownerRow = this.db.select({ description: gbpLocations.description }).from(gbpLocations).where(and74(eq88(gbpLocations.projectId, projectId), eq88(gbpLocations.locationName, locationName))).get();
|
|
90851
91107
|
const descriptionMissing = !(ownerRow?.description ?? "").trim();
|
|
90852
|
-
const placeRow = this.db.select({ attributes: gbpPlaceDetails.attributes }).from(gbpPlaceDetails).where(
|
|
91108
|
+
const placeRow = this.db.select({ attributes: gbpPlaceDetails.attributes }).from(gbpPlaceDetails).where(and74(eq88(gbpPlaceDetails.projectId, projectId), eq88(gbpPlaceDetails.locationName, locationName))).orderBy(desc40(gbpPlaceDetails.syncedAt)).limit(1).get();
|
|
90853
91109
|
const placesAmenities = placeRow ? extractPlaceAmenities(placeRow.attributes) : [];
|
|
90854
91110
|
const summary3 = buildGbpSummary({
|
|
90855
91111
|
locationName,
|
|
@@ -90885,7 +91141,7 @@ var IntelligenceService = class {
|
|
|
90885
91141
|
/** Build the month-over-month keyword series for a location from the
|
|
90886
91142
|
* accumulating gbp_keyword_monthly table (latest complete month vs prior). */
|
|
90887
91143
|
buildGbpKeywordTrend(projectId, locationName) {
|
|
90888
|
-
const rows = this.db.select({ month: gbpKeywordMonthly.month, keyword: gbpKeywordMonthly.keyword, valueCount: gbpKeywordMonthly.valueCount }).from(gbpKeywordMonthly).where(
|
|
91144
|
+
const rows = this.db.select({ month: gbpKeywordMonthly.month, keyword: gbpKeywordMonthly.keyword, valueCount: gbpKeywordMonthly.valueCount }).from(gbpKeywordMonthly).where(and74(eq88(gbpKeywordMonthly.projectId, projectId), eq88(gbpKeywordMonthly.locationName, locationName))).all();
|
|
90889
91145
|
if (rows.length === 0) return { recentMonth: null, priorMonth: null, points: [] };
|
|
90890
91146
|
const months = [...new Set(rows.map((r) => r.month))].sort().reverse();
|
|
90891
91147
|
const recentMonth = months[0] ?? null;
|
|
@@ -90916,7 +91172,7 @@ var IntelligenceService = class {
|
|
|
90916
91172
|
*/
|
|
90917
91173
|
persistGbpInsights(runId, projectId, gbpInsights, coveredLocationNames) {
|
|
90918
91174
|
const covered = new Set(coveredLocationNames);
|
|
90919
|
-
const existing = this.db.select({ id: insights.id, dismissed: insights.dismissed }).from(insights).where(
|
|
91175
|
+
const existing = this.db.select({ id: insights.id, dismissed: insights.dismissed }).from(insights).where(and74(eq88(insights.projectId, projectId), eq88(insights.provider, GBP_INSIGHT_PROVIDER))).all();
|
|
90920
91176
|
const staleIds = [];
|
|
90921
91177
|
const dismissedSlots = /* @__PURE__ */ new Set();
|
|
90922
91178
|
for (const row of existing) {
|
|
@@ -90927,7 +91183,7 @@ var IntelligenceService = class {
|
|
|
90927
91183
|
}
|
|
90928
91184
|
this.db.transaction((tx) => {
|
|
90929
91185
|
for (const id of staleIds) {
|
|
90930
|
-
tx.delete(insights).where(
|
|
91186
|
+
tx.delete(insights).where(eq88(insights.id, id)).run();
|
|
90931
91187
|
}
|
|
90932
91188
|
for (const insight of gbpInsights) {
|
|
90933
91189
|
const parsed = parseGbpInsightId(insight.id);
|
|
@@ -91014,7 +91270,7 @@ var IntelligenceService = class {
|
|
|
91014
91270
|
* create per run + aggregate). DB is left untouched.
|
|
91015
91271
|
*/
|
|
91016
91272
|
backfill(projectName, opts, onProgress) {
|
|
91017
|
-
const project = this.db.select().from(projects).where(
|
|
91273
|
+
const project = this.db.select().from(projects).where(eq88(projects.name, projectName)).get();
|
|
91018
91274
|
if (!project) {
|
|
91019
91275
|
throw new Error(`Project "${projectName}" not found`);
|
|
91020
91276
|
}
|
|
@@ -91026,7 +91282,7 @@ var IntelligenceService = class {
|
|
|
91026
91282
|
}
|
|
91027
91283
|
sinceTimestamp = parsed;
|
|
91028
91284
|
}
|
|
91029
|
-
const allRuns = this.db.select().from(runs).where(
|
|
91285
|
+
const allRuns = this.db.select().from(runs).where(and74(...analyzableRunPredicates(project.id))).orderBy(asc17(runs.finishedAt)).all();
|
|
91030
91286
|
const measuredRunIds = this.runIdsWithSnapshots(allRuns.map((r) => r.id));
|
|
91031
91287
|
const measuredRuns = allRuns.filter((r) => measuredRunIds.has(r.id));
|
|
91032
91288
|
let startIdx = 0;
|
|
@@ -91050,14 +91306,14 @@ var IntelligenceService = class {
|
|
|
91050
91306
|
});
|
|
91051
91307
|
}
|
|
91052
91308
|
let processed = 0;
|
|
91053
|
-
let
|
|
91309
|
+
let skipped4 = 0;
|
|
91054
91310
|
let totalInsights = 0;
|
|
91055
91311
|
const isDryRun = opts?.dryRun === true;
|
|
91056
91312
|
const perRunDelta = [];
|
|
91057
91313
|
let wouldDeleteTotal = 0;
|
|
91058
91314
|
const existingByRunId = /* @__PURE__ */ new Map();
|
|
91059
91315
|
if (isDryRun && targetRuns.length > 0) {
|
|
91060
|
-
const rows = this.db.select({ runId: insights.runId }).from(insights).where(
|
|
91316
|
+
const rows = this.db.select({ runId: insights.runId }).from(insights).where(inArray36(insights.runId, targetRuns.map((r) => r.id))).all();
|
|
91061
91317
|
for (const r of rows) {
|
|
91062
91318
|
if (r.runId == null) continue;
|
|
91063
91319
|
existingByRunId.set(r.runId, (existingByRunId.get(r.runId) ?? 0) + 1);
|
|
@@ -91082,14 +91338,14 @@ var IntelligenceService = class {
|
|
|
91082
91338
|
}
|
|
91083
91339
|
onProgress?.({ runId: run.id, index: i + 1, total: targetRuns.length, insights: result.insights.length });
|
|
91084
91340
|
} else {
|
|
91085
|
-
|
|
91341
|
+
skipped4++;
|
|
91086
91342
|
onProgress?.({ runId: run.id, index: i + 1, total: targetRuns.length, insights: 0 });
|
|
91087
91343
|
}
|
|
91088
91344
|
}
|
|
91089
91345
|
if (isDryRun) {
|
|
91090
91346
|
return {
|
|
91091
91347
|
processed,
|
|
91092
|
-
skipped:
|
|
91348
|
+
skipped: skipped4,
|
|
91093
91349
|
totalInsights,
|
|
91094
91350
|
dryRun: true,
|
|
91095
91351
|
delta: {
|
|
@@ -91100,7 +91356,7 @@ var IntelligenceService = class {
|
|
|
91100
91356
|
}
|
|
91101
91357
|
};
|
|
91102
91358
|
}
|
|
91103
|
-
return { processed, skipped:
|
|
91359
|
+
return { processed, skipped: skipped4, totalInsights };
|
|
91104
91360
|
}
|
|
91105
91361
|
/**
|
|
91106
91362
|
* The project's tracked competitors with every name they answer to
|
|
@@ -91108,7 +91364,7 @@ var IntelligenceService = class {
|
|
|
91108
91364
|
* signals match the same identities the dashboard and reports do.
|
|
91109
91365
|
*/
|
|
91110
91366
|
loadTrackedCompetitors(projectId) {
|
|
91111
|
-
return this.db.select(competitorIdentityColumns).from(competitors).where(
|
|
91367
|
+
return this.db.select(competitorIdentityColumns).from(competitors).where(eq88(competitors.projectId, projectId)).orderBy(competitors.domain).all();
|
|
91112
91368
|
}
|
|
91113
91369
|
/**
|
|
91114
91370
|
* Which of `runIds` actually wrote query snapshots. One grouped read, so
|
|
@@ -91117,7 +91373,7 @@ var IntelligenceService = class {
|
|
|
91117
91373
|
*/
|
|
91118
91374
|
runIdsWithSnapshots(runIds) {
|
|
91119
91375
|
if (runIds.length === 0) return /* @__PURE__ */ new Set();
|
|
91120
|
-
const rows = this.db.select({ runId: querySnapshots.runId }).from(querySnapshots).where(
|
|
91376
|
+
const rows = this.db.select({ runId: querySnapshots.runId }).from(querySnapshots).where(inArray36(querySnapshots.runId, [...runIds])).groupBy(querySnapshots.runId).all();
|
|
91121
91377
|
return new Set(rows.map((r) => r.runId));
|
|
91122
91378
|
}
|
|
91123
91379
|
/**
|
|
@@ -91139,15 +91395,15 @@ var IntelligenceService = class {
|
|
|
91139
91395
|
}
|
|
91140
91396
|
persistResult(result, runId, projectId) {
|
|
91141
91397
|
const previouslyDismissed = /* @__PURE__ */ new Set();
|
|
91142
|
-
const existingInsights = this.db.select({ query: insights.query, provider: insights.provider, type: insights.type, dismissed: insights.dismissed }).from(insights).where(
|
|
91398
|
+
const existingInsights = this.db.select({ query: insights.query, provider: insights.provider, type: insights.type, dismissed: insights.dismissed }).from(insights).where(eq88(insights.runId, runId)).all();
|
|
91143
91399
|
for (const row of existingInsights) {
|
|
91144
91400
|
if (row.dismissed) {
|
|
91145
91401
|
previouslyDismissed.add(`${row.query}:${row.provider}:${row.type}`);
|
|
91146
91402
|
}
|
|
91147
91403
|
}
|
|
91148
91404
|
this.db.transaction((tx) => {
|
|
91149
|
-
tx.delete(insights).where(
|
|
91150
|
-
tx.delete(healthSnapshots).where(
|
|
91405
|
+
tx.delete(insights).where(eq88(insights.runId, runId)).run();
|
|
91406
|
+
tx.delete(healthSnapshots).where(eq88(healthSnapshots.runId, runId)).run();
|
|
91151
91407
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
91152
91408
|
for (const insight of result.insights) {
|
|
91153
91409
|
const wasDismissed = previouslyDismissed.has(`${insight.query}:${insight.provider}:${insight.type}`);
|
|
@@ -91194,9 +91450,9 @@ var IntelligenceService = class {
|
|
|
91194
91450
|
clicks: gscSearchData.clicks,
|
|
91195
91451
|
impressions: gscSearchData.impressions,
|
|
91196
91452
|
position: gscSearchData.position
|
|
91197
|
-
}).from(gscSearchData).where(
|
|
91198
|
-
|
|
91199
|
-
|
|
91453
|
+
}).from(gscSearchData).where(and74(
|
|
91454
|
+
eq88(gscSearchData.projectId, projectId),
|
|
91455
|
+
gte24(gscSearchData.date, window.startDate),
|
|
91200
91456
|
lte20(gscSearchData.date, window.endDate)
|
|
91201
91457
|
)).all();
|
|
91202
91458
|
const byDay = /* @__PURE__ */ new Map();
|
|
@@ -91248,21 +91504,21 @@ var IntelligenceService = class {
|
|
|
91248
91504
|
const key = row.query.toLowerCase();
|
|
91249
91505
|
gscImpressionsByQuery.set(key, (gscImpressionsByQuery.get(key) ?? 0) + row.impressions);
|
|
91250
91506
|
}
|
|
91251
|
-
const projectRow = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
91507
|
+
const projectRow = this.db.select({ locations: projects.locations }).from(projects).where(eq88(projects.id, projectId)).get();
|
|
91252
91508
|
const locationCount = Math.max(
|
|
91253
91509
|
1,
|
|
91254
91510
|
(projectRow?.locations ?? []).length
|
|
91255
91511
|
);
|
|
91256
91512
|
const ROWS_PER_GROUP_BUDGET = Math.max(2, locationCount);
|
|
91257
91513
|
const recentRunRows = this.db.select({ id: runs.id, createdAt: runs.createdAt }).from(runs).where(
|
|
91258
|
-
|
|
91259
|
-
|
|
91260
|
-
|
|
91261
|
-
|
|
91514
|
+
and74(
|
|
91515
|
+
eq88(runs.projectId, projectId),
|
|
91516
|
+
eq88(runs.kind, RunKinds["answer-visibility"]),
|
|
91517
|
+
or22(eq88(runs.status, "completed"), eq88(runs.status, "partial")),
|
|
91262
91518
|
// Defensive — see top of file.
|
|
91263
91519
|
ne16(runs.trigger, RunTriggers.probe)
|
|
91264
91520
|
)
|
|
91265
|
-
).orderBy(
|
|
91521
|
+
).orderBy(desc40(runs.createdAt), desc40(runs.id)).limit((RECURRENCE_LOOKBACK_RUNS + 1) * ROWS_PER_GROUP_BUDGET).all();
|
|
91266
91522
|
const recentGroups = groupRunsByCreatedAt(recentRunRows);
|
|
91267
91523
|
const recentRunIds = [];
|
|
91268
91524
|
const recentRunIdToCreatedAt = /* @__PURE__ */ new Map();
|
|
@@ -91278,7 +91534,7 @@ var IntelligenceService = class {
|
|
|
91278
91534
|
const haveHistory = recentRunIds.length > 0;
|
|
91279
91535
|
const priorRegressionsByPair = /* @__PURE__ */ new Map();
|
|
91280
91536
|
if (haveHistory) {
|
|
91281
|
-
const priorRows = this.db.select({ query: insights.query, provider: insights.provider, runId: insights.runId }).from(insights).where(
|
|
91537
|
+
const priorRows = this.db.select({ query: insights.query, provider: insights.provider, runId: insights.runId }).from(insights).where(and74(eq88(insights.type, "regression"), inArray36(insights.runId, recentRunIds))).all();
|
|
91282
91538
|
const regressionGroups = /* @__PURE__ */ new Map();
|
|
91283
91539
|
for (const row of priorRows) {
|
|
91284
91540
|
if (!row.runId) continue;
|
|
@@ -91307,7 +91563,7 @@ var IntelligenceService = class {
|
|
|
91307
91563
|
});
|
|
91308
91564
|
}
|
|
91309
91565
|
buildRunData(runId, projectId, completedAt, location = null, trackedCompetitors = []) {
|
|
91310
|
-
const projectDomainRow = this.db.select({ canonicalDomain: projects.canonicalDomain, ownedDomains: projects.ownedDomains }).from(projects).where(
|
|
91566
|
+
const projectDomainRow = this.db.select({ canonicalDomain: projects.canonicalDomain, ownedDomains: projects.ownedDomains }).from(projects).where(eq88(projects.id, projectId)).get();
|
|
91311
91567
|
const projectDomains = projectDomainRow ? effectiveDomains({
|
|
91312
91568
|
canonicalDomain: projectDomainRow.canonicalDomain,
|
|
91313
91569
|
ownedDomains: projectDomainRow.ownedDomains
|
|
@@ -91325,7 +91581,7 @@ var IntelligenceService = class {
|
|
|
91325
91581
|
citedDomains: querySnapshots.citedDomains,
|
|
91326
91582
|
rawResponse: querySnapshots.rawResponse,
|
|
91327
91583
|
snapshotLocation: querySnapshots.location
|
|
91328
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
91584
|
+
}).from(querySnapshots).leftJoin(queries, eq88(querySnapshots.queryId, queries.id)).where(eq88(querySnapshots.runId, runId)).all();
|
|
91329
91585
|
const snapshots = [];
|
|
91330
91586
|
let orphanCount = 0;
|
|
91331
91587
|
for (const r of rows) {
|