@canonry/canonry 7.2.0 → 7.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agent-workspace/skills/aero/references/aeo-discovery.md +3 -2
- package/assets/agent-workspace/skills/aero/references/orchestration.md +1 -1
- package/assets/agent-workspace/skills/canonry/references/aeo-analysis.md +1 -1
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +33 -4
- package/assets/assets/{AuditHistoryPanel-Duvyi8vx.js → AuditHistoryPanel-DIL5oTHF.js} +1 -1
- package/assets/assets/{BacklinksPage-CY5AllQu.js → BacklinksPage-BpCURZh8.js} +1 -1
- package/assets/assets/{HistoryPage-C0Gr0gcj.js → HistoryPage-TPbexxzA.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-DrAB6VKX.js → MeasurementPropertyPage-H991GW9h.js} +1 -1
- package/assets/assets/ProjectPage-l1NAl-9k.js +8 -0
- package/assets/assets/{RunsPage-DMxOrtfl.js → RunsPage-Dhu7Qe7B.js} +1 -1
- package/assets/assets/{SettingsPage-sxiUNt2A.js → SettingsPage-BCG1QJ07.js} +1 -1
- package/assets/assets/SiteHealthSection-BXueYSiu.js +4 -0
- package/assets/assets/{TrafficPage-CvSQDNTi.js → TrafficPage-ua9wsccA.js} +3 -3
- package/assets/assets/{TrafficSourceDetailPage-BaYeIBmc.js → TrafficSourceDetailPage-pwATnkp8.js} +1 -1
- package/assets/assets/{extract-error-message-76IWhqQF.js → extract-error-message-DJfNdKqh.js} +1 -1
- package/assets/assets/{index-BlrH2zRW.js → index-DvWWyhGp.js} +21 -21
- package/assets/assets/{react-sigma_core.esm.min-CkAcKW2k.js → react-sigma_core.esm.min-DnbWY5j_.js} +1 -1
- package/assets/assets/{v2-overview-adapter-DnEE4E9w.js → v2-overview-adapter-BIOvZAlo.js} +1 -1
- package/assets/index.html +1 -1
- package/dist/{chunk-UIDHHZH3.js → chunk-GEKMHZS2.js} +293 -1267
- package/dist/{chunk-BYSSJEL7.js → chunk-JU6TPMCJ.js} +33 -11
- package/dist/{chunk-TY3CJKKR.js → chunk-PXRUWG75.js} +11327 -10807
- package/dist/{chunk-RCQJI3UX.js → chunk-SWSWCUHS.js} +3215 -2723
- package/dist/{chunk-FDBQOQLP.js → chunk-TNLOOXUP.js} +29 -2
- package/dist/{chunk-SVTVBPIT.js → chunk-XFVJ6QVE.js} +2 -2
- package/dist/{chunk-EQBYK77W.js → chunk-ZP6GF32R.js} +3 -3
- package/dist/cli.js +1151 -68
- package/dist/{demo-server-GWRYDPJC.js → demo-server-INMOE4BA.js} +6 -6
- package/dist/index.js +6 -6
- package/dist/{intelligence-service-3L2665AK.js → intelligence-service-IBUS2IND.js} +2 -2
- package/dist/mcp.js +4 -4
- package/package.json +13 -13
- package/assets/assets/ProjectPage-DSxvvnAd.js +0 -8
- package/assets/assets/SiteHealthSection--6tZe3hM.js +0 -4
package/dist/cli.js
CHANGED
|
@@ -1,15 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node --import tsx
|
|
2
2
|
import {
|
|
3
3
|
AERO_TOOL_PROFILES,
|
|
4
|
-
|
|
5
|
-
backfillAiReferralPathsCommand,
|
|
6
|
-
backfillAnswerMentionsCommand,
|
|
7
|
-
backfillAnswerVisibilityCommand,
|
|
8
|
-
backfillInsightsCommand,
|
|
9
|
-
backfillNormalizedPaths,
|
|
10
|
-
backfillNormalizedPathsCommand,
|
|
11
|
-
backfillSnapshotAttributionCommand,
|
|
12
|
-
backfillTrafficClassificationCommand,
|
|
4
|
+
OPENAI_RETRIEVAL_CONTRACT,
|
|
13
5
|
checkLatestVersionForCli,
|
|
14
6
|
cliRuntimeContext,
|
|
15
7
|
coerceAgentProvider,
|
|
@@ -28,6 +20,11 @@ import {
|
|
|
28
20
|
listAgentProviders,
|
|
29
21
|
notePrintedUpdateAvailable,
|
|
30
22
|
readCachedUpdateAvailable,
|
|
23
|
+
reparseStoredResult,
|
|
24
|
+
reparseStoredResult2,
|
|
25
|
+
reparseStoredResult3,
|
|
26
|
+
reparseStoredResult4,
|
|
27
|
+
reparseStoredResult5,
|
|
31
28
|
setGoogleAuthConfig,
|
|
32
29
|
setTelemetryPreference,
|
|
33
30
|
setTelemetrySource,
|
|
@@ -35,11 +32,11 @@ import {
|
|
|
35
32
|
trackCliCommandFinished,
|
|
36
33
|
trackEvent,
|
|
37
34
|
waitForServerRuntimeStartup
|
|
38
|
-
} from "./chunk-
|
|
35
|
+
} from "./chunk-GEKMHZS2.js";
|
|
39
36
|
import {
|
|
40
37
|
autoSyncSkills,
|
|
41
38
|
formatAutoSyncNotice
|
|
42
|
-
} from "./chunk-
|
|
39
|
+
} from "./chunk-ZP6GF32R.js";
|
|
43
40
|
import {
|
|
44
41
|
configExists,
|
|
45
42
|
createApiClient,
|
|
@@ -56,13 +53,13 @@ import {
|
|
|
56
53
|
saveConfig,
|
|
57
54
|
saveConfigPatch,
|
|
58
55
|
shouldPrintServeSkillsNudge
|
|
59
|
-
} from "./chunk-
|
|
56
|
+
} from "./chunk-TNLOOXUP.js";
|
|
60
57
|
import {
|
|
61
58
|
DEFAULT_DEMO_TRUSTED_PROXIES,
|
|
62
59
|
trustedProxiesProblem,
|
|
63
60
|
trustedProxyProblem
|
|
64
61
|
} from "./chunk-RB2F4D3Z.js";
|
|
65
|
-
import "./chunk-
|
|
62
|
+
import "./chunk-XFVJ6QVE.js";
|
|
66
63
|
import {
|
|
67
64
|
CliError,
|
|
68
65
|
EXIT_SYSTEM_ERROR,
|
|
@@ -74,22 +71,36 @@ import {
|
|
|
74
71
|
runMeasurementDraftAction,
|
|
75
72
|
systemError,
|
|
76
73
|
usageError
|
|
77
|
-
} from "./chunk-
|
|
74
|
+
} from "./chunk-JU6TPMCJ.js";
|
|
78
75
|
import {
|
|
79
76
|
CLOUDFLARE_WORKER_BINDINGS,
|
|
80
77
|
CLOUDFLARE_WORKER_GENERATED_MARKER,
|
|
81
78
|
CLOUDFLARE_WRANGLER_GENERATED_MARKER,
|
|
82
79
|
DEFAULT_BOT_LIST,
|
|
80
|
+
aiUserFetchEventsHourly,
|
|
83
81
|
apiKeys,
|
|
82
|
+
auditLog,
|
|
83
|
+
backfillProjectAnswerMentions,
|
|
84
|
+
classifyAiUserFetch,
|
|
85
|
+
classifyCrawler,
|
|
86
|
+
competitors,
|
|
87
|
+
crawlerEventsHourly,
|
|
84
88
|
createClient,
|
|
89
|
+
createRunCompetitorResolver,
|
|
85
90
|
dataRepairCompletions,
|
|
91
|
+
determineCitationState,
|
|
92
|
+
gaAiReferrals,
|
|
93
|
+
gaTrafficSnapshots,
|
|
86
94
|
generateWorkerScript,
|
|
87
95
|
generateWranglerToml,
|
|
88
96
|
migrate,
|
|
97
|
+
parseJsonColumn,
|
|
89
98
|
projects,
|
|
90
99
|
queries,
|
|
100
|
+
querySnapshots,
|
|
101
|
+
rawEventSamples,
|
|
91
102
|
runs
|
|
92
|
-
} from "./chunk-
|
|
103
|
+
} from "./chunk-SWSWCUHS.js";
|
|
93
104
|
import {
|
|
94
105
|
AdsDeliverySnapshotStatuses,
|
|
95
106
|
AdsHistoricalCampaignRollupStatuses,
|
|
@@ -102,19 +113,26 @@ import {
|
|
|
102
113
|
CheckScopes,
|
|
103
114
|
CheckStatuses,
|
|
104
115
|
CitationStates,
|
|
116
|
+
DiscoveryPromoteCompetitorSkipReasons,
|
|
105
117
|
FEEDBACK_KINDS,
|
|
106
118
|
GaMoverChangeBases,
|
|
107
119
|
MeasurementEvidenceShapes,
|
|
108
120
|
OUTSTANDING_PROVIDER_BATCH_STATUSES,
|
|
109
121
|
ProviderBatchStatuses,
|
|
122
|
+
ProviderNames,
|
|
110
123
|
READ_ONLY_SCOPE,
|
|
111
124
|
RatioUnits,
|
|
112
125
|
ResearchRunStatuses,
|
|
126
|
+
RetrievalContracts,
|
|
113
127
|
RunKinds,
|
|
114
128
|
RunStatuses,
|
|
115
129
|
RunTriggers,
|
|
130
|
+
SITE_AUDIT_MAX_PAGE_LIMIT,
|
|
131
|
+
TrafficEventConfidences,
|
|
116
132
|
TrafficEventKinds,
|
|
133
|
+
TrafficEvidenceKinds,
|
|
117
134
|
TrafficSeriesGranularities,
|
|
135
|
+
TrafficSourceTypes,
|
|
118
136
|
UNATTRIBUTED_MENTION_REASON,
|
|
119
137
|
URL_PATH_NORMALIZATION_VERSION,
|
|
120
138
|
USER_PASSWORD_MIN_LENGTH,
|
|
@@ -137,13 +155,17 @@ import {
|
|
|
137
155
|
buildModelChangeNotice,
|
|
138
156
|
canonicalizeGtmAccountId,
|
|
139
157
|
canonicalizeGtmResourceSelection,
|
|
158
|
+
computeCompetitorOverlap,
|
|
140
159
|
contentTargetDismissRequestSchema,
|
|
141
160
|
conversionTrackingContractWriteRequestSchema,
|
|
142
161
|
deduplicateResearchQueries,
|
|
143
162
|
describeError,
|
|
163
|
+
determineAnswerMentioned,
|
|
144
164
|
discoveryBucketSchema,
|
|
145
165
|
discoveryCompetitorTypeSchema,
|
|
166
|
+
effectiveBrandNames,
|
|
146
167
|
effectiveDomains,
|
|
168
|
+
extractRecommendedCompetitors,
|
|
147
169
|
factorStatusFromScore,
|
|
148
170
|
feedbackSubmissionSchema,
|
|
149
171
|
formatGbpMetricLabel,
|
|
@@ -152,6 +174,7 @@ import {
|
|
|
152
174
|
formatPercent,
|
|
153
175
|
formatRunErrorOneLine,
|
|
154
176
|
formatSignedPointDelta,
|
|
177
|
+
formatSiteAuditPageBudget,
|
|
155
178
|
gbpNegativeReviewMaxStarsSchema,
|
|
156
179
|
googleAdsMetricsWindowSchema,
|
|
157
180
|
isRetiredAgentProviderId,
|
|
@@ -172,8 +195,11 @@ import {
|
|
|
172
195
|
measurementQueryTemplateApplyRequestSchema,
|
|
173
196
|
measurementQueryTemplateUpsertRequestSchema,
|
|
174
197
|
measurementQuestionResultQuerySchema,
|
|
198
|
+
normalizeCompetitorAliases,
|
|
199
|
+
normalizeCompetitorDomain,
|
|
175
200
|
normalizeProjectAliases,
|
|
176
201
|
normalizeTelemetryStatus,
|
|
202
|
+
normalizeUrlPath,
|
|
177
203
|
notificationEventSchema,
|
|
178
204
|
parseTrafficAnalyticsPeriod,
|
|
179
205
|
providerDispatchModeSchema,
|
|
@@ -195,12 +221,13 @@ import {
|
|
|
195
221
|
sentimentSummaryRequestSchema,
|
|
196
222
|
shareOfVoiceLabel,
|
|
197
223
|
shareOfVoiceReason,
|
|
224
|
+
siteAuditPageBudgetSchema,
|
|
198
225
|
snapshotProviderModeSchema,
|
|
199
226
|
unattributedAnswersLabel,
|
|
200
227
|
visibilityCompareSelectionSchema,
|
|
201
228
|
visibilityReportRequestSchema,
|
|
202
229
|
winnabilityClassSchema
|
|
203
|
-
} from "./chunk-
|
|
230
|
+
} from "./chunk-PXRUWG75.js";
|
|
204
231
|
|
|
205
232
|
// src/cli.ts
|
|
206
233
|
import { pathToFileURL } from "url";
|
|
@@ -750,7 +777,7 @@ function parseDemoTrustedProxies(values) {
|
|
|
750
777
|
async function demoCommand(options) {
|
|
751
778
|
const { host, port } = parseDemoListenOptions(options);
|
|
752
779
|
const trustProxy = parseDemoTrustedProxies(options.trustProxy);
|
|
753
|
-
const { createDemoServer } = await import("./demo-server-
|
|
780
|
+
const { createDemoServer } = await import("./demo-server-INMOE4BA.js");
|
|
754
781
|
const app = await createDemoServer({ trustProxy });
|
|
755
782
|
let stopping = false;
|
|
756
783
|
const stop = () => {
|
|
@@ -796,6 +823,908 @@ var DEMO_CLI_COMMANDS = [{
|
|
|
796
823
|
}
|
|
797
824
|
}];
|
|
798
825
|
|
|
826
|
+
// src/commands/backfill.ts
|
|
827
|
+
import { setImmediate as yieldToEventLoop } from "timers/promises";
|
|
828
|
+
import { and, asc, desc, eq, gt, inArray, isNull, lte, sql } from "drizzle-orm";
|
|
829
|
+
var SNAPSHOT_BATCH_SIZE = 500;
|
|
830
|
+
var PATH_REPAIR_PAGE_SIZE = 128;
|
|
831
|
+
async function backfillAnswerVisibilityCommand(opts) {
|
|
832
|
+
const config = loadConfig();
|
|
833
|
+
const db = createClient(config.database);
|
|
834
|
+
migrate(db);
|
|
835
|
+
const projectFilter = opts?.project?.trim();
|
|
836
|
+
const isDryRun = opts?.dryRun === true;
|
|
837
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
838
|
+
let examined = 0;
|
|
839
|
+
let updated = 0;
|
|
840
|
+
let wouldUpdate = 0;
|
|
841
|
+
let mentioned = 0;
|
|
842
|
+
let reparsed = 0;
|
|
843
|
+
let providerErrors = 0;
|
|
844
|
+
let retrievalRelabeled = 0;
|
|
845
|
+
if (scopedProjects.length > 0) {
|
|
846
|
+
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId, planVersionId: runs.measurementPlanVersionId }).from(runs).where(and(
|
|
847
|
+
eq(runs.kind, RunKinds["answer-visibility"]),
|
|
848
|
+
inArray(runs.projectId, scopedProjects.map((project2) => project2.id))
|
|
849
|
+
)).all() : db.select({ id: runs.id, projectId: runs.projectId, planVersionId: runs.measurementPlanVersionId }).from(runs).where(eq(runs.kind, RunKinds["answer-visibility"])).all();
|
|
850
|
+
const planVersionByRun = new Map(runRows.map((run) => [run.id, run.planVersionId]));
|
|
851
|
+
const runIdsByProject = /* @__PURE__ */ new Map();
|
|
852
|
+
for (const run of runRows) {
|
|
853
|
+
const existing = runIdsByProject.get(run.projectId);
|
|
854
|
+
if (existing) existing.push(run.id);
|
|
855
|
+
else runIdsByProject.set(run.projectId, [run.id]);
|
|
856
|
+
}
|
|
857
|
+
for (const project2 of scopedProjects) {
|
|
858
|
+
const competitorsForRun = createRunCompetitorResolver(db, db.select({ domain: competitors.domain, aliases: competitors.aliases }).from(competitors).where(eq(competitors.projectId, project2.id)).all());
|
|
859
|
+
const runIds = runIdsByProject.get(project2.id) ?? [];
|
|
860
|
+
if (runIds.length === 0) continue;
|
|
861
|
+
const projectDomains = effectiveDomains({
|
|
862
|
+
canonicalDomain: project2.canonicalDomain,
|
|
863
|
+
ownedDomains: project2.ownedDomains
|
|
864
|
+
});
|
|
865
|
+
const projectBrandNames = effectiveBrandNames({
|
|
866
|
+
displayName: project2.displayName,
|
|
867
|
+
aliases: project2.aliases
|
|
868
|
+
});
|
|
869
|
+
for (let offset = 0; offset < runIds.length; offset += SNAPSHOT_BATCH_SIZE) {
|
|
870
|
+
const batchRunIds = runIds.slice(offset, offset + SNAPSHOT_BATCH_SIZE);
|
|
871
|
+
const snapshotRows = db.select({
|
|
872
|
+
id: querySnapshots.id,
|
|
873
|
+
runId: querySnapshots.runId,
|
|
874
|
+
executionId: querySnapshots.measurementExecutionId,
|
|
875
|
+
provider: querySnapshots.provider,
|
|
876
|
+
citationState: querySnapshots.citationState,
|
|
877
|
+
answerMentioned: querySnapshots.answerMentioned,
|
|
878
|
+
answerText: querySnapshots.answerText,
|
|
879
|
+
citedDomains: querySnapshots.citedDomains,
|
|
880
|
+
competitorOverlap: querySnapshots.competitorOverlap,
|
|
881
|
+
recommendedCompetitors: querySnapshots.recommendedCompetitors,
|
|
882
|
+
rawResponse: querySnapshots.rawResponse,
|
|
883
|
+
retrievalStatus: querySnapshots.retrievalStatus,
|
|
884
|
+
retrievalContract: querySnapshots.retrievalContract
|
|
885
|
+
}).from(querySnapshots).where(inArray(querySnapshots.runId, batchRunIds)).all();
|
|
886
|
+
const pendingUpdates = [];
|
|
887
|
+
for (const snapshot of snapshotRows) {
|
|
888
|
+
examined++;
|
|
889
|
+
const reparsedResult = reparseProviderSnapshot(snapshot.provider, snapshot.rawResponse);
|
|
890
|
+
if (reparsedResult) reparsed++;
|
|
891
|
+
if (reparsedResult?.providerError) providerErrors++;
|
|
892
|
+
const answerText = reparsedResult?.answerText ?? snapshot.answerText ?? "";
|
|
893
|
+
const nextValue = determineAnswerMentioned(answerText, projectBrandNames, projectDomains);
|
|
894
|
+
if (nextValue) mentioned++;
|
|
895
|
+
const nextPatch = {};
|
|
896
|
+
if (snapshot.answerMentioned !== nextValue) {
|
|
897
|
+
nextPatch.answerMentioned = nextValue;
|
|
898
|
+
}
|
|
899
|
+
if ((snapshot.answerText ?? "") !== answerText) {
|
|
900
|
+
nextPatch.answerText = answerText;
|
|
901
|
+
}
|
|
902
|
+
if (reparsedResult) {
|
|
903
|
+
const normalized = {
|
|
904
|
+
provider: snapshot.provider,
|
|
905
|
+
answerText,
|
|
906
|
+
citedDomains: reparsedResult.citedDomains,
|
|
907
|
+
groundingSources: reparsedResult.groundingSources,
|
|
908
|
+
searchQueries: reparsedResult.searchQueries,
|
|
909
|
+
// Only feeds the citation/mention helpers below, which ignore
|
|
910
|
+
// it. Retrieval is written back solely by the narrow OpenAI
|
|
911
|
+
// contract correction after this block.
|
|
912
|
+
retrievalStatus: "unknown"
|
|
913
|
+
};
|
|
914
|
+
const nextCitationState = determineCitationState(normalized, projectDomains);
|
|
915
|
+
const nextCitedDomains = reparsedResult.citedDomains;
|
|
916
|
+
const runCompetitors = competitorsForRun(planVersionByRun.get(snapshot.runId), snapshot.executionId);
|
|
917
|
+
const nextCompetitorOverlap = computeCompetitorOverlap(normalized, runCompetitors.domains, runCompetitors.aliases);
|
|
918
|
+
const nextRecommendedCompetitors = extractRecommendedCompetitors(
|
|
919
|
+
normalized.answerText,
|
|
920
|
+
projectDomains,
|
|
921
|
+
normalized.citedDomains,
|
|
922
|
+
runCompetitors.domains,
|
|
923
|
+
projectBrandNames,
|
|
924
|
+
runCompetitors.aliases
|
|
925
|
+
);
|
|
926
|
+
const nextRawResponse = stringifyStoredSnapshotEnvelope(
|
|
927
|
+
snapshot.rawResponse,
|
|
928
|
+
reparsedResult
|
|
929
|
+
);
|
|
930
|
+
if (snapshot.citationState !== nextCitationState) {
|
|
931
|
+
nextPatch.citationState = nextCitationState;
|
|
932
|
+
}
|
|
933
|
+
if (JSON.stringify(snapshot.citedDomains) !== JSON.stringify(nextCitedDomains)) {
|
|
934
|
+
nextPatch.citedDomains = nextCitedDomains;
|
|
935
|
+
}
|
|
936
|
+
if (JSON.stringify(snapshot.competitorOverlap) !== JSON.stringify(nextCompetitorOverlap)) {
|
|
937
|
+
nextPatch.competitorOverlap = nextCompetitorOverlap;
|
|
938
|
+
}
|
|
939
|
+
if (JSON.stringify(snapshot.recommendedCompetitors) !== JSON.stringify(nextRecommendedCompetitors)) {
|
|
940
|
+
nextPatch.recommendedCompetitors = nextRecommendedCompetitors;
|
|
941
|
+
}
|
|
942
|
+
if (snapshot.rawResponse !== nextRawResponse) {
|
|
943
|
+
nextPatch.rawResponse = nextRawResponse;
|
|
944
|
+
}
|
|
945
|
+
}
|
|
946
|
+
const retrievalCorrection = correctStoredOpenAIRetrieval(snapshot, reparsedResult);
|
|
947
|
+
if (retrievalCorrection) {
|
|
948
|
+
retrievalRelabeled++;
|
|
949
|
+
Object.assign(nextPatch, retrievalCorrection);
|
|
950
|
+
}
|
|
951
|
+
if (Object.keys(nextPatch).length > 0) {
|
|
952
|
+
pendingUpdates.push({ id: snapshot.id, patch: nextPatch });
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
if (pendingUpdates.length > 0) {
|
|
956
|
+
if (isDryRun) {
|
|
957
|
+
wouldUpdate += pendingUpdates.length;
|
|
958
|
+
} else {
|
|
959
|
+
db.transaction((tx) => {
|
|
960
|
+
for (const update of pendingUpdates) {
|
|
961
|
+
tx.update(querySnapshots).set(update.patch).where(eq(querySnapshots.id, update.id)).run();
|
|
962
|
+
}
|
|
963
|
+
});
|
|
964
|
+
updated += pendingUpdates.length;
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
const result = {
|
|
971
|
+
project: projectFilter ?? null,
|
|
972
|
+
projects: scopedProjects.length,
|
|
973
|
+
examined,
|
|
974
|
+
updated,
|
|
975
|
+
mentioned,
|
|
976
|
+
reparsed,
|
|
977
|
+
providerErrors,
|
|
978
|
+
retrievalRelabeled
|
|
979
|
+
};
|
|
980
|
+
if (isDryRun) {
|
|
981
|
+
result.dryRun = true;
|
|
982
|
+
result.wouldUpdate = wouldUpdate;
|
|
983
|
+
}
|
|
984
|
+
if (isMachineFormat(opts?.format)) {
|
|
985
|
+
console.log(JSON.stringify(result, null, 2));
|
|
986
|
+
return;
|
|
987
|
+
}
|
|
988
|
+
console.log(`Answer visibility backfill ${isDryRun ? "preview" : "complete"}.
|
|
989
|
+
`);
|
|
990
|
+
if (projectFilter) {
|
|
991
|
+
console.log(` Project: ${projectFilter}`);
|
|
992
|
+
}
|
|
993
|
+
console.log(` Projects: ${scopedProjects.length}`);
|
|
994
|
+
console.log(` Examined: ${examined}`);
|
|
995
|
+
if (isDryRun) {
|
|
996
|
+
console.log(` Would update: ${wouldUpdate}`);
|
|
997
|
+
} else {
|
|
998
|
+
console.log(` Updated: ${updated}`);
|
|
999
|
+
}
|
|
1000
|
+
console.log(` Mentioned: ${mentioned}`);
|
|
1001
|
+
console.log(` Reparsed: ${reparsed}`);
|
|
1002
|
+
console.log(` Errors: ${providerErrors}`);
|
|
1003
|
+
console.log(` Relabeled: ${retrievalRelabeled} (OpenAI retrieval contract)`);
|
|
1004
|
+
if (isDryRun) {
|
|
1005
|
+
console.log(`
|
|
1006
|
+
No DB writes performed. Re-run without --dry-run to apply.`);
|
|
1007
|
+
}
|
|
1008
|
+
}
|
|
1009
|
+
function backfillNormalizedPaths(db, opts) {
|
|
1010
|
+
return repairStoredPaths(db, gaTrafficSnapshots, opts);
|
|
1011
|
+
}
|
|
1012
|
+
async function repairStoredPaths(db, table, opts) {
|
|
1013
|
+
const pageSize = opts?.pageSize ?? PATH_REPAIR_PAGE_SIZE;
|
|
1014
|
+
const projectScope = opts?.projectId ? sql`+${table.projectId} = ${opts.projectId}` : void 0;
|
|
1015
|
+
const lastId = db.select({ id: table.id }).from(table).where(projectScope).orderBy(desc(table.id)).limit(1).get()?.id;
|
|
1016
|
+
if (lastId === void 0) return { examined: 0, updated: 0, unchanged: 0 };
|
|
1017
|
+
let cursor;
|
|
1018
|
+
let examined = 0;
|
|
1019
|
+
let updated = 0;
|
|
1020
|
+
for (; ; ) {
|
|
1021
|
+
const rows = db.select({
|
|
1022
|
+
id: table.id,
|
|
1023
|
+
projectId: table.projectId,
|
|
1024
|
+
landingPage: table.landingPage,
|
|
1025
|
+
landingPageNormalized: table.landingPageNormalized
|
|
1026
|
+
}).from(table).where(and(
|
|
1027
|
+
projectScope,
|
|
1028
|
+
lte(table.id, lastId),
|
|
1029
|
+
cursor === void 0 ? void 0 : gt(table.id, cursor)
|
|
1030
|
+
)).orderBy(asc(table.id)).limit(pageSize).all();
|
|
1031
|
+
examined += rows.length;
|
|
1032
|
+
const changes = rows.flatMap((row) => {
|
|
1033
|
+
const next = normalizeUrlPath(row.landingPage);
|
|
1034
|
+
return next === null || row.landingPageNormalized === next ? [] : [{ row, next }];
|
|
1035
|
+
});
|
|
1036
|
+
if (changes.length > 0) {
|
|
1037
|
+
const written = db.transaction((tx) => {
|
|
1038
|
+
let written2 = 0;
|
|
1039
|
+
for (const { row, next } of changes) {
|
|
1040
|
+
written2 += tx.update(table).set({ landingPageNormalized: next }).where(and(
|
|
1041
|
+
eq(table.id, row.id),
|
|
1042
|
+
eq(table.projectId, row.projectId),
|
|
1043
|
+
eq(table.landingPage, row.landingPage),
|
|
1044
|
+
row.landingPageNormalized === null ? isNull(table.landingPageNormalized) : eq(table.landingPageNormalized, row.landingPageNormalized)
|
|
1045
|
+
)).run().changes;
|
|
1046
|
+
}
|
|
1047
|
+
return written2;
|
|
1048
|
+
});
|
|
1049
|
+
updated += written;
|
|
1050
|
+
if (written < changes.length) opts?.onConflict?.();
|
|
1051
|
+
}
|
|
1052
|
+
if (rows.length < pageSize) break;
|
|
1053
|
+
cursor = rows.at(-1).id;
|
|
1054
|
+
if (cursor === lastId) break;
|
|
1055
|
+
await yieldToEventLoop();
|
|
1056
|
+
}
|
|
1057
|
+
return { examined, updated, unchanged: examined - updated };
|
|
1058
|
+
}
|
|
1059
|
+
async function backfillNormalizedPathsCommand(opts) {
|
|
1060
|
+
const config = loadConfig();
|
|
1061
|
+
const db = createClient(config.database);
|
|
1062
|
+
migrate(db);
|
|
1063
|
+
const projectFilter = opts?.project?.trim();
|
|
1064
|
+
let projectId;
|
|
1065
|
+
if (projectFilter) {
|
|
1066
|
+
const project2 = db.select({ id: projects.id }).from(projects).where(eq(projects.name, projectFilter)).get();
|
|
1067
|
+
if (!project2) {
|
|
1068
|
+
const result2 = {
|
|
1069
|
+
project: projectFilter,
|
|
1070
|
+
examined: 0,
|
|
1071
|
+
updated: 0,
|
|
1072
|
+
unchanged: 0
|
|
1073
|
+
};
|
|
1074
|
+
if (isMachineFormat(opts?.format)) {
|
|
1075
|
+
console.log(JSON.stringify(result2, null, 2));
|
|
1076
|
+
return;
|
|
1077
|
+
}
|
|
1078
|
+
console.log(`Backfill normalized-paths: project "${projectFilter}" not found.`);
|
|
1079
|
+
return;
|
|
1080
|
+
}
|
|
1081
|
+
projectId = project2.id;
|
|
1082
|
+
}
|
|
1083
|
+
const { examined, updated, unchanged } = await backfillNormalizedPaths(db, { projectId });
|
|
1084
|
+
const result = {
|
|
1085
|
+
project: projectFilter ?? null,
|
|
1086
|
+
examined,
|
|
1087
|
+
updated,
|
|
1088
|
+
unchanged
|
|
1089
|
+
};
|
|
1090
|
+
if (isMachineFormat(opts?.format)) {
|
|
1091
|
+
console.log(JSON.stringify(result, null, 2));
|
|
1092
|
+
return;
|
|
1093
|
+
}
|
|
1094
|
+
console.log("Normalized-path backfill complete.\n");
|
|
1095
|
+
if (projectFilter) console.log(` Project: ${projectFilter}`);
|
|
1096
|
+
console.log(` Examined: ${examined}`);
|
|
1097
|
+
console.log(` Updated: ${updated}`);
|
|
1098
|
+
console.log(` Unchanged: ${unchanged}`);
|
|
1099
|
+
}
|
|
1100
|
+
function backfillAiReferralPaths(db, opts) {
|
|
1101
|
+
return repairStoredPaths(db, gaAiReferrals, opts);
|
|
1102
|
+
}
|
|
1103
|
+
async function backfillAiReferralPathsCommand(opts) {
|
|
1104
|
+
const config = loadConfig();
|
|
1105
|
+
const db = createClient(config.database);
|
|
1106
|
+
migrate(db);
|
|
1107
|
+
const projectFilter = opts?.project?.trim();
|
|
1108
|
+
let projectId;
|
|
1109
|
+
if (projectFilter) {
|
|
1110
|
+
const project2 = db.select({ id: projects.id }).from(projects).where(eq(projects.name, projectFilter)).get();
|
|
1111
|
+
if (!project2) {
|
|
1112
|
+
const result2 = {
|
|
1113
|
+
project: projectFilter,
|
|
1114
|
+
examined: 0,
|
|
1115
|
+
updated: 0,
|
|
1116
|
+
unchanged: 0
|
|
1117
|
+
};
|
|
1118
|
+
if (isMachineFormat(opts?.format)) {
|
|
1119
|
+
console.log(JSON.stringify(result2, null, 2));
|
|
1120
|
+
return;
|
|
1121
|
+
}
|
|
1122
|
+
console.log(`Backfill ai-referral-paths: project "${projectFilter}" not found.`);
|
|
1123
|
+
return;
|
|
1124
|
+
}
|
|
1125
|
+
projectId = project2.id;
|
|
1126
|
+
}
|
|
1127
|
+
const { examined, updated, unchanged } = await backfillAiReferralPaths(db, { projectId });
|
|
1128
|
+
const result = {
|
|
1129
|
+
project: projectFilter ?? null,
|
|
1130
|
+
examined,
|
|
1131
|
+
updated,
|
|
1132
|
+
unchanged
|
|
1133
|
+
};
|
|
1134
|
+
if (isMachineFormat(opts?.format)) {
|
|
1135
|
+
console.log(JSON.stringify(result, null, 2));
|
|
1136
|
+
return;
|
|
1137
|
+
}
|
|
1138
|
+
console.log("AI referral landing-page backfill complete.\n");
|
|
1139
|
+
if (projectFilter) console.log(` Project: ${projectFilter}`);
|
|
1140
|
+
console.log(` Examined: ${examined}`);
|
|
1141
|
+
console.log(` Updated: ${updated}`);
|
|
1142
|
+
console.log(` Unchanged: ${unchanged}`);
|
|
1143
|
+
}
|
|
1144
|
+
async function backfillAnswerMentionsCommand(opts) {
|
|
1145
|
+
const config = loadConfig();
|
|
1146
|
+
const db = createClient(config.database);
|
|
1147
|
+
migrate(db);
|
|
1148
|
+
const projectFilter = opts?.project?.trim();
|
|
1149
|
+
const isDryRun = opts?.dryRun === true;
|
|
1150
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
1151
|
+
let examined = 0;
|
|
1152
|
+
let updated = 0;
|
|
1153
|
+
let wouldUpdate = 0;
|
|
1154
|
+
let mentioned = 0;
|
|
1155
|
+
for (const project2 of scopedProjects) {
|
|
1156
|
+
const result2 = backfillProjectAnswerMentions(db, project2.id, { dryRun: isDryRun });
|
|
1157
|
+
examined += result2.examined;
|
|
1158
|
+
updated += result2.updated;
|
|
1159
|
+
wouldUpdate += result2.wouldUpdate ?? 0;
|
|
1160
|
+
mentioned += result2.mentioned;
|
|
1161
|
+
}
|
|
1162
|
+
const result = {
|
|
1163
|
+
project: projectFilter ?? null,
|
|
1164
|
+
projects: scopedProjects.length,
|
|
1165
|
+
examined,
|
|
1166
|
+
updated,
|
|
1167
|
+
mentioned
|
|
1168
|
+
};
|
|
1169
|
+
if (isDryRun) {
|
|
1170
|
+
result.dryRun = true;
|
|
1171
|
+
result.wouldUpdate = wouldUpdate;
|
|
1172
|
+
}
|
|
1173
|
+
if (isMachineFormat(opts?.format)) {
|
|
1174
|
+
console.log(JSON.stringify(result, null, 2));
|
|
1175
|
+
return;
|
|
1176
|
+
}
|
|
1177
|
+
console.log(`Answer mentions backfill ${isDryRun ? "preview" : "complete"}.
|
|
1178
|
+
`);
|
|
1179
|
+
if (projectFilter) console.log(` Project: ${projectFilter}`);
|
|
1180
|
+
console.log(` Projects: ${scopedProjects.length}`);
|
|
1181
|
+
console.log(` Examined: ${examined}`);
|
|
1182
|
+
if (isDryRun) {
|
|
1183
|
+
console.log(` Would update: ${wouldUpdate}`);
|
|
1184
|
+
} else {
|
|
1185
|
+
console.log(` Updated: ${updated}`);
|
|
1186
|
+
}
|
|
1187
|
+
console.log(` Mentioned: ${mentioned}`);
|
|
1188
|
+
if (isDryRun) {
|
|
1189
|
+
console.log(`
|
|
1190
|
+
No DB writes performed. Re-run without --dry-run to apply.`);
|
|
1191
|
+
}
|
|
1192
|
+
}
|
|
1193
|
+
async function backfillInsightsCommand(project2, opts) {
|
|
1194
|
+
const { IntelligenceService } = await import("./intelligence-service-IBUS2IND.js");
|
|
1195
|
+
const config = loadConfig();
|
|
1196
|
+
const db = createClient(config.database);
|
|
1197
|
+
migrate(db);
|
|
1198
|
+
const service = new IntelligenceService(db);
|
|
1199
|
+
const isJson = isMachineFormat(opts?.format);
|
|
1200
|
+
const isDryRun = opts?.dryRun === true;
|
|
1201
|
+
if (!isJson) {
|
|
1202
|
+
const scope = opts?.since ? ` (since ${opts.since})` : "";
|
|
1203
|
+
const mode = isDryRun ? " [DRY RUN \u2014 no writes]" : "";
|
|
1204
|
+
process.stderr.write(`Backfilling insights for "${project2}"${scope}${mode}...
|
|
1205
|
+
`);
|
|
1206
|
+
}
|
|
1207
|
+
const result = service.backfill(project2, {
|
|
1208
|
+
fromRunId: opts?.fromRun,
|
|
1209
|
+
toRunId: opts?.toRun,
|
|
1210
|
+
since: opts?.since,
|
|
1211
|
+
dryRun: isDryRun
|
|
1212
|
+
}, (info) => {
|
|
1213
|
+
if (!isJson) {
|
|
1214
|
+
process.stderr.write(` [${info.index}/${info.total}] ${info.runId} \u2014 ${info.insights} insights
|
|
1215
|
+
`);
|
|
1216
|
+
}
|
|
1217
|
+
});
|
|
1218
|
+
const output = {
|
|
1219
|
+
project: project2,
|
|
1220
|
+
processed: result.processed,
|
|
1221
|
+
skipped: result.skipped,
|
|
1222
|
+
totalInsights: result.totalInsights
|
|
1223
|
+
};
|
|
1224
|
+
if (result.dryRun) {
|
|
1225
|
+
output.dryRun = true;
|
|
1226
|
+
output.delta = result.delta;
|
|
1227
|
+
}
|
|
1228
|
+
if (isJson) {
|
|
1229
|
+
console.log(JSON.stringify(output, null, 2));
|
|
1230
|
+
return;
|
|
1231
|
+
}
|
|
1232
|
+
console.log(`
|
|
1233
|
+
Backfill ${isDryRun ? "preview" : "complete"}.`);
|
|
1234
|
+
console.log(` Processed: ${result.processed}`);
|
|
1235
|
+
console.log(` Skipped: ${result.skipped}`);
|
|
1236
|
+
console.log(` Insights: ${result.totalInsights}`);
|
|
1237
|
+
if (result.delta) {
|
|
1238
|
+
console.log(` Delta: -${result.delta.wouldDelete} existing +${result.delta.wouldCreate} new (net ${result.delta.netChange >= 0 ? "+" : ""}${result.delta.netChange})`);
|
|
1239
|
+
console.log(` No DB writes performed. Re-run without --dry-run to apply.`);
|
|
1240
|
+
}
|
|
1241
|
+
}
|
|
1242
|
+
function replayQueryAuditLog(events) {
|
|
1243
|
+
const active = [];
|
|
1244
|
+
for (const ev of events) {
|
|
1245
|
+
let diff;
|
|
1246
|
+
try {
|
|
1247
|
+
diff = ev.diff ? JSON.parse(ev.diff) : {};
|
|
1248
|
+
} catch {
|
|
1249
|
+
continue;
|
|
1250
|
+
}
|
|
1251
|
+
if (ev.action === "keywords.appended" || ev.action === "queries.appended") {
|
|
1252
|
+
const added = Array.isArray(diff.added) ? diff.added : [];
|
|
1253
|
+
for (const q of added) {
|
|
1254
|
+
active.push({ text: q, addedAt: ev.createdAt, deletedAt: null });
|
|
1255
|
+
}
|
|
1256
|
+
} else if (ev.action === "keywords.deleted" || ev.action === "queries.deleted") {
|
|
1257
|
+
const deleted = Array.isArray(diff.deleted) ? diff.deleted : [];
|
|
1258
|
+
for (const q of deleted) {
|
|
1259
|
+
for (let i = active.length - 1; i >= 0; i--) {
|
|
1260
|
+
if (active[i].text === q && active[i].deletedAt === null) {
|
|
1261
|
+
active[i].deletedAt = ev.createdAt;
|
|
1262
|
+
break;
|
|
1263
|
+
}
|
|
1264
|
+
}
|
|
1265
|
+
}
|
|
1266
|
+
} else if (ev.action === "queries.replaced") {
|
|
1267
|
+
const newSet = Array.isArray(diff.queries) ? diff.queries : [];
|
|
1268
|
+
for (const e of active) {
|
|
1269
|
+
if (e.deletedAt === null) e.deletedAt = ev.createdAt;
|
|
1270
|
+
}
|
|
1271
|
+
for (const q of newSet) {
|
|
1272
|
+
active.push({ text: q, addedAt: ev.createdAt, deletedAt: null });
|
|
1273
|
+
}
|
|
1274
|
+
}
|
|
1275
|
+
}
|
|
1276
|
+
return active;
|
|
1277
|
+
}
|
|
1278
|
+
function activeQueriesAt(history, t) {
|
|
1279
|
+
return history.filter((e) => e.addedAt <= t && (e.deletedAt === null || e.deletedAt > t)).map((e) => e.text);
|
|
1280
|
+
}
|
|
1281
|
+
var CONTENT_MATCH_STOPWORDS = /* @__PURE__ */ new Set([
|
|
1282
|
+
"the",
|
|
1283
|
+
"a",
|
|
1284
|
+
"an",
|
|
1285
|
+
"and",
|
|
1286
|
+
"or",
|
|
1287
|
+
"but",
|
|
1288
|
+
"is",
|
|
1289
|
+
"are",
|
|
1290
|
+
"was",
|
|
1291
|
+
"were",
|
|
1292
|
+
"be",
|
|
1293
|
+
"been",
|
|
1294
|
+
"being",
|
|
1295
|
+
"have",
|
|
1296
|
+
"has",
|
|
1297
|
+
"had",
|
|
1298
|
+
"do",
|
|
1299
|
+
"does",
|
|
1300
|
+
"did",
|
|
1301
|
+
"will",
|
|
1302
|
+
"would",
|
|
1303
|
+
"could",
|
|
1304
|
+
"should",
|
|
1305
|
+
"may",
|
|
1306
|
+
"might",
|
|
1307
|
+
"can",
|
|
1308
|
+
"this",
|
|
1309
|
+
"that",
|
|
1310
|
+
"these",
|
|
1311
|
+
"those",
|
|
1312
|
+
"i",
|
|
1313
|
+
"you",
|
|
1314
|
+
"he",
|
|
1315
|
+
"she",
|
|
1316
|
+
"it",
|
|
1317
|
+
"we",
|
|
1318
|
+
"they",
|
|
1319
|
+
"what",
|
|
1320
|
+
"which",
|
|
1321
|
+
"who",
|
|
1322
|
+
"how",
|
|
1323
|
+
"why",
|
|
1324
|
+
"when",
|
|
1325
|
+
"where",
|
|
1326
|
+
"with",
|
|
1327
|
+
"for",
|
|
1328
|
+
"from",
|
|
1329
|
+
"of",
|
|
1330
|
+
"in",
|
|
1331
|
+
"on",
|
|
1332
|
+
"at",
|
|
1333
|
+
"to",
|
|
1334
|
+
"by",
|
|
1335
|
+
"as",
|
|
1336
|
+
"best",
|
|
1337
|
+
"most"
|
|
1338
|
+
]);
|
|
1339
|
+
function tokenizeForMatch(s) {
|
|
1340
|
+
return s.toLowerCase().replace(/[^a-z0-9\s]/g, " ").split(/\s+/).filter((t) => t.length >= 3 && !CONTENT_MATCH_STOPWORDS.has(t));
|
|
1341
|
+
}
|
|
1342
|
+
function contentMatchScore(queryText, answerText) {
|
|
1343
|
+
const queryTokens = [...new Set(tokenizeForMatch(queryText))];
|
|
1344
|
+
if (queryTokens.length === 0) return 0;
|
|
1345
|
+
const answerHead = answerText.slice(0, 300).toLowerCase();
|
|
1346
|
+
const hit = queryTokens.filter((t) => answerHead.includes(t)).length;
|
|
1347
|
+
return hit / queryTokens.length;
|
|
1348
|
+
}
|
|
1349
|
+
async function backfillSnapshotAttributionCommand(opts) {
|
|
1350
|
+
const config = loadConfig();
|
|
1351
|
+
const db = createClient(config.database);
|
|
1352
|
+
migrate(db);
|
|
1353
|
+
const project2 = db.select().from(projects).where(eq(projects.name, opts.project)).get();
|
|
1354
|
+
if (!project2) {
|
|
1355
|
+
throw new Error(`Project "${opts.project}" not found`);
|
|
1356
|
+
}
|
|
1357
|
+
const isJson = isMachineFormat(opts.format);
|
|
1358
|
+
const isDryRun = opts.dryRun === true;
|
|
1359
|
+
if (!isJson) {
|
|
1360
|
+
const mode = isDryRun ? " [DRY RUN \u2014 no writes]" : "";
|
|
1361
|
+
process.stderr.write(`Recovering orphan snapshot attribution for "${project2.name}"${mode}...
|
|
1362
|
+
`);
|
|
1363
|
+
}
|
|
1364
|
+
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(and(
|
|
1365
|
+
eq(auditLog.projectId, project2.id),
|
|
1366
|
+
inArray(auditLog.action, ["keywords.appended", "keywords.deleted", "queries.appended", "queries.deleted", "queries.replaced"])
|
|
1367
|
+
)).orderBy(auditLog.createdAt).all();
|
|
1368
|
+
const history = replayQueryAuditLog(events);
|
|
1369
|
+
const orphanRuns = db.select({
|
|
1370
|
+
runId: runs.id,
|
|
1371
|
+
createdAt: runs.createdAt,
|
|
1372
|
+
location: runs.location
|
|
1373
|
+
}).from(runs).innerJoin(querySnapshots, eq(querySnapshots.runId, runs.id)).where(and(
|
|
1374
|
+
eq(runs.projectId, project2.id),
|
|
1375
|
+
isNull(querySnapshots.queryId),
|
|
1376
|
+
isNull(querySnapshots.queryText)
|
|
1377
|
+
)).groupBy(runs.id).orderBy(runs.createdAt).all();
|
|
1378
|
+
const result = {
|
|
1379
|
+
project: project2.name,
|
|
1380
|
+
examinedRuns: orphanRuns.length,
|
|
1381
|
+
orphanSnapshots: 0,
|
|
1382
|
+
recoveredByPosition: 0,
|
|
1383
|
+
recoveredByContent: 0,
|
|
1384
|
+
unrecovered: 0,
|
|
1385
|
+
dryRun: isDryRun,
|
|
1386
|
+
perRun: []
|
|
1387
|
+
};
|
|
1388
|
+
const updates = [];
|
|
1389
|
+
for (const run of orphanRuns) {
|
|
1390
|
+
const activeAt = activeQueriesAt(history, run.createdAt);
|
|
1391
|
+
const orphanSnaps = db.select({
|
|
1392
|
+
id: querySnapshots.id,
|
|
1393
|
+
provider: querySnapshots.provider,
|
|
1394
|
+
createdAt: querySnapshots.createdAt,
|
|
1395
|
+
answerText: querySnapshots.answerText
|
|
1396
|
+
}).from(querySnapshots).where(and(
|
|
1397
|
+
eq(querySnapshots.runId, run.runId),
|
|
1398
|
+
isNull(querySnapshots.queryId),
|
|
1399
|
+
isNull(querySnapshots.queryText)
|
|
1400
|
+
)).orderBy(querySnapshots.provider, querySnapshots.createdAt).all();
|
|
1401
|
+
const byProvider = /* @__PURE__ */ new Map();
|
|
1402
|
+
for (const s of orphanSnaps) {
|
|
1403
|
+
const arr = byProvider.get(s.provider);
|
|
1404
|
+
if (arr) arr.push(s);
|
|
1405
|
+
else byProvider.set(s.provider, [s]);
|
|
1406
|
+
}
|
|
1407
|
+
let runPosition = 0;
|
|
1408
|
+
let runContent = 0;
|
|
1409
|
+
let runUnrecovered = 0;
|
|
1410
|
+
for (const [, snaps] of byProvider) {
|
|
1411
|
+
if (snaps.length === activeAt.length) {
|
|
1412
|
+
for (let i = 0; i < snaps.length; i++) {
|
|
1413
|
+
updates.push({ id: snaps[i].id, queryText: activeAt[i] });
|
|
1414
|
+
runPosition++;
|
|
1415
|
+
}
|
|
1416
|
+
} else if (snaps.length < activeAt.length) {
|
|
1417
|
+
const LOOKAHEAD_LIMIT = 5;
|
|
1418
|
+
let cidx = 0;
|
|
1419
|
+
for (const snap of snaps) {
|
|
1420
|
+
const answerText = snap.answerText ?? "";
|
|
1421
|
+
let matchedIdx = -1;
|
|
1422
|
+
for (let i = cidx; i < Math.min(activeAt.length, cidx + LOOKAHEAD_LIMIT); i++) {
|
|
1423
|
+
if (contentMatchScore(activeAt[i], answerText) >= 1) {
|
|
1424
|
+
matchedIdx = i;
|
|
1425
|
+
break;
|
|
1426
|
+
}
|
|
1427
|
+
}
|
|
1428
|
+
if (matchedIdx >= 0) {
|
|
1429
|
+
updates.push({ id: snap.id, queryText: activeAt[matchedIdx] });
|
|
1430
|
+
runContent++;
|
|
1431
|
+
cidx = matchedIdx + 1;
|
|
1432
|
+
} else {
|
|
1433
|
+
runUnrecovered++;
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
} else {
|
|
1437
|
+
runUnrecovered += snaps.length;
|
|
1438
|
+
}
|
|
1439
|
+
}
|
|
1440
|
+
result.orphanSnapshots += orphanSnaps.length;
|
|
1441
|
+
result.recoveredByPosition += runPosition;
|
|
1442
|
+
result.recoveredByContent += runContent;
|
|
1443
|
+
result.unrecovered += runUnrecovered;
|
|
1444
|
+
result.perRun.push({
|
|
1445
|
+
runId: run.runId,
|
|
1446
|
+
runCreatedAt: run.createdAt,
|
|
1447
|
+
activeQueryCount: activeAt.length,
|
|
1448
|
+
orphanSnaps: orphanSnaps.length,
|
|
1449
|
+
recoveredPosition: runPosition,
|
|
1450
|
+
recoveredContent: runContent,
|
|
1451
|
+
unrecovered: runUnrecovered
|
|
1452
|
+
});
|
|
1453
|
+
if (!isJson) {
|
|
1454
|
+
process.stderr.write(
|
|
1455
|
+
` ${run.createdAt} loc=${run.location ?? "-"} \u2192 ${orphanSnaps.length} orphan; ${runPosition} position-matched, ${runContent} content-matched, ${runUnrecovered} unrecovered
|
|
1456
|
+
`
|
|
1457
|
+
);
|
|
1458
|
+
}
|
|
1459
|
+
}
|
|
1460
|
+
if (!isDryRun && updates.length > 0) {
|
|
1461
|
+
db.transaction((tx) => {
|
|
1462
|
+
for (const u of updates) {
|
|
1463
|
+
tx.update(querySnapshots).set({ queryText: u.queryText }).where(eq(querySnapshots.id, u.id)).run();
|
|
1464
|
+
}
|
|
1465
|
+
});
|
|
1466
|
+
}
|
|
1467
|
+
if (isJson) {
|
|
1468
|
+
console.log(JSON.stringify(result, null, 2));
|
|
1469
|
+
return;
|
|
1470
|
+
}
|
|
1471
|
+
console.log(`
|
|
1472
|
+
Snapshot attribution ${isDryRun ? "preview" : "recovery"} complete.`);
|
|
1473
|
+
console.log(` Examined runs: ${result.examinedRuns}`);
|
|
1474
|
+
console.log(` Orphan snapshots: ${result.orphanSnapshots}`);
|
|
1475
|
+
console.log(` Position-matched: ${result.recoveredByPosition}`);
|
|
1476
|
+
console.log(` Content-matched: ${result.recoveredByContent}`);
|
|
1477
|
+
console.log(` Unrecovered: ${result.unrecovered}`);
|
|
1478
|
+
if (isDryRun) {
|
|
1479
|
+
console.log(`
|
|
1480
|
+
No DB writes performed. Re-run without --dry-run to apply.`);
|
|
1481
|
+
}
|
|
1482
|
+
}
|
|
1483
|
+
function correctStoredOpenAIRetrieval(snapshot, reparsed) {
|
|
1484
|
+
if (snapshot.provider !== ProviderNames.openai) return null;
|
|
1485
|
+
if (snapshot.retrievalContract !== RetrievalContracts["native-auto-v1"]) return null;
|
|
1486
|
+
return {
|
|
1487
|
+
retrievalContract: OPENAI_RETRIEVAL_CONTRACT,
|
|
1488
|
+
...reparsed?.retrievalStatus === void 0 ? {} : { retrievalStatus: reparsed.retrievalStatus }
|
|
1489
|
+
};
|
|
1490
|
+
}
|
|
1491
|
+
function reparseProviderSnapshot(provider, rawResponse) {
|
|
1492
|
+
const envelope = parseJsonColumn(rawResponse, {});
|
|
1493
|
+
const apiResponse = resolveStoredApiResponse(envelope);
|
|
1494
|
+
if (!apiResponse) return null;
|
|
1495
|
+
switch (provider) {
|
|
1496
|
+
case ProviderNames.openai:
|
|
1497
|
+
return reparseStoredResult(apiResponse);
|
|
1498
|
+
case ProviderNames.claude:
|
|
1499
|
+
return reparseStoredResult2(apiResponse);
|
|
1500
|
+
case ProviderNames.gemini:
|
|
1501
|
+
return reparseStoredResult3(apiResponse);
|
|
1502
|
+
case ProviderNames.perplexity:
|
|
1503
|
+
return reparseStoredResult4(apiResponse);
|
|
1504
|
+
case ProviderNames.muse:
|
|
1505
|
+
return reparseStoredResult5(apiResponse);
|
|
1506
|
+
default:
|
|
1507
|
+
return null;
|
|
1508
|
+
}
|
|
1509
|
+
}
|
|
1510
|
+
function resolveStoredApiResponse(parsed) {
|
|
1511
|
+
const nested = parsed.apiResponse;
|
|
1512
|
+
if (nested !== null && typeof nested === "object" && !Array.isArray(nested)) {
|
|
1513
|
+
return nested;
|
|
1514
|
+
}
|
|
1515
|
+
if (looksLikeProviderApiResponse(parsed)) {
|
|
1516
|
+
return parsed;
|
|
1517
|
+
}
|
|
1518
|
+
return null;
|
|
1519
|
+
}
|
|
1520
|
+
function looksLikeProviderApiResponse(value) {
|
|
1521
|
+
return Array.isArray(value.output) || Array.isArray(value.content) || Array.isArray(value.candidates) || Array.isArray(value.choices);
|
|
1522
|
+
}
|
|
1523
|
+
function stringifyStoredSnapshotEnvelope(rawResponse, reparsed) {
|
|
1524
|
+
const parsed = parseJsonColumn(rawResponse, {});
|
|
1525
|
+
const apiResponse = resolveStoredApiResponse(parsed);
|
|
1526
|
+
const envelope = apiResponse === parsed ? {} : { ...parsed };
|
|
1527
|
+
delete envelope.answerText;
|
|
1528
|
+
delete envelope.citedDomains;
|
|
1529
|
+
delete envelope.competitorOverlap;
|
|
1530
|
+
delete envelope.recommendedCompetitors;
|
|
1531
|
+
delete envelope.providerError;
|
|
1532
|
+
return JSON.stringify({
|
|
1533
|
+
...envelope,
|
|
1534
|
+
groundingSources: reparsed.groundingSources,
|
|
1535
|
+
searchQueries: reparsed.searchQueries,
|
|
1536
|
+
...reparsed.providerError ? { providerError: reparsed.providerError } : {},
|
|
1537
|
+
...apiResponse ? { apiResponse } : {}
|
|
1538
|
+
});
|
|
1539
|
+
}
|
|
1540
|
+
async function backfillTrafficClassificationCommand(opts) {
|
|
1541
|
+
const config = loadConfig();
|
|
1542
|
+
const db = createClient(config.database);
|
|
1543
|
+
migrate(db);
|
|
1544
|
+
const projectFilter = opts?.project?.trim();
|
|
1545
|
+
const isDryRun = opts?.dryRun === true;
|
|
1546
|
+
const isJson = isMachineFormat(opts?.format);
|
|
1547
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
1548
|
+
if (scopedProjects.length === 0) {
|
|
1549
|
+
if (projectFilter && !isJson) {
|
|
1550
|
+
process.stderr.write(`No project named "${projectFilter}".
|
|
1551
|
+
`);
|
|
1552
|
+
}
|
|
1553
|
+
if (isJson) console.log(JSON.stringify({ project: projectFilter ?? null, examined: 0, reclassified: 0 }));
|
|
1554
|
+
return;
|
|
1555
|
+
}
|
|
1556
|
+
if (!isJson) {
|
|
1557
|
+
const mode = isDryRun ? " [DRY RUN \u2014 no writes]" : "";
|
|
1558
|
+
const scope = projectFilter ? `"${projectFilter}"` : "all projects";
|
|
1559
|
+
process.stderr.write(`Re-classifying unknown traffic samples for ${scope}${mode}...
|
|
1560
|
+
`);
|
|
1561
|
+
}
|
|
1562
|
+
const projectIds = scopedProjects.map((p) => p.id);
|
|
1563
|
+
const result = {
|
|
1564
|
+
project: projectFilter ?? null,
|
|
1565
|
+
examined: 0,
|
|
1566
|
+
reclassified: 0,
|
|
1567
|
+
unknownBefore: 0,
|
|
1568
|
+
unknownAfter: 0,
|
|
1569
|
+
dryRun: isDryRun,
|
|
1570
|
+
byBot: {}
|
|
1571
|
+
};
|
|
1572
|
+
const unknownCountRow = db.select({ n: sql`count(*)` }).from(rawEventSamples).where(and(
|
|
1573
|
+
eq(rawEventSamples.eventType, "unknown"),
|
|
1574
|
+
inArray(rawEventSamples.projectId, projectIds)
|
|
1575
|
+
)).get();
|
|
1576
|
+
result.unknownBefore = Number(unknownCountRow?.n ?? 0);
|
|
1577
|
+
const unknownSamples = db.select({
|
|
1578
|
+
id: rawEventSamples.id,
|
|
1579
|
+
projectId: rawEventSamples.projectId,
|
|
1580
|
+
sourceId: rawEventSamples.sourceId,
|
|
1581
|
+
ts: rawEventSamples.ts,
|
|
1582
|
+
userAgent: rawEventSamples.userAgent,
|
|
1583
|
+
pathNormalized: rawEventSamples.pathNormalized,
|
|
1584
|
+
status: rawEventSamples.status
|
|
1585
|
+
}).from(rawEventSamples).where(and(
|
|
1586
|
+
eq(rawEventSamples.eventType, "unknown"),
|
|
1587
|
+
inArray(rawEventSamples.projectId, projectIds)
|
|
1588
|
+
)).all();
|
|
1589
|
+
result.examined = unknownSamples.length;
|
|
1590
|
+
if (unknownSamples.length === 0) {
|
|
1591
|
+
result.unknownAfter = result.unknownBefore;
|
|
1592
|
+
emitTrafficClassificationReport(result, isJson);
|
|
1593
|
+
return;
|
|
1594
|
+
}
|
|
1595
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1596
|
+
for (const snap of unknownSamples) {
|
|
1597
|
+
if (!snap.userAgent) continue;
|
|
1598
|
+
const probe = {
|
|
1599
|
+
sourceType: TrafficSourceTypes["cloud-run"],
|
|
1600
|
+
evidenceKind: TrafficEvidenceKinds["raw-request"],
|
|
1601
|
+
confidence: TrafficEventConfidences.observed,
|
|
1602
|
+
eventId: snap.id,
|
|
1603
|
+
observedAt: snap.ts,
|
|
1604
|
+
method: "GET",
|
|
1605
|
+
requestUrl: `https://placeholder${snap.pathNormalized}`,
|
|
1606
|
+
host: "placeholder",
|
|
1607
|
+
path: snap.pathNormalized,
|
|
1608
|
+
queryString: null,
|
|
1609
|
+
status: snap.status ?? 200,
|
|
1610
|
+
userAgent: snap.userAgent,
|
|
1611
|
+
remoteIp: null,
|
|
1612
|
+
referer: null,
|
|
1613
|
+
latencyMs: null,
|
|
1614
|
+
requestSizeBytes: null,
|
|
1615
|
+
responseSizeBytes: null,
|
|
1616
|
+
providerResource: { type: "cloud_run_revision", labels: {} },
|
|
1617
|
+
providerLabels: {}
|
|
1618
|
+
};
|
|
1619
|
+
const userFetch = classifyAiUserFetch(probe);
|
|
1620
|
+
const classified = userFetch ?? classifyCrawler(probe);
|
|
1621
|
+
if (!classified) continue;
|
|
1622
|
+
result.reclassified++;
|
|
1623
|
+
result.byBot[classified.botId] = (result.byBot[classified.botId] ?? 0) + 1;
|
|
1624
|
+
if (isDryRun) continue;
|
|
1625
|
+
const tsHour = new Date(snap.ts);
|
|
1626
|
+
tsHour.setUTCMinutes(0, 0, 0);
|
|
1627
|
+
const tsHourIso = tsHour.toISOString();
|
|
1628
|
+
const status = snap.status ?? 200;
|
|
1629
|
+
db.transaction((tx) => {
|
|
1630
|
+
tx.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(eq(rawEventSamples.id, snap.id)).run();
|
|
1631
|
+
if (userFetch) {
|
|
1632
|
+
tx.insert(aiUserFetchEventsHourly).values({
|
|
1633
|
+
projectId: snap.projectId,
|
|
1634
|
+
sourceId: snap.sourceId,
|
|
1635
|
+
tsHour: tsHourIso,
|
|
1636
|
+
botId: userFetch.botId,
|
|
1637
|
+
operator: userFetch.operator,
|
|
1638
|
+
verificationStatus: userFetch.verificationStatus,
|
|
1639
|
+
pathNormalized: snap.pathNormalized,
|
|
1640
|
+
status,
|
|
1641
|
+
hits: 1,
|
|
1642
|
+
sampledUserAgent: snap.userAgent,
|
|
1643
|
+
createdAt: now,
|
|
1644
|
+
updatedAt: now
|
|
1645
|
+
}).onConflictDoUpdate({
|
|
1646
|
+
target: [
|
|
1647
|
+
aiUserFetchEventsHourly.projectId,
|
|
1648
|
+
aiUserFetchEventsHourly.sourceId,
|
|
1649
|
+
aiUserFetchEventsHourly.tsHour,
|
|
1650
|
+
aiUserFetchEventsHourly.botId,
|
|
1651
|
+
aiUserFetchEventsHourly.verificationStatus,
|
|
1652
|
+
aiUserFetchEventsHourly.pathNormalized,
|
|
1653
|
+
aiUserFetchEventsHourly.status
|
|
1654
|
+
],
|
|
1655
|
+
set: {
|
|
1656
|
+
hits: sql`${aiUserFetchEventsHourly.hits} + 1`,
|
|
1657
|
+
updatedAt: now
|
|
1658
|
+
}
|
|
1659
|
+
}).run();
|
|
1660
|
+
} else {
|
|
1661
|
+
tx.insert(crawlerEventsHourly).values({
|
|
1662
|
+
projectId: snap.projectId,
|
|
1663
|
+
sourceId: snap.sourceId,
|
|
1664
|
+
tsHour: tsHourIso,
|
|
1665
|
+
botId: classified.botId,
|
|
1666
|
+
operator: classified.operator,
|
|
1667
|
+
verificationStatus: classified.verificationStatus,
|
|
1668
|
+
pathNormalized: snap.pathNormalized,
|
|
1669
|
+
status,
|
|
1670
|
+
hits: 1,
|
|
1671
|
+
sampledUserAgent: snap.userAgent,
|
|
1672
|
+
createdAt: now,
|
|
1673
|
+
updatedAt: now
|
|
1674
|
+
}).onConflictDoUpdate({
|
|
1675
|
+
target: [
|
|
1676
|
+
crawlerEventsHourly.projectId,
|
|
1677
|
+
crawlerEventsHourly.sourceId,
|
|
1678
|
+
crawlerEventsHourly.tsHour,
|
|
1679
|
+
crawlerEventsHourly.botId,
|
|
1680
|
+
crawlerEventsHourly.verificationStatus,
|
|
1681
|
+
crawlerEventsHourly.pathNormalized,
|
|
1682
|
+
crawlerEventsHourly.status
|
|
1683
|
+
],
|
|
1684
|
+
set: {
|
|
1685
|
+
hits: sql`${crawlerEventsHourly.hits} + 1`,
|
|
1686
|
+
updatedAt: now
|
|
1687
|
+
}
|
|
1688
|
+
}).run();
|
|
1689
|
+
}
|
|
1690
|
+
});
|
|
1691
|
+
}
|
|
1692
|
+
if (!isDryRun) {
|
|
1693
|
+
const afterRow = db.select({ n: sql`count(*)` }).from(rawEventSamples).where(and(
|
|
1694
|
+
eq(rawEventSamples.eventType, "unknown"),
|
|
1695
|
+
inArray(rawEventSamples.projectId, projectIds)
|
|
1696
|
+
)).get();
|
|
1697
|
+
result.unknownAfter = Number(afterRow?.n ?? 0);
|
|
1698
|
+
} else {
|
|
1699
|
+
result.unknownAfter = result.unknownBefore - result.reclassified;
|
|
1700
|
+
}
|
|
1701
|
+
emitTrafficClassificationReport(result, isJson);
|
|
1702
|
+
}
|
|
1703
|
+
function emitTrafficClassificationReport(result, isJson) {
|
|
1704
|
+
if (isJson) {
|
|
1705
|
+
console.log(JSON.stringify(result, null, 2));
|
|
1706
|
+
return;
|
|
1707
|
+
}
|
|
1708
|
+
const verb = result.dryRun ? "Would re-classify" : "Re-classified";
|
|
1709
|
+
console.log(`
|
|
1710
|
+
Traffic classification ${result.dryRun ? "preview" : "recovery"} complete.`);
|
|
1711
|
+
console.log(` Examined unknowns: ${result.examined}`);
|
|
1712
|
+
console.log(` ${verb}: ${result.reclassified}`);
|
|
1713
|
+
console.log(` Unknown before: ${result.unknownBefore}`);
|
|
1714
|
+
console.log(` Unknown after: ${result.unknownAfter}`);
|
|
1715
|
+
if (result.reclassified > 0) {
|
|
1716
|
+
console.log(` By bot:`);
|
|
1717
|
+
const sorted = Object.entries(result.byBot).sort(([, a], [, b]) => b - a);
|
|
1718
|
+
for (const [bot, count] of sorted) {
|
|
1719
|
+
console.log(` ${count.toString().padStart(5)} ${bot}`);
|
|
1720
|
+
}
|
|
1721
|
+
}
|
|
1722
|
+
if (result.dryRun) {
|
|
1723
|
+
console.log(`
|
|
1724
|
+
No DB writes performed. Re-run without --dry-run to apply.`);
|
|
1725
|
+
}
|
|
1726
|
+
}
|
|
1727
|
+
|
|
799
1728
|
// src/cli-commands/backfill.ts
|
|
800
1729
|
var BACKFILL_CLI_COMMANDS = [
|
|
801
1730
|
{
|
|
@@ -2333,10 +3262,16 @@ async function discoverPromotePreview(project2, sessionId, opts) {
|
|
|
2333
3262
|
if (preview.suggestedCompetitors.length > 0) {
|
|
2334
3263
|
console.log(` Suggested new competitors:`);
|
|
2335
3264
|
for (const c of preview.suggestedCompetitors) {
|
|
2336
|
-
console.log(` - ${c.domain} (${c.hits} hits, ${c.competitorType})`);
|
|
3265
|
+
console.log(` - ${c.domain} (${c.hits} hits, ${c.competitorType})${mergedHostsNote(c)}`);
|
|
2337
3266
|
}
|
|
2338
3267
|
console.log(" Only direct-competitor is promoted by default \u2014 pass --competitor-types to include other types.");
|
|
2339
3268
|
}
|
|
3269
|
+
const skippedCompetitors = preview.skippedCompetitors ?? [];
|
|
3270
|
+
const held = skippedCompetitors.filter((c) => c.reason !== DiscoveryPromoteCompetitorSkipReasons["already-tracked"]);
|
|
3271
|
+
if (held.length > 0) {
|
|
3272
|
+
console.log(` Competitors a promote leaves out:`);
|
|
3273
|
+
for (const c of held) console.log(` ! ${c.domain} (${c.reason}): ${c.message}`);
|
|
3274
|
+
}
|
|
2340
3275
|
console.log(`
|
|
2341
3276
|
Run \`canonry discover promote ${project2} ${sessionId}\` to merge cited + aspirational queries.`);
|
|
2342
3277
|
console.log(" Add `--bucket wasted-surface` only when off-ICP competitor gaps should be tracked.");
|
|
@@ -2353,15 +3288,24 @@ async function discoverPromote(project2, sessionId, opts) {
|
|
|
2353
3288
|
return;
|
|
2354
3289
|
}
|
|
2355
3290
|
const { promoted, skipped } = result;
|
|
3291
|
+
const competitorDetails = result.competitorDetails;
|
|
3292
|
+
const promotedCompetitors = competitorDetails?.promoted ?? promoted.competitors.map((domain) => ({ domain }));
|
|
3293
|
+
const held = (competitorDetails?.skipped ?? []).filter((c) => c.reason !== DiscoveryPromoteCompetitorSkipReasons["already-tracked"]);
|
|
2356
3294
|
console.log(`Promoted discovery session ${sessionId} into "${project2}":`);
|
|
2357
3295
|
console.log(` Queries: ${promoted.queries.length} added, ${skipped.queries.length} already tracked`);
|
|
2358
3296
|
for (const q of promoted.queries) console.log(` + ${q}`);
|
|
2359
|
-
console.log(` Competitors: ${promoted.competitors.length} added, ${skipped.competitors.length} already tracked`);
|
|
2360
|
-
for (const c of
|
|
3297
|
+
console.log(` Competitors: ${promoted.competitors.length} added, ${skipped.competitors.length - held.length} already tracked, ${held.length} left out`);
|
|
3298
|
+
for (const c of promotedCompetitors) console.log(` + ${c.domain}${mergedHostsNote(c)}`);
|
|
3299
|
+
for (const c of held) console.log(` ! ${c.domain} (${c.reason}): ${c.message}`);
|
|
2361
3300
|
if (promoted.queries.length === 0 && promoted.competitors.length === 0) {
|
|
2362
3301
|
console.log(` Nothing new \u2014 the project's basket already covers this session.`);
|
|
2363
3302
|
}
|
|
2364
3303
|
}
|
|
3304
|
+
function mergedHostsNote(competitor) {
|
|
3305
|
+
const sources = competitor.sources ?? [];
|
|
3306
|
+
const hosts = sources.map((source) => source.domain).filter((host) => host !== competitor.domain);
|
|
3307
|
+
return hosts.length > 0 ? ` (from ${sources.map((source) => source.domain).join(", ")})` : "";
|
|
3308
|
+
}
|
|
2365
3309
|
function printSessionDetail(session) {
|
|
2366
3310
|
console.log(`Discovery session: ${session.id}`);
|
|
2367
3311
|
console.log(` Status: ${session.status}`);
|
|
@@ -8149,21 +9093,28 @@ var TRAFFIC_CLI_COMMANDS = [
|
|
|
8149
9093
|
function getClient10() {
|
|
8150
9094
|
return createApiClient();
|
|
8151
9095
|
}
|
|
8152
|
-
async function addCompetitors(project2, domains, format) {
|
|
9096
|
+
async function addCompetitors(project2, domains, format, aliases = []) {
|
|
8153
9097
|
const client = getClient10();
|
|
8154
9098
|
const existing = await client.listCompetitors(project2);
|
|
8155
9099
|
const existingDomains = existing.map((c) => c.domain);
|
|
8156
9100
|
const existingSet = new Set(existingDomains);
|
|
8157
|
-
const requested = new Set(uniqueStrings(domains));
|
|
8158
|
-
const current = await client.appendCompetitors(
|
|
9101
|
+
const requested = new Set(uniqueStrings(domains).map((domain) => normalizeCompetitorDomain(domain.trim())));
|
|
9102
|
+
const current = await client.appendCompetitors(
|
|
9103
|
+
project2,
|
|
9104
|
+
aliases.length > 0 ? domains.map((domain) => ({ domain, aliases })) : domains
|
|
9105
|
+
);
|
|
8159
9106
|
const currentDomains = current.map((c) => c.domain);
|
|
8160
9107
|
const addedDomains = currentDomains.filter((domain) => requested.has(domain) && !existingSet.has(domain));
|
|
9108
|
+
const aliasTarget = aliases.length > 0 ? normalizeCompetitorDomain(domains[0].trim()) : null;
|
|
9109
|
+
const aliasRow = aliasTarget ? current.find((c) => normalizeCompetitorDomain(c.domain) === aliasTarget) : void 0;
|
|
8161
9110
|
if (isMachineFormat(format)) {
|
|
8162
9111
|
console.log(JSON.stringify({
|
|
8163
9112
|
project: project2,
|
|
8164
9113
|
domains: currentDomains,
|
|
8165
9114
|
addedDomains,
|
|
8166
|
-
addedCount: addedDomains.length
|
|
9115
|
+
addedCount: addedDomains.length,
|
|
9116
|
+
// The aliased competitor as the API returns it (`CompetitorDto`).
|
|
9117
|
+
...aliasRow ? { competitor: aliasRow } : {}
|
|
8167
9118
|
}, null, 2));
|
|
8168
9119
|
return;
|
|
8169
9120
|
}
|
|
@@ -8172,15 +9123,75 @@ async function addCompetitors(project2, domains, format) {
|
|
|
8172
9123
|
} else {
|
|
8173
9124
|
console.log(`Added ${addedDomains.length} competitor(s) to "${project2}".`);
|
|
8174
9125
|
}
|
|
9126
|
+
if (aliasRow) console.log(`Aliases for ${aliasRow.domain}: ${formatAliases(storedAliases(aliasRow))}`);
|
|
9127
|
+
}
|
|
9128
|
+
function storedAliases(competitor) {
|
|
9129
|
+
return competitor.aliases ?? [];
|
|
9130
|
+
}
|
|
9131
|
+
function formatAliases(aliases) {
|
|
9132
|
+
return aliases.length > 0 ? aliases.join(", ") : "(none)";
|
|
9133
|
+
}
|
|
9134
|
+
function sameAliasName(a, b) {
|
|
9135
|
+
return a.trim().toLowerCase() === b.trim().toLowerCase();
|
|
9136
|
+
}
|
|
9137
|
+
async function competitorAliases(project2, domain, options) {
|
|
9138
|
+
const client = getClient10();
|
|
9139
|
+
let result;
|
|
9140
|
+
if (options.set !== void 0 || options.clear === true) {
|
|
9141
|
+
result = await client.setCompetitorAliases(project2, domain, options.clear ? [] : options.set ?? []);
|
|
9142
|
+
} else {
|
|
9143
|
+
const target = normalizeCompetitorDomain(domain.trim());
|
|
9144
|
+
const matches = (await client.listCompetitors(project2)).filter((c) => normalizeCompetitorDomain(c.domain) === target);
|
|
9145
|
+
if (matches.length > 1) {
|
|
9146
|
+
const rows = [...matches].sort((a, b) => a.domain.localeCompare(b.domain));
|
|
9147
|
+
const aliases = normalizeCompetitorAliases(rows.flatMap(storedAliases));
|
|
9148
|
+
const addAgain = `canonry competitor add ${project2} ${target}${aliases.map((alias) => ` --alias ${JSON.stringify(alias)}`).join("")}`;
|
|
9149
|
+
const stored = `${matches.length} rows (${rows.map((c) => c.domain).join(", ")})`;
|
|
9150
|
+
throw new CliError({
|
|
9151
|
+
code: "VALIDATION_ERROR",
|
|
9152
|
+
message: `Competitor ${target} is stored as ${stored}; remove the competitor (every row goes) and add it again: ${addAgain}`,
|
|
9153
|
+
displayMessage: `Error: competitor ${target} is stored as ${stored}. Remove it with: canonry competitor remove ${project2} ${target}, then add it again with: ${addAgain}`,
|
|
9154
|
+
details: { project: project2, domain: target, matches: rows.map((c) => ({ id: c.id, domain: c.domain })), aliases }
|
|
9155
|
+
});
|
|
9156
|
+
}
|
|
9157
|
+
const current = matches.at(0);
|
|
9158
|
+
if (!current) {
|
|
9159
|
+
throw new CliError({
|
|
9160
|
+
code: "NOT_FOUND",
|
|
9161
|
+
message: `Competitor "${target}" is not tracked by project "${project2}"`,
|
|
9162
|
+
displayMessage: `Error: competitor "${target}" is not tracked by "${project2}". Add it with: canonry competitor add ${project2} ${target}`,
|
|
9163
|
+
details: { project: project2, domain: target }
|
|
9164
|
+
});
|
|
9165
|
+
}
|
|
9166
|
+
const removed = options.remove ?? [];
|
|
9167
|
+
const added = options.add ?? [];
|
|
9168
|
+
if (added.length === 0 && removed.length === 0) {
|
|
9169
|
+
result = current;
|
|
9170
|
+
} else if (removed.length === 0) {
|
|
9171
|
+
const listed = await client.appendCompetitors(project2, [{ domain: current.domain, aliases: added }]);
|
|
9172
|
+
result = listed.find((c) => c.domain === current.domain) ?? current;
|
|
9173
|
+
} else {
|
|
9174
|
+
const next = [
|
|
9175
|
+
...storedAliases(current).filter((alias) => !removed.some((name) => sameAliasName(name, alias))),
|
|
9176
|
+
...added
|
|
9177
|
+
];
|
|
9178
|
+
result = await client.setCompetitorAliases(project2, current.domain, next);
|
|
9179
|
+
}
|
|
9180
|
+
}
|
|
9181
|
+
if (isMachineFormat(options.format)) {
|
|
9182
|
+
console.log(JSON.stringify(result, null, 2));
|
|
9183
|
+
return;
|
|
9184
|
+
}
|
|
9185
|
+
console.log(`Aliases for ${result.domain}: ${formatAliases(storedAliases(result))}`);
|
|
8175
9186
|
}
|
|
8176
9187
|
async function removeCompetitors(project2, domains, format) {
|
|
8177
9188
|
const client = getClient10();
|
|
8178
9189
|
const existing = await client.listCompetitors(project2);
|
|
8179
9190
|
const existingDomains = existing.map((c) => c.domain);
|
|
8180
|
-
const requested = new Set(uniqueStrings(domains));
|
|
9191
|
+
const requested = new Set(uniqueStrings(domains).map((domain) => normalizeCompetitorDomain(domain.trim())));
|
|
8181
9192
|
const current = await client.deleteCompetitors(project2, domains);
|
|
8182
9193
|
const currentSet = new Set(current.map((c) => c.domain));
|
|
8183
|
-
const removedDomains = existingDomains.filter((domain) => requested.has(domain) && !currentSet.has(domain));
|
|
9194
|
+
const removedDomains = existingDomains.filter((domain) => requested.has(normalizeCompetitorDomain(domain)) && !currentSet.has(domain));
|
|
8184
9195
|
if (isMachineFormat(format)) {
|
|
8185
9196
|
console.log(JSON.stringify({
|
|
8186
9197
|
project: project2,
|
|
@@ -8219,7 +9230,8 @@ async function listCompetitors(project2, format) {
|
|
|
8219
9230
|
console.log(`Competitors for "${project2}" (${comps.length}):
|
|
8220
9231
|
`);
|
|
8221
9232
|
for (const c of comps) {
|
|
8222
|
-
|
|
9233
|
+
const aliases = storedAliases(c);
|
|
9234
|
+
console.log(aliases.length > 0 ? ` ${c.domain} (aliases: ${aliases.join(", ")})` : ` ${c.domain}`);
|
|
8223
9235
|
}
|
|
8224
9236
|
}
|
|
8225
9237
|
async function showCompetitorLandscape(project2, options) {
|
|
@@ -8309,6 +9321,8 @@ function printLandscapeRows(heading, rows, showSampleCount = false, context, que
|
|
|
8309
9321
|
}
|
|
8310
9322
|
|
|
8311
9323
|
// src/cli-commands/competitor.ts
|
|
9324
|
+
var ADD_USAGE = "canonry competitor add <project> <domain...> [--alias <name>]... [--format json]";
|
|
9325
|
+
var ALIASES_USAGE = "canonry competitor aliases <project> <domain> [--set <name>]... [--add <name>]... [--remove <name>]... [--clear] [--format json]";
|
|
8312
9326
|
var LANDSCAPE_USAGE = "canonry competitor landscape <project> [--window 7d|30d|90d|all] [--group-key <key>|--scope all-markets] [--by-model] [--provider <provider> [--model <id>]] [--query-class all|branded|non-brand] [--location <label>] [--run-id <id>] [--format json|jsonl]";
|
|
8313
9327
|
function parseLandscapeScope(value) {
|
|
8314
9328
|
if (value === void 0 || value === "") return void 0;
|
|
@@ -8321,20 +9335,63 @@ function parseLandscapeScope(value) {
|
|
|
8321
9335
|
var COMPETITOR_CLI_COMMANDS = [
|
|
8322
9336
|
{
|
|
8323
9337
|
path: ["competitor", "add"],
|
|
8324
|
-
usage:
|
|
9338
|
+
usage: ADD_USAGE,
|
|
9339
|
+
options: {
|
|
9340
|
+
alias: multiStringOption()
|
|
9341
|
+
},
|
|
8325
9342
|
run: async (input) => {
|
|
8326
|
-
const project2 = requireProject(input, "competitor.add",
|
|
9343
|
+
const project2 = requireProject(input, "competitor.add", ADD_USAGE);
|
|
8327
9344
|
const domains = input.positionals.slice(1);
|
|
8328
9345
|
if (domains.length === 0) {
|
|
8329
|
-
throw usageError(
|
|
9346
|
+
throw usageError(`Error: project name and at least one domain required
|
|
9347
|
+
Usage: ${ADD_USAGE}`, {
|
|
8330
9348
|
message: "project name and at least one domain required",
|
|
8331
9349
|
details: {
|
|
8332
9350
|
command: "competitor.add",
|
|
8333
|
-
usage:
|
|
9351
|
+
usage: ADD_USAGE
|
|
8334
9352
|
}
|
|
8335
9353
|
});
|
|
8336
9354
|
}
|
|
8337
|
-
|
|
9355
|
+
const aliases = getStringArray(input.values, "alias") ?? [];
|
|
9356
|
+
if (aliases.length > 0 && domains.length !== 1) {
|
|
9357
|
+
throw usageError(`Error: --alias names one competitor, so pass exactly one domain
|
|
9358
|
+
Usage: ${ADD_USAGE}`, {
|
|
9359
|
+
message: "--alias names one competitor, so pass exactly one domain",
|
|
9360
|
+
details: { command: "competitor.add", usage: ADD_USAGE }
|
|
9361
|
+
});
|
|
9362
|
+
}
|
|
9363
|
+
await addCompetitors(project2, domains, input.format, aliases);
|
|
9364
|
+
}
|
|
9365
|
+
},
|
|
9366
|
+
{
|
|
9367
|
+
path: ["competitor", "aliases"],
|
|
9368
|
+
usage: ALIASES_USAGE,
|
|
9369
|
+
options: {
|
|
9370
|
+
set: multiStringOption(),
|
|
9371
|
+
add: multiStringOption(),
|
|
9372
|
+
remove: multiStringOption(),
|
|
9373
|
+
clear: { type: "boolean" }
|
|
9374
|
+
},
|
|
9375
|
+
run: async (input) => {
|
|
9376
|
+
const project2 = requireProject(input, "competitor.aliases", ALIASES_USAGE);
|
|
9377
|
+
const domain = requirePositional(input, 1, {
|
|
9378
|
+
command: "competitor.aliases",
|
|
9379
|
+
usage: ALIASES_USAGE,
|
|
9380
|
+
message: "competitor domain is required"
|
|
9381
|
+
});
|
|
9382
|
+
const set = getStringArray(input.values, "set");
|
|
9383
|
+
const add = getStringArray(input.values, "add");
|
|
9384
|
+
const remove = getStringArray(input.values, "remove");
|
|
9385
|
+
const clear = getBoolean(input.values, "clear");
|
|
9386
|
+
const exclusive = [set !== void 0, clear].filter(Boolean).length;
|
|
9387
|
+
if (exclusive > 1 || exclusive === 1 && (add !== void 0 || remove !== void 0)) {
|
|
9388
|
+
throw usageError(`Error: --set and --clear replace the whole list, so they cannot be combined with each other or with --add/--remove
|
|
9389
|
+
Usage: ${ALIASES_USAGE}`, {
|
|
9390
|
+
message: "--set and --clear cannot be combined with each other or with --add/--remove",
|
|
9391
|
+
details: { command: "competitor.aliases", usage: ALIASES_USAGE }
|
|
9392
|
+
});
|
|
9393
|
+
}
|
|
9394
|
+
await competitorAliases(project2, domain, { set, add, remove, clear, format: input.format });
|
|
8338
9395
|
}
|
|
8339
9396
|
},
|
|
8340
9397
|
{
|
|
@@ -8429,12 +9486,12 @@ var COMPETITOR_CLI_COMMANDS = [
|
|
|
8429
9486
|
},
|
|
8430
9487
|
{
|
|
8431
9488
|
path: ["competitor"],
|
|
8432
|
-
usage: "canonry competitor <add|remove|delete|list|landscape> <project> [args]",
|
|
9489
|
+
usage: "canonry competitor <add|aliases|remove|delete|list|landscape> <project> [args]",
|
|
8433
9490
|
run: async (input) => {
|
|
8434
9491
|
unknownSubcommand(input.positionals[0], {
|
|
8435
9492
|
command: "competitor",
|
|
8436
|
-
usage: "canonry competitor <add|remove|delete|list|landscape> <project> [args]",
|
|
8437
|
-
available: ["add", "remove", "delete", "list", "landscape"]
|
|
9493
|
+
usage: "canonry competitor <add|aliases|remove|delete|list|landscape> <project> [args]",
|
|
9494
|
+
available: ["add", "aliases", "remove", "delete", "list", "landscape"]
|
|
8438
9495
|
});
|
|
8439
9496
|
}
|
|
8440
9497
|
}
|
|
@@ -11969,10 +13026,10 @@ Brand Gap Analysis`);
|
|
|
11969
13026
|
console.log(`
|
|
11970
13027
|
Opportunity Gaps (competitors cited, you're not):`);
|
|
11971
13028
|
for (const q of data.gap) {
|
|
11972
|
-
const
|
|
13029
|
+
const competitors2 = q.competitorsCiting.join(", ");
|
|
11973
13030
|
const cons = q.consistency.totalRuns > 0 ? ` [cited ${q.consistency.citedRuns}/${q.consistency.totalRuns} runs]` : "";
|
|
11974
13031
|
console.log(` \u2022 ${q.query}${cons}`);
|
|
11975
|
-
console.log(` Competitors: ${
|
|
13032
|
+
console.log(` Competitors: ${competitors2}`);
|
|
11976
13033
|
}
|
|
11977
13034
|
}
|
|
11978
13035
|
if (data.cited.length > 0) {
|
|
@@ -12517,7 +13574,10 @@ async function createProject(name, opts) {
|
|
|
12517
13574
|
language: opts.language,
|
|
12518
13575
|
providers: opts.providers ?? [],
|
|
12519
13576
|
providerModels: opts.providerModels ?? {},
|
|
12520
|
-
...opts.providerDispatchModes && Object.keys(opts.providerDispatchModes).length > 0 ? { providerDispatchModes: opts.providerDispatchModes } : {}
|
|
13577
|
+
...opts.providerDispatchModes && Object.keys(opts.providerDispatchModes).length > 0 ? { providerDispatchModes: opts.providerDispatchModes } : {},
|
|
13578
|
+
// Sent only when given: this PUT also re-runs over an existing project,
|
|
13579
|
+
// and an omitted budget keeps the stored one.
|
|
13580
|
+
...opts.siteAuditMaxPages !== void 0 ? { siteAuditMaxPages: opts.siteAuditMaxPages } : {}
|
|
12521
13581
|
});
|
|
12522
13582
|
warnDroppedDispatchModes(opts.providerDispatchModes ?? {}, result);
|
|
12523
13583
|
if (isMachineFormat(opts.format)) {
|
|
@@ -12601,6 +13661,9 @@ async function showProject(name, format) {
|
|
|
12601
13661
|
const dispatchModes = Object.entries(project2.providerDispatchModes ?? {}).sort(([left], [right]) => left.localeCompare(right));
|
|
12602
13662
|
console.log(` Dispatch modes: ${dispatchModes.length > 0 ? `${dispatchModes.map(([provider, mode]) => `${provider}=${mode}`).join(", ")} (scheduled sweeps; others sync)` : "(none; every provider runs sync)"}`);
|
|
12603
13663
|
console.log(` Negative reviews: ${resolveNegativeReviewMaxStars(project2.negativeReviewMaxStars)} stars or fewer${project2.negativeReviewMaxStars == null ? " (default)" : ""}`);
|
|
13664
|
+
if (project2.siteAuditMaxPages !== void 0) {
|
|
13665
|
+
console.log(` Site Health page budget: ${formatSiteAuditPageBudget(project2.siteAuditMaxPages)}`);
|
|
13666
|
+
}
|
|
12604
13667
|
console.log(` Tags: ${project2.tags.length > 0 ? project2.tags.join(", ") : "(none)"}`);
|
|
12605
13668
|
const labelEntries = Object.entries(project2.labels);
|
|
12606
13669
|
console.log(` Labels: ${labelEntries.length > 0 ? labelEntries.map(([k, v]) => `${k}=${v}`).join(", ") : "(none)"}`);
|
|
@@ -12684,7 +13747,10 @@ async function updateProjectSettings(name, opts) {
|
|
|
12684
13747
|
autoExtractBacklinks: project2.autoExtractBacklinks,
|
|
12685
13748
|
// Sent only when asked for: the server keeps the stored threshold when the
|
|
12686
13749
|
// field is absent, so a stale read cannot overwrite a newer setting.
|
|
12687
|
-
...opts.negativeReviewMaxStars !== void 0 ? { negativeReviewMaxStars: opts.negativeReviewMaxStars } : {}
|
|
13750
|
+
...opts.negativeReviewMaxStars !== void 0 ? { negativeReviewMaxStars: opts.negativeReviewMaxStars } : {},
|
|
13751
|
+
// Same for the page budget: never echoed from the read above, so an edit
|
|
13752
|
+
// that does not touch it cannot overwrite a newer one or send a stray null.
|
|
13753
|
+
...opts.siteAuditMaxPages !== void 0 ? { siteAuditMaxPages: opts.siteAuditMaxPages } : {}
|
|
12688
13754
|
});
|
|
12689
13755
|
warnDroppedDispatchModes(touchesDispatch ? providerDispatchModes : project2.providerDispatchModes ?? {}, result);
|
|
12690
13756
|
if (isMachineFormat(opts.format)) {
|
|
@@ -12799,7 +13865,7 @@ async function setDefaultLocation(project2, label, format) {
|
|
|
12799
13865
|
var PROJECT_CLI_COMMANDS = [
|
|
12800
13866
|
{
|
|
12801
13867
|
path: ["project", "create"],
|
|
12802
|
-
usage: "canonry project create <name> [--domain <domain>] [--owned-domain <domain>...] [--alias <name>...] [--qualified-alias <name>...] [--country <code>] [--language <lang>] [--display-name <name>] [--provider <name>...] [--provider-model provider=model...] [--dispatch-mode provider=sync|batch...] [--format json]",
|
|
13868
|
+
usage: "canonry project create <name> [--domain <domain>] [--owned-domain <domain>...] [--alias <name>...] [--qualified-alias <name>...] [--country <code>] [--language <lang>] [--display-name <name>] [--provider <name>...] [--provider-model provider=model...] [--dispatch-mode provider=sync|batch...] [--site-audit-max-pages <1-50000|full>] [--format json]",
|
|
12803
13869
|
help: "Create a project. Pass --domain for the public site to scan. Provider credentials are not required for Page Health.",
|
|
12804
13870
|
options: {
|
|
12805
13871
|
domain: { type: "string", short: "d" },
|
|
@@ -12811,13 +13877,14 @@ var PROJECT_CLI_COMMANDS = [
|
|
|
12811
13877
|
"display-name": stringOption(),
|
|
12812
13878
|
provider: multiStringOption(),
|
|
12813
13879
|
"provider-model": multiStringOption(),
|
|
12814
|
-
"dispatch-mode": multiStringOption()
|
|
13880
|
+
"dispatch-mode": multiStringOption(),
|
|
13881
|
+
"site-audit-max-pages": stringOption()
|
|
12815
13882
|
},
|
|
12816
13883
|
run: async (input) => {
|
|
12817
13884
|
const name = requireProject(
|
|
12818
13885
|
input,
|
|
12819
13886
|
"project.create",
|
|
12820
|
-
"canonry project create <name> [--domain <domain>] [--owned-domain <domain>...] [--alias <name>...] [--qualified-alias <name>...] [--country <code>] [--language <lang>] [--display-name <name>] [--provider <name>...] [--provider-model provider=model...] [--dispatch-mode provider=sync|batch...] [--format json]"
|
|
13887
|
+
"canonry project create <name> [--domain <domain>] [--owned-domain <domain>...] [--alias <name>...] [--qualified-alias <name>...] [--country <code>] [--language <lang>] [--display-name <name>] [--provider <name>...] [--provider-model provider=model...] [--dispatch-mode provider=sync|batch...] [--site-audit-max-pages <1-50000|full>] [--format json]"
|
|
12821
13888
|
);
|
|
12822
13889
|
await createProject(name, {
|
|
12823
13890
|
domain: getString(input.values, "domain") ?? name,
|
|
@@ -12830,13 +13897,14 @@ var PROJECT_CLI_COMMANDS = [
|
|
|
12830
13897
|
providers: getStringArray(input.values, "provider") ?? [],
|
|
12831
13898
|
providerModels: parseProviderModelAssignments(getStringArray(input.values, "provider-model")),
|
|
12832
13899
|
providerDispatchModes: parseDispatchModeAssignments(getStringArray(input.values, "dispatch-mode")),
|
|
13900
|
+
siteAuditMaxPages: parseSiteAuditMaxPages(getString(input.values, "site-audit-max-pages"), "project.create"),
|
|
12833
13901
|
format: input.format
|
|
12834
13902
|
});
|
|
12835
13903
|
}
|
|
12836
13904
|
},
|
|
12837
13905
|
{
|
|
12838
13906
|
path: ["project", "update"],
|
|
12839
|
-
usage: "canonry project update <name> [--domain <domain>] [--owned-domain <domain>...] [--add-domain <domain>...] [--remove-domain <domain>...] [--alias <name>...] [--add-alias <name>...] [--remove-alias <name>...] [--add-qualified-alias <name>...] [--remove-qualified-alias <name>...] [--country <code>] [--language <lang>] [--display-name <name>] [--provider <name>...] [--all-providers] [--provider-model provider=model...] [--clear-provider-model <provider>...] [--dispatch-mode provider=sync|batch...] [--clear-dispatch-mode <provider>...] [--negative-review-max-stars <1-4|default>] [--format json]",
|
|
13907
|
+
usage: "canonry project update <name> [--domain <domain>] [--owned-domain <domain>...] [--add-domain <domain>...] [--remove-domain <domain>...] [--alias <name>...] [--add-alias <name>...] [--remove-alias <name>...] [--add-qualified-alias <name>...] [--remove-qualified-alias <name>...] [--country <code>] [--language <lang>] [--display-name <name>] [--provider <name>...] [--all-providers] [--provider-model provider=model...] [--clear-provider-model <provider>...] [--dispatch-mode provider=sync|batch...] [--clear-dispatch-mode <provider>...] [--negative-review-max-stars <1-4|default>] [--site-audit-max-pages <1-50000|full>] [--format json]",
|
|
12840
13908
|
options: {
|
|
12841
13909
|
domain: { type: "string", short: "d" },
|
|
12842
13910
|
"owned-domain": multiStringOption(),
|
|
@@ -12856,13 +13924,14 @@ var PROJECT_CLI_COMMANDS = [
|
|
|
12856
13924
|
"clear-provider-model": multiStringOption(),
|
|
12857
13925
|
"dispatch-mode": multiStringOption(),
|
|
12858
13926
|
"clear-dispatch-mode": multiStringOption(),
|
|
12859
|
-
"negative-review-max-stars": stringOption()
|
|
13927
|
+
"negative-review-max-stars": stringOption(),
|
|
13928
|
+
"site-audit-max-pages": stringOption()
|
|
12860
13929
|
},
|
|
12861
13930
|
run: async (input) => {
|
|
12862
13931
|
const name = requireProject(
|
|
12863
13932
|
input,
|
|
12864
13933
|
"project.update",
|
|
12865
|
-
"canonry project update <name> [--domain <domain>] [--owned-domain <domain>...] [--add-domain <domain>...] [--remove-domain <domain>...] [--alias <name>...] [--add-alias <name>...] [--remove-alias <name>...] [--add-qualified-alias <name>...] [--remove-qualified-alias <name>...] [--country <code>] [--language <lang>] [--display-name <name>] [--provider <name>...] [--all-providers] [--provider-model provider=model...] [--clear-provider-model <provider>...] [--dispatch-mode provider=sync|batch...] [--clear-dispatch-mode <provider>...] [--negative-review-max-stars <1-4|default>] [--format json]"
|
|
13934
|
+
"canonry project update <name> [--domain <domain>] [--owned-domain <domain>...] [--add-domain <domain>...] [--remove-domain <domain>...] [--alias <name>...] [--add-alias <name>...] [--remove-alias <name>...] [--add-qualified-alias <name>...] [--remove-qualified-alias <name>...] [--country <code>] [--language <lang>] [--display-name <name>] [--provider <name>...] [--all-providers] [--provider-model provider=model...] [--clear-provider-model <provider>...] [--dispatch-mode provider=sync|batch...] [--clear-dispatch-mode <provider>...] [--negative-review-max-stars <1-4|default>] [--site-audit-max-pages <1-50000|full>] [--format json]"
|
|
12866
13935
|
);
|
|
12867
13936
|
const providers = getStringArray(input.values, "provider");
|
|
12868
13937
|
const allProviders = getBoolean(input.values, "all-providers");
|
|
@@ -12896,6 +13965,7 @@ var PROJECT_CLI_COMMANDS = [
|
|
|
12896
13965
|
dispatchModes,
|
|
12897
13966
|
clearDispatchModes,
|
|
12898
13967
|
negativeReviewMaxStars: parseNegativeReviewMaxStars(getString(input.values, "negative-review-max-stars")),
|
|
13968
|
+
siteAuditMaxPages: parseSiteAuditMaxPages(getString(input.values, "site-audit-max-pages"), "project.update"),
|
|
12899
13969
|
format: input.format
|
|
12900
13970
|
});
|
|
12901
13971
|
}
|
|
@@ -13047,6 +14117,19 @@ function parseNegativeReviewMaxStars(raw) {
|
|
|
13047
14117
|
}
|
|
13048
14118
|
return parsed.data;
|
|
13049
14119
|
}
|
|
14120
|
+
function parseSiteAuditMaxPages(raw, command) {
|
|
14121
|
+
if (raw === void 0) return void 0;
|
|
14122
|
+
const value = raw.trim();
|
|
14123
|
+
if (value.toLowerCase() === "full") return null;
|
|
14124
|
+
const parsed = /^\d+$/.test(value) ? siteAuditPageBudgetSchema.safeParse(Number(value)) : void 0;
|
|
14125
|
+
if (!parsed?.success) {
|
|
14126
|
+
throw usageError(`Error: --site-audit-max-pages must be a whole number of pages from 1 to ${SITE_AUDIT_MAX_PAGE_LIMIT}, or "full" for the full site`, {
|
|
14127
|
+
message: `--site-audit-max-pages must be 1-${SITE_AUDIT_MAX_PAGE_LIMIT} or "full"`,
|
|
14128
|
+
details: { command, option: "site-audit-max-pages", value: raw }
|
|
14129
|
+
});
|
|
14130
|
+
}
|
|
14131
|
+
return parsed.data;
|
|
14132
|
+
}
|
|
13050
14133
|
|
|
13051
14134
|
// src/commands/organic-evidence.ts
|
|
13052
14135
|
async function showOrganicEvidence(project2, opts) {
|
|
@@ -14647,8 +15730,8 @@ function renderQueries(pdf, report) {
|
|
|
14647
15730
|
for (const result of queryResult.providerResults) {
|
|
14648
15731
|
const status = result.error ? "error" : result.mentioned ? result.cited ? "mentioned and cited" : "mentioned" : "not mentioned";
|
|
14649
15732
|
const accuracy = result.describedAccurately === "not-mentioned" ? "" : `; accuracy: ${result.describedAccurately}`;
|
|
14650
|
-
const
|
|
14651
|
-
const line = `${result.displayName}: ${status}${accuracy}${
|
|
15733
|
+
const competitors2 = result.recommendedCompetitors.length > 0 ? `; recommended instead: ${result.recommendedCompetitors.join(", ")}` : "";
|
|
15734
|
+
const line = `${result.displayName}: ${status}${accuracy}${competitors2}`;
|
|
14652
15735
|
pdf.bullet(line);
|
|
14653
15736
|
if (result.error) {
|
|
14654
15737
|
pdf.paragraph(`Error: ${result.error}`, { size: 9, color: FAIL, lineHeight: 12 });
|
|
@@ -14797,8 +15880,8 @@ function formatSnapshotMarkdown(report) {
|
|
|
14797
15880
|
const mentioned = result.mentioned ? "Yes" : "No";
|
|
14798
15881
|
const cited = result.cited ? "Yes" : "No";
|
|
14799
15882
|
const accuracy = result.describedAccurately === "not-mentioned" ? "-" : result.describedAccurately;
|
|
14800
|
-
const
|
|
14801
|
-
lines.push(`| ${result.displayName} | ${mentioned} | ${cited} | ${accuracy} | ${
|
|
15883
|
+
const competitors2 = result.recommendedCompetitors.length > 0 ? result.recommendedCompetitors.join(", ") : "-";
|
|
15884
|
+
lines.push(`| ${result.displayName} | ${mentioned} | ${cited} | ${accuracy} | ${competitors2} |`);
|
|
14802
15885
|
}
|
|
14803
15886
|
lines.push("");
|
|
14804
15887
|
}
|
|
@@ -15097,7 +16180,7 @@ function renderHuman2(overview) {
|
|
|
15097
16180
|
citationMovement,
|
|
15098
16181
|
mentionMovement,
|
|
15099
16182
|
movementComparison,
|
|
15100
|
-
competitors,
|
|
16183
|
+
competitors: competitors2,
|
|
15101
16184
|
providerScores,
|
|
15102
16185
|
attentionItems,
|
|
15103
16186
|
runHistory,
|
|
@@ -15160,9 +16243,9 @@ function renderHuman2(overview) {
|
|
|
15160
16243
|
console.log(`
|
|
15161
16244
|
Transitions since ${transitions.since}: +${transitions.gained} gained, -${transitions.lost} lost, ${transitions.emerging} emerging`);
|
|
15162
16245
|
}
|
|
15163
|
-
if (
|
|
16246
|
+
if (competitors2.length > 0) {
|
|
15164
16247
|
console.log("\n Competitors:");
|
|
15165
|
-
for (const c of
|
|
16248
|
+
for (const c of competitors2) {
|
|
15166
16249
|
console.log(` ${c.domain.padEnd(28)} ${c.citationCount}/${c.totalQueries} ${c.pressureLabel}`);
|
|
15167
16250
|
}
|
|
15168
16251
|
}
|
|
@@ -15212,13 +16295,13 @@ function printMentionShareBreakdown(mentionShare) {
|
|
|
15212
16295
|
const ranking = legacyCompatible.ranking ?? [];
|
|
15213
16296
|
const project2 = ranking.find((row) => row.kind === "project");
|
|
15214
16297
|
if (!project2) return;
|
|
15215
|
-
const
|
|
16298
|
+
const competitors2 = ranking.filter((row) => row.kind === "competitor");
|
|
15216
16299
|
console.log(` you${" ".repeat(28)} ${project2.mentionSnapshots} mentions (${formatPercent(project2.share)} of combined)`);
|
|
15217
|
-
for (const row of
|
|
16300
|
+
for (const row of competitors2.slice(0, 3)) {
|
|
15218
16301
|
console.log(` ${(row.domain ?? "").padEnd(30)} ${row.mentionSnapshots} mentions (${formatPercent(row.share)} of combined)`);
|
|
15219
16302
|
}
|
|
15220
|
-
if (
|
|
15221
|
-
console.log(` + ${
|
|
16303
|
+
if (competitors2.length > 3) {
|
|
16304
|
+
console.log(` + ${competitors2.length - 3} more competitor${competitors2.length - 3 === 1 ? "" : "s"} (--format json for full breakdown)`);
|
|
15222
16305
|
}
|
|
15223
16306
|
}
|
|
15224
16307
|
|
|
@@ -15811,7 +16894,7 @@ var CONTENT_CLI_COMMANDS = [
|
|
|
15811
16894
|
import crypto2 from "crypto";
|
|
15812
16895
|
import path8 from "path";
|
|
15813
16896
|
import { isDeepStrictEqual } from "util";
|
|
15814
|
-
import { eq } from "drizzle-orm";
|
|
16897
|
+
import { eq as eq2 } from "drizzle-orm";
|
|
15815
16898
|
function persistedValue(value) {
|
|
15816
16899
|
if (Array.isArray(value)) return value.map(persistedValue);
|
|
15817
16900
|
if (value && typeof value === "object") {
|
|
@@ -15871,7 +16954,7 @@ async function bootstrapCommand(opts) {
|
|
|
15871
16954
|
projectId: apiKeys.projectId,
|
|
15872
16955
|
lastUsedAt: apiKeys.lastUsedAt,
|
|
15873
16956
|
revokedAt: apiKeys.revokedAt
|
|
15874
|
-
}).from(apiKeys).where(
|
|
16957
|
+
}).from(apiKeys).where(eq2(apiKeys.name, "default")).all();
|
|
15875
16958
|
const existingDefault = existingDefaults.find((key) => key.keyHash === existingConfigKeyHash) ?? existingDefaults.at(0);
|
|
15876
16959
|
if (existingDefault) {
|
|
15877
16960
|
const rotating = existingDefault.keyHash !== keyHash;
|
|
@@ -15884,13 +16967,13 @@ async function bootstrapCommand(opts) {
|
|
|
15884
16967
|
projectId: null,
|
|
15885
16968
|
...rotating ? { lastUsedAt: null } : {},
|
|
15886
16969
|
revokedAt: null
|
|
15887
|
-
}).where(
|
|
16970
|
+
}).where(eq2(apiKeys.id, existingDefault.id)).run();
|
|
15888
16971
|
changed = true;
|
|
15889
16972
|
}
|
|
15890
16973
|
for (const duplicate of existingDefaults) {
|
|
15891
16974
|
if (duplicate.id === existingDefault.id) continue;
|
|
15892
16975
|
if (duplicate.revokedAt === null) {
|
|
15893
|
-
tx.update(apiKeys).set({ revokedAt: rotatedAt }).where(
|
|
16976
|
+
tx.update(apiKeys).set({ revokedAt: rotatedAt }).where(eq2(apiKeys.id, duplicate.id)).run();
|
|
15894
16977
|
changed = true;
|
|
15895
16978
|
}
|
|
15896
16979
|
}
|
|
@@ -16541,7 +17624,7 @@ function encodeLegacySetupState(state) {
|
|
|
16541
17624
|
}
|
|
16542
17625
|
|
|
16543
17626
|
// src/commands/serve.ts
|
|
16544
|
-
import { and, asc, eq as
|
|
17627
|
+
import { and as and2, asc as asc2, eq as eq4, ne } from "drizzle-orm";
|
|
16545
17628
|
|
|
16546
17629
|
// src/server-shutdown.ts
|
|
16547
17630
|
async function closeWithIdleSweep(app, intervalMs = 250) {
|
|
@@ -16555,7 +17638,7 @@ async function closeWithIdleSweep(app, intervalMs = 250) {
|
|
|
16555
17638
|
}
|
|
16556
17639
|
|
|
16557
17640
|
// src/startup-path-repairs.ts
|
|
16558
|
-
import { eq as
|
|
17641
|
+
import { eq as eq3, lt } from "drizzle-orm";
|
|
16559
17642
|
var STARTUP_REPAIR_PAGE_SIZE = 4096;
|
|
16560
17643
|
function repairNormalizedPathsOnStartup(db) {
|
|
16561
17644
|
return repairPathsOnce(db, "ga-traffic-paths", backfillNormalizedPaths);
|
|
@@ -16564,7 +17647,7 @@ function repairAiReferralPathsOnStartup(db) {
|
|
|
16564
17647
|
return repairPathsOnce(db, "ga-ai-referral-paths", backfillAiReferralPaths);
|
|
16565
17648
|
}
|
|
16566
17649
|
async function repairPathsOnce(db, name, repair) {
|
|
16567
|
-
const completed = db.select({ version: dataRepairCompletions.version }).from(dataRepairCompletions).where(
|
|
17650
|
+
const completed = db.select({ version: dataRepairCompletions.version }).from(dataRepairCompletions).where(eq3(dataRepairCompletions.name, name)).get();
|
|
16568
17651
|
if (completed && completed.version >= URL_PATH_NORMALIZATION_VERSION) return null;
|
|
16569
17652
|
let conflicted = false;
|
|
16570
17653
|
const result = await repair(db, { pageSize: STARTUP_REPAIR_PAGE_SIZE, onConflict: () => {
|
|
@@ -16785,15 +17868,15 @@ function readServeOpenState(db) {
|
|
|
16785
17868
|
id: projects.id,
|
|
16786
17869
|
name: projects.name,
|
|
16787
17870
|
createdAt: projects.createdAt
|
|
16788
|
-
}).from(projects).orderBy(
|
|
17871
|
+
}).from(projects).orderBy(asc2(projects.createdAt), asc2(projects.name)).all();
|
|
16789
17872
|
if (rows.length === 0) return { projectCount: 0, hasSiteAudit: false };
|
|
16790
17873
|
let scanned;
|
|
16791
17874
|
try {
|
|
16792
17875
|
const audits = db.select({
|
|
16793
17876
|
projectId: runs.projectId,
|
|
16794
17877
|
status: runs.status
|
|
16795
|
-
}).from(runs).where(
|
|
16796
|
-
|
|
17878
|
+
}).from(runs).where(and2(
|
|
17879
|
+
eq4(runs.kind, RunKinds["site-audit"]),
|
|
16797
17880
|
ne(runs.trigger, RunTriggers.probe)
|
|
16798
17881
|
)).all();
|
|
16799
17882
|
scanned = new Set(
|
|
@@ -17855,7 +18938,7 @@ var TECHNICAL_AEO_CLI_COMMANDS_BASE = [
|
|
|
17855
18938
|
{
|
|
17856
18939
|
path: ["technical-aeo", "run"],
|
|
17857
18940
|
usage: "canonry technical-aeo run <project> [--sitemap-url <url>] [--max-pages <n>] [--max-edges <n>] [--max-depth <n>] [--check-dead-links] [--wait] [--format json]",
|
|
17858
|
-
help: "Start a Site Health crawl. No answer-engine provider is required. --wait polls for up to 15 minutes
|
|
18941
|
+
help: "Start a Site Health crawl. No answer-engine provider is required. Without --max-pages the scan uses the project's saved page budget (canonry project update <project> --site-audit-max-pages <n|full>), else the full site, up to 50,000 pages. --wait polls for up to 15 minutes, so a large full-site scan can outlast it; follow it with canonry technical-aeo progress <project> --run-id <id>. Dead-link checking is off unless --check-dead-links is set.",
|
|
17859
18942
|
options: {
|
|
17860
18943
|
"sitemap-url": stringOption(),
|
|
17861
18944
|
limit: stringOption(),
|
|
@@ -20592,7 +21675,7 @@ Page Health:
|
|
|
20592
21675
|
Projects:
|
|
20593
21676
|
project Create, update, list, show, delete projects
|
|
20594
21677
|
query Add, replace, remove, list, import, generate queries
|
|
20595
|
-
competitor Add, remove, list competitors
|
|
21678
|
+
competitor Add, remove, list competitors and their aliases
|
|
20596
21679
|
|
|
20597
21680
|
Monitoring:
|
|
20598
21681
|
run Trigger visibility sweeps
|