@ainyc/canonry 4.78.0 → 4.81.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +38 -12
- package/assets/assets/{BacklinksPage-CwXveumn.js → BacklinksPage-DHShKKpo.js} +1 -1
- package/assets/assets/{ChartPrimitives-DntKGI5J.js → ChartPrimitives-udHScxjY.js} +1 -1
- package/assets/assets/ProjectPage-BsS1anh7.js +6 -0
- package/assets/assets/{RunRow-DMtYXaxG.js → RunRow-CXyPHMVQ.js} +1 -1
- package/assets/assets/{RunsPage-Cz-YlucO.js → RunsPage-BpQ_NpFt.js} +1 -1
- package/assets/assets/{SettingsPage-BCuG3C-0.js → SettingsPage-1ep4ch7n.js} +1 -1
- package/assets/assets/{TrafficPage-DV8X47wa.js → TrafficPage-C3Hx-sE7.js} +1 -1
- package/assets/assets/TrafficSourceDetailPage-B26n2R6G.js +1 -0
- package/assets/assets/{arrow-left-CUmHyNnF.js → arrow-left-Dc_IPJxw.js} +1 -1
- package/assets/assets/{extract-error-message-DFjy9_zi.js → extract-error-message-B3PoKkHW.js} +1 -1
- package/assets/assets/{index-D9smxU6R.js → index-DhdFTQkU.js} +86 -86
- package/assets/assets/{trash-2-B_UtEEm8.js → trash-2-BQ69cGl0.js} +1 -1
- package/assets/index.html +1 -1
- package/dist/{chunk-XI6YSTGE.js → chunk-6XOZSS3Y.js} +258 -8
- package/dist/{chunk-KPN22EWK.js → chunk-GMT3YPLT.js} +214 -4
- package/dist/{chunk-3WXARKUE.js → chunk-UAQ42NVJ.js} +1346 -357
- package/dist/{chunk-QKTIP6GC.js → chunk-VX5C7DK7.js} +902 -313
- package/dist/cli.js +468 -152
- package/dist/index.d.ts +17 -0
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-CDVUUG7O.js → intelligence-service-CAAQAKPN.js} +2 -2
- package/dist/mcp.js +9 -3
- package/package.json +9 -8
- package/assets/assets/ProjectPage-CVudiU8X.js +0 -6
- package/assets/assets/TrafficSourceDetailPage-BmYhK9jm.js +0 -1
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
loadConfig,
|
|
10
10
|
loadConfigRaw,
|
|
11
11
|
saveConfigPatch
|
|
12
|
-
} from "./chunk-
|
|
12
|
+
} from "./chunk-6XOZSS3Y.js";
|
|
13
13
|
import {
|
|
14
14
|
CC_CACHE_DIR,
|
|
15
15
|
DUCKDB_SPEC,
|
|
@@ -17,6 +17,11 @@ import {
|
|
|
17
17
|
GSC_DATA_LAG_DAYS,
|
|
18
18
|
IntelligenceService,
|
|
19
19
|
PLUGIN_DIR,
|
|
20
|
+
adsAdGroups,
|
|
21
|
+
adsAds,
|
|
22
|
+
adsCampaigns,
|
|
23
|
+
adsConnections,
|
|
24
|
+
adsInsightsDaily,
|
|
20
25
|
agentMemory,
|
|
21
26
|
agentSessions,
|
|
22
27
|
aiUserFetchEventsHourly,
|
|
@@ -57,9 +62,11 @@ import {
|
|
|
57
62
|
gbpPlaceActions,
|
|
58
63
|
gbpPlaceDetails,
|
|
59
64
|
getCrawlIssues,
|
|
65
|
+
getLinkCounts,
|
|
60
66
|
getLodging,
|
|
61
67
|
getPlaceDetails,
|
|
62
68
|
getUrlInfo,
|
|
69
|
+
getUrlLinks,
|
|
63
70
|
groupRunsByCreatedAt,
|
|
64
71
|
gscCoverageSnapshots,
|
|
65
72
|
gscSearchData,
|
|
@@ -97,13 +104,14 @@ import {
|
|
|
97
104
|
siteAuditPages,
|
|
98
105
|
siteAuditSnapshots,
|
|
99
106
|
usageCounters
|
|
100
|
-
} from "./chunk-
|
|
107
|
+
} from "./chunk-UAQ42NVJ.js";
|
|
101
108
|
import {
|
|
102
109
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
103
110
|
AGENT_PROVIDER_IDS,
|
|
104
111
|
AI_ENGINE_DOMAINS,
|
|
105
112
|
AI_ENGINE_SELF_DOMAINS,
|
|
106
113
|
AgentProviderIds,
|
|
114
|
+
BacklinkSources,
|
|
107
115
|
CcReleaseSyncStatuses,
|
|
108
116
|
CodingAgents,
|
|
109
117
|
DiscoveryCompetitorTypes,
|
|
@@ -131,12 +139,15 @@ import {
|
|
|
131
139
|
buildRunErrorFromMessages,
|
|
132
140
|
classifySkillFile,
|
|
133
141
|
coerceSkillManifest,
|
|
142
|
+
computeBacklinkSummaryMetrics,
|
|
134
143
|
contentActionLabel,
|
|
135
144
|
contentBriefDtoSchema,
|
|
136
145
|
determineAnswerMentioned,
|
|
146
|
+
dollarsToMicros,
|
|
137
147
|
effectiveBrandNames,
|
|
138
148
|
effectiveDomains,
|
|
139
149
|
factorStatusFromScore,
|
|
150
|
+
hostOf,
|
|
140
151
|
isAgentProviderId,
|
|
141
152
|
isBrowserProvider,
|
|
142
153
|
isRetryableHttpError,
|
|
@@ -149,7 +160,7 @@ import {
|
|
|
149
160
|
validationError,
|
|
150
161
|
winnabilityClassLabel,
|
|
151
162
|
withRetry
|
|
152
|
-
} from "./chunk-
|
|
163
|
+
} from "./chunk-GMT3YPLT.js";
|
|
153
164
|
|
|
154
165
|
// src/telemetry.ts
|
|
155
166
|
import crypto from "crypto";
|
|
@@ -437,11 +448,11 @@ function checkLatestVersionForServer(opts) {
|
|
|
437
448
|
|
|
438
449
|
// src/server.ts
|
|
439
450
|
import { createRequire as createRequire3 } from "module";
|
|
440
|
-
import
|
|
451
|
+
import crypto20 from "crypto";
|
|
441
452
|
import fs8 from "fs";
|
|
442
453
|
import path9 from "path";
|
|
443
454
|
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
444
|
-
import { and as
|
|
455
|
+
import { and as and14, eq as eq20 } from "drizzle-orm";
|
|
445
456
|
import Fastify from "fastify";
|
|
446
457
|
import os5 from "os";
|
|
447
458
|
|
|
@@ -4077,12 +4088,361 @@ function monthKey(m) {
|
|
|
4077
4088
|
return `${m.year}-${String(m.month).padStart(2, "0")}`;
|
|
4078
4089
|
}
|
|
4079
4090
|
|
|
4080
|
-
// src/
|
|
4091
|
+
// src/ads-sync.ts
|
|
4081
4092
|
import crypto6 from "crypto";
|
|
4082
|
-
import { eq as eq4
|
|
4093
|
+
import { eq as eq4 } from "drizzle-orm";
|
|
4094
|
+
|
|
4095
|
+
// ../integration-openai-ads/src/constants.ts
|
|
4096
|
+
var OPENAI_ADS_API_BASE = "https://api.ads.openai.com/v1";
|
|
4097
|
+
var OPENAI_ADS_REQUEST_TIMEOUT_MS = 3e4;
|
|
4098
|
+
var OPENAI_ADS_MAX_PAGES = 100;
|
|
4099
|
+
|
|
4100
|
+
// ../integration-openai-ads/src/types.ts
|
|
4101
|
+
var OpenAiAdsApiError = class extends Error {
|
|
4102
|
+
status;
|
|
4103
|
+
code;
|
|
4104
|
+
constructor(message, status, code = null) {
|
|
4105
|
+
super(message);
|
|
4106
|
+
this.name = "OpenAiAdsApiError";
|
|
4107
|
+
this.status = status;
|
|
4108
|
+
this.code = code;
|
|
4109
|
+
}
|
|
4110
|
+
};
|
|
4111
|
+
function parseErrorEnvelope(body) {
|
|
4112
|
+
try {
|
|
4113
|
+
const parsed = JSON.parse(body);
|
|
4114
|
+
return {
|
|
4115
|
+
message: parsed.error?.message ?? null,
|
|
4116
|
+
code: parsed.error?.code ?? null
|
|
4117
|
+
};
|
|
4118
|
+
} catch {
|
|
4119
|
+
return { message: null, code: null };
|
|
4120
|
+
}
|
|
4121
|
+
}
|
|
4122
|
+
|
|
4123
|
+
// ../integration-openai-ads/src/ads-client.ts
|
|
4124
|
+
function validateApiKey(apiKey) {
|
|
4125
|
+
if (!apiKey || typeof apiKey !== "string" || apiKey.trim().length === 0) {
|
|
4126
|
+
throw new OpenAiAdsApiError("API key is required and must be a non-empty string", 400);
|
|
4127
|
+
}
|
|
4128
|
+
}
|
|
4129
|
+
function validateId(value, label) {
|
|
4130
|
+
if (!value || typeof value !== "string" || value.trim().length === 0) {
|
|
4131
|
+
throw new OpenAiAdsApiError(`${label} is required and must be a non-empty string`, 400);
|
|
4132
|
+
}
|
|
4133
|
+
}
|
|
4134
|
+
function adsClientLog(level, action, ctx) {
|
|
4135
|
+
const entry = {
|
|
4136
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4137
|
+
level,
|
|
4138
|
+
module: "OpenAiAdsClient",
|
|
4139
|
+
action,
|
|
4140
|
+
...ctx
|
|
4141
|
+
};
|
|
4142
|
+
if (entry.apiKey) entry.apiKey = "***";
|
|
4143
|
+
const stream = level === "error" ? process.stderr : process.stdout;
|
|
4144
|
+
stream.write(JSON.stringify(entry) + "\n");
|
|
4145
|
+
}
|
|
4146
|
+
function buildUrl(path10, queryPairs) {
|
|
4147
|
+
const qs = queryPairs.join("&");
|
|
4148
|
+
return qs ? `${OPENAI_ADS_API_BASE}/${path10}?${qs}` : `${OPENAI_ADS_API_BASE}/${path10}`;
|
|
4149
|
+
}
|
|
4150
|
+
async function adsFetch(apiKey, path10, queryPairs = []) {
|
|
4151
|
+
const url = buildUrl(path10, queryPairs);
|
|
4152
|
+
const res = await fetch(url, {
|
|
4153
|
+
method: "GET",
|
|
4154
|
+
headers: {
|
|
4155
|
+
Authorization: `Bearer ${apiKey}`,
|
|
4156
|
+
"Content-Type": "application/json"
|
|
4157
|
+
},
|
|
4158
|
+
signal: AbortSignal.timeout(OPENAI_ADS_REQUEST_TIMEOUT_MS)
|
|
4159
|
+
});
|
|
4160
|
+
if (res.status === 401 || res.status === 403) {
|
|
4161
|
+
const { code } = parseErrorEnvelope(await res.text());
|
|
4162
|
+
adsClientLog("error", "http.auth-failed", { path: path10, httpStatus: res.status, code });
|
|
4163
|
+
throw new OpenAiAdsApiError("OpenAI Ads API key is invalid or unauthorized", res.status, code);
|
|
4164
|
+
}
|
|
4165
|
+
if (res.status === 429) {
|
|
4166
|
+
const { code } = parseErrorEnvelope(await res.text());
|
|
4167
|
+
adsClientLog("error", "http.rate-limited", { path: path10, httpStatus: 429, code });
|
|
4168
|
+
throw new OpenAiAdsApiError("OpenAI Ads API rate limit exceeded", 429, code);
|
|
4169
|
+
}
|
|
4170
|
+
if (!res.ok) {
|
|
4171
|
+
const body = await res.text();
|
|
4172
|
+
const { message, code } = parseErrorEnvelope(body);
|
|
4173
|
+
adsClientLog("error", "http.error", { path: path10, httpStatus: res.status, code });
|
|
4174
|
+
const detail = message ?? (body.length <= 500 ? body : `${body.slice(0, 500)}... [truncated]`);
|
|
4175
|
+
throw new OpenAiAdsApiError(`OpenAI Ads API error (${res.status}): ${detail}`, res.status, code);
|
|
4176
|
+
}
|
|
4177
|
+
const text = await res.text();
|
|
4178
|
+
try {
|
|
4179
|
+
return JSON.parse(text);
|
|
4180
|
+
} catch {
|
|
4181
|
+
throw new OpenAiAdsApiError("OpenAI Ads API returned invalid JSON", 502);
|
|
4182
|
+
}
|
|
4183
|
+
}
|
|
4184
|
+
async function fetchAllPages(apiKey, path10, queryPairs) {
|
|
4185
|
+
const items = [];
|
|
4186
|
+
let after = null;
|
|
4187
|
+
for (let page = 0; page < OPENAI_ADS_MAX_PAGES; page++) {
|
|
4188
|
+
const pairs = after ? [...queryPairs, `after=${encodeURIComponent(after)}`] : [...queryPairs];
|
|
4189
|
+
const response = await adsFetch(apiKey, path10, pairs);
|
|
4190
|
+
items.push(...response.data);
|
|
4191
|
+
if (!response.has_more || !response.last_id) {
|
|
4192
|
+
return items;
|
|
4193
|
+
}
|
|
4194
|
+
after = response.last_id;
|
|
4195
|
+
}
|
|
4196
|
+
adsClientLog("error", "pagination.cap-reached", { path: path10, pages: OPENAI_ADS_MAX_PAGES, items: items.length });
|
|
4197
|
+
return items;
|
|
4198
|
+
}
|
|
4199
|
+
function insightsPairs(opts) {
|
|
4200
|
+
return (opts?.fields ?? []).map((field) => `fields[]=${encodeURIComponent(field)}`);
|
|
4201
|
+
}
|
|
4202
|
+
async function getAdAccount(apiKey) {
|
|
4203
|
+
validateApiKey(apiKey);
|
|
4204
|
+
return adsFetch(apiKey, "ad_account");
|
|
4205
|
+
}
|
|
4206
|
+
async function listCampaigns(apiKey) {
|
|
4207
|
+
validateApiKey(apiKey);
|
|
4208
|
+
return fetchAllPages(apiKey, "campaigns", []);
|
|
4209
|
+
}
|
|
4210
|
+
async function listAdGroups(apiKey, campaignId) {
|
|
4211
|
+
validateApiKey(apiKey);
|
|
4212
|
+
validateId(campaignId, "Campaign id");
|
|
4213
|
+
return fetchAllPages(apiKey, "ad_groups", [`campaign_id=${encodeURIComponent(campaignId)}`]);
|
|
4214
|
+
}
|
|
4215
|
+
async function listAds(apiKey, adGroupId) {
|
|
4216
|
+
validateApiKey(apiKey);
|
|
4217
|
+
validateId(adGroupId, "Ad group id");
|
|
4218
|
+
return fetchAllPages(apiKey, "ads", [`ad_group_id=${encodeURIComponent(adGroupId)}`]);
|
|
4219
|
+
}
|
|
4220
|
+
async function getCampaignInsights(apiKey, campaignId, opts) {
|
|
4221
|
+
validateApiKey(apiKey);
|
|
4222
|
+
validateId(campaignId, "Campaign id");
|
|
4223
|
+
return fetchAllPages(
|
|
4224
|
+
apiKey,
|
|
4225
|
+
`campaigns/${encodeURIComponent(campaignId)}/insights`,
|
|
4226
|
+
insightsPairs(opts)
|
|
4227
|
+
);
|
|
4228
|
+
}
|
|
4229
|
+
async function getAdGroupInsights(apiKey, adGroupId, opts) {
|
|
4230
|
+
validateApiKey(apiKey);
|
|
4231
|
+
validateId(adGroupId, "Ad group id");
|
|
4232
|
+
return fetchAllPages(
|
|
4233
|
+
apiKey,
|
|
4234
|
+
`ad_groups/${encodeURIComponent(adGroupId)}/insights`,
|
|
4235
|
+
insightsPairs(opts)
|
|
4236
|
+
);
|
|
4237
|
+
}
|
|
4238
|
+
|
|
4239
|
+
// src/ads-config.ts
|
|
4240
|
+
function ensureConnections7(config) {
|
|
4241
|
+
if (!config.openaiAds) config.openaiAds = {};
|
|
4242
|
+
if (!config.openaiAds.connections) config.openaiAds.connections = [];
|
|
4243
|
+
return config.openaiAds.connections;
|
|
4244
|
+
}
|
|
4245
|
+
function getOpenAiAdsConnection(config, projectName) {
|
|
4246
|
+
return (config.openaiAds?.connections ?? []).find((c) => c.projectName === projectName);
|
|
4247
|
+
}
|
|
4248
|
+
function upsertOpenAiAdsConnection(config, connection) {
|
|
4249
|
+
const connections = ensureConnections7(config);
|
|
4250
|
+
const index = connections.findIndex((c) => c.projectName === connection.projectName);
|
|
4251
|
+
if (index === -1) {
|
|
4252
|
+
connections.push(connection);
|
|
4253
|
+
return connection;
|
|
4254
|
+
}
|
|
4255
|
+
connections[index] = connection;
|
|
4256
|
+
return connection;
|
|
4257
|
+
}
|
|
4258
|
+
function removeOpenAiAdsConnection(config, projectName) {
|
|
4259
|
+
const connections = config.openaiAds?.connections;
|
|
4260
|
+
if (!connections?.length) return false;
|
|
4261
|
+
const next = connections.filter((c) => c.projectName !== projectName);
|
|
4262
|
+
if (next.length === connections.length) return false;
|
|
4263
|
+
if (!config.openaiAds) return false;
|
|
4264
|
+
config.openaiAds.connections = next;
|
|
4265
|
+
if (next.length === 0) {
|
|
4266
|
+
delete config.openaiAds;
|
|
4267
|
+
}
|
|
4268
|
+
return true;
|
|
4269
|
+
}
|
|
4270
|
+
|
|
4271
|
+
// src/ads-sync.ts
|
|
4272
|
+
var log4 = createLogger("AdsSync");
|
|
4273
|
+
var CAMPAIGN_INSIGHT_FIELDS = ["campaign.impressions", "campaign.clicks", "campaign.spend", "metadata.readable_time"];
|
|
4274
|
+
var AD_GROUP_INSIGHT_FIELDS = ["ad_group.impressions", "ad_group.clicks", "ad_group.spend", "metadata.readable_time"];
|
|
4275
|
+
function toInsightUpserts(level, entityId, rows) {
|
|
4276
|
+
const upserts = [];
|
|
4277
|
+
for (const row of rows) {
|
|
4278
|
+
if (!row.readable_time) {
|
|
4279
|
+
log4.warn("insights.row-missing-date", { level, entityId, rowId: row.id });
|
|
4280
|
+
continue;
|
|
4281
|
+
}
|
|
4282
|
+
upserts.push({
|
|
4283
|
+
level,
|
|
4284
|
+
entityId,
|
|
4285
|
+
date: row.readable_time,
|
|
4286
|
+
impressions: row.impressions ?? 0,
|
|
4287
|
+
clicks: row.clicks ?? 0,
|
|
4288
|
+
spendMicros: dollarsToMicros(row.spend ?? 0)
|
|
4289
|
+
});
|
|
4290
|
+
}
|
|
4291
|
+
return upserts;
|
|
4292
|
+
}
|
|
4293
|
+
async function executeAdsSync(db, runId, projectId, opts) {
|
|
4294
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4295
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq4(runs.id, runId)).run();
|
|
4296
|
+
try {
|
|
4297
|
+
const project = db.select().from(projects).where(eq4(projects.id, projectId)).get();
|
|
4298
|
+
if (!project) throw new Error(`Project not found: ${projectId}`);
|
|
4299
|
+
const connRow = db.select().from(adsConnections).where(eq4(adsConnections.projectId, projectId)).get();
|
|
4300
|
+
if (!connRow) {
|
|
4301
|
+
throw new Error('No ads connection found for this project. Run "canonry ads connect" first.');
|
|
4302
|
+
}
|
|
4303
|
+
const cfgConn = getOpenAiAdsConnection(opts.config, project.name);
|
|
4304
|
+
if (!cfgConn?.apiKey) {
|
|
4305
|
+
throw new Error('No OpenAI Ads API key in the local Canonry config. Run "canonry ads connect" first.');
|
|
4306
|
+
}
|
|
4307
|
+
const apiKey = cfgConn.apiKey;
|
|
4308
|
+
log4.info("sync.start", { runId, projectId, adAccountId: connRow.adAccountId });
|
|
4309
|
+
const account = await getAdAccount(apiKey);
|
|
4310
|
+
const campaigns = await listCampaigns(apiKey);
|
|
4311
|
+
const errors = /* @__PURE__ */ new Map();
|
|
4312
|
+
const adGroupsByCampaign = /* @__PURE__ */ new Map();
|
|
4313
|
+
const adsByGroup = /* @__PURE__ */ new Map();
|
|
4314
|
+
const insightUpserts = [];
|
|
4315
|
+
const syncedCampaigns = [];
|
|
4316
|
+
for (const campaign of campaigns) {
|
|
4317
|
+
try {
|
|
4318
|
+
const [adGroups, campaignInsights] = await Promise.all([
|
|
4319
|
+
listAdGroups(apiKey, campaign.id),
|
|
4320
|
+
getCampaignInsights(apiKey, campaign.id, { fields: CAMPAIGN_INSIGHT_FIELDS })
|
|
4321
|
+
]);
|
|
4322
|
+
const groupResults = await Promise.all(adGroups.map(async (group) => ({
|
|
4323
|
+
group,
|
|
4324
|
+
ads: await listAds(apiKey, group.id),
|
|
4325
|
+
insights: await getAdGroupInsights(apiKey, group.id, { fields: AD_GROUP_INSIGHT_FIELDS })
|
|
4326
|
+
})));
|
|
4327
|
+
syncedCampaigns.push(campaign);
|
|
4328
|
+
adGroupsByCampaign.set(campaign.id, adGroups);
|
|
4329
|
+
insightUpserts.push(...toInsightUpserts("campaign", campaign.id, campaignInsights));
|
|
4330
|
+
for (const { group, ads, insights: insights2 } of groupResults) {
|
|
4331
|
+
adsByGroup.set(group.id, ads);
|
|
4332
|
+
insightUpserts.push(...toInsightUpserts("ad_group", group.id, insights2));
|
|
4333
|
+
}
|
|
4334
|
+
} catch (err) {
|
|
4335
|
+
errors.set(campaign.name, err instanceof Error ? err.message : String(err));
|
|
4336
|
+
log4.error("campaign.failed", { runId, campaignId: campaign.id, error: err instanceof Error ? err.message : String(err) });
|
|
4337
|
+
}
|
|
4338
|
+
}
|
|
4339
|
+
const insertNow = (/* @__PURE__ */ new Date()).toISOString();
|
|
4340
|
+
db.transaction((tx) => {
|
|
4341
|
+
tx.delete(adsCampaigns).where(eq4(adsCampaigns.projectId, projectId)).run();
|
|
4342
|
+
for (const campaign of syncedCampaigns) {
|
|
4343
|
+
tx.insert(adsCampaigns).values({
|
|
4344
|
+
id: campaign.id,
|
|
4345
|
+
projectId,
|
|
4346
|
+
name: campaign.name,
|
|
4347
|
+
status: campaign.status,
|
|
4348
|
+
biddingType: campaign.bidding_type,
|
|
4349
|
+
dailySpendLimitMicros: campaign.budget?.daily_spend_limit_micros ?? null,
|
|
4350
|
+
lifetimeSpendLimitMicros: campaign.budget?.lifetime_spend_limit_micros ?? null,
|
|
4351
|
+
targeting: campaign.targeting,
|
|
4352
|
+
upstreamCreatedAt: campaign.created_at,
|
|
4353
|
+
upstreamUpdatedAt: campaign.updated_at,
|
|
4354
|
+
syncRunId: runId,
|
|
4355
|
+
syncedAt: insertNow
|
|
4356
|
+
}).run();
|
|
4357
|
+
for (const group of adGroupsByCampaign.get(campaign.id) ?? []) {
|
|
4358
|
+
tx.insert(adsAdGroups).values({
|
|
4359
|
+
id: group.id,
|
|
4360
|
+
projectId,
|
|
4361
|
+
campaignId: campaign.id,
|
|
4362
|
+
name: group.name,
|
|
4363
|
+
status: group.status,
|
|
4364
|
+
billingEventType: group.bidding_config?.billing_event_type ?? null,
|
|
4365
|
+
maxBidMicros: group.bidding_config?.max_bid_micros ?? null,
|
|
4366
|
+
contextHints: group.context_hints,
|
|
4367
|
+
upstreamCreatedAt: group.created_at,
|
|
4368
|
+
upstreamUpdatedAt: group.updated_at,
|
|
4369
|
+
syncRunId: runId,
|
|
4370
|
+
syncedAt: insertNow
|
|
4371
|
+
}).run();
|
|
4372
|
+
for (const ad of adsByGroup.get(group.id) ?? []) {
|
|
4373
|
+
tx.insert(adsAds).values({
|
|
4374
|
+
id: ad.id,
|
|
4375
|
+
projectId,
|
|
4376
|
+
adGroupId: group.id,
|
|
4377
|
+
name: ad.name,
|
|
4378
|
+
status: ad.status,
|
|
4379
|
+
creative: ad.creative,
|
|
4380
|
+
reviewStatus: ad.review_status ?? ad.review?.status ?? null,
|
|
4381
|
+
upstreamCreatedAt: ad.created_at,
|
|
4382
|
+
upstreamUpdatedAt: ad.updated_at,
|
|
4383
|
+
syncRunId: runId,
|
|
4384
|
+
syncedAt: insertNow
|
|
4385
|
+
}).run();
|
|
4386
|
+
}
|
|
4387
|
+
}
|
|
4388
|
+
}
|
|
4389
|
+
for (const upsert of insightUpserts) {
|
|
4390
|
+
tx.insert(adsInsightsDaily).values({
|
|
4391
|
+
id: crypto6.randomUUID(),
|
|
4392
|
+
projectId,
|
|
4393
|
+
...upsert,
|
|
4394
|
+
syncRunId: runId
|
|
4395
|
+
}).onConflictDoUpdate({
|
|
4396
|
+
target: [adsInsightsDaily.projectId, adsInsightsDaily.level, adsInsightsDaily.entityId, adsInsightsDaily.date],
|
|
4397
|
+
set: {
|
|
4398
|
+
impressions: upsert.impressions,
|
|
4399
|
+
clicks: upsert.clicks,
|
|
4400
|
+
spendMicros: upsert.spendMicros,
|
|
4401
|
+
syncRunId: runId
|
|
4402
|
+
}
|
|
4403
|
+
}).run();
|
|
4404
|
+
}
|
|
4405
|
+
tx.update(adsConnections).set({
|
|
4406
|
+
adAccountId: account.id,
|
|
4407
|
+
displayName: account.name,
|
|
4408
|
+
currencyCode: account.currency_code,
|
|
4409
|
+
timezone: account.timezone,
|
|
4410
|
+
status: account.status,
|
|
4411
|
+
lastSyncedAt: insertNow,
|
|
4412
|
+
updatedAt: insertNow
|
|
4413
|
+
}).where(eq4(adsConnections.projectId, projectId)).run();
|
|
4414
|
+
});
|
|
4415
|
+
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
4416
|
+
if (errors.size === 0) {
|
|
4417
|
+
db.update(runs).set({ status: "completed", finishedAt }).where(eq4(runs.id, runId)).run();
|
|
4418
|
+
} else if (syncedCampaigns.length > 0) {
|
|
4419
|
+
db.update(runs).set({
|
|
4420
|
+
status: "partial",
|
|
4421
|
+
error: serializeRunError(buildRunErrorFromMessages(errors)),
|
|
4422
|
+
finishedAt
|
|
4423
|
+
}).where(eq4(runs.id, runId)).run();
|
|
4424
|
+
} else {
|
|
4425
|
+
db.update(runs).set({
|
|
4426
|
+
status: "failed",
|
|
4427
|
+
error: serializeRunError(buildRunErrorFromMessages(errors)),
|
|
4428
|
+
finishedAt
|
|
4429
|
+
}).where(eq4(runs.id, runId)).run();
|
|
4430
|
+
}
|
|
4431
|
+
log4.info("sync.done", { runId, projectId, campaigns: syncedCampaigns.length, insightRows: insightUpserts.length, failed: errors.size });
|
|
4432
|
+
} catch (err) {
|
|
4433
|
+
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
4434
|
+
db.update(runs).set({ status: "failed", error: serializeRunError({ message: errorMsg }), finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq4(runs.id, runId)).run();
|
|
4435
|
+
log4.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
4436
|
+
throw err;
|
|
4437
|
+
}
|
|
4438
|
+
}
|
|
4439
|
+
|
|
4440
|
+
// src/gsc-inspect-sitemap.ts
|
|
4441
|
+
import crypto7 from "crypto";
|
|
4442
|
+
import { eq as eq5, and as and4 } from "drizzle-orm";
|
|
4083
4443
|
|
|
4084
4444
|
// src/sitemap-parser.ts
|
|
4085
|
-
var
|
|
4445
|
+
var log5 = createLogger("SitemapParser");
|
|
4086
4446
|
var LOC_REGEX = /<loc>([^<]+)<\/loc>/gi;
|
|
4087
4447
|
var SITEMAP_TAG_REGEX = /<sitemap>[\s\S]*?<\/sitemap>/gi;
|
|
4088
4448
|
async function validateSitemapUrl(url) {
|
|
@@ -4124,7 +4484,7 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
4124
4484
|
res = await fetch(url);
|
|
4125
4485
|
} catch (err) {
|
|
4126
4486
|
if (!isChild) throw err;
|
|
4127
|
-
|
|
4487
|
+
log5.warn("child-sitemap.fetch-failed", {
|
|
4128
4488
|
url,
|
|
4129
4489
|
error: err instanceof Error ? err.message : String(err)
|
|
4130
4490
|
});
|
|
@@ -4134,7 +4494,7 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
4134
4494
|
if (!isChild) {
|
|
4135
4495
|
throw new Error(`Failed to fetch sitemap at ${url}: ${res.status} ${res.statusText}`);
|
|
4136
4496
|
}
|
|
4137
|
-
|
|
4497
|
+
log5.warn("child-sitemap.http-error", { url, status: res.status, statusText: res.statusText });
|
|
4138
4498
|
return;
|
|
4139
4499
|
}
|
|
4140
4500
|
let xml;
|
|
@@ -4142,7 +4502,7 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
4142
4502
|
xml = await readSitemapBody(res);
|
|
4143
4503
|
} catch (err) {
|
|
4144
4504
|
if (!isChild) throw err;
|
|
4145
|
-
|
|
4505
|
+
log5.warn("child-sitemap.parse-failed", {
|
|
4146
4506
|
url,
|
|
4147
4507
|
error: err instanceof Error ? err.message : String(err)
|
|
4148
4508
|
});
|
|
@@ -4178,16 +4538,16 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
4178
4538
|
}
|
|
4179
4539
|
|
|
4180
4540
|
// src/gsc-inspect-sitemap.ts
|
|
4181
|
-
var
|
|
4541
|
+
var log6 = createLogger("InspectSitemap");
|
|
4182
4542
|
async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
4183
4543
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4184
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
4544
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq5(runs.id, runId)).run();
|
|
4185
4545
|
try {
|
|
4186
4546
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
4187
4547
|
if (!googleClientId || !googleClientSecret) {
|
|
4188
4548
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
4189
4549
|
}
|
|
4190
|
-
const project = db.select().from(projects).where(
|
|
4550
|
+
const project = db.select().from(projects).where(eq5(projects.id, projectId)).get();
|
|
4191
4551
|
if (!project) {
|
|
4192
4552
|
throw new Error(`Project not found: ${projectId}`);
|
|
4193
4553
|
}
|
|
@@ -4212,9 +4572,9 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
4212
4572
|
saveConfigPatch(opts.config);
|
|
4213
4573
|
}
|
|
4214
4574
|
const sitemapUrl = opts.sitemapUrl || conn.sitemapUrl || `https://${project.canonicalDomain}/sitemap.xml`;
|
|
4215
|
-
|
|
4575
|
+
log6.info("sitemap.fetch", { runId, projectId, sitemapUrl });
|
|
4216
4576
|
const urls = await fetchAndParseSitemap(sitemapUrl);
|
|
4217
|
-
|
|
4577
|
+
log6.info("sitemap.parsed", { runId, projectId, urlCount: urls.length, sitemapUrl });
|
|
4218
4578
|
if (urls.length === 0) {
|
|
4219
4579
|
throw new Error("No URLs found in sitemap");
|
|
4220
4580
|
}
|
|
@@ -4229,7 +4589,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
4229
4589
|
const rich = ir.richResultsResult;
|
|
4230
4590
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
4231
4591
|
db.insert(gscUrlInspections).values({
|
|
4232
|
-
id:
|
|
4592
|
+
id: crypto7.randomUUID(),
|
|
4233
4593
|
projectId,
|
|
4234
4594
|
syncRunId: runId,
|
|
4235
4595
|
url: pageUrl,
|
|
@@ -4246,16 +4606,16 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
4246
4606
|
inspectedAt,
|
|
4247
4607
|
createdAt: inspectedAt
|
|
4248
4608
|
}).run();
|
|
4249
|
-
|
|
4609
|
+
log6.info("inspect.url-done", { runId, projectId, url: pageUrl, progress: `${index + 1}/${urls.length}` });
|
|
4250
4610
|
},
|
|
4251
4611
|
onError: (pageUrl, err) => {
|
|
4252
|
-
|
|
4612
|
+
log6.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
|
|
4253
4613
|
}
|
|
4254
4614
|
},
|
|
4255
4615
|
{
|
|
4256
4616
|
log: {
|
|
4257
|
-
info: (action, ctx) =>
|
|
4258
|
-
error: (action, ctx) =>
|
|
4617
|
+
info: (action, ctx) => log6.info(action, { runId, projectId, ...ctx }),
|
|
4618
|
+
error: (action, ctx) => log6.error(action, { runId, projectId, ...ctx })
|
|
4259
4619
|
}
|
|
4260
4620
|
}
|
|
4261
4621
|
);
|
|
@@ -4265,7 +4625,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
4265
4625
|
`URL inspection aborted after ${INSPECT_FAILFAST_THRESHOLD} consecutive rate/access failures (likely GSC URL Inspection quota exhaustion or property access loss). Last error: ${detail}`
|
|
4266
4626
|
);
|
|
4267
4627
|
}
|
|
4268
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
4628
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq5(gscUrlInspections.projectId, projectId)).all();
|
|
4269
4629
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
4270
4630
|
for (const row of allInspections) {
|
|
4271
4631
|
const existing = latestByUrl.get(row.url);
|
|
@@ -4286,9 +4646,9 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
4286
4646
|
}
|
|
4287
4647
|
}
|
|
4288
4648
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
4289
|
-
db.delete(gscCoverageSnapshots).where(and4(
|
|
4649
|
+
db.delete(gscCoverageSnapshots).where(and4(eq5(gscCoverageSnapshots.projectId, projectId), eq5(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
4290
4650
|
db.insert(gscCoverageSnapshots).values({
|
|
4291
|
-
id:
|
|
4651
|
+
id: crypto7.randomUUID(),
|
|
4292
4652
|
projectId,
|
|
4293
4653
|
syncRunId: runId,
|
|
4294
4654
|
date: snapshotDate,
|
|
@@ -4298,20 +4658,20 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
4298
4658
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
4299
4659
|
}).run();
|
|
4300
4660
|
const status = errors > 0 && inspected > 0 ? "partial" : errors === urls.length ? "failed" : "completed";
|
|
4301
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
4302
|
-
|
|
4661
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq5(runs.id, runId)).run();
|
|
4662
|
+
log6.info("inspect.completed", { runId, projectId, inspected, errors, total: urls.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
4303
4663
|
} catch (err) {
|
|
4304
4664
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
4305
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
4306
|
-
|
|
4665
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq5(runs.id, runId)).run();
|
|
4666
|
+
log6.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
4307
4667
|
throw err;
|
|
4308
4668
|
}
|
|
4309
4669
|
}
|
|
4310
4670
|
|
|
4311
4671
|
// src/bing-inspect-sitemap.ts
|
|
4312
|
-
import
|
|
4313
|
-
import { eq as
|
|
4314
|
-
var
|
|
4672
|
+
import crypto8 from "crypto";
|
|
4673
|
+
import { eq as eq6, desc as desc2 } from "drizzle-orm";
|
|
4674
|
+
var log7 = createLogger("BingInspectSitemap");
|
|
4315
4675
|
function parseBingDate(value) {
|
|
4316
4676
|
if (!value) return null;
|
|
4317
4677
|
const match = /\/Date\((-?\d+)(?:[-+]\d+)?\)\//.exec(value);
|
|
@@ -4328,9 +4688,9 @@ function isBlockingIssueType(issueType) {
|
|
|
4328
4688
|
}
|
|
4329
4689
|
async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
4330
4690
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
4331
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
4691
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq6(runs.id, runId)).run();
|
|
4332
4692
|
try {
|
|
4333
|
-
const project = db.select().from(projects).where(
|
|
4693
|
+
const project = db.select().from(projects).where(eq6(projects.id, projectId)).get();
|
|
4334
4694
|
if (!project) {
|
|
4335
4695
|
throw new Error(`Project not found: ${projectId}`);
|
|
4336
4696
|
}
|
|
@@ -4342,16 +4702,16 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4342
4702
|
throw new Error('No Bing site configured. Run "canonry bing set-site <project> <url>" first.');
|
|
4343
4703
|
}
|
|
4344
4704
|
const sitemapUrl = opts.sitemapUrl ?? `https://${project.canonicalDomain}/sitemap.xml`;
|
|
4345
|
-
|
|
4705
|
+
log7.info("sitemap.fetch", { runId, projectId, sitemapUrl });
|
|
4346
4706
|
const sitemapUrls = await fetchAndParseSitemap(sitemapUrl);
|
|
4347
|
-
|
|
4707
|
+
log7.info("sitemap.parsed", { runId, projectId, urlCount: sitemapUrls.length, sitemapUrl });
|
|
4348
4708
|
if (sitemapUrls.length === 0) {
|
|
4349
4709
|
throw new Error("No URLs found in sitemap");
|
|
4350
4710
|
}
|
|
4351
|
-
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(
|
|
4711
|
+
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(eq6(bingUrlInspections.projectId, projectId)).all();
|
|
4352
4712
|
const trackedUrls = new Set(trackedRows.map((r) => r.url));
|
|
4353
4713
|
const discovered = sitemapUrls.filter((u) => !trackedUrls.has(u));
|
|
4354
|
-
|
|
4714
|
+
log7.info("sitemap.diff", {
|
|
4355
4715
|
runId,
|
|
4356
4716
|
projectId,
|
|
4357
4717
|
sitemapTotal: sitemapUrls.length,
|
|
@@ -4366,9 +4726,9 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4366
4726
|
blockedUrls.add(issue.Url);
|
|
4367
4727
|
}
|
|
4368
4728
|
}
|
|
4369
|
-
|
|
4729
|
+
log7.info("crawl-issues.loaded", { runId, projectId, blockedCount: blockedUrls.size });
|
|
4370
4730
|
} catch (err) {
|
|
4371
|
-
|
|
4731
|
+
log7.warn("crawl-issues.lookup-failed", {
|
|
4372
4732
|
runId,
|
|
4373
4733
|
projectId,
|
|
4374
4734
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -4397,7 +4757,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4397
4757
|
derivedInIndex = false;
|
|
4398
4758
|
}
|
|
4399
4759
|
db.insert(bingUrlInspections).values({
|
|
4400
|
-
id:
|
|
4760
|
+
id: crypto8.randomUUID(),
|
|
4401
4761
|
projectId,
|
|
4402
4762
|
url: pageUrl,
|
|
4403
4763
|
httpCode,
|
|
@@ -4412,7 +4772,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4412
4772
|
discoveryDate
|
|
4413
4773
|
}).run();
|
|
4414
4774
|
inspected++;
|
|
4415
|
-
|
|
4775
|
+
log7.info("inspect.url-done", {
|
|
4416
4776
|
runId,
|
|
4417
4777
|
projectId,
|
|
4418
4778
|
url: pageUrl,
|
|
@@ -4420,7 +4780,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4420
4780
|
});
|
|
4421
4781
|
} catch (err) {
|
|
4422
4782
|
errors++;
|
|
4423
|
-
|
|
4783
|
+
log7.error("inspect.url-failed", {
|
|
4424
4784
|
runId,
|
|
4425
4785
|
projectId,
|
|
4426
4786
|
url: pageUrl,
|
|
@@ -4431,7 +4791,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4431
4791
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
4432
4792
|
}
|
|
4433
4793
|
}
|
|
4434
|
-
const allInspections = db.select().from(bingUrlInspections).where(
|
|
4794
|
+
const allInspections = db.select().from(bingUrlInspections).where(eq6(bingUrlInspections.projectId, projectId)).orderBy(desc2(bingUrlInspections.inspectedAt)).all();
|
|
4435
4795
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
4436
4796
|
const definitiveByUrl = /* @__PURE__ */ new Map();
|
|
4437
4797
|
for (const row of allInspections) {
|
|
@@ -4455,7 +4815,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4455
4815
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
4456
4816
|
const snapNow = (/* @__PURE__ */ new Date()).toISOString();
|
|
4457
4817
|
db.insert(bingCoverageSnapshots).values({
|
|
4458
|
-
id:
|
|
4818
|
+
id: crypto8.randomUUID(),
|
|
4459
4819
|
projectId,
|
|
4460
4820
|
syncRunId: runId,
|
|
4461
4821
|
date: snapshotDate,
|
|
@@ -4474,8 +4834,8 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4474
4834
|
}
|
|
4475
4835
|
}).run();
|
|
4476
4836
|
const status = errors === sitemapUrls.length ? RunStatuses.failed : errors > 0 ? RunStatuses.partial : RunStatuses.completed;
|
|
4477
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
4478
|
-
|
|
4837
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq6(runs.id, runId)).run();
|
|
4838
|
+
log7.info("inspect.completed", {
|
|
4479
4839
|
runId,
|
|
4480
4840
|
projectId,
|
|
4481
4841
|
inspected,
|
|
@@ -4488,16 +4848,16 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4488
4848
|
});
|
|
4489
4849
|
} catch (err) {
|
|
4490
4850
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
4491
|
-
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
4492
|
-
|
|
4851
|
+
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq6(runs.id, runId)).run();
|
|
4852
|
+
log7.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
4493
4853
|
throw err;
|
|
4494
4854
|
}
|
|
4495
4855
|
}
|
|
4496
4856
|
|
|
4497
4857
|
// src/coverage-refresh.ts
|
|
4498
|
-
import
|
|
4499
|
-
import { and as and5, desc as desc3, eq as
|
|
4500
|
-
var
|
|
4858
|
+
import crypto9 from "crypto";
|
|
4859
|
+
import { and as and5, desc as desc3, eq as eq7, inArray as inArray3 } from "drizzle-orm";
|
|
4860
|
+
var log8 = createLogger("CoverageRefresh");
|
|
4501
4861
|
var COVERAGE_REFRESH_MIN_INTERVAL_MS = 60 * 60 * 1e3;
|
|
4502
4862
|
var ACTIVE_OR_DONE_STATUSES = [
|
|
4503
4863
|
RunStatuses.queued,
|
|
@@ -4507,7 +4867,7 @@ var ACTIVE_OR_DONE_STATUSES = [
|
|
|
4507
4867
|
];
|
|
4508
4868
|
var defaultDeps = { executeInspectSitemap };
|
|
4509
4869
|
async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps, nowMs = Date.now()) {
|
|
4510
|
-
const project = db.select({ canonicalDomain: projects.canonicalDomain }).from(projects).where(
|
|
4870
|
+
const project = db.select({ canonicalDomain: projects.canonicalDomain }).from(projects).where(eq7(projects.id, projectId)).get();
|
|
4511
4871
|
if (!project) return null;
|
|
4512
4872
|
const { clientId, clientSecret } = getGoogleAuthConfig(config);
|
|
4513
4873
|
if (!clientId || !clientSecret) return null;
|
|
@@ -4515,19 +4875,19 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
4515
4875
|
if (!conn?.refreshToken || !conn.propertyId) return null;
|
|
4516
4876
|
const recent = db.select({ createdAt: runs.createdAt }).from(runs).where(
|
|
4517
4877
|
and5(
|
|
4518
|
-
|
|
4519
|
-
|
|
4878
|
+
eq7(runs.projectId, projectId),
|
|
4879
|
+
eq7(runs.kind, RunKinds["inspect-sitemap"]),
|
|
4520
4880
|
inArray3(runs.status, ACTIVE_OR_DONE_STATUSES)
|
|
4521
4881
|
)
|
|
4522
4882
|
).orderBy(desc3(runs.createdAt)).limit(1).get();
|
|
4523
4883
|
if (recent) {
|
|
4524
4884
|
const ageMs = nowMs - Date.parse(recent.createdAt);
|
|
4525
4885
|
if (Number.isFinite(ageMs) && ageMs < COVERAGE_REFRESH_MIN_INTERVAL_MS) {
|
|
4526
|
-
|
|
4886
|
+
log8.info("skip.recent", { projectId, ageMs });
|
|
4527
4887
|
return null;
|
|
4528
4888
|
}
|
|
4529
4889
|
}
|
|
4530
|
-
const runId =
|
|
4890
|
+
const runId = crypto9.randomUUID();
|
|
4531
4891
|
db.insert(runs).values({
|
|
4532
4892
|
id: runId,
|
|
4533
4893
|
projectId,
|
|
@@ -4536,11 +4896,11 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
4536
4896
|
trigger: RunTriggers.scheduled,
|
|
4537
4897
|
createdAt: new Date(nowMs).toISOString()
|
|
4538
4898
|
}).run();
|
|
4539
|
-
|
|
4899
|
+
log8.info("refresh.start", { projectId, runId });
|
|
4540
4900
|
try {
|
|
4541
4901
|
await deps.executeInspectSitemap(db, runId, projectId, { config });
|
|
4542
4902
|
} catch (err) {
|
|
4543
|
-
|
|
4903
|
+
log8.error("refresh.failed", {
|
|
4544
4904
|
projectId,
|
|
4545
4905
|
runId,
|
|
4546
4906
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -4550,10 +4910,10 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
4550
4910
|
}
|
|
4551
4911
|
|
|
4552
4912
|
// src/commoncrawl-sync.ts
|
|
4553
|
-
import
|
|
4913
|
+
import crypto10 from "crypto";
|
|
4554
4914
|
import path4 from "path";
|
|
4555
|
-
import { and as and6, eq as
|
|
4556
|
-
var
|
|
4915
|
+
import { and as and6, eq as eq8, sql as sql3 } from "drizzle-orm";
|
|
4916
|
+
var log9 = createLogger("CommonCrawlSync");
|
|
4557
4917
|
var INSERT_CHUNK_SIZE = 1e4;
|
|
4558
4918
|
function defaultDeps2() {
|
|
4559
4919
|
return {
|
|
@@ -4578,7 +4938,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4578
4938
|
phaseDetail: "downloading vertices + edges",
|
|
4579
4939
|
updatedAt: downloadStartedAt,
|
|
4580
4940
|
error: null
|
|
4581
|
-
}).where(
|
|
4941
|
+
}).where(eq8(ccReleaseSyncs.id, syncId)).run();
|
|
4582
4942
|
const paths = ccReleasePaths(release);
|
|
4583
4943
|
const releaseCacheDir = path4.join(deps.cacheDir, release);
|
|
4584
4944
|
const vertexPath = path4.join(releaseCacheDir, paths.vertexFilename);
|
|
@@ -4601,7 +4961,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4601
4961
|
vertexSha256: vertex.sha256,
|
|
4602
4962
|
edgesSha256: edges.sha256,
|
|
4603
4963
|
updatedAt: downloadFinishedAt
|
|
4604
|
-
}).where(
|
|
4964
|
+
}).where(eq8(ccReleaseSyncs.id, syncId)).run();
|
|
4605
4965
|
const allProjects = db.select().from(projects).all();
|
|
4606
4966
|
const targets = Array.from(new Set(allProjects.map((p) => p.canonicalDomain)));
|
|
4607
4967
|
let rows = [];
|
|
@@ -4617,15 +4977,15 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4617
4977
|
}
|
|
4618
4978
|
const queriedAt = deps.now().toISOString();
|
|
4619
4979
|
db.transaction((tx) => {
|
|
4620
|
-
tx.delete(backlinkDomains).where(
|
|
4621
|
-
tx.delete(backlinkSummaries).where(
|
|
4980
|
+
tx.delete(backlinkDomains).where(eq8(backlinkDomains.releaseSyncId, syncId)).run();
|
|
4981
|
+
tx.delete(backlinkSummaries).where(eq8(backlinkSummaries.releaseSyncId, syncId)).run();
|
|
4622
4982
|
const expanded = [];
|
|
4623
4983
|
for (const r of rows) {
|
|
4624
4984
|
const projectIds = projectsByDomain.get(r.targetDomain);
|
|
4625
4985
|
if (!projectIds) continue;
|
|
4626
4986
|
for (const projectId of projectIds) {
|
|
4627
4987
|
expanded.push({
|
|
4628
|
-
id:
|
|
4988
|
+
id: crypto10.randomUUID(),
|
|
4629
4989
|
projectId,
|
|
4630
4990
|
releaseSyncId: syncId,
|
|
4631
4991
|
release,
|
|
@@ -4645,9 +5005,10 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4645
5005
|
const projectRows = rowsByProject.get(p.id) ?? [];
|
|
4646
5006
|
const summary = computeSummary(projectRows);
|
|
4647
5007
|
tx.insert(backlinkSummaries).values({
|
|
4648
|
-
id:
|
|
5008
|
+
id: crypto10.randomUUID(),
|
|
4649
5009
|
projectId: p.id,
|
|
4650
5010
|
releaseSyncId: syncId,
|
|
5011
|
+
source: BacklinkSources.commoncrawl,
|
|
4651
5012
|
release,
|
|
4652
5013
|
targetDomain: p.canonicalDomain,
|
|
4653
5014
|
totalLinkingDomains: summary.totalLinkingDomains,
|
|
@@ -4656,7 +5017,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4656
5017
|
queriedAt,
|
|
4657
5018
|
createdAt: queriedAt
|
|
4658
5019
|
}).onConflictDoUpdate({
|
|
4659
|
-
target: [backlinkSummaries.projectId, backlinkSummaries.release],
|
|
5020
|
+
target: [backlinkSummaries.projectId, backlinkSummaries.source, backlinkSummaries.release],
|
|
4660
5021
|
set: {
|
|
4661
5022
|
releaseSyncId: syncId,
|
|
4662
5023
|
targetDomain: p.canonicalDomain,
|
|
@@ -4677,8 +5038,8 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4677
5038
|
domainsDiscovered: rows.length,
|
|
4678
5039
|
updatedAt: finishedAt,
|
|
4679
5040
|
error: null
|
|
4680
|
-
}).where(
|
|
4681
|
-
|
|
5041
|
+
}).where(eq8(ccReleaseSyncs.id, syncId)).run();
|
|
5042
|
+
log9.info("sync.completed", {
|
|
4682
5043
|
syncId,
|
|
4683
5044
|
release,
|
|
4684
5045
|
projectsProcessed: allProjects.length,
|
|
@@ -4690,7 +5051,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4690
5051
|
try {
|
|
4691
5052
|
deps.enqueueAutoExtract({ projectId: p.id, release });
|
|
4692
5053
|
} catch (err) {
|
|
4693
|
-
|
|
5054
|
+
log9.error("auto-extract.enqueue-failed", {
|
|
4694
5055
|
syncId,
|
|
4695
5056
|
release,
|
|
4696
5057
|
projectId: p.id,
|
|
@@ -4707,8 +5068,8 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4707
5068
|
error: errorMsg,
|
|
4708
5069
|
phaseDetail: null,
|
|
4709
5070
|
updatedAt: finishedAt
|
|
4710
|
-
}).where(
|
|
4711
|
-
|
|
5071
|
+
}).where(eq8(ccReleaseSyncs.id, syncId)).run();
|
|
5072
|
+
log9.error("sync.failed", { syncId, release, error: errorMsg });
|
|
4712
5073
|
throw err;
|
|
4713
5074
|
}
|
|
4714
5075
|
}
|
|
@@ -4741,10 +5102,10 @@ function computeSummary(rows) {
|
|
|
4741
5102
|
}
|
|
4742
5103
|
|
|
4743
5104
|
// src/backlink-extract.ts
|
|
4744
|
-
import
|
|
5105
|
+
import crypto11 from "crypto";
|
|
4745
5106
|
import fs3 from "fs";
|
|
4746
|
-
import { and as and7, desc as desc4, eq as
|
|
4747
|
-
var
|
|
5107
|
+
import { and as and7, desc as desc4, eq as eq9 } from "drizzle-orm";
|
|
5108
|
+
var log10 = createLogger("BacklinkExtract");
|
|
4748
5109
|
function defaultDeps3() {
|
|
4749
5110
|
return {
|
|
4750
5111
|
queryBacklinks,
|
|
@@ -4755,13 +5116,13 @@ function defaultDeps3() {
|
|
|
4755
5116
|
async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
4756
5117
|
const deps = { ...defaultDeps3(), ...opts.deps };
|
|
4757
5118
|
const startedAt = deps.now().toISOString();
|
|
4758
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
5119
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq9(runs.id, runId)).run();
|
|
4759
5120
|
try {
|
|
4760
|
-
const project = db.select().from(projects).where(
|
|
5121
|
+
const project = db.select().from(projects).where(eq9(projects.id, projectId)).get();
|
|
4761
5122
|
if (!project) {
|
|
4762
5123
|
throw new Error(`Project not found: ${projectId}`);
|
|
4763
5124
|
}
|
|
4764
|
-
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(
|
|
5125
|
+
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(eq9(ccReleaseSyncs.release, opts.release)).get() : db.select().from(ccReleaseSyncs).where(eq9(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).orderBy(desc4(ccReleaseSyncs.createdAt)).limit(1).get();
|
|
4765
5126
|
if (!sync) {
|
|
4766
5127
|
throw new Error("No ready release sync available \u2014 run `canonry backlinks sync` first");
|
|
4767
5128
|
}
|
|
@@ -4789,13 +5150,18 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
4789
5150
|
const targetDomain = project.canonicalDomain;
|
|
4790
5151
|
db.transaction((tx) => {
|
|
4791
5152
|
tx.delete(backlinkDomains).where(
|
|
4792
|
-
and7(
|
|
5153
|
+
and7(
|
|
5154
|
+
eq9(backlinkDomains.projectId, projectId),
|
|
5155
|
+
eq9(backlinkDomains.source, BacklinkSources.commoncrawl),
|
|
5156
|
+
eq9(backlinkDomains.release, release)
|
|
5157
|
+
)
|
|
4793
5158
|
).run();
|
|
4794
5159
|
if (rows.length > 0) {
|
|
4795
5160
|
const values = rows.map((r) => ({
|
|
4796
|
-
id:
|
|
5161
|
+
id: crypto11.randomUUID(),
|
|
4797
5162
|
projectId,
|
|
4798
5163
|
releaseSyncId: syncId,
|
|
5164
|
+
source: BacklinkSources.commoncrawl,
|
|
4799
5165
|
release,
|
|
4800
5166
|
targetDomain,
|
|
4801
5167
|
linkingDomain: r.linkingDomain,
|
|
@@ -4806,9 +5172,10 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
4806
5172
|
}
|
|
4807
5173
|
const summary = computeSummary2(rows);
|
|
4808
5174
|
tx.insert(backlinkSummaries).values({
|
|
4809
|
-
id:
|
|
5175
|
+
id: crypto11.randomUUID(),
|
|
4810
5176
|
projectId,
|
|
4811
5177
|
releaseSyncId: syncId,
|
|
5178
|
+
source: BacklinkSources.commoncrawl,
|
|
4812
5179
|
release,
|
|
4813
5180
|
targetDomain,
|
|
4814
5181
|
totalLinkingDomains: summary.totalLinkingDomains,
|
|
@@ -4817,7 +5184,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
4817
5184
|
queriedAt,
|
|
4818
5185
|
createdAt: queriedAt
|
|
4819
5186
|
}).onConflictDoUpdate({
|
|
4820
|
-
target: [backlinkSummaries.projectId, backlinkSummaries.release],
|
|
5187
|
+
target: [backlinkSummaries.projectId, backlinkSummaries.source, backlinkSummaries.release],
|
|
4821
5188
|
set: {
|
|
4822
5189
|
releaseSyncId: syncId,
|
|
4823
5190
|
targetDomain,
|
|
@@ -4829,8 +5196,8 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
4829
5196
|
}).run();
|
|
4830
5197
|
});
|
|
4831
5198
|
const finishedAt = deps.now().toISOString();
|
|
4832
|
-
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
4833
|
-
|
|
5199
|
+
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq9(runs.id, runId)).run();
|
|
5200
|
+
log10.info("extract.completed", { runId, projectId, release, rows: rows.length });
|
|
4834
5201
|
} catch (err) {
|
|
4835
5202
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
4836
5203
|
const finishedAt = deps.now().toISOString();
|
|
@@ -4838,39 +5205,173 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
4838
5205
|
status: RunStatuses.failed,
|
|
4839
5206
|
error: errorMsg,
|
|
4840
5207
|
finishedAt
|
|
4841
|
-
}).where(
|
|
4842
|
-
|
|
5208
|
+
}).where(eq9(runs.id, runId)).run();
|
|
5209
|
+
log10.error("extract.failed", { runId, projectId, error: errorMsg });
|
|
4843
5210
|
throw err;
|
|
4844
5211
|
}
|
|
4845
5212
|
}
|
|
4846
5213
|
function computeSummary2(rows) {
|
|
4847
|
-
|
|
4848
|
-
|
|
5214
|
+
return computeBacklinkSummaryMetrics(rows);
|
|
5215
|
+
}
|
|
5216
|
+
|
|
5217
|
+
// src/bing-backlinks-sync.ts
|
|
5218
|
+
import crypto12 from "crypto";
|
|
5219
|
+
import { and as and8, eq as eq10 } from "drizzle-orm";
|
|
5220
|
+
var log11 = createLogger("BingBacklinkSync");
|
|
5221
|
+
var DEFAULT_MAX_PAGES = 200;
|
|
5222
|
+
var LINK_COUNTS_MAX_PAGES = 50;
|
|
5223
|
+
var URL_LINKS_MAX_PAGES = 20;
|
|
5224
|
+
function bingReleaseId(date) {
|
|
5225
|
+
return `bing-${date.toISOString().slice(0, 10)}`;
|
|
5226
|
+
}
|
|
5227
|
+
function aggregateInboundLinksByDomain(links, targetDomain) {
|
|
5228
|
+
const target = hostOf(targetDomain);
|
|
5229
|
+
const byHost = /* @__PURE__ */ new Map();
|
|
5230
|
+
for (const link of links) {
|
|
5231
|
+
const host = hostOf(link.Url);
|
|
5232
|
+
if (!host) continue;
|
|
5233
|
+
if (target && (host === target || host.endsWith(`.${target}`))) continue;
|
|
5234
|
+
let urls = byHost.get(host);
|
|
5235
|
+
if (!urls) {
|
|
5236
|
+
urls = /* @__PURE__ */ new Set();
|
|
5237
|
+
byHost.set(host, urls);
|
|
5238
|
+
}
|
|
5239
|
+
urls.add(link.Url);
|
|
5240
|
+
}
|
|
5241
|
+
return [...byHost.entries()].map(([linkingDomain, urls]) => ({ linkingDomain, numHosts: urls.size })).sort((a, b) => b.numHosts - a.numHosts || a.linkingDomain.localeCompare(b.linkingDomain));
|
|
5242
|
+
}
|
|
5243
|
+
function computeBingSummary(rows) {
|
|
5244
|
+
return computeBacklinkSummaryMetrics(rows);
|
|
5245
|
+
}
|
|
5246
|
+
function defaultDeps4() {
|
|
5247
|
+
return { getLinkCounts, getUrlLinks, now: () => /* @__PURE__ */ new Date() };
|
|
5248
|
+
}
|
|
5249
|
+
async function executeBingBacklinkSync(db, runId, projectId, opts) {
|
|
5250
|
+
const deps = { ...defaultDeps4(), ...opts.deps };
|
|
5251
|
+
const startedAt = deps.now().toISOString();
|
|
5252
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq10(runs.id, runId)).run();
|
|
5253
|
+
try {
|
|
5254
|
+
const project = db.select().from(projects).where(eq10(projects.id, projectId)).get();
|
|
5255
|
+
if (!project) throw new Error(`Project not found: ${projectId}`);
|
|
5256
|
+
const conn = opts.resolveConnection(project.canonicalDomain);
|
|
5257
|
+
if (!conn) throw new Error(`No Bing Webmaster connection for ${project.canonicalDomain}`);
|
|
5258
|
+
const siteUrl = conn.siteUrl;
|
|
5259
|
+
if (!siteUrl) {
|
|
5260
|
+
throw new Error(`Bing connection for ${project.canonicalDomain} has no verified site selected`);
|
|
5261
|
+
}
|
|
5262
|
+
const pages = await deps.getLinkCounts(conn.apiKey, siteUrl, { maxPages: LINK_COUNTS_MAX_PAGES });
|
|
5263
|
+
const maxPages = Math.max(1, opts.maxPages ?? DEFAULT_MAX_PAGES);
|
|
5264
|
+
const targetPages = [...pages].sort((a, b) => b.Count - a.Count).slice(0, maxPages);
|
|
5265
|
+
if (pages.length > targetPages.length) {
|
|
5266
|
+
log11.info("bing-sync.pages-capped", {
|
|
5267
|
+
runId,
|
|
5268
|
+
projectId,
|
|
5269
|
+
sitePagesFound: pages.length,
|
|
5270
|
+
sitePagesPulled: targetPages.length
|
|
5271
|
+
});
|
|
5272
|
+
}
|
|
5273
|
+
const allLinks = [];
|
|
5274
|
+
let pageFailures = 0;
|
|
5275
|
+
for (const page of targetPages) {
|
|
5276
|
+
try {
|
|
5277
|
+
const links = await deps.getUrlLinks(conn.apiKey, siteUrl, page.Url, { maxPages: URL_LINKS_MAX_PAGES });
|
|
5278
|
+
allLinks.push(...links);
|
|
5279
|
+
} catch (err) {
|
|
5280
|
+
pageFailures++;
|
|
5281
|
+
log11.warn("bing-sync.page-failed", {
|
|
5282
|
+
runId,
|
|
5283
|
+
projectId,
|
|
5284
|
+
page: page.Url,
|
|
5285
|
+
error: err instanceof Error ? err.message : String(err)
|
|
5286
|
+
});
|
|
5287
|
+
}
|
|
5288
|
+
}
|
|
5289
|
+
if (targetPages.length > 0 && pageFailures === targetPages.length) {
|
|
5290
|
+
throw new Error(`All ${targetPages.length} Bing inbound-link page fetch(es) failed`);
|
|
5291
|
+
}
|
|
5292
|
+
const rows = aggregateInboundLinksByDomain(allLinks, project.canonicalDomain);
|
|
5293
|
+
const summary = computeBingSummary(rows);
|
|
5294
|
+
const queriedAt = deps.now().toISOString();
|
|
5295
|
+
const release = bingReleaseId(deps.now());
|
|
5296
|
+
const targetDomain = project.canonicalDomain;
|
|
5297
|
+
const source = BacklinkSources["bing-webmaster"];
|
|
5298
|
+
const existing = db.select({ id: backlinkSummaries.id }).from(backlinkSummaries).where(and8(
|
|
5299
|
+
eq10(backlinkSummaries.projectId, projectId),
|
|
5300
|
+
eq10(backlinkSummaries.source, source),
|
|
5301
|
+
eq10(backlinkSummaries.release, release)
|
|
5302
|
+
)).get();
|
|
5303
|
+
const preserveExisting = pageFailures > 0 && !!existing;
|
|
5304
|
+
if (!preserveExisting) {
|
|
5305
|
+
db.transaction((tx) => {
|
|
5306
|
+
tx.delete(backlinkDomains).where(and8(
|
|
5307
|
+
eq10(backlinkDomains.projectId, projectId),
|
|
5308
|
+
eq10(backlinkDomains.source, source),
|
|
5309
|
+
eq10(backlinkDomains.release, release)
|
|
5310
|
+
)).run();
|
|
5311
|
+
if (rows.length > 0) {
|
|
5312
|
+
tx.insert(backlinkDomains).values(rows.map((r) => ({
|
|
5313
|
+
id: crypto12.randomUUID(),
|
|
5314
|
+
projectId,
|
|
5315
|
+
releaseSyncId: null,
|
|
5316
|
+
source,
|
|
5317
|
+
release,
|
|
5318
|
+
targetDomain,
|
|
5319
|
+
linkingDomain: r.linkingDomain,
|
|
5320
|
+
numHosts: r.numHosts,
|
|
5321
|
+
createdAt: queriedAt
|
|
5322
|
+
}))).run();
|
|
5323
|
+
}
|
|
5324
|
+
tx.insert(backlinkSummaries).values({
|
|
5325
|
+
id: crypto12.randomUUID(),
|
|
5326
|
+
projectId,
|
|
5327
|
+
releaseSyncId: null,
|
|
5328
|
+
source,
|
|
5329
|
+
release,
|
|
5330
|
+
targetDomain,
|
|
5331
|
+
totalLinkingDomains: summary.totalLinkingDomains,
|
|
5332
|
+
totalHosts: summary.totalHosts,
|
|
5333
|
+
top10HostsShare: summary.top10HostsShare,
|
|
5334
|
+
queriedAt,
|
|
5335
|
+
createdAt: queriedAt
|
|
5336
|
+
}).onConflictDoUpdate({
|
|
5337
|
+
target: [backlinkSummaries.projectId, backlinkSummaries.source, backlinkSummaries.release],
|
|
5338
|
+
set: {
|
|
5339
|
+
targetDomain,
|
|
5340
|
+
totalLinkingDomains: summary.totalLinkingDomains,
|
|
5341
|
+
totalHosts: summary.totalHosts,
|
|
5342
|
+
top10HostsShare: summary.top10HostsShare,
|
|
5343
|
+
queriedAt
|
|
5344
|
+
}
|
|
5345
|
+
}).run();
|
|
5346
|
+
});
|
|
5347
|
+
}
|
|
5348
|
+
const finishedAt = deps.now().toISOString();
|
|
5349
|
+
const status = pageFailures > 0 ? RunStatuses.partial : RunStatuses.completed;
|
|
5350
|
+
const error = preserveExisting ? `Kept existing ${release} snapshot; ${pageFailures} of ${targetPages.length} inbound-link page fetches failed` : pageFailures > 0 ? `${pageFailures} of ${targetPages.length} inbound-link page fetches failed` : null;
|
|
5351
|
+
db.update(runs).set({ status, error, finishedAt }).where(eq10(runs.id, runId)).run();
|
|
5352
|
+
log11.info("bing-sync.completed", { runId, projectId, release, rows: rows.length, status, pageFailures, preserveExisting });
|
|
5353
|
+
} catch (err) {
|
|
5354
|
+
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
5355
|
+
const finishedAt = deps.now().toISOString();
|
|
5356
|
+
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt }).where(eq10(runs.id, runId)).run();
|
|
5357
|
+
log11.error("bing-sync.failed", { runId, projectId, error: errorMsg });
|
|
5358
|
+
throw err;
|
|
4849
5359
|
}
|
|
4850
|
-
const sorted = [...rows].sort((a, b) => b.numHosts - a.numHosts);
|
|
4851
|
-
const totalHosts = sorted.reduce((acc, r) => acc + r.numHosts, 0);
|
|
4852
|
-
const top10Hosts = sorted.slice(0, 10).reduce((acc, r) => acc + r.numHosts, 0);
|
|
4853
|
-
const share = totalHosts > 0 ? top10Hosts / totalHosts : 0;
|
|
4854
|
-
return {
|
|
4855
|
-
totalLinkingDomains: rows.length,
|
|
4856
|
-
totalHosts,
|
|
4857
|
-
top10HostsShare: share.toFixed(6)
|
|
4858
|
-
};
|
|
4859
5360
|
}
|
|
4860
5361
|
|
|
4861
5362
|
// src/discovery-run.ts
|
|
4862
|
-
import
|
|
4863
|
-
import { and as
|
|
4864
|
-
var
|
|
5363
|
+
import crypto13 from "crypto";
|
|
5364
|
+
import { and as and9, eq as eq11 } from "drizzle-orm";
|
|
5365
|
+
var log12 = createLogger("DiscoveryRun");
|
|
4865
5366
|
var DEFAULT_SEED_COUNT = 30;
|
|
4866
5367
|
var QUERIES_PER_INTENT_BUCKET = 6;
|
|
4867
5368
|
async function executeDiscoveryRun(opts) {
|
|
4868
5369
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
4869
|
-
opts.db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
5370
|
+
opts.db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq11(runs.id, opts.runId)).run();
|
|
4870
5371
|
try {
|
|
4871
|
-
const projectRow = opts.db.select().from(projects).where(
|
|
5372
|
+
const projectRow = opts.db.select().from(projects).where(eq11(projects.id, opts.projectId)).get();
|
|
4872
5373
|
if (!projectRow) throw new Error(`Project ${opts.projectId} not found`);
|
|
4873
|
-
const projectCompetitors = opts.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
5374
|
+
const projectCompetitors = opts.db.select({ domain: competitors.domain }).from(competitors).where(eq11(competitors.projectId, opts.projectId)).all().map((r) => r.domain.toLowerCase());
|
|
4874
5375
|
const canonicalDomains = effectiveDomains({
|
|
4875
5376
|
canonicalDomain: projectRow.canonicalDomain,
|
|
4876
5377
|
ownedDomains: projectRow.ownedDomains
|
|
@@ -4900,8 +5401,8 @@ async function executeDiscoveryRun(opts) {
|
|
|
4900
5401
|
seedProvider: result.seedProvider,
|
|
4901
5402
|
result
|
|
4902
5403
|
});
|
|
4903
|
-
opts.db.update(runs).set({ status: RunStatuses.completed, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
4904
|
-
|
|
5404
|
+
opts.db.update(runs).set({ status: RunStatuses.completed, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq11(runs.id, opts.runId)).run();
|
|
5405
|
+
log12.info("discovery.completed", {
|
|
4905
5406
|
runId: opts.runId,
|
|
4906
5407
|
sessionId: opts.sessionId,
|
|
4907
5408
|
buckets: result.buckets,
|
|
@@ -4909,13 +5410,13 @@ async function executeDiscoveryRun(opts) {
|
|
|
4909
5410
|
});
|
|
4910
5411
|
} catch (err) {
|
|
4911
5412
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
4912
|
-
|
|
5413
|
+
log12.error("discovery.failed", { runId: opts.runId, sessionId: opts.sessionId, error: errorMsg });
|
|
4913
5414
|
markSessionFailed(opts.db, opts.sessionId, errorMsg);
|
|
4914
5415
|
opts.db.update(runs).set({
|
|
4915
5416
|
status: RunStatuses.failed,
|
|
4916
5417
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4917
5418
|
error: errorMsg
|
|
4918
|
-
}).where(
|
|
5419
|
+
}).where(eq11(runs.id, opts.runId)).run();
|
|
4919
5420
|
}
|
|
4920
5421
|
}
|
|
4921
5422
|
function buildDefaultDeps(registry) {
|
|
@@ -5120,13 +5621,13 @@ function writeDiscoveryInsight(db, input) {
|
|
|
5120
5621
|
totalProbes
|
|
5121
5622
|
});
|
|
5122
5623
|
db.transaction((tx) => {
|
|
5123
|
-
tx.update(insights).set({ dismissed: true }).where(
|
|
5124
|
-
|
|
5125
|
-
|
|
5126
|
-
|
|
5624
|
+
tx.update(insights).set({ dismissed: true }).where(and9(
|
|
5625
|
+
eq11(insights.projectId, input.projectId),
|
|
5626
|
+
eq11(insights.type, "discovery.basket-divergence"),
|
|
5627
|
+
eq11(insights.dismissed, false)
|
|
5127
5628
|
)).run();
|
|
5128
5629
|
tx.insert(insights).values({
|
|
5129
|
-
id:
|
|
5630
|
+
id: crypto13.randomUUID(),
|
|
5130
5631
|
projectId: input.projectId,
|
|
5131
5632
|
runId: input.runId,
|
|
5132
5633
|
type: "discovery.basket-divergence",
|
|
@@ -5162,10 +5663,10 @@ function buildDiscoveryInsightTitle(input) {
|
|
|
5162
5663
|
}
|
|
5163
5664
|
|
|
5164
5665
|
// src/execute-site-audit.ts
|
|
5165
|
-
import
|
|
5166
|
-
import { eq as
|
|
5666
|
+
import crypto14 from "crypto";
|
|
5667
|
+
import { eq as eq12 } from "drizzle-orm";
|
|
5167
5668
|
import { runSitemapAudit } from "@ainyc/aeo-audit";
|
|
5168
|
-
var
|
|
5669
|
+
var log13 = createLogger("SiteAudit");
|
|
5169
5670
|
var SITE_AUDIT_DEFAULT_PAGE_LIMIT = 500;
|
|
5170
5671
|
var SITE_AUDIT_MAX_PAGE_LIMIT = 2e3;
|
|
5171
5672
|
function toHomepageUrl(canonicalDomain) {
|
|
@@ -5226,15 +5727,15 @@ function computeFactorAverages(pages) {
|
|
|
5226
5727
|
}
|
|
5227
5728
|
async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
5228
5729
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
5229
|
-
db.update(runs).set({ status: "running", startedAt }).where(
|
|
5730
|
+
db.update(runs).set({ status: "running", startedAt }).where(eq12(runs.id, runId)).run();
|
|
5230
5731
|
try {
|
|
5231
|
-
const project = db.select().from(projects).where(
|
|
5732
|
+
const project = db.select().from(projects).where(eq12(projects.id, projectId)).get();
|
|
5232
5733
|
if (!project) {
|
|
5233
5734
|
throw new Error(`Project not found: ${projectId}`);
|
|
5234
5735
|
}
|
|
5235
5736
|
const homepageUrl = toHomepageUrl(project.canonicalDomain);
|
|
5236
5737
|
const limit = clampSiteAuditLimit(opts.limit);
|
|
5237
|
-
|
|
5738
|
+
log13.info("start", { runId, projectId, homepageUrl, sitemapUrl: opts.sitemapUrl ?? null, limit });
|
|
5238
5739
|
await assertSiteAuditUrlAllowed(homepageUrl, "canonicalDomain");
|
|
5239
5740
|
if (opts.sitemapUrl) await assertSiteAuditUrlAllowed(opts.sitemapUrl, "sitemapUrl");
|
|
5240
5741
|
const report = await runSitemapAudit(homepageUrl, { sitemapUrl: opts.sitemapUrl, limit });
|
|
@@ -5242,7 +5743,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
5242
5743
|
const pagesErrored = report.pages.filter((page) => page.status === "error").length;
|
|
5243
5744
|
const auditable = report.pagesDiscovered - report.pagesSkipped;
|
|
5244
5745
|
if (auditable > report.pagesAudited) {
|
|
5245
|
-
|
|
5746
|
+
log13.info("truncated", {
|
|
5246
5747
|
runId,
|
|
5247
5748
|
projectId,
|
|
5248
5749
|
auditable,
|
|
@@ -5261,7 +5762,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
5261
5762
|
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
5262
5763
|
db.transaction((tx) => {
|
|
5263
5764
|
tx.insert(siteAuditSnapshots).values({
|
|
5264
|
-
id:
|
|
5765
|
+
id: crypto14.randomUUID(),
|
|
5265
5766
|
projectId,
|
|
5266
5767
|
runId,
|
|
5267
5768
|
sitemapUrl: report.sitemapUrl,
|
|
@@ -5290,7 +5791,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
5290
5791
|
}).run();
|
|
5291
5792
|
for (const page of report.pages) {
|
|
5292
5793
|
tx.insert(siteAuditPages).values({
|
|
5293
|
-
id:
|
|
5794
|
+
id: crypto14.randomUUID(),
|
|
5294
5795
|
projectId,
|
|
5295
5796
|
runId,
|
|
5296
5797
|
url: page.url,
|
|
@@ -5301,9 +5802,9 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
5301
5802
|
createdAt: finishedAt
|
|
5302
5803
|
}).run();
|
|
5303
5804
|
}
|
|
5304
|
-
tx.update(runs).set({ status, finishedAt }).where(
|
|
5805
|
+
tx.update(runs).set({ status, finishedAt }).where(eq12(runs.id, runId)).run();
|
|
5305
5806
|
});
|
|
5306
|
-
|
|
5807
|
+
log13.info("completed", {
|
|
5307
5808
|
runId,
|
|
5308
5809
|
projectId,
|
|
5309
5810
|
status,
|
|
@@ -5313,14 +5814,14 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
5313
5814
|
});
|
|
5314
5815
|
} catch (err) {
|
|
5315
5816
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
5316
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
5317
|
-
|
|
5817
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq12(runs.id, runId)).run();
|
|
5818
|
+
log13.error("failed", { runId, projectId, error: errorMsg });
|
|
5318
5819
|
throw err;
|
|
5319
5820
|
}
|
|
5320
5821
|
}
|
|
5321
5822
|
|
|
5322
5823
|
// src/commands/backfill.ts
|
|
5323
|
-
import { and as
|
|
5824
|
+
import { and as and10, eq as eq13, inArray as inArray4, isNull, sql as sql4 } from "drizzle-orm";
|
|
5324
5825
|
var SNAPSHOT_BATCH_SIZE = 500;
|
|
5325
5826
|
async function backfillAnswerVisibilityCommand(opts) {
|
|
5326
5827
|
const config = loadConfig();
|
|
@@ -5328,7 +5829,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
5328
5829
|
migrate(db);
|
|
5329
5830
|
const projectFilter = opts?.project?.trim();
|
|
5330
5831
|
const isDryRun = opts?.dryRun === true;
|
|
5331
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
5832
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq13(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
5332
5833
|
let examined = 0;
|
|
5333
5834
|
let updated = 0;
|
|
5334
5835
|
let wouldUpdate = 0;
|
|
@@ -5336,10 +5837,10 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
5336
5837
|
let reparsed = 0;
|
|
5337
5838
|
let providerErrors = 0;
|
|
5338
5839
|
if (scopedProjects.length > 0) {
|
|
5339
|
-
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
5340
|
-
|
|
5840
|
+
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(and10(
|
|
5841
|
+
eq13(runs.kind, RunKinds["answer-visibility"]),
|
|
5341
5842
|
inArray4(runs.projectId, scopedProjects.map((project) => project.id))
|
|
5342
|
-
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
5843
|
+
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(eq13(runs.kind, RunKinds["answer-visibility"])).all();
|
|
5343
5844
|
const runIdsByProject = /* @__PURE__ */ new Map();
|
|
5344
5845
|
for (const run of runRows) {
|
|
5345
5846
|
const existing = runIdsByProject.get(run.projectId);
|
|
@@ -5347,7 +5848,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
5347
5848
|
else runIdsByProject.set(run.projectId, [run.id]);
|
|
5348
5849
|
}
|
|
5349
5850
|
for (const project of scopedProjects) {
|
|
5350
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
5851
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq13(competitors.projectId, project.id)).all().map((row) => row.domain);
|
|
5351
5852
|
const runIds = runIdsByProject.get(project.id) ?? [];
|
|
5352
5853
|
if (runIds.length === 0) continue;
|
|
5353
5854
|
const projectDomains = effectiveDomains({
|
|
@@ -5435,7 +5936,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
5435
5936
|
} else {
|
|
5436
5937
|
db.transaction((tx) => {
|
|
5437
5938
|
for (const update of pendingUpdates) {
|
|
5438
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
5939
|
+
tx.update(querySnapshots).set(update.patch).where(eq13(querySnapshots.id, update.id)).run();
|
|
5439
5940
|
}
|
|
5440
5941
|
});
|
|
5441
5942
|
updated += pendingUpdates.length;
|
|
@@ -5484,13 +5985,13 @@ No DB writes performed. Re-run without --dry-run to apply.`);
|
|
|
5484
5985
|
function backfillNormalizedPaths(db, opts) {
|
|
5485
5986
|
const baseConditions = [];
|
|
5486
5987
|
if (opts?.projectId) {
|
|
5487
|
-
baseConditions.push(
|
|
5988
|
+
baseConditions.push(eq13(gaTrafficSnapshots.projectId, opts.projectId));
|
|
5488
5989
|
}
|
|
5489
5990
|
const rows = db.select({
|
|
5490
5991
|
id: gaTrafficSnapshots.id,
|
|
5491
5992
|
landingPage: gaTrafficSnapshots.landingPage,
|
|
5492
5993
|
landingPageNormalized: gaTrafficSnapshots.landingPageNormalized
|
|
5493
|
-
}).from(gaTrafficSnapshots).where(baseConditions.length > 0 ?
|
|
5994
|
+
}).from(gaTrafficSnapshots).where(baseConditions.length > 0 ? and10(...baseConditions) : void 0).all();
|
|
5494
5995
|
let updated = 0;
|
|
5495
5996
|
let unchanged = 0;
|
|
5496
5997
|
if (rows.length > 0) {
|
|
@@ -5505,7 +6006,7 @@ function backfillNormalizedPaths(db, opts) {
|
|
|
5505
6006
|
unchanged++;
|
|
5506
6007
|
continue;
|
|
5507
6008
|
}
|
|
5508
|
-
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(
|
|
6009
|
+
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(eq13(gaTrafficSnapshots.id, row.id)).run();
|
|
5509
6010
|
updated++;
|
|
5510
6011
|
}
|
|
5511
6012
|
});
|
|
@@ -5519,7 +6020,7 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
5519
6020
|
const projectFilter = opts?.project?.trim();
|
|
5520
6021
|
let projectId;
|
|
5521
6022
|
if (projectFilter) {
|
|
5522
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
6023
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq13(projects.name, projectFilter)).get();
|
|
5523
6024
|
if (!project) {
|
|
5524
6025
|
const result2 = {
|
|
5525
6026
|
project: projectFilter,
|
|
@@ -5556,13 +6057,13 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
5556
6057
|
function backfillAiReferralPaths(db, opts) {
|
|
5557
6058
|
const baseConditions = [];
|
|
5558
6059
|
if (opts?.projectId) {
|
|
5559
|
-
baseConditions.push(
|
|
6060
|
+
baseConditions.push(eq13(gaAiReferrals.projectId, opts.projectId));
|
|
5560
6061
|
}
|
|
5561
6062
|
const rows = db.select({
|
|
5562
6063
|
id: gaAiReferrals.id,
|
|
5563
6064
|
landingPage: gaAiReferrals.landingPage,
|
|
5564
6065
|
landingPageNormalized: gaAiReferrals.landingPageNormalized
|
|
5565
|
-
}).from(gaAiReferrals).where(baseConditions.length > 0 ?
|
|
6066
|
+
}).from(gaAiReferrals).where(baseConditions.length > 0 ? and10(...baseConditions) : void 0).all();
|
|
5566
6067
|
let updated = 0;
|
|
5567
6068
|
let unchanged = 0;
|
|
5568
6069
|
if (rows.length > 0) {
|
|
@@ -5577,7 +6078,7 @@ function backfillAiReferralPaths(db, opts) {
|
|
|
5577
6078
|
unchanged++;
|
|
5578
6079
|
continue;
|
|
5579
6080
|
}
|
|
5580
|
-
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(
|
|
6081
|
+
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(eq13(gaAiReferrals.id, row.id)).run();
|
|
5581
6082
|
updated++;
|
|
5582
6083
|
}
|
|
5583
6084
|
});
|
|
@@ -5591,7 +6092,7 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
5591
6092
|
const projectFilter = opts?.project?.trim();
|
|
5592
6093
|
let projectId;
|
|
5593
6094
|
if (projectFilter) {
|
|
5594
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
6095
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq13(projects.name, projectFilter)).get();
|
|
5595
6096
|
if (!project) {
|
|
5596
6097
|
const result2 = {
|
|
5597
6098
|
project: projectFilter,
|
|
@@ -5627,10 +6128,10 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
5627
6128
|
}
|
|
5628
6129
|
function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
5629
6130
|
const isDryRun = opts?.dryRun === true;
|
|
5630
|
-
const project = db.select().from(projects).where(
|
|
6131
|
+
const project = db.select().from(projects).where(eq13(projects.id, projectId)).get();
|
|
5631
6132
|
if (!project) return { examined: 0, updated: 0, mentioned: 0 };
|
|
5632
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
5633
|
-
const runRows = db.select({ id: runs.id }).from(runs).where(
|
|
6133
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq13(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
6134
|
+
const runRows = db.select({ id: runs.id }).from(runs).where(and10(eq13(runs.kind, RunKinds["answer-visibility"]), eq13(runs.projectId, projectId))).all();
|
|
5634
6135
|
const runIds = runRows.map((r) => r.id);
|
|
5635
6136
|
let examined = 0;
|
|
5636
6137
|
let updated = 0;
|
|
@@ -5702,7 +6203,7 @@ function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
|
5702
6203
|
} else {
|
|
5703
6204
|
db.transaction((tx) => {
|
|
5704
6205
|
for (const update of pendingUpdates) {
|
|
5705
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
6206
|
+
tx.update(querySnapshots).set(update.patch).where(eq13(querySnapshots.id, update.id)).run();
|
|
5706
6207
|
}
|
|
5707
6208
|
});
|
|
5708
6209
|
updated += pendingUpdates.length;
|
|
@@ -5717,7 +6218,7 @@ async function backfillAnswerMentionsCommand(opts) {
|
|
|
5717
6218
|
migrate(db);
|
|
5718
6219
|
const projectFilter = opts?.project?.trim();
|
|
5719
6220
|
const isDryRun = opts?.dryRun === true;
|
|
5720
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
6221
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq13(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
5721
6222
|
let examined = 0;
|
|
5722
6223
|
let updated = 0;
|
|
5723
6224
|
let wouldUpdate = 0;
|
|
@@ -5777,7 +6278,7 @@ function readStoredGroundingSources(rawResponse) {
|
|
|
5777
6278
|
return result;
|
|
5778
6279
|
}
|
|
5779
6280
|
async function backfillInsightsCommand(project, opts) {
|
|
5780
|
-
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-
|
|
6281
|
+
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-CAAQAKPN.js");
|
|
5781
6282
|
const config = loadConfig();
|
|
5782
6283
|
const db = createClient(config.database);
|
|
5783
6284
|
migrate(db);
|
|
@@ -5936,7 +6437,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
5936
6437
|
const config = loadConfig();
|
|
5937
6438
|
const db = createClient(config.database);
|
|
5938
6439
|
migrate(db);
|
|
5939
|
-
const project = db.select().from(projects).where(
|
|
6440
|
+
const project = db.select().from(projects).where(eq13(projects.name, opts.project)).get();
|
|
5940
6441
|
if (!project) {
|
|
5941
6442
|
throw new Error(`Project "${opts.project}" not found`);
|
|
5942
6443
|
}
|
|
@@ -5947,8 +6448,8 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
5947
6448
|
process.stderr.write(`Recovering orphan snapshot attribution for "${project.name}"${mode}...
|
|
5948
6449
|
`);
|
|
5949
6450
|
}
|
|
5950
|
-
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(
|
|
5951
|
-
|
|
6451
|
+
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(and10(
|
|
6452
|
+
eq13(auditLog.projectId, project.id),
|
|
5952
6453
|
inArray4(auditLog.action, ["keywords.appended", "keywords.deleted", "queries.appended", "queries.deleted", "queries.replaced"])
|
|
5953
6454
|
)).orderBy(auditLog.createdAt).all();
|
|
5954
6455
|
const history = replayQueryAuditLog(events);
|
|
@@ -5956,8 +6457,8 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
5956
6457
|
runId: runs.id,
|
|
5957
6458
|
createdAt: runs.createdAt,
|
|
5958
6459
|
location: runs.location
|
|
5959
|
-
}).from(runs).innerJoin(querySnapshots,
|
|
5960
|
-
|
|
6460
|
+
}).from(runs).innerJoin(querySnapshots, eq13(querySnapshots.runId, runs.id)).where(and10(
|
|
6461
|
+
eq13(runs.projectId, project.id),
|
|
5961
6462
|
isNull(querySnapshots.queryId),
|
|
5962
6463
|
isNull(querySnapshots.queryText)
|
|
5963
6464
|
)).groupBy(runs.id).orderBy(runs.createdAt).all();
|
|
@@ -5979,8 +6480,8 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
5979
6480
|
provider: querySnapshots.provider,
|
|
5980
6481
|
createdAt: querySnapshots.createdAt,
|
|
5981
6482
|
answerText: querySnapshots.answerText
|
|
5982
|
-
}).from(querySnapshots).where(
|
|
5983
|
-
|
|
6483
|
+
}).from(querySnapshots).where(and10(
|
|
6484
|
+
eq13(querySnapshots.runId, run.runId),
|
|
5984
6485
|
isNull(querySnapshots.queryId),
|
|
5985
6486
|
isNull(querySnapshots.queryText)
|
|
5986
6487
|
)).orderBy(querySnapshots.provider, querySnapshots.createdAt).all();
|
|
@@ -6046,7 +6547,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
6046
6547
|
if (!isDryRun && updates.length > 0) {
|
|
6047
6548
|
db.transaction((tx) => {
|
|
6048
6549
|
for (const u of updates) {
|
|
6049
|
-
tx.update(querySnapshots).set({ queryText: u.queryText }).where(
|
|
6550
|
+
tx.update(querySnapshots).set({ queryText: u.queryText }).where(eq13(querySnapshots.id, u.id)).run();
|
|
6050
6551
|
}
|
|
6051
6552
|
});
|
|
6052
6553
|
}
|
|
@@ -6120,7 +6621,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
6120
6621
|
const projectFilter = opts?.project?.trim();
|
|
6121
6622
|
const isDryRun = opts?.dryRun === true;
|
|
6122
6623
|
const isJson = isMachineFormat(opts?.format);
|
|
6123
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
6624
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq13(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
6124
6625
|
if (scopedProjects.length === 0) {
|
|
6125
6626
|
if (projectFilter && !isJson) {
|
|
6126
6627
|
process.stderr.write(`No project named "${projectFilter}".
|
|
@@ -6145,8 +6646,8 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
6145
6646
|
dryRun: isDryRun,
|
|
6146
6647
|
byBot: {}
|
|
6147
6648
|
};
|
|
6148
|
-
const unknownCountRow = db.select({ n: sql4`count(*)` }).from(rawEventSamples).where(
|
|
6149
|
-
|
|
6649
|
+
const unknownCountRow = db.select({ n: sql4`count(*)` }).from(rawEventSamples).where(and10(
|
|
6650
|
+
eq13(rawEventSamples.eventType, "unknown"),
|
|
6150
6651
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
6151
6652
|
)).get();
|
|
6152
6653
|
result.unknownBefore = Number(unknownCountRow?.n ?? 0);
|
|
@@ -6158,8 +6659,8 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
6158
6659
|
userAgent: rawEventSamples.userAgent,
|
|
6159
6660
|
pathNormalized: rawEventSamples.pathNormalized,
|
|
6160
6661
|
status: rawEventSamples.status
|
|
6161
|
-
}).from(rawEventSamples).where(
|
|
6162
|
-
|
|
6662
|
+
}).from(rawEventSamples).where(and10(
|
|
6663
|
+
eq13(rawEventSamples.eventType, "unknown"),
|
|
6163
6664
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
6164
6665
|
)).all();
|
|
6165
6666
|
result.examined = unknownSamples.length;
|
|
@@ -6198,7 +6699,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
6198
6699
|
result.reclassified++;
|
|
6199
6700
|
result.byBot[classified.botId] = (result.byBot[classified.botId] ?? 0) + 1;
|
|
6200
6701
|
if (isDryRun) continue;
|
|
6201
|
-
db.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(
|
|
6702
|
+
db.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(eq13(rawEventSamples.id, snap.id)).run();
|
|
6202
6703
|
const tsHour = new Date(snap.ts);
|
|
6203
6704
|
tsHour.setUTCMinutes(0, 0, 0);
|
|
6204
6705
|
if (userFetch) {
|
|
@@ -6262,8 +6763,8 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
6262
6763
|
}
|
|
6263
6764
|
}
|
|
6264
6765
|
if (!isDryRun) {
|
|
6265
|
-
const afterRow = db.select({ n: sql4`count(*)` }).from(rawEventSamples).where(
|
|
6266
|
-
|
|
6766
|
+
const afterRow = db.select({ n: sql4`count(*)` }).from(rawEventSamples).where(and10(
|
|
6767
|
+
eq13(rawEventSamples.eventType, "unknown"),
|
|
6267
6768
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
6268
6769
|
)).get();
|
|
6269
6770
|
result.unknownAfter = Number(afterRow?.n ?? 0);
|
|
@@ -6298,7 +6799,7 @@ No DB writes performed. Re-run without --dry-run to apply.`);
|
|
|
6298
6799
|
}
|
|
6299
6800
|
|
|
6300
6801
|
// src/commands/skills.ts
|
|
6301
|
-
import
|
|
6802
|
+
import crypto15 from "crypto";
|
|
6302
6803
|
import fs4 from "fs";
|
|
6303
6804
|
import os4 from "os";
|
|
6304
6805
|
import path5 from "path";
|
|
@@ -6353,7 +6854,7 @@ function walkRelative(dir, prefix = "") {
|
|
|
6353
6854
|
return out.sort();
|
|
6354
6855
|
}
|
|
6355
6856
|
function sha256File(filePath) {
|
|
6356
|
-
return
|
|
6857
|
+
return crypto15.createHash("sha256").update(fs4.readFileSync(filePath)).digest("hex");
|
|
6357
6858
|
}
|
|
6358
6859
|
function readSkillManifest(skillDir) {
|
|
6359
6860
|
try {
|
|
@@ -6676,10 +7177,10 @@ var ProviderRegistry = class {
|
|
|
6676
7177
|
};
|
|
6677
7178
|
|
|
6678
7179
|
// src/scheduler.ts
|
|
6679
|
-
import
|
|
7180
|
+
import crypto16 from "crypto";
|
|
6680
7181
|
import cron from "node-cron";
|
|
6681
|
-
import { and as
|
|
6682
|
-
var
|
|
7182
|
+
import { and as and11, eq as eq14, inArray as inArray5 } from "drizzle-orm";
|
|
7183
|
+
var log14 = createLogger("Scheduler");
|
|
6683
7184
|
function taskKey(projectId, kind) {
|
|
6684
7185
|
return `${projectId}::${kind}`;
|
|
6685
7186
|
}
|
|
@@ -6693,16 +7194,16 @@ var Scheduler = class {
|
|
|
6693
7194
|
}
|
|
6694
7195
|
/** Load all enabled schedules from DB and register cron jobs. */
|
|
6695
7196
|
start() {
|
|
6696
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
7197
|
+
const allSchedules = this.db.select().from(schedules).where(eq14(schedules.enabled, true)).all();
|
|
6697
7198
|
for (const schedule of allSchedules) {
|
|
6698
7199
|
const missedRunAt = schedule.nextRunAt;
|
|
6699
7200
|
this.registerCronTask(schedule);
|
|
6700
7201
|
if (missedRunAt && new Date(missedRunAt) < /* @__PURE__ */ new Date()) {
|
|
6701
|
-
|
|
7202
|
+
log14.info("run.catch-up", { projectId: schedule.projectId, kind: schedule.kind, missedRunAt });
|
|
6702
7203
|
this.triggerRun(schedule.id, schedule.projectId, schedule.kind);
|
|
6703
7204
|
}
|
|
6704
7205
|
}
|
|
6705
|
-
|
|
7206
|
+
log14.info("started", { scheduleCount: allSchedules.length });
|
|
6706
7207
|
}
|
|
6707
7208
|
/** Stop all cron tasks for graceful shutdown. */
|
|
6708
7209
|
stop() {
|
|
@@ -6723,7 +7224,7 @@ var Scheduler = class {
|
|
|
6723
7224
|
this.stopTask(key, existing, "Stopped");
|
|
6724
7225
|
this.tasks.delete(key);
|
|
6725
7226
|
}
|
|
6726
|
-
const schedule = this.db.select().from(schedules).where(
|
|
7227
|
+
const schedule = this.db.select().from(schedules).where(and11(eq14(schedules.projectId, projectId), eq14(schedules.kind, kind))).get();
|
|
6727
7228
|
if (schedule && schedule.enabled) {
|
|
6728
7229
|
this.registerCronTask(schedule);
|
|
6729
7230
|
}
|
|
@@ -6746,13 +7247,13 @@ var Scheduler = class {
|
|
|
6746
7247
|
stopTask(key, task, verb) {
|
|
6747
7248
|
void task.stop();
|
|
6748
7249
|
void task.destroy();
|
|
6749
|
-
|
|
7250
|
+
log14.info(`task.${verb.toLowerCase()}`, { key });
|
|
6750
7251
|
}
|
|
6751
7252
|
registerCronTask(schedule) {
|
|
6752
7253
|
const { id: scheduleId, projectId, cronExpr, timezone } = schedule;
|
|
6753
7254
|
const kind = schedule.kind;
|
|
6754
7255
|
if (!cron.validate(cronExpr)) {
|
|
6755
|
-
|
|
7256
|
+
log14.error("cron.invalid", { projectId, kind, cronExpr });
|
|
6756
7257
|
return;
|
|
6757
7258
|
}
|
|
6758
7259
|
const task = cron.schedule(cronExpr, () => {
|
|
@@ -6764,51 +7265,51 @@ var Scheduler = class {
|
|
|
6764
7265
|
this.db.update(schedules).set({
|
|
6765
7266
|
nextRunAt: nextRunFromCron(cronExpr, timezone),
|
|
6766
7267
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
6767
|
-
}).where(
|
|
7268
|
+
}).where(eq14(schedules.id, scheduleId)).run();
|
|
6768
7269
|
const label = schedule.preset ?? cronExpr;
|
|
6769
|
-
|
|
7270
|
+
log14.info("cron.registered", { projectId, kind, schedule: label, timezone });
|
|
6770
7271
|
}
|
|
6771
7272
|
triggerRun(scheduleId, projectId, kind) {
|
|
6772
7273
|
try {
|
|
6773
7274
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
6774
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
7275
|
+
const currentSchedule = this.db.select().from(schedules).where(eq14(schedules.id, scheduleId)).get();
|
|
6775
7276
|
if (!currentSchedule || !currentSchedule.enabled) {
|
|
6776
|
-
|
|
7277
|
+
log14.warn("schedule.stale", { scheduleId, projectId, kind, msg: "schedule no longer exists or is disabled" });
|
|
6777
7278
|
this.remove(projectId, kind);
|
|
6778
7279
|
return;
|
|
6779
7280
|
}
|
|
6780
7281
|
const nextRunAt = nextRunFromCron(currentSchedule.cronExpr, currentSchedule.timezone);
|
|
6781
|
-
const project = this.db.select().from(projects).where(
|
|
7282
|
+
const project = this.db.select().from(projects).where(eq14(projects.id, projectId)).get();
|
|
6782
7283
|
if (!project) {
|
|
6783
|
-
|
|
7284
|
+
log14.error("project.not-found", { projectId, kind, msg: "skipping scheduled run" });
|
|
6784
7285
|
this.remove(projectId, kind);
|
|
6785
7286
|
return;
|
|
6786
7287
|
}
|
|
6787
7288
|
if (kind === SchedulableRunKinds["traffic-sync"]) {
|
|
6788
7289
|
const sourceId = currentSchedule.sourceId;
|
|
6789
7290
|
if (!sourceId) {
|
|
6790
|
-
|
|
7291
|
+
log14.warn("traffic-sync.missing-source", { scheduleId, projectId });
|
|
6791
7292
|
return;
|
|
6792
7293
|
}
|
|
6793
7294
|
if (!this.callbacks.onTrafficSyncRequested) {
|
|
6794
|
-
|
|
7295
|
+
log14.warn("traffic-sync.no-callback", { scheduleId, projectId, msg: "host did not register onTrafficSyncRequested" });
|
|
6795
7296
|
return;
|
|
6796
7297
|
}
|
|
6797
7298
|
this.db.update(schedules).set({
|
|
6798
7299
|
lastRunAt: now,
|
|
6799
7300
|
nextRunAt,
|
|
6800
7301
|
updatedAt: now
|
|
6801
|
-
}).where(
|
|
6802
|
-
|
|
7302
|
+
}).where(eq14(schedules.id, currentSchedule.id)).run();
|
|
7303
|
+
log14.info("traffic-sync.triggered", { projectName: project.name, sourceId });
|
|
6803
7304
|
this.callbacks.onTrafficSyncRequested(project.name, sourceId);
|
|
6804
7305
|
return;
|
|
6805
7306
|
}
|
|
6806
7307
|
if (kind === SchedulableRunKinds["gbp-sync"]) {
|
|
6807
7308
|
if (!this.callbacks.onGbpSyncRequested) {
|
|
6808
|
-
|
|
7309
|
+
log14.warn("gbp-sync.no-callback", { scheduleId, projectId, msg: "host did not register onGbpSyncRequested" });
|
|
6809
7310
|
return;
|
|
6810
7311
|
}
|
|
6811
|
-
const runId2 =
|
|
7312
|
+
const runId2 = crypto16.randomUUID();
|
|
6812
7313
|
this.db.insert(runs).values({
|
|
6813
7314
|
id: runId2,
|
|
6814
7315
|
projectId,
|
|
@@ -6821,55 +7322,88 @@ var Scheduler = class {
|
|
|
6821
7322
|
lastRunAt: now,
|
|
6822
7323
|
nextRunAt,
|
|
6823
7324
|
updatedAt: now
|
|
6824
|
-
}).where(
|
|
6825
|
-
|
|
7325
|
+
}).where(eq14(schedules.id, currentSchedule.id)).run();
|
|
7326
|
+
log14.info("gbp-sync.triggered", { runId: runId2, projectName: project.name });
|
|
6826
7327
|
this.callbacks.onGbpSyncRequested(runId2, projectId);
|
|
6827
7328
|
return;
|
|
6828
7329
|
}
|
|
7330
|
+
if (kind === SchedulableRunKinds["ads-sync"]) {
|
|
7331
|
+
if (!this.callbacks.onAdsSyncRequested) {
|
|
7332
|
+
log14.warn("ads-sync.no-callback", { scheduleId, projectId, msg: "host did not register onAdsSyncRequested" });
|
|
7333
|
+
return;
|
|
7334
|
+
}
|
|
7335
|
+
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(and11(
|
|
7336
|
+
eq14(runs.projectId, projectId),
|
|
7337
|
+
eq14(runs.kind, RunKinds["ads-sync"]),
|
|
7338
|
+
inArray5(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
7339
|
+
)).get();
|
|
7340
|
+
if (activeAdsRun) {
|
|
7341
|
+
log14.info("ads-sync.skipped-active", { projectName: project.name, activeRunId: activeAdsRun.id });
|
|
7342
|
+
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq14(schedules.id, currentSchedule.id)).run();
|
|
7343
|
+
return;
|
|
7344
|
+
}
|
|
7345
|
+
const runId2 = crypto16.randomUUID();
|
|
7346
|
+
this.db.insert(runs).values({
|
|
7347
|
+
id: runId2,
|
|
7348
|
+
projectId,
|
|
7349
|
+
kind: RunKinds["ads-sync"],
|
|
7350
|
+
status: RunStatuses.queued,
|
|
7351
|
+
trigger: RunTriggers.scheduled,
|
|
7352
|
+
createdAt: now
|
|
7353
|
+
}).run();
|
|
7354
|
+
this.db.update(schedules).set({
|
|
7355
|
+
lastRunAt: now,
|
|
7356
|
+
nextRunAt,
|
|
7357
|
+
updatedAt: now
|
|
7358
|
+
}).where(eq14(schedules.id, currentSchedule.id)).run();
|
|
7359
|
+
log14.info("ads-sync.triggered", { runId: runId2, projectName: project.name });
|
|
7360
|
+
this.callbacks.onAdsSyncRequested(runId2, projectId);
|
|
7361
|
+
return;
|
|
7362
|
+
}
|
|
6829
7363
|
if (kind === SchedulableRunKinds["data-refresh"]) {
|
|
6830
7364
|
if (!this.callbacks.onDataRefreshRequested) {
|
|
6831
|
-
|
|
7365
|
+
log14.warn("data-refresh.no-callback", { scheduleId, projectId, msg: "host did not register onDataRefreshRequested" });
|
|
6832
7366
|
return;
|
|
6833
7367
|
}
|
|
6834
7368
|
this.db.update(schedules).set({
|
|
6835
7369
|
lastRunAt: now,
|
|
6836
7370
|
nextRunAt,
|
|
6837
7371
|
updatedAt: now
|
|
6838
|
-
}).where(
|
|
6839
|
-
|
|
7372
|
+
}).where(eq14(schedules.id, currentSchedule.id)).run();
|
|
7373
|
+
log14.info("data-refresh.triggered", { projectName: project.name });
|
|
6840
7374
|
this.callbacks.onDataRefreshRequested(project.name);
|
|
6841
7375
|
return;
|
|
6842
7376
|
}
|
|
6843
7377
|
if (kind === SchedulableRunKinds["backlinks-sync"]) {
|
|
6844
7378
|
if (!this.callbacks.onBacklinksSyncRequested) {
|
|
6845
|
-
|
|
7379
|
+
log14.warn("backlinks-sync.no-callback", { scheduleId, projectId, msg: "host did not register onBacklinksSyncRequested" });
|
|
6846
7380
|
return;
|
|
6847
7381
|
}
|
|
6848
7382
|
this.db.update(schedules).set({
|
|
6849
7383
|
lastRunAt: now,
|
|
6850
7384
|
nextRunAt,
|
|
6851
7385
|
updatedAt: now
|
|
6852
|
-
}).where(
|
|
6853
|
-
|
|
7386
|
+
}).where(eq14(schedules.id, currentSchedule.id)).run();
|
|
7387
|
+
log14.info("backlinks-sync.triggered", { projectName: project.name });
|
|
6854
7388
|
this.callbacks.onBacklinksSyncRequested(project.name);
|
|
6855
7389
|
return;
|
|
6856
7390
|
}
|
|
6857
7391
|
if (kind === SchedulableRunKinds["site-audit"]) {
|
|
6858
7392
|
if (!this.callbacks.onSiteAuditRequested) {
|
|
6859
|
-
|
|
7393
|
+
log14.warn("site-audit.no-callback", { scheduleId, projectId, msg: "host did not register onSiteAuditRequested" });
|
|
6860
7394
|
return;
|
|
6861
7395
|
}
|
|
6862
|
-
const active = this.db.select({ id: runs.id }).from(runs).where(
|
|
6863
|
-
|
|
6864
|
-
|
|
7396
|
+
const active = this.db.select({ id: runs.id }).from(runs).where(and11(
|
|
7397
|
+
eq14(runs.projectId, projectId),
|
|
7398
|
+
eq14(runs.kind, RunKinds["site-audit"]),
|
|
6865
7399
|
inArray5(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
6866
7400
|
)).get();
|
|
6867
7401
|
if (active) {
|
|
6868
|
-
|
|
6869
|
-
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(
|
|
7402
|
+
log14.info("site-audit.skipped-active", { projectName: project.name, activeRunId: active.id });
|
|
7403
|
+
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq14(schedules.id, currentSchedule.id)).run();
|
|
6870
7404
|
return;
|
|
6871
7405
|
}
|
|
6872
|
-
const runId2 =
|
|
7406
|
+
const runId2 = crypto16.randomUUID();
|
|
6873
7407
|
this.db.insert(runs).values({
|
|
6874
7408
|
id: runId2,
|
|
6875
7409
|
projectId,
|
|
@@ -6882,8 +7416,8 @@ var Scheduler = class {
|
|
|
6882
7416
|
lastRunAt: now,
|
|
6883
7417
|
nextRunAt,
|
|
6884
7418
|
updatedAt: now
|
|
6885
|
-
}).where(
|
|
6886
|
-
|
|
7419
|
+
}).where(eq14(schedules.id, currentSchedule.id)).run();
|
|
7420
|
+
log14.info("site-audit.triggered", { runId: runId2, projectName: project.name });
|
|
6887
7421
|
this.callbacks.onSiteAuditRequested(runId2, projectId);
|
|
6888
7422
|
return;
|
|
6889
7423
|
}
|
|
@@ -6892,7 +7426,7 @@ var Scheduler = class {
|
|
|
6892
7426
|
if (project.defaultLocation) {
|
|
6893
7427
|
const loc = projectLocations.find((l) => l.label === project.defaultLocation);
|
|
6894
7428
|
if (!loc) {
|
|
6895
|
-
|
|
7429
|
+
log14.warn("default-location.stale", { scheduleId, projectId, label: project.defaultLocation });
|
|
6896
7430
|
return;
|
|
6897
7431
|
}
|
|
6898
7432
|
resolvedLocation = loc;
|
|
@@ -6906,11 +7440,11 @@ var Scheduler = class {
|
|
|
6906
7440
|
location: locationLabel
|
|
6907
7441
|
});
|
|
6908
7442
|
if (queueResult.conflict) {
|
|
6909
|
-
|
|
7443
|
+
log14.info("run.skipped-active", { projectName: project.name, activeRunId: queueResult.activeRunId });
|
|
6910
7444
|
this.db.update(schedules).set({
|
|
6911
7445
|
nextRunAt,
|
|
6912
7446
|
updatedAt: now
|
|
6913
|
-
}).where(
|
|
7447
|
+
}).where(eq14(schedules.id, currentSchedule.id)).run();
|
|
6914
7448
|
return;
|
|
6915
7449
|
}
|
|
6916
7450
|
const runId = queueResult.runId;
|
|
@@ -6918,43 +7452,44 @@ var Scheduler = class {
|
|
|
6918
7452
|
lastRunAt: now,
|
|
6919
7453
|
nextRunAt,
|
|
6920
7454
|
updatedAt: now
|
|
6921
|
-
}).where(
|
|
7455
|
+
}).where(eq14(schedules.id, currentSchedule.id)).run();
|
|
6922
7456
|
const scheduleProviders = currentSchedule.providers;
|
|
6923
7457
|
const providers = scheduleProviders.length > 0 ? scheduleProviders : void 0;
|
|
6924
|
-
|
|
7458
|
+
log14.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
6925
7459
|
this.callbacks.onRunCreated(runId, projectId, providers, resolvedLocation);
|
|
6926
7460
|
} catch (err) {
|
|
6927
|
-
|
|
7461
|
+
log14.error("trigger.error", { scheduleId, projectId, kind, error: err instanceof Error ? err.message : String(err) });
|
|
6928
7462
|
}
|
|
6929
7463
|
}
|
|
6930
7464
|
};
|
|
6931
7465
|
|
|
6932
7466
|
// src/data-refresh.ts
|
|
6933
|
-
var
|
|
7467
|
+
var log15 = createLogger("DataRefresh");
|
|
6934
7468
|
async function refreshAllIntegrations(client, projectName) {
|
|
6935
7469
|
const integrations = [
|
|
6936
7470
|
{ name: "gsc", run: () => client.gscSync(projectName, {}) },
|
|
6937
7471
|
{ name: "bing", run: () => client.bingInspectSitemap(projectName, {}) },
|
|
6938
7472
|
{ name: "ga", run: () => client.gaSync(projectName, { days: 30 }) },
|
|
6939
|
-
{ name: "gbp", run: () => client.triggerGbpSync(projectName, {}) }
|
|
7473
|
+
{ name: "gbp", run: () => client.triggerGbpSync(projectName, {}) },
|
|
7474
|
+
{ name: "ads", run: () => client.triggerAdsSync(projectName) }
|
|
6940
7475
|
];
|
|
6941
7476
|
const results = await Promise.allSettled(integrations.map((i) => i.run()));
|
|
6942
7477
|
results.forEach((result, idx) => {
|
|
6943
7478
|
const integration = integrations[idx].name;
|
|
6944
7479
|
if (result.status === "fulfilled") {
|
|
6945
|
-
|
|
7480
|
+
log15.info("integration.refreshed", { projectName, integration });
|
|
6946
7481
|
} else {
|
|
6947
7482
|
const reason = result.reason;
|
|
6948
7483
|
const message = reason instanceof Error ? reason.message : String(reason);
|
|
6949
|
-
|
|
7484
|
+
log15.warn("integration.refresh-failed", { projectName, integration, error: message });
|
|
6950
7485
|
}
|
|
6951
7486
|
});
|
|
6952
7487
|
}
|
|
6953
7488
|
|
|
6954
7489
|
// src/notifier.ts
|
|
6955
|
-
import { eq as
|
|
6956
|
-
import
|
|
6957
|
-
var
|
|
7490
|
+
import { eq as eq15, desc as desc5, and as and12, inArray as inArray6, or } from "drizzle-orm";
|
|
7491
|
+
import crypto17 from "crypto";
|
|
7492
|
+
var log16 = createLogger("Notifier");
|
|
6958
7493
|
var Notifier = class {
|
|
6959
7494
|
db;
|
|
6960
7495
|
serverUrl;
|
|
@@ -6964,26 +7499,26 @@ var Notifier = class {
|
|
|
6964
7499
|
}
|
|
6965
7500
|
/** Called after a run completes (success, partial, or failed). */
|
|
6966
7501
|
async onRunCompleted(runId, projectId) {
|
|
6967
|
-
|
|
6968
|
-
const notifs = this.db.select().from(notifications).where(
|
|
7502
|
+
log16.info("run.completed", { runId, projectId });
|
|
7503
|
+
const notifs = this.db.select().from(notifications).where(eq15(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
6969
7504
|
if (notifs.length === 0) {
|
|
6970
|
-
|
|
7505
|
+
log16.info("notifications.none-enabled", { projectId });
|
|
6971
7506
|
return;
|
|
6972
7507
|
}
|
|
6973
|
-
|
|
6974
|
-
const run = this.db.select().from(runs).where(
|
|
7508
|
+
log16.info("notifications.found", { projectId, count: notifs.length });
|
|
7509
|
+
const run = this.db.select().from(runs).where(eq15(runs.id, runId)).get();
|
|
6975
7510
|
if (!run) {
|
|
6976
|
-
|
|
7511
|
+
log16.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
6977
7512
|
return;
|
|
6978
7513
|
}
|
|
6979
|
-
const project = this.db.select().from(projects).where(
|
|
7514
|
+
const project = this.db.select().from(projects).where(eq15(projects.id, projectId)).get();
|
|
6980
7515
|
if (!project) {
|
|
6981
|
-
|
|
7516
|
+
log16.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
6982
7517
|
return;
|
|
6983
7518
|
}
|
|
6984
7519
|
const transitions = this.computeTransitions(runId, projectId);
|
|
6985
7520
|
const events = [];
|
|
6986
|
-
|
|
7521
|
+
log16.info("run.status", { runId: run.id, status: run.status, projectId });
|
|
6987
7522
|
if (run.status === "completed" || run.status === "partial") {
|
|
6988
7523
|
events.push("run.completed");
|
|
6989
7524
|
}
|
|
@@ -6999,7 +7534,7 @@ var Notifier = class {
|
|
|
6999
7534
|
if (!config.url) continue;
|
|
7000
7535
|
const subscribedEvents = config.events;
|
|
7001
7536
|
const matchingEvents = events.filter((e) => subscribedEvents.includes(e));
|
|
7002
|
-
|
|
7537
|
+
log16.info("notification.match", { notificationId: notif.id, subscribedEvents, matchedEvents: matchingEvents });
|
|
7003
7538
|
if (matchingEvents.length === 0) continue;
|
|
7004
7539
|
for (const event of matchingEvents) {
|
|
7005
7540
|
const relevantTransitions = event === "citation.lost" ? lostTransitions : event === "citation.gained" ? gainedTransitions : transitions;
|
|
@@ -7023,11 +7558,11 @@ var Notifier = class {
|
|
|
7023
7558
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
7024
7559
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
7025
7560
|
if (insightEvents.length === 0) return;
|
|
7026
|
-
const notifs = this.db.select().from(notifications).where(
|
|
7561
|
+
const notifs = this.db.select().from(notifications).where(eq15(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
7027
7562
|
if (notifs.length === 0) return;
|
|
7028
|
-
const run = this.db.select().from(runs).where(
|
|
7563
|
+
const run = this.db.select().from(runs).where(eq15(runs.id, runId)).get();
|
|
7029
7564
|
if (!run) return;
|
|
7030
|
-
const project = this.db.select().from(projects).where(
|
|
7565
|
+
const project = this.db.select().from(projects).where(eq15(projects.id, projectId)).get();
|
|
7031
7566
|
if (!project) return;
|
|
7032
7567
|
for (const notif of notifs) {
|
|
7033
7568
|
const config = notif.config;
|
|
@@ -7057,12 +7592,12 @@ var Notifier = class {
|
|
|
7057
7592
|
}
|
|
7058
7593
|
}
|
|
7059
7594
|
computeTransitions(runId, projectId) {
|
|
7060
|
-
const thisRun = this.db.select().from(runs).where(
|
|
7595
|
+
const thisRun = this.db.select().from(runs).where(eq15(runs.id, runId)).get();
|
|
7061
7596
|
if (!thisRun) return [];
|
|
7062
|
-
const groupSiblings = this.db.select().from(runs).where(
|
|
7063
|
-
|
|
7064
|
-
|
|
7065
|
-
|
|
7597
|
+
const groupSiblings = this.db.select().from(runs).where(and12(
|
|
7598
|
+
eq15(runs.projectId, projectId),
|
|
7599
|
+
eq15(runs.kind, thisRun.kind),
|
|
7600
|
+
eq15(runs.createdAt, thisRun.createdAt)
|
|
7066
7601
|
)).all();
|
|
7067
7602
|
const stillPending = groupSiblings.some((r) => r.status === "queued" || r.status === "running");
|
|
7068
7603
|
if (stillPending) return [];
|
|
@@ -7078,17 +7613,17 @@ var Notifier = class {
|
|
|
7078
7613
|
return candidate.id > best.id ? candidate : best;
|
|
7079
7614
|
});
|
|
7080
7615
|
if (winner.id !== runId) return [];
|
|
7081
|
-
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
7616
|
+
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(eq15(projects.id, projectId)).get();
|
|
7082
7617
|
const locationCount = Math.max(
|
|
7083
7618
|
1,
|
|
7084
7619
|
(projectLocations?.locations ?? []).length
|
|
7085
7620
|
);
|
|
7086
7621
|
const RECENT_FETCH_LIMIT = Math.max(8, locationCount * 4);
|
|
7087
7622
|
const recentRuns = this.db.select().from(runs).where(
|
|
7088
|
-
|
|
7089
|
-
|
|
7090
|
-
|
|
7091
|
-
or(
|
|
7623
|
+
and12(
|
|
7624
|
+
eq15(runs.projectId, projectId),
|
|
7625
|
+
eq15(runs.kind, thisRun.kind),
|
|
7626
|
+
or(eq15(runs.status, "completed"), eq15(runs.status, "partial"))
|
|
7092
7627
|
)
|
|
7093
7628
|
).orderBy(desc5(runs.createdAt), desc5(runs.id)).limit(RECENT_FETCH_LIMIT).all();
|
|
7094
7629
|
const groups = groupRunsByCreatedAt(recentRuns);
|
|
@@ -7105,7 +7640,7 @@ var Notifier = class {
|
|
|
7105
7640
|
provider: querySnapshots.provider,
|
|
7106
7641
|
location: querySnapshots.location,
|
|
7107
7642
|
citationState: querySnapshots.citationState
|
|
7108
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
7643
|
+
}).from(querySnapshots).leftJoin(queries, eq15(querySnapshots.queryId, queries.id)).where(inArray6(querySnapshots.runId, currentRunIds)).all();
|
|
7109
7644
|
const previousSnapshots = this.db.select({
|
|
7110
7645
|
queryId: querySnapshots.queryId,
|
|
7111
7646
|
provider: querySnapshots.provider,
|
|
@@ -7138,23 +7673,23 @@ var Notifier = class {
|
|
|
7138
7673
|
const targetLabel = redactNotificationUrl(url).urlDisplay;
|
|
7139
7674
|
const targetCheck = await resolveWebhookTarget(url);
|
|
7140
7675
|
if (!targetCheck.ok) {
|
|
7141
|
-
|
|
7676
|
+
log16.error("webhook.ssrf-blocked", { url: targetLabel, reason: targetCheck.message });
|
|
7142
7677
|
this.logDelivery(projectId, notificationId, payload.event, "failed", `SSRF: ${targetCheck.message}`);
|
|
7143
7678
|
return;
|
|
7144
7679
|
}
|
|
7145
|
-
|
|
7680
|
+
log16.info("webhook.send", { event: payload.event, url: targetLabel });
|
|
7146
7681
|
const maxRetries = 3;
|
|
7147
7682
|
const delays = [1e3, 4e3, 16e3];
|
|
7148
7683
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
7149
7684
|
try {
|
|
7150
7685
|
const response = await deliverWebhook(targetCheck.target, payload, webhookSecret);
|
|
7151
7686
|
if (response.status >= 200 && response.status < 300) {
|
|
7152
|
-
|
|
7687
|
+
log16.info("webhook.delivered", { event: payload.event, url: targetLabel, httpStatus: response.status });
|
|
7153
7688
|
this.logDelivery(projectId, notificationId, payload.event, "sent", null);
|
|
7154
7689
|
return;
|
|
7155
7690
|
}
|
|
7156
7691
|
const errorDetail = response.error ?? `HTTP ${response.status}`;
|
|
7157
|
-
|
|
7692
|
+
log16.warn("webhook.attempt-failed", { event: payload.event, url: targetLabel, attempt: attempt + 1, maxRetries, httpStatus: response.status, error: errorDetail });
|
|
7158
7693
|
if (attempt === maxRetries - 1) {
|
|
7159
7694
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
7160
7695
|
}
|
|
@@ -7162,7 +7697,7 @@ var Notifier = class {
|
|
|
7162
7697
|
const errorDetail = err instanceof Error ? err.message : String(err);
|
|
7163
7698
|
if (attempt === maxRetries - 1) {
|
|
7164
7699
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
7165
|
-
|
|
7700
|
+
log16.error("webhook.exhausted", { event: payload.event, url: targetLabel, maxRetries, error: errorDetail });
|
|
7166
7701
|
}
|
|
7167
7702
|
}
|
|
7168
7703
|
if (attempt < maxRetries - 1) {
|
|
@@ -7172,7 +7707,7 @@ var Notifier = class {
|
|
|
7172
7707
|
}
|
|
7173
7708
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
7174
7709
|
this.db.insert(auditLog).values({
|
|
7175
|
-
id:
|
|
7710
|
+
id: crypto17.randomUUID(),
|
|
7176
7711
|
projectId,
|
|
7177
7712
|
actor: "scheduler",
|
|
7178
7713
|
action: `notification.${status}`,
|
|
@@ -7185,8 +7720,8 @@ var Notifier = class {
|
|
|
7185
7720
|
};
|
|
7186
7721
|
|
|
7187
7722
|
// src/run-coordinator.ts
|
|
7188
|
-
import { eq as
|
|
7189
|
-
var
|
|
7723
|
+
import { eq as eq16 } from "drizzle-orm";
|
|
7724
|
+
var log17 = createLogger("RunCoordinator");
|
|
7190
7725
|
var RunCoordinator = class {
|
|
7191
7726
|
constructor(db, notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
7192
7727
|
this.db = db;
|
|
@@ -7201,10 +7736,10 @@ var RunCoordinator = class {
|
|
|
7201
7736
|
onInsightsGenerated;
|
|
7202
7737
|
onAeroEvent;
|
|
7203
7738
|
async onRunCompleted(runId, projectId) {
|
|
7204
|
-
const runRow = this.db.select().from(runs).where(
|
|
7739
|
+
const runRow = this.db.select().from(runs).where(eq16(runs.id, runId)).get();
|
|
7205
7740
|
const kind = runRow?.kind ?? RunKinds["answer-visibility"];
|
|
7206
7741
|
if (runRow?.trigger === RunTriggers.probe) {
|
|
7207
|
-
|
|
7742
|
+
log17.info("probe.skip-side-effects", { runId, projectId, kind });
|
|
7208
7743
|
return;
|
|
7209
7744
|
}
|
|
7210
7745
|
let insightCount = 0;
|
|
@@ -7221,12 +7756,12 @@ var RunCoordinator = class {
|
|
|
7221
7756
|
try {
|
|
7222
7757
|
await this.onInsightsGenerated(runId, projectId, result);
|
|
7223
7758
|
} catch (err) {
|
|
7224
|
-
|
|
7759
|
+
log17.error("insight-webhook.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
7225
7760
|
}
|
|
7226
7761
|
}
|
|
7227
7762
|
}
|
|
7228
7763
|
} catch (err) {
|
|
7229
|
-
|
|
7764
|
+
log17.error("intelligence.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
7230
7765
|
}
|
|
7231
7766
|
} else if (kind === RunKinds["gbp-sync"]) {
|
|
7232
7767
|
try {
|
|
@@ -7239,17 +7774,17 @@ var RunCoordinator = class {
|
|
|
7239
7774
|
try {
|
|
7240
7775
|
await this.onInsightsGenerated(runId, projectId, analysisResultFromInsights(gbpInsights));
|
|
7241
7776
|
} catch (err) {
|
|
7242
|
-
|
|
7777
|
+
log17.error("gbp-insight-webhook.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
7243
7778
|
}
|
|
7244
7779
|
}
|
|
7245
7780
|
} catch (err) {
|
|
7246
|
-
|
|
7781
|
+
log17.error("gbp-intelligence.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
7247
7782
|
}
|
|
7248
7783
|
}
|
|
7249
7784
|
try {
|
|
7250
7785
|
await this.notifier.onRunCompleted(runId, projectId);
|
|
7251
7786
|
} catch (err) {
|
|
7252
|
-
|
|
7787
|
+
log17.error("notifier.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
7253
7788
|
}
|
|
7254
7789
|
if (this.onAeroEvent) {
|
|
7255
7790
|
try {
|
|
@@ -7262,7 +7797,7 @@ var RunCoordinator = class {
|
|
|
7262
7797
|
};
|
|
7263
7798
|
await this.onAeroEvent(ctx);
|
|
7264
7799
|
} catch (err) {
|
|
7265
|
-
|
|
7800
|
+
log17.error("aero.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
7266
7801
|
}
|
|
7267
7802
|
}
|
|
7268
7803
|
}
|
|
@@ -7277,7 +7812,7 @@ var RunCoordinator = class {
|
|
|
7277
7812
|
* so the Aero queue is never starved of a follow-up.
|
|
7278
7813
|
*/
|
|
7279
7814
|
buildDiscoveryAeroContext(runId, projectId, status, error) {
|
|
7280
|
-
const session = this.db.select().from(discoverySessions).where(
|
|
7815
|
+
const session = this.db.select().from(discoverySessions).where(eq16(discoverySessions.runId, runId)).get();
|
|
7281
7816
|
const competitorMap = session ? session.competitorMap : [];
|
|
7282
7817
|
return {
|
|
7283
7818
|
kind: RunKinds["aeo-discover-probe"],
|
|
@@ -7317,8 +7852,8 @@ function analysisResultFromInsights(insights2) {
|
|
|
7317
7852
|
}
|
|
7318
7853
|
|
|
7319
7854
|
// src/agent/session-registry.ts
|
|
7320
|
-
import
|
|
7321
|
-
import { eq as
|
|
7855
|
+
import crypto19 from "crypto";
|
|
7856
|
+
import { eq as eq18 } from "drizzle-orm";
|
|
7322
7857
|
|
|
7323
7858
|
// src/agent/session.ts
|
|
7324
7859
|
import fs7 from "fs";
|
|
@@ -7706,8 +8241,8 @@ function resolveSessionProviderAndModel(config, opts) {
|
|
|
7706
8241
|
}
|
|
7707
8242
|
|
|
7708
8243
|
// src/agent/memory-store.ts
|
|
7709
|
-
import
|
|
7710
|
-
import { and as
|
|
8244
|
+
import crypto18 from "crypto";
|
|
8245
|
+
import { and as and13, desc as desc6, eq as eq17, like, sql as sql5 } from "drizzle-orm";
|
|
7711
8246
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
7712
8247
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
7713
8248
|
function rowToDto(row) {
|
|
@@ -7721,7 +8256,7 @@ function rowToDto(row) {
|
|
|
7721
8256
|
};
|
|
7722
8257
|
}
|
|
7723
8258
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
7724
|
-
const query = db.select().from(agentMemory).where(
|
|
8259
|
+
const query = db.select().from(agentMemory).where(eq17(agentMemory.projectId, projectId)).orderBy(desc6(agentMemory.updatedAt));
|
|
7725
8260
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
7726
8261
|
return rows.map(rowToDto);
|
|
7727
8262
|
}
|
|
@@ -7735,7 +8270,7 @@ function upsertMemoryEntry(db, args) {
|
|
|
7735
8270
|
throw new Error(`memory key prefix "${COMPACTION_KEY_PREFIX}" is reserved for compaction notes`);
|
|
7736
8271
|
}
|
|
7737
8272
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7738
|
-
const id =
|
|
8273
|
+
const id = crypto18.randomUUID();
|
|
7739
8274
|
db.insert(agentMemory).values({
|
|
7740
8275
|
id,
|
|
7741
8276
|
projectId: args.projectId,
|
|
@@ -7752,12 +8287,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
7752
8287
|
updatedAt: now
|
|
7753
8288
|
}
|
|
7754
8289
|
}).run();
|
|
7755
|
-
const row = db.select().from(agentMemory).where(
|
|
8290
|
+
const row = db.select().from(agentMemory).where(and13(eq17(agentMemory.projectId, args.projectId), eq17(agentMemory.key, args.key))).get();
|
|
7756
8291
|
if (!row) throw new Error("memory upsert produced no row");
|
|
7757
8292
|
return rowToDto(row);
|
|
7758
8293
|
}
|
|
7759
8294
|
function deleteMemoryEntry(db, projectId, key) {
|
|
7760
|
-
const result = db.delete(agentMemory).where(
|
|
8295
|
+
const result = db.delete(agentMemory).where(and13(eq17(agentMemory.projectId, projectId), eq17(agentMemory.key, key))).run();
|
|
7761
8296
|
const changes = result.changes ?? 0;
|
|
7762
8297
|
return changes > 0;
|
|
7763
8298
|
}
|
|
@@ -7772,7 +8307,7 @@ function writeCompactionNote(db, args) {
|
|
|
7772
8307
|
}
|
|
7773
8308
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7774
8309
|
const key = `${COMPACTION_KEY_PREFIX}${args.sessionId}:${now}`;
|
|
7775
|
-
const id =
|
|
8310
|
+
const id = crypto18.randomUUID();
|
|
7776
8311
|
let inserted;
|
|
7777
8312
|
db.transaction((tx) => {
|
|
7778
8313
|
tx.insert(agentMemory).values({
|
|
@@ -7786,8 +8321,8 @@ function writeCompactionNote(db, args) {
|
|
|
7786
8321
|
}).run();
|
|
7787
8322
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
7788
8323
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
7789
|
-
|
|
7790
|
-
|
|
8324
|
+
and13(
|
|
8325
|
+
eq17(agentMemory.projectId, args.projectId),
|
|
7791
8326
|
like(agentMemory.key, `${sessionPrefix}%`)
|
|
7792
8327
|
)
|
|
7793
8328
|
).orderBy(desc6(agentMemory.updatedAt)).all();
|
|
@@ -7795,7 +8330,7 @@ function writeCompactionNote(db, args) {
|
|
|
7795
8330
|
if (stale.length > 0) {
|
|
7796
8331
|
tx.delete(agentMemory).where(sql5`${agentMemory.id} IN (${sql5.join(stale.map((s) => sql5`${s}`), sql5`, `)})`).run();
|
|
7797
8332
|
}
|
|
7798
|
-
const row = tx.select().from(agentMemory).where(
|
|
8333
|
+
const row = tx.select().from(agentMemory).where(and13(eq17(agentMemory.projectId, args.projectId), eq17(agentMemory.key, key))).get();
|
|
7799
8334
|
if (row) inserted = rowToDto(row);
|
|
7800
8335
|
});
|
|
7801
8336
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -7928,7 +8463,7 @@ async function compactMessages(args) {
|
|
|
7928
8463
|
}
|
|
7929
8464
|
|
|
7930
8465
|
// src/agent/session-registry.ts
|
|
7931
|
-
var
|
|
8466
|
+
var log18 = createLogger("SessionRegistry");
|
|
7932
8467
|
var MAX_HYDRATE_NOTES = 20;
|
|
7933
8468
|
var MAX_HYDRATE_BYTES = 32 * 1024;
|
|
7934
8469
|
function escapeMemoryFragment(value) {
|
|
@@ -7977,7 +8512,7 @@ var SessionRegistry = class {
|
|
|
7977
8512
|
modelProvider: effectiveProvider,
|
|
7978
8513
|
modelId: effectiveModelId,
|
|
7979
8514
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
7980
|
-
}).where(
|
|
8515
|
+
}).where(eq18(agentSessions.projectId, projectId)).run();
|
|
7981
8516
|
}
|
|
7982
8517
|
const agent2 = createAeroSession({
|
|
7983
8518
|
projectName,
|
|
@@ -8155,13 +8690,13 @@ ${lines.join("\n")}
|
|
|
8155
8690
|
agent.state.messages = result.messages;
|
|
8156
8691
|
agent.state.systemPrompt = this.buildHydratedSystemPrompt(projectId, row.systemPrompt);
|
|
8157
8692
|
this.save(projectName);
|
|
8158
|
-
|
|
8693
|
+
log18.info("compaction.completed", {
|
|
8159
8694
|
projectName,
|
|
8160
8695
|
removedCount: result.removedCount,
|
|
8161
8696
|
summaryBytes: Buffer.byteLength(result.summary, "utf8")
|
|
8162
8697
|
});
|
|
8163
8698
|
} catch (err) {
|
|
8164
|
-
|
|
8699
|
+
log18.error("compaction.failed", {
|
|
8165
8700
|
projectName,
|
|
8166
8701
|
error: err instanceof Error ? err.message : String(err)
|
|
8167
8702
|
});
|
|
@@ -8191,7 +8726,7 @@ ${lines.join("\n")}
|
|
|
8191
8726
|
modelProvider: nextProvider,
|
|
8192
8727
|
modelId: nextModelId,
|
|
8193
8728
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
8194
|
-
}).where(
|
|
8729
|
+
}).where(eq18(agentSessions.projectId, projectId)).run();
|
|
8195
8730
|
}
|
|
8196
8731
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
8197
8732
|
save(projectName) {
|
|
@@ -8258,7 +8793,7 @@ ${lines.join("\n")}
|
|
|
8258
8793
|
await agent.prompt(msgs);
|
|
8259
8794
|
this.save(projectName);
|
|
8260
8795
|
} catch (err) {
|
|
8261
|
-
|
|
8796
|
+
log18.error("drain.failed", {
|
|
8262
8797
|
projectName,
|
|
8263
8798
|
error: err instanceof Error ? err.message : String(err)
|
|
8264
8799
|
});
|
|
@@ -8353,17 +8888,17 @@ ${lines.join("\n")}
|
|
|
8353
8888
|
return id;
|
|
8354
8889
|
}
|
|
8355
8890
|
tryResolveProjectId(projectName) {
|
|
8356
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
8891
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq18(projects.name, projectName)).get();
|
|
8357
8892
|
return row?.id;
|
|
8358
8893
|
}
|
|
8359
8894
|
loadRow(projectId) {
|
|
8360
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
8895
|
+
const row = this.opts.db.select().from(agentSessions).where(eq18(agentSessions.projectId, projectId)).get();
|
|
8361
8896
|
return row ?? null;
|
|
8362
8897
|
}
|
|
8363
8898
|
insertRow(params) {
|
|
8364
8899
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8365
8900
|
this.opts.db.insert(agentSessions).values({
|
|
8366
|
-
id:
|
|
8901
|
+
id: crypto19.randomUUID(),
|
|
8367
8902
|
projectId: params.projectId,
|
|
8368
8903
|
systemPrompt: params.systemPrompt,
|
|
8369
8904
|
modelProvider: params.provider ?? params.modelProvider ?? AgentProviderIds.claude,
|
|
@@ -8376,14 +8911,14 @@ ${lines.join("\n")}
|
|
|
8376
8911
|
}
|
|
8377
8912
|
updateRow(projectId, patch) {
|
|
8378
8913
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8379
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
8914
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq18(agentSessions.projectId, projectId)).run();
|
|
8380
8915
|
}
|
|
8381
8916
|
};
|
|
8382
8917
|
|
|
8383
8918
|
// src/agent/agent-routes.ts
|
|
8384
|
-
import { eq as
|
|
8919
|
+
import { eq as eq19 } from "drizzle-orm";
|
|
8385
8920
|
function resolveProject(db, name) {
|
|
8386
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
8921
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq19(projects.name, name)).get();
|
|
8387
8922
|
if (!row) throw notFound("project", name);
|
|
8388
8923
|
return row;
|
|
8389
8924
|
}
|
|
@@ -8392,7 +8927,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
8392
8927
|
"/projects/:name/agent/transcript",
|
|
8393
8928
|
async (request) => {
|
|
8394
8929
|
const project = resolveProject(opts.db, request.params.name);
|
|
8395
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
8930
|
+
const row = opts.db.select().from(agentSessions).where(eq19(agentSessions.projectId, project.id)).get();
|
|
8396
8931
|
if (!row) {
|
|
8397
8932
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
8398
8933
|
}
|
|
@@ -8416,7 +8951,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
8416
8951
|
async (request) => {
|
|
8417
8952
|
const project = resolveProject(opts.db, request.params.name);
|
|
8418
8953
|
opts.sessionRegistry.reset(project.name);
|
|
8419
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
8954
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq19(agentSessions.projectId, project.id)).run();
|
|
8420
8955
|
return { status: "reset" };
|
|
8421
8956
|
}
|
|
8422
8957
|
);
|
|
@@ -8856,7 +9391,7 @@ function formatAuditFactorScore(factor) {
|
|
|
8856
9391
|
}
|
|
8857
9392
|
|
|
8858
9393
|
// src/snapshot-service.ts
|
|
8859
|
-
var
|
|
9394
|
+
var log19 = createLogger("Snapshot");
|
|
8860
9395
|
var ANALYSIS_PROVIDER_PRIORITY = ["openai", "claude", "gemini", "perplexity", "local"];
|
|
8861
9396
|
var SNAPSHOT_QUERY_COUNT = 6;
|
|
8862
9397
|
var ProviderExecutionGate2 = class {
|
|
@@ -9002,7 +9537,7 @@ var SnapshotService = class {
|
|
|
9002
9537
|
return mapAuditReport(report);
|
|
9003
9538
|
} catch (err) {
|
|
9004
9539
|
const message = err instanceof Error ? err.message : String(err);
|
|
9005
|
-
|
|
9540
|
+
log19.warn("audit.failed", { homepageUrl, error: message });
|
|
9006
9541
|
return {
|
|
9007
9542
|
url: homepageUrl,
|
|
9008
9543
|
finalUrl: homepageUrl,
|
|
@@ -9031,7 +9566,7 @@ var SnapshotService = class {
|
|
|
9031
9566
|
queries: parsedQueries
|
|
9032
9567
|
};
|
|
9033
9568
|
} catch (err) {
|
|
9034
|
-
|
|
9569
|
+
log19.warn("profile.generation-failed", {
|
|
9035
9570
|
domain: ctx.domain,
|
|
9036
9571
|
provider: ctx.analysisProvider.adapter.name,
|
|
9037
9572
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -9173,7 +9708,7 @@ var SnapshotService = class {
|
|
|
9173
9708
|
recommendedActions: uniqueStrings(parsed.recommendedActions ?? []).slice(0, 4)
|
|
9174
9709
|
};
|
|
9175
9710
|
} catch (err) {
|
|
9176
|
-
|
|
9711
|
+
log19.warn("response.analysis-failed", {
|
|
9177
9712
|
provider: ctx.analysisProvider.adapter.name,
|
|
9178
9713
|
error: err instanceof Error ? err.message : String(err)
|
|
9179
9714
|
});
|
|
@@ -9455,7 +9990,7 @@ function clipText(value, length) {
|
|
|
9455
9990
|
// src/server.ts
|
|
9456
9991
|
var _require3 = createRequire3(import.meta.url);
|
|
9457
9992
|
var { version: PKG_VERSION2 } = _require3("../package.json");
|
|
9458
|
-
var
|
|
9993
|
+
var log20 = createLogger("Server");
|
|
9459
9994
|
var DEFAULT_QUOTA = {
|
|
9460
9995
|
maxConcurrency: 2,
|
|
9461
9996
|
maxRequestsPerMinute: 10,
|
|
@@ -9490,14 +10025,14 @@ function summarizeProviderConfig(config) {
|
|
|
9490
10025
|
};
|
|
9491
10026
|
}
|
|
9492
10027
|
function hashApiKey(key) {
|
|
9493
|
-
return
|
|
10028
|
+
return crypto20.createHash("sha256").update(key).digest("hex");
|
|
9494
10029
|
}
|
|
9495
10030
|
var DASHBOARD_SCRYPT_KEYLEN = 64;
|
|
9496
10031
|
var DASHBOARD_SCRYPT_COST = 1 << 15;
|
|
9497
10032
|
var DASHBOARD_SCRYPT_MAXMEM = 64 * 1024 * 1024;
|
|
9498
10033
|
function hashDashboardPassword(password) {
|
|
9499
|
-
const salt =
|
|
9500
|
-
const derived =
|
|
10034
|
+
const salt = crypto20.randomBytes(16);
|
|
10035
|
+
const derived = crypto20.scryptSync(password, salt, DASHBOARD_SCRYPT_KEYLEN, {
|
|
9501
10036
|
N: DASHBOARD_SCRYPT_COST,
|
|
9502
10037
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
9503
10038
|
});
|
|
@@ -9518,18 +10053,18 @@ function verifyDashboardPassword(password, storedHash) {
|
|
|
9518
10053
|
} catch {
|
|
9519
10054
|
return { ok: false, needsRehash: false };
|
|
9520
10055
|
}
|
|
9521
|
-
const derived =
|
|
10056
|
+
const derived = crypto20.scryptSync(password, salt, expected.length, {
|
|
9522
10057
|
N: DASHBOARD_SCRYPT_COST,
|
|
9523
10058
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
9524
10059
|
});
|
|
9525
10060
|
if (derived.length !== expected.length) return { ok: false, needsRehash: false };
|
|
9526
|
-
return { ok:
|
|
10061
|
+
return { ok: crypto20.timingSafeEqual(derived, expected), needsRehash: false };
|
|
9527
10062
|
}
|
|
9528
10063
|
if (/^[a-f0-9]{64}$/i.test(storedHash)) {
|
|
9529
10064
|
const candidate = Buffer.from(hashApiKey(password), "hex");
|
|
9530
10065
|
const expected = Buffer.from(storedHash, "hex");
|
|
9531
10066
|
if (candidate.length !== expected.length) return { ok: false, needsRehash: false };
|
|
9532
|
-
const ok =
|
|
10067
|
+
const ok = crypto20.timingSafeEqual(candidate, expected);
|
|
9533
10068
|
return { ok, needsRehash: ok };
|
|
9534
10069
|
}
|
|
9535
10070
|
return { ok: false, needsRehash: false };
|
|
@@ -9588,7 +10123,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
9588
10123
|
}
|
|
9589
10124
|
if (migratedGoogle > 0) {
|
|
9590
10125
|
saveConfigPatch({ google: config.google });
|
|
9591
|
-
|
|
10126
|
+
log20.info("credentials.migrated", { type: "google", count: migratedGoogle });
|
|
9592
10127
|
}
|
|
9593
10128
|
let migratedGa4 = 0;
|
|
9594
10129
|
for (const row of rows.ga4) {
|
|
@@ -9606,7 +10141,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
9606
10141
|
}
|
|
9607
10142
|
if (migratedGa4 > 0) {
|
|
9608
10143
|
saveConfigPatch({ ga4: config.ga4 });
|
|
9609
|
-
|
|
10144
|
+
log20.info("credentials.migrated", { type: "ga4", count: migratedGa4 });
|
|
9610
10145
|
}
|
|
9611
10146
|
}
|
|
9612
10147
|
function isLoopbackBindHost(host) {
|
|
@@ -9645,11 +10180,11 @@ async function createServer(opts) {
|
|
|
9645
10180
|
applyLegacyCredentials(legacyRows, opts.config);
|
|
9646
10181
|
dropLegacyCredentialColumns(opts.db);
|
|
9647
10182
|
} catch (err) {
|
|
9648
|
-
|
|
10183
|
+
log20.warn("credentials.migration.failed", {
|
|
9649
10184
|
error: err instanceof Error ? err.message : String(err)
|
|
9650
10185
|
});
|
|
9651
10186
|
}
|
|
9652
|
-
|
|
10187
|
+
log20.info("providers.configured", { providers: Object.keys(providers).filter((k) => {
|
|
9653
10188
|
const p = providers[k];
|
|
9654
10189
|
return p?.apiKey || p?.baseUrl || p?.vertexProject;
|
|
9655
10190
|
}) });
|
|
@@ -9698,7 +10233,7 @@ async function createServer(opts) {
|
|
|
9698
10233
|
intelligenceService,
|
|
9699
10234
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
9700
10235
|
async (ctx) => {
|
|
9701
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
10236
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq20(projects.id, ctx.projectId)).get();
|
|
9702
10237
|
if (!project) return;
|
|
9703
10238
|
let content;
|
|
9704
10239
|
if (ctx.kind === RunKinds["aeo-discover-probe"]) {
|
|
@@ -9741,11 +10276,41 @@ async function createServer(opts) {
|
|
|
9741
10276
|
app.log.error({ runId, err }, "GBP sync failed");
|
|
9742
10277
|
});
|
|
9743
10278
|
};
|
|
10279
|
+
const runAdsSync = (runId, projectId) => {
|
|
10280
|
+
executeAdsSync(opts.db, runId, projectId, { config: opts.config }).then(() => runCoordinator.onRunCompleted(runId, projectId)).catch((err) => {
|
|
10281
|
+
app.log.error({ runId, err }, "Ads sync failed");
|
|
10282
|
+
});
|
|
10283
|
+
};
|
|
9744
10284
|
const runSiteAudit = (runId, projectId, auditOpts) => {
|
|
9745
10285
|
executeSiteAudit(opts.db, runId, projectId, auditOpts ?? {}).then(() => runCoordinator.onRunCompleted(runId, projectId)).catch((err) => {
|
|
9746
10286
|
app.log.error({ runId, err }, "Site audit failed");
|
|
9747
10287
|
});
|
|
9748
10288
|
};
|
|
10289
|
+
const adsCredentialStore = {
|
|
10290
|
+
getConnection: (projectName) => {
|
|
10291
|
+
return getOpenAiAdsConnection(opts.config, projectName);
|
|
10292
|
+
},
|
|
10293
|
+
upsertConnection: (connection) => {
|
|
10294
|
+
const updated = upsertOpenAiAdsConnection(opts.config, connection);
|
|
10295
|
+
saveConfigPatch(opts.config);
|
|
10296
|
+
return updated;
|
|
10297
|
+
},
|
|
10298
|
+
removeConnection: (projectName) => {
|
|
10299
|
+
const removed = removeOpenAiAdsConnection(opts.config, projectName);
|
|
10300
|
+
if (removed) saveConfigPatch(opts.config);
|
|
10301
|
+
return removed;
|
|
10302
|
+
}
|
|
10303
|
+
};
|
|
10304
|
+
const verifyAdsAccount = async (apiKey) => {
|
|
10305
|
+
const account = await getAdAccount(apiKey);
|
|
10306
|
+
return {
|
|
10307
|
+
id: account.id,
|
|
10308
|
+
name: account.name,
|
|
10309
|
+
status: account.status,
|
|
10310
|
+
currencyCode: account.currency_code ?? null,
|
|
10311
|
+
timezone: account.timezone ?? null
|
|
10312
|
+
};
|
|
10313
|
+
};
|
|
9749
10314
|
const scheduler = new Scheduler(opts.db, {
|
|
9750
10315
|
onRunCreated: (runId, projectId, providers2, location) => {
|
|
9751
10316
|
jobRunner.executeRun(runId, projectId, providers2, location).catch((err) => {
|
|
@@ -9760,6 +10325,9 @@ async function createServer(opts) {
|
|
|
9760
10325
|
onGbpSyncRequested: (runId, projectId) => {
|
|
9761
10326
|
runGbpSync(runId, projectId);
|
|
9762
10327
|
},
|
|
10328
|
+
onAdsSyncRequested: (runId, projectId) => {
|
|
10329
|
+
runAdsSync(runId, projectId);
|
|
10330
|
+
},
|
|
9763
10331
|
onDataRefreshRequested: (projectName) => {
|
|
9764
10332
|
void refreshAllIntegrations(aeroClient, projectName);
|
|
9765
10333
|
},
|
|
@@ -9770,9 +10338,9 @@ async function createServer(opts) {
|
|
|
9770
10338
|
return null;
|
|
9771
10339
|
});
|
|
9772
10340
|
if (!probed) return;
|
|
9773
|
-
const alreadySynced = opts.db.select().from(ccReleaseSyncs).where(
|
|
9774
|
-
|
|
9775
|
-
|
|
10341
|
+
const alreadySynced = opts.db.select().from(ccReleaseSyncs).where(and14(
|
|
10342
|
+
eq20(ccReleaseSyncs.release, probed.release),
|
|
10343
|
+
eq20(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)
|
|
9776
10344
|
)).limit(1).get();
|
|
9777
10345
|
if (alreadySynced) {
|
|
9778
10346
|
app.log.info({ projectName, release: probed.release }, "Scheduled backlinks sync: already up to date, skipping");
|
|
@@ -9785,6 +10353,15 @@ async function createServer(opts) {
|
|
|
9785
10353
|
);
|
|
9786
10354
|
});
|
|
9787
10355
|
})();
|
|
10356
|
+
const project = opts.db.select().from(projects).where(eq20(projects.name, projectName)).get();
|
|
10357
|
+
if (project && bingConnectionStore.getConnection(project.canonicalDomain)) {
|
|
10358
|
+
aeroClient.backlinksBingSync(projectName).catch((err) => {
|
|
10359
|
+
app.log.error(
|
|
10360
|
+
{ projectName, err: err instanceof Error ? err.message : String(err) },
|
|
10361
|
+
"Scheduled Bing backlinks sync failed"
|
|
10362
|
+
);
|
|
10363
|
+
});
|
|
10364
|
+
}
|
|
9788
10365
|
},
|
|
9789
10366
|
onSiteAuditRequested: (runId, projectId) => {
|
|
9790
10367
|
runSiteAudit(runId, projectId);
|
|
@@ -9905,7 +10482,7 @@ async function createServer(opts) {
|
|
|
9905
10482
|
return removed;
|
|
9906
10483
|
}
|
|
9907
10484
|
};
|
|
9908
|
-
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ??
|
|
10485
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto20.randomBytes(32).toString("hex");
|
|
9909
10486
|
const googleConnectionStore = {
|
|
9910
10487
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
9911
10488
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -9951,11 +10528,11 @@ async function createServer(opts) {
|
|
|
9951
10528
|
const apiPrefix = basePath ? `${basePath}api/v1` : "/api/v1";
|
|
9952
10529
|
if (opts.config.apiKey) {
|
|
9953
10530
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
9954
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
10531
|
+
const existing = opts.db.select().from(apiKeys).where(eq20(apiKeys.keyHash, keyHash)).get();
|
|
9955
10532
|
if (!existing) {
|
|
9956
10533
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
9957
10534
|
opts.db.insert(apiKeys).values({
|
|
9958
|
-
id: `key_${
|
|
10535
|
+
id: `key_${crypto20.randomBytes(8).toString("hex")}`,
|
|
9959
10536
|
name: "default",
|
|
9960
10537
|
keyHash,
|
|
9961
10538
|
keyPrefix: prefix,
|
|
@@ -9979,7 +10556,7 @@ async function createServer(opts) {
|
|
|
9979
10556
|
};
|
|
9980
10557
|
const createSession = (apiKeyId) => {
|
|
9981
10558
|
pruneExpiredSessions();
|
|
9982
|
-
const sessionId =
|
|
10559
|
+
const sessionId = crypto20.randomBytes(32).toString("hex");
|
|
9983
10560
|
sessions.set(sessionId, {
|
|
9984
10561
|
apiKeyId,
|
|
9985
10562
|
expiresAt: Date.now() + SESSION_TTL_MS
|
|
@@ -10003,7 +10580,7 @@ async function createServer(opts) {
|
|
|
10003
10580
|
};
|
|
10004
10581
|
const getDefaultApiKey = () => {
|
|
10005
10582
|
if (!opts.config.apiKey) return void 0;
|
|
10006
|
-
return opts.db.select().from(apiKeys).where(
|
|
10583
|
+
return opts.db.select().from(apiKeys).where(eq20(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
10007
10584
|
};
|
|
10008
10585
|
const createPasswordSession = (reply) => {
|
|
10009
10586
|
const key = getDefaultApiKey();
|
|
@@ -10024,7 +10601,7 @@ async function createServer(opts) {
|
|
|
10024
10601
|
if (!header) return false;
|
|
10025
10602
|
const parts = header.split(" ");
|
|
10026
10603
|
if (parts.length !== 2 || parts[0] !== "Bearer") return false;
|
|
10027
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
10604
|
+
const key = opts.db.select().from(apiKeys).where(eq20(apiKeys.keyHash, hashApiKey(parts[1]))).get();
|
|
10028
10605
|
return Boolean(key && !key.revokedAt);
|
|
10029
10606
|
};
|
|
10030
10607
|
app.get(apiPrefix + "/session", async (request, reply) => {
|
|
@@ -10078,12 +10655,12 @@ async function createServer(opts) {
|
|
|
10078
10655
|
return reply.send({ authenticated: true });
|
|
10079
10656
|
}
|
|
10080
10657
|
if (apiKey) {
|
|
10081
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
10658
|
+
const key = opts.db.select().from(apiKeys).where(eq20(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
10082
10659
|
if (!key || key.revokedAt) {
|
|
10083
10660
|
const err2 = authInvalid();
|
|
10084
10661
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
10085
10662
|
}
|
|
10086
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
10663
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq20(apiKeys.id, key.id)).run();
|
|
10087
10664
|
const sessionId = createSession(key.id);
|
|
10088
10665
|
reply.header("set-cookie", serializeSessionCookie({
|
|
10089
10666
|
name: SESSION_COOKIE_NAME,
|
|
@@ -10216,6 +10793,11 @@ async function createServer(opts) {
|
|
|
10216
10793
|
onGbpSyncRequested: (runId, projectId, syncOpts) => {
|
|
10217
10794
|
runGbpSync(runId, projectId, syncOpts);
|
|
10218
10795
|
},
|
|
10796
|
+
adsCredentialStore,
|
|
10797
|
+
verifyAdsAccount,
|
|
10798
|
+
onAdsSyncRequested: (runId, projectId) => {
|
|
10799
|
+
runAdsSync(runId, projectId);
|
|
10800
|
+
},
|
|
10219
10801
|
getBacklinksStatus: () => ({
|
|
10220
10802
|
duckdbInstalled: isDuckdbInstalled(),
|
|
10221
10803
|
duckdbVersion: readInstalledVersion() ?? void 0,
|
|
@@ -10237,7 +10819,7 @@ async function createServer(opts) {
|
|
|
10237
10819
|
deps: {
|
|
10238
10820
|
enqueueAutoExtract: ({ projectId, release: r }) => {
|
|
10239
10821
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
10240
|
-
const runId =
|
|
10822
|
+
const runId = crypto20.randomUUID();
|
|
10241
10823
|
opts.db.insert(runs).values({
|
|
10242
10824
|
id: runId,
|
|
10243
10825
|
projectId,
|
|
@@ -10260,6 +10842,13 @@ async function createServer(opts) {
|
|
|
10260
10842
|
app.log.error({ runId, err }, "Backlink extract failed");
|
|
10261
10843
|
});
|
|
10262
10844
|
},
|
|
10845
|
+
onBingBacklinkSyncRequested: (runId, projectId) => {
|
|
10846
|
+
executeBingBacklinkSync(opts.db, runId, projectId, {
|
|
10847
|
+
resolveConnection: (domain) => bingConnectionStore.getConnection(domain)
|
|
10848
|
+
}).catch((err) => {
|
|
10849
|
+
app.log.error({ runId, err }, "Bing backlink sync failed");
|
|
10850
|
+
});
|
|
10851
|
+
},
|
|
10263
10852
|
onDiscoveryRunRequested: (input) => {
|
|
10264
10853
|
executeDiscoveryRun({
|
|
10265
10854
|
db: opts.db,
|
|
@@ -10323,7 +10912,7 @@ async function createServer(opts) {
|
|
|
10323
10912
|
...inspectOpts,
|
|
10324
10913
|
config: opts.config
|
|
10325
10914
|
}).then(() => {
|
|
10326
|
-
const finished = opts.db.select({ status: runs.status }).from(runs).where(
|
|
10915
|
+
const finished = opts.db.select({ status: runs.status }).from(runs).where(eq20(runs.id, runId)).get();
|
|
10327
10916
|
if (finished?.status === RunStatuses.completed || finished?.status === RunStatuses.partial) {
|
|
10328
10917
|
return maybeRefreshGscCoverage(opts.db, opts.config, projectId);
|
|
10329
10918
|
}
|
|
@@ -10411,7 +11000,7 @@ async function createServer(opts) {
|
|
|
10411
11000
|
const targetProjectIds = affectedProjectIds.length > 0 ? affectedProjectIds : [null];
|
|
10412
11001
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
10413
11002
|
opts.db.insert(auditLog).values(targetProjectIds.map((projectId) => ({
|
|
10414
|
-
id:
|
|
11003
|
+
id: crypto20.randomUUID(),
|
|
10415
11004
|
projectId,
|
|
10416
11005
|
actor: "api",
|
|
10417
11006
|
action: existing ? "provider.updated" : "provider.created",
|