@ainyc/canonry 4.78.0 → 4.80.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +18 -0
- package/assets/assets/{BacklinksPage-CwXveumn.js → BacklinksPage-dRc62jAY.js} +1 -1
- package/assets/assets/{ChartPrimitives-DntKGI5J.js → ChartPrimitives-D2_IvTkk.js} +1 -1
- package/assets/assets/{ProjectPage-CVudiU8X.js → ProjectPage-DSuvRUIf.js} +1 -1
- package/assets/assets/{RunRow-DMtYXaxG.js → RunRow-C0MA3yuQ.js} +1 -1
- package/assets/assets/{RunsPage-Cz-YlucO.js → RunsPage-4uxTYgGy.js} +1 -1
- package/assets/assets/{SettingsPage-BCuG3C-0.js → SettingsPage-3-SLhcJ7.js} +1 -1
- package/assets/assets/{TrafficPage-DV8X47wa.js → TrafficPage-DZ50qwme.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-BmYhK9jm.js → TrafficSourceDetailPage-CzK5TMFp.js} +1 -1
- package/assets/assets/{arrow-left-CUmHyNnF.js → arrow-left-BaZIkAXX.js} +1 -1
- package/assets/assets/{extract-error-message-DFjy9_zi.js → extract-error-message-cpvfuFqW.js} +1 -1
- package/assets/assets/{index-D9smxU6R.js → index-EnY_OBRd.js} +70 -70
- package/assets/assets/{trash-2-B_UtEEm8.js → trash-2-JpcztiS5.js} +1 -1
- package/assets/index.html +1 -1
- package/dist/{chunk-XI6YSTGE.js → chunk-2QBSRHSN.js} +187 -1
- package/dist/{chunk-KPN22EWK.js → chunk-AVN6Q6LM.js} +138 -2
- package/dist/{chunk-3WXARKUE.js → chunk-CXIGHPBE.js} +996 -324
- package/dist/{chunk-QKTIP6GC.js → chunk-LCABGFYN.js} +713 -286
- package/dist/cli.js +369 -148
- package/dist/index.d.ts +17 -0
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-CDVUUG7O.js → intelligence-service-ZWW3I3NL.js} +2 -2
- package/dist/mcp.js +9 -3
- package/package.json +11 -10
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
loadConfig,
|
|
10
10
|
loadConfigRaw,
|
|
11
11
|
saveConfigPatch
|
|
12
|
-
} from "./chunk-
|
|
12
|
+
} from "./chunk-2QBSRHSN.js";
|
|
13
13
|
import {
|
|
14
14
|
CC_CACHE_DIR,
|
|
15
15
|
DUCKDB_SPEC,
|
|
@@ -17,6 +17,11 @@ import {
|
|
|
17
17
|
GSC_DATA_LAG_DAYS,
|
|
18
18
|
IntelligenceService,
|
|
19
19
|
PLUGIN_DIR,
|
|
20
|
+
adsAdGroups,
|
|
21
|
+
adsAds,
|
|
22
|
+
adsCampaigns,
|
|
23
|
+
adsConnections,
|
|
24
|
+
adsInsightsDaily,
|
|
20
25
|
agentMemory,
|
|
21
26
|
agentSessions,
|
|
22
27
|
aiUserFetchEventsHourly,
|
|
@@ -97,7 +102,7 @@ import {
|
|
|
97
102
|
siteAuditPages,
|
|
98
103
|
siteAuditSnapshots,
|
|
99
104
|
usageCounters
|
|
100
|
-
} from "./chunk-
|
|
105
|
+
} from "./chunk-CXIGHPBE.js";
|
|
101
106
|
import {
|
|
102
107
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
103
108
|
AGENT_PROVIDER_IDS,
|
|
@@ -134,6 +139,7 @@ import {
|
|
|
134
139
|
contentActionLabel,
|
|
135
140
|
contentBriefDtoSchema,
|
|
136
141
|
determineAnswerMentioned,
|
|
142
|
+
dollarsToMicros,
|
|
137
143
|
effectiveBrandNames,
|
|
138
144
|
effectiveDomains,
|
|
139
145
|
factorStatusFromScore,
|
|
@@ -149,7 +155,7 @@ import {
|
|
|
149
155
|
validationError,
|
|
150
156
|
winnabilityClassLabel,
|
|
151
157
|
withRetry
|
|
152
|
-
} from "./chunk-
|
|
158
|
+
} from "./chunk-AVN6Q6LM.js";
|
|
153
159
|
|
|
154
160
|
// src/telemetry.ts
|
|
155
161
|
import crypto from "crypto";
|
|
@@ -437,11 +443,11 @@ function checkLatestVersionForServer(opts) {
|
|
|
437
443
|
|
|
438
444
|
// src/server.ts
|
|
439
445
|
import { createRequire as createRequire3 } from "module";
|
|
440
|
-
import
|
|
446
|
+
import crypto19 from "crypto";
|
|
441
447
|
import fs8 from "fs";
|
|
442
448
|
import path9 from "path";
|
|
443
449
|
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
444
|
-
import { and as and13, eq as
|
|
450
|
+
import { and as and13, eq as eq19 } from "drizzle-orm";
|
|
445
451
|
import Fastify from "fastify";
|
|
446
452
|
import os5 from "os";
|
|
447
453
|
|
|
@@ -4077,12 +4083,361 @@ function monthKey(m) {
|
|
|
4077
4083
|
return `${m.year}-${String(m.month).padStart(2, "0")}`;
|
|
4078
4084
|
}
|
|
4079
4085
|
|
|
4080
|
-
// src/
|
|
4086
|
+
// src/ads-sync.ts
|
|
4081
4087
|
import crypto6 from "crypto";
|
|
4082
|
-
import { eq as eq4
|
|
4088
|
+
import { eq as eq4 } from "drizzle-orm";
|
|
4089
|
+
|
|
4090
|
+
// ../integration-openai-ads/src/constants.ts
|
|
4091
|
+
var OPENAI_ADS_API_BASE = "https://api.ads.openai.com/v1";
|
|
4092
|
+
var OPENAI_ADS_REQUEST_TIMEOUT_MS = 3e4;
|
|
4093
|
+
var OPENAI_ADS_MAX_PAGES = 100;
|
|
4094
|
+
|
|
4095
|
+
// ../integration-openai-ads/src/types.ts
|
|
4096
|
+
var OpenAiAdsApiError = class extends Error {
|
|
4097
|
+
status;
|
|
4098
|
+
code;
|
|
4099
|
+
constructor(message, status, code = null) {
|
|
4100
|
+
super(message);
|
|
4101
|
+
this.name = "OpenAiAdsApiError";
|
|
4102
|
+
this.status = status;
|
|
4103
|
+
this.code = code;
|
|
4104
|
+
}
|
|
4105
|
+
};
|
|
4106
|
+
function parseErrorEnvelope(body) {
|
|
4107
|
+
try {
|
|
4108
|
+
const parsed = JSON.parse(body);
|
|
4109
|
+
return {
|
|
4110
|
+
message: parsed.error?.message ?? null,
|
|
4111
|
+
code: parsed.error?.code ?? null
|
|
4112
|
+
};
|
|
4113
|
+
} catch {
|
|
4114
|
+
return { message: null, code: null };
|
|
4115
|
+
}
|
|
4116
|
+
}
|
|
4117
|
+
|
|
4118
|
+
// ../integration-openai-ads/src/ads-client.ts
|
|
4119
|
+
function validateApiKey(apiKey) {
|
|
4120
|
+
if (!apiKey || typeof apiKey !== "string" || apiKey.trim().length === 0) {
|
|
4121
|
+
throw new OpenAiAdsApiError("API key is required and must be a non-empty string", 400);
|
|
4122
|
+
}
|
|
4123
|
+
}
|
|
4124
|
+
function validateId(value, label) {
|
|
4125
|
+
if (!value || typeof value !== "string" || value.trim().length === 0) {
|
|
4126
|
+
throw new OpenAiAdsApiError(`${label} is required and must be a non-empty string`, 400);
|
|
4127
|
+
}
|
|
4128
|
+
}
|
|
4129
|
+
function adsClientLog(level, action, ctx) {
|
|
4130
|
+
const entry = {
|
|
4131
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4132
|
+
level,
|
|
4133
|
+
module: "OpenAiAdsClient",
|
|
4134
|
+
action,
|
|
4135
|
+
...ctx
|
|
4136
|
+
};
|
|
4137
|
+
if (entry.apiKey) entry.apiKey = "***";
|
|
4138
|
+
const stream = level === "error" ? process.stderr : process.stdout;
|
|
4139
|
+
stream.write(JSON.stringify(entry) + "\n");
|
|
4140
|
+
}
|
|
4141
|
+
function buildUrl(path10, queryPairs) {
|
|
4142
|
+
const qs = queryPairs.join("&");
|
|
4143
|
+
return qs ? `${OPENAI_ADS_API_BASE}/${path10}?${qs}` : `${OPENAI_ADS_API_BASE}/${path10}`;
|
|
4144
|
+
}
|
|
4145
|
+
async function adsFetch(apiKey, path10, queryPairs = []) {
|
|
4146
|
+
const url = buildUrl(path10, queryPairs);
|
|
4147
|
+
const res = await fetch(url, {
|
|
4148
|
+
method: "GET",
|
|
4149
|
+
headers: {
|
|
4150
|
+
Authorization: `Bearer ${apiKey}`,
|
|
4151
|
+
"Content-Type": "application/json"
|
|
4152
|
+
},
|
|
4153
|
+
signal: AbortSignal.timeout(OPENAI_ADS_REQUEST_TIMEOUT_MS)
|
|
4154
|
+
});
|
|
4155
|
+
if (res.status === 401 || res.status === 403) {
|
|
4156
|
+
const { code } = parseErrorEnvelope(await res.text());
|
|
4157
|
+
adsClientLog("error", "http.auth-failed", { path: path10, httpStatus: res.status, code });
|
|
4158
|
+
throw new OpenAiAdsApiError("OpenAI Ads API key is invalid or unauthorized", res.status, code);
|
|
4159
|
+
}
|
|
4160
|
+
if (res.status === 429) {
|
|
4161
|
+
const { code } = parseErrorEnvelope(await res.text());
|
|
4162
|
+
adsClientLog("error", "http.rate-limited", { path: path10, httpStatus: 429, code });
|
|
4163
|
+
throw new OpenAiAdsApiError("OpenAI Ads API rate limit exceeded", 429, code);
|
|
4164
|
+
}
|
|
4165
|
+
if (!res.ok) {
|
|
4166
|
+
const body = await res.text();
|
|
4167
|
+
const { message, code } = parseErrorEnvelope(body);
|
|
4168
|
+
adsClientLog("error", "http.error", { path: path10, httpStatus: res.status, code });
|
|
4169
|
+
const detail = message ?? (body.length <= 500 ? body : `${body.slice(0, 500)}... [truncated]`);
|
|
4170
|
+
throw new OpenAiAdsApiError(`OpenAI Ads API error (${res.status}): ${detail}`, res.status, code);
|
|
4171
|
+
}
|
|
4172
|
+
const text = await res.text();
|
|
4173
|
+
try {
|
|
4174
|
+
return JSON.parse(text);
|
|
4175
|
+
} catch {
|
|
4176
|
+
throw new OpenAiAdsApiError("OpenAI Ads API returned invalid JSON", 502);
|
|
4177
|
+
}
|
|
4178
|
+
}
|
|
4179
|
+
async function fetchAllPages(apiKey, path10, queryPairs) {
|
|
4180
|
+
const items = [];
|
|
4181
|
+
let after = null;
|
|
4182
|
+
for (let page = 0; page < OPENAI_ADS_MAX_PAGES; page++) {
|
|
4183
|
+
const pairs = after ? [...queryPairs, `after=${encodeURIComponent(after)}`] : [...queryPairs];
|
|
4184
|
+
const response = await adsFetch(apiKey, path10, pairs);
|
|
4185
|
+
items.push(...response.data);
|
|
4186
|
+
if (!response.has_more || !response.last_id) {
|
|
4187
|
+
return items;
|
|
4188
|
+
}
|
|
4189
|
+
after = response.last_id;
|
|
4190
|
+
}
|
|
4191
|
+
adsClientLog("error", "pagination.cap-reached", { path: path10, pages: OPENAI_ADS_MAX_PAGES, items: items.length });
|
|
4192
|
+
return items;
|
|
4193
|
+
}
|
|
4194
|
+
function insightsPairs(opts) {
|
|
4195
|
+
return (opts?.fields ?? []).map((field) => `fields[]=${encodeURIComponent(field)}`);
|
|
4196
|
+
}
|
|
4197
|
+
async function getAdAccount(apiKey) {
|
|
4198
|
+
validateApiKey(apiKey);
|
|
4199
|
+
return adsFetch(apiKey, "ad_account");
|
|
4200
|
+
}
|
|
4201
|
+
async function listCampaigns(apiKey) {
|
|
4202
|
+
validateApiKey(apiKey);
|
|
4203
|
+
return fetchAllPages(apiKey, "campaigns", []);
|
|
4204
|
+
}
|
|
4205
|
+
async function listAdGroups(apiKey, campaignId) {
|
|
4206
|
+
validateApiKey(apiKey);
|
|
4207
|
+
validateId(campaignId, "Campaign id");
|
|
4208
|
+
return fetchAllPages(apiKey, "ad_groups", [`campaign_id=${encodeURIComponent(campaignId)}`]);
|
|
4209
|
+
}
|
|
4210
|
+
async function listAds(apiKey, adGroupId) {
|
|
4211
|
+
validateApiKey(apiKey);
|
|
4212
|
+
validateId(adGroupId, "Ad group id");
|
|
4213
|
+
return fetchAllPages(apiKey, "ads", [`ad_group_id=${encodeURIComponent(adGroupId)}`]);
|
|
4214
|
+
}
|
|
4215
|
+
async function getCampaignInsights(apiKey, campaignId, opts) {
|
|
4216
|
+
validateApiKey(apiKey);
|
|
4217
|
+
validateId(campaignId, "Campaign id");
|
|
4218
|
+
return fetchAllPages(
|
|
4219
|
+
apiKey,
|
|
4220
|
+
`campaigns/${encodeURIComponent(campaignId)}/insights`,
|
|
4221
|
+
insightsPairs(opts)
|
|
4222
|
+
);
|
|
4223
|
+
}
|
|
4224
|
+
async function getAdGroupInsights(apiKey, adGroupId, opts) {
|
|
4225
|
+
validateApiKey(apiKey);
|
|
4226
|
+
validateId(adGroupId, "Ad group id");
|
|
4227
|
+
return fetchAllPages(
|
|
4228
|
+
apiKey,
|
|
4229
|
+
`ad_groups/${encodeURIComponent(adGroupId)}/insights`,
|
|
4230
|
+
insightsPairs(opts)
|
|
4231
|
+
);
|
|
4232
|
+
}
|
|
4233
|
+
|
|
4234
|
+
// src/ads-config.ts
|
|
4235
|
+
function ensureConnections7(config) {
|
|
4236
|
+
if (!config.openaiAds) config.openaiAds = {};
|
|
4237
|
+
if (!config.openaiAds.connections) config.openaiAds.connections = [];
|
|
4238
|
+
return config.openaiAds.connections;
|
|
4239
|
+
}
|
|
4240
|
+
function getOpenAiAdsConnection(config, projectName) {
|
|
4241
|
+
return (config.openaiAds?.connections ?? []).find((c) => c.projectName === projectName);
|
|
4242
|
+
}
|
|
4243
|
+
function upsertOpenAiAdsConnection(config, connection) {
|
|
4244
|
+
const connections = ensureConnections7(config);
|
|
4245
|
+
const index = connections.findIndex((c) => c.projectName === connection.projectName);
|
|
4246
|
+
if (index === -1) {
|
|
4247
|
+
connections.push(connection);
|
|
4248
|
+
return connection;
|
|
4249
|
+
}
|
|
4250
|
+
connections[index] = connection;
|
|
4251
|
+
return connection;
|
|
4252
|
+
}
|
|
4253
|
+
function removeOpenAiAdsConnection(config, projectName) {
|
|
4254
|
+
const connections = config.openaiAds?.connections;
|
|
4255
|
+
if (!connections?.length) return false;
|
|
4256
|
+
const next = connections.filter((c) => c.projectName !== projectName);
|
|
4257
|
+
if (next.length === connections.length) return false;
|
|
4258
|
+
if (!config.openaiAds) return false;
|
|
4259
|
+
config.openaiAds.connections = next;
|
|
4260
|
+
if (next.length === 0) {
|
|
4261
|
+
delete config.openaiAds;
|
|
4262
|
+
}
|
|
4263
|
+
return true;
|
|
4264
|
+
}
|
|
4265
|
+
|
|
4266
|
+
// src/ads-sync.ts
|
|
4267
|
+
var log4 = createLogger("AdsSync");
|
|
4268
|
+
var CAMPAIGN_INSIGHT_FIELDS = ["campaign.impressions", "campaign.clicks", "campaign.spend", "metadata.readable_time"];
|
|
4269
|
+
var AD_GROUP_INSIGHT_FIELDS = ["ad_group.impressions", "ad_group.clicks", "ad_group.spend", "metadata.readable_time"];
|
|
4270
|
+
function toInsightUpserts(level, entityId, rows) {
|
|
4271
|
+
const upserts = [];
|
|
4272
|
+
for (const row of rows) {
|
|
4273
|
+
if (!row.readable_time) {
|
|
4274
|
+
log4.warn("insights.row-missing-date", { level, entityId, rowId: row.id });
|
|
4275
|
+
continue;
|
|
4276
|
+
}
|
|
4277
|
+
upserts.push({
|
|
4278
|
+
level,
|
|
4279
|
+
entityId,
|
|
4280
|
+
date: row.readable_time,
|
|
4281
|
+
impressions: row.impressions ?? 0,
|
|
4282
|
+
clicks: row.clicks ?? 0,
|
|
4283
|
+
spendMicros: dollarsToMicros(row.spend ?? 0)
|
|
4284
|
+
});
|
|
4285
|
+
}
|
|
4286
|
+
return upserts;
|
|
4287
|
+
}
|
|
4288
|
+
async function executeAdsSync(db, runId, projectId, opts) {
|
|
4289
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4290
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq4(runs.id, runId)).run();
|
|
4291
|
+
try {
|
|
4292
|
+
const project = db.select().from(projects).where(eq4(projects.id, projectId)).get();
|
|
4293
|
+
if (!project) throw new Error(`Project not found: ${projectId}`);
|
|
4294
|
+
const connRow = db.select().from(adsConnections).where(eq4(adsConnections.projectId, projectId)).get();
|
|
4295
|
+
if (!connRow) {
|
|
4296
|
+
throw new Error('No ads connection found for this project. Run "canonry ads connect" first.');
|
|
4297
|
+
}
|
|
4298
|
+
const cfgConn = getOpenAiAdsConnection(opts.config, project.name);
|
|
4299
|
+
if (!cfgConn?.apiKey) {
|
|
4300
|
+
throw new Error('No OpenAI Ads API key in the local Canonry config. Run "canonry ads connect" first.');
|
|
4301
|
+
}
|
|
4302
|
+
const apiKey = cfgConn.apiKey;
|
|
4303
|
+
log4.info("sync.start", { runId, projectId, adAccountId: connRow.adAccountId });
|
|
4304
|
+
const account = await getAdAccount(apiKey);
|
|
4305
|
+
const campaigns = await listCampaigns(apiKey);
|
|
4306
|
+
const errors = /* @__PURE__ */ new Map();
|
|
4307
|
+
const adGroupsByCampaign = /* @__PURE__ */ new Map();
|
|
4308
|
+
const adsByGroup = /* @__PURE__ */ new Map();
|
|
4309
|
+
const insightUpserts = [];
|
|
4310
|
+
const syncedCampaigns = [];
|
|
4311
|
+
for (const campaign of campaigns) {
|
|
4312
|
+
try {
|
|
4313
|
+
const [adGroups, campaignInsights] = await Promise.all([
|
|
4314
|
+
listAdGroups(apiKey, campaign.id),
|
|
4315
|
+
getCampaignInsights(apiKey, campaign.id, { fields: CAMPAIGN_INSIGHT_FIELDS })
|
|
4316
|
+
]);
|
|
4317
|
+
const groupResults = await Promise.all(adGroups.map(async (group) => ({
|
|
4318
|
+
group,
|
|
4319
|
+
ads: await listAds(apiKey, group.id),
|
|
4320
|
+
insights: await getAdGroupInsights(apiKey, group.id, { fields: AD_GROUP_INSIGHT_FIELDS })
|
|
4321
|
+
})));
|
|
4322
|
+
syncedCampaigns.push(campaign);
|
|
4323
|
+
adGroupsByCampaign.set(campaign.id, adGroups);
|
|
4324
|
+
insightUpserts.push(...toInsightUpserts("campaign", campaign.id, campaignInsights));
|
|
4325
|
+
for (const { group, ads, insights: insights2 } of groupResults) {
|
|
4326
|
+
adsByGroup.set(group.id, ads);
|
|
4327
|
+
insightUpserts.push(...toInsightUpserts("ad_group", group.id, insights2));
|
|
4328
|
+
}
|
|
4329
|
+
} catch (err) {
|
|
4330
|
+
errors.set(campaign.name, err instanceof Error ? err.message : String(err));
|
|
4331
|
+
log4.error("campaign.failed", { runId, campaignId: campaign.id, error: err instanceof Error ? err.message : String(err) });
|
|
4332
|
+
}
|
|
4333
|
+
}
|
|
4334
|
+
const insertNow = (/* @__PURE__ */ new Date()).toISOString();
|
|
4335
|
+
db.transaction((tx) => {
|
|
4336
|
+
tx.delete(adsCampaigns).where(eq4(adsCampaigns.projectId, projectId)).run();
|
|
4337
|
+
for (const campaign of syncedCampaigns) {
|
|
4338
|
+
tx.insert(adsCampaigns).values({
|
|
4339
|
+
id: campaign.id,
|
|
4340
|
+
projectId,
|
|
4341
|
+
name: campaign.name,
|
|
4342
|
+
status: campaign.status,
|
|
4343
|
+
biddingType: campaign.bidding_type,
|
|
4344
|
+
dailySpendLimitMicros: campaign.budget?.daily_spend_limit_micros ?? null,
|
|
4345
|
+
lifetimeSpendLimitMicros: campaign.budget?.lifetime_spend_limit_micros ?? null,
|
|
4346
|
+
targeting: campaign.targeting,
|
|
4347
|
+
upstreamCreatedAt: campaign.created_at,
|
|
4348
|
+
upstreamUpdatedAt: campaign.updated_at,
|
|
4349
|
+
syncRunId: runId,
|
|
4350
|
+
syncedAt: insertNow
|
|
4351
|
+
}).run();
|
|
4352
|
+
for (const group of adGroupsByCampaign.get(campaign.id) ?? []) {
|
|
4353
|
+
tx.insert(adsAdGroups).values({
|
|
4354
|
+
id: group.id,
|
|
4355
|
+
projectId,
|
|
4356
|
+
campaignId: campaign.id,
|
|
4357
|
+
name: group.name,
|
|
4358
|
+
status: group.status,
|
|
4359
|
+
billingEventType: group.bidding_config?.billing_event_type ?? null,
|
|
4360
|
+
maxBidMicros: group.bidding_config?.max_bid_micros ?? null,
|
|
4361
|
+
contextHints: group.context_hints,
|
|
4362
|
+
upstreamCreatedAt: group.created_at,
|
|
4363
|
+
upstreamUpdatedAt: group.updated_at,
|
|
4364
|
+
syncRunId: runId,
|
|
4365
|
+
syncedAt: insertNow
|
|
4366
|
+
}).run();
|
|
4367
|
+
for (const ad of adsByGroup.get(group.id) ?? []) {
|
|
4368
|
+
tx.insert(adsAds).values({
|
|
4369
|
+
id: ad.id,
|
|
4370
|
+
projectId,
|
|
4371
|
+
adGroupId: group.id,
|
|
4372
|
+
name: ad.name,
|
|
4373
|
+
status: ad.status,
|
|
4374
|
+
creative: ad.creative,
|
|
4375
|
+
reviewStatus: ad.review_status ?? ad.review?.status ?? null,
|
|
4376
|
+
upstreamCreatedAt: ad.created_at,
|
|
4377
|
+
upstreamUpdatedAt: ad.updated_at,
|
|
4378
|
+
syncRunId: runId,
|
|
4379
|
+
syncedAt: insertNow
|
|
4380
|
+
}).run();
|
|
4381
|
+
}
|
|
4382
|
+
}
|
|
4383
|
+
}
|
|
4384
|
+
for (const upsert of insightUpserts) {
|
|
4385
|
+
tx.insert(adsInsightsDaily).values({
|
|
4386
|
+
id: crypto6.randomUUID(),
|
|
4387
|
+
projectId,
|
|
4388
|
+
...upsert,
|
|
4389
|
+
syncRunId: runId
|
|
4390
|
+
}).onConflictDoUpdate({
|
|
4391
|
+
target: [adsInsightsDaily.projectId, adsInsightsDaily.level, adsInsightsDaily.entityId, adsInsightsDaily.date],
|
|
4392
|
+
set: {
|
|
4393
|
+
impressions: upsert.impressions,
|
|
4394
|
+
clicks: upsert.clicks,
|
|
4395
|
+
spendMicros: upsert.spendMicros,
|
|
4396
|
+
syncRunId: runId
|
|
4397
|
+
}
|
|
4398
|
+
}).run();
|
|
4399
|
+
}
|
|
4400
|
+
tx.update(adsConnections).set({
|
|
4401
|
+
adAccountId: account.id,
|
|
4402
|
+
displayName: account.name,
|
|
4403
|
+
currencyCode: account.currency_code,
|
|
4404
|
+
timezone: account.timezone,
|
|
4405
|
+
status: account.status,
|
|
4406
|
+
lastSyncedAt: insertNow,
|
|
4407
|
+
updatedAt: insertNow
|
|
4408
|
+
}).where(eq4(adsConnections.projectId, projectId)).run();
|
|
4409
|
+
});
|
|
4410
|
+
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
4411
|
+
if (errors.size === 0) {
|
|
4412
|
+
db.update(runs).set({ status: "completed", finishedAt }).where(eq4(runs.id, runId)).run();
|
|
4413
|
+
} else if (syncedCampaigns.length > 0) {
|
|
4414
|
+
db.update(runs).set({
|
|
4415
|
+
status: "partial",
|
|
4416
|
+
error: serializeRunError(buildRunErrorFromMessages(errors)),
|
|
4417
|
+
finishedAt
|
|
4418
|
+
}).where(eq4(runs.id, runId)).run();
|
|
4419
|
+
} else {
|
|
4420
|
+
db.update(runs).set({
|
|
4421
|
+
status: "failed",
|
|
4422
|
+
error: serializeRunError(buildRunErrorFromMessages(errors)),
|
|
4423
|
+
finishedAt
|
|
4424
|
+
}).where(eq4(runs.id, runId)).run();
|
|
4425
|
+
}
|
|
4426
|
+
log4.info("sync.done", { runId, projectId, campaigns: syncedCampaigns.length, insightRows: insightUpserts.length, failed: errors.size });
|
|
4427
|
+
} catch (err) {
|
|
4428
|
+
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
4429
|
+
db.update(runs).set({ status: "failed", error: serializeRunError({ message: errorMsg }), finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq4(runs.id, runId)).run();
|
|
4430
|
+
log4.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
4431
|
+
throw err;
|
|
4432
|
+
}
|
|
4433
|
+
}
|
|
4434
|
+
|
|
4435
|
+
// src/gsc-inspect-sitemap.ts
|
|
4436
|
+
import crypto7 from "crypto";
|
|
4437
|
+
import { eq as eq5, and as and4 } from "drizzle-orm";
|
|
4083
4438
|
|
|
4084
4439
|
// src/sitemap-parser.ts
|
|
4085
|
-
var
|
|
4440
|
+
var log5 = createLogger("SitemapParser");
|
|
4086
4441
|
var LOC_REGEX = /<loc>([^<]+)<\/loc>/gi;
|
|
4087
4442
|
var SITEMAP_TAG_REGEX = /<sitemap>[\s\S]*?<\/sitemap>/gi;
|
|
4088
4443
|
async function validateSitemapUrl(url) {
|
|
@@ -4124,7 +4479,7 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
4124
4479
|
res = await fetch(url);
|
|
4125
4480
|
} catch (err) {
|
|
4126
4481
|
if (!isChild) throw err;
|
|
4127
|
-
|
|
4482
|
+
log5.warn("child-sitemap.fetch-failed", {
|
|
4128
4483
|
url,
|
|
4129
4484
|
error: err instanceof Error ? err.message : String(err)
|
|
4130
4485
|
});
|
|
@@ -4134,7 +4489,7 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
4134
4489
|
if (!isChild) {
|
|
4135
4490
|
throw new Error(`Failed to fetch sitemap at ${url}: ${res.status} ${res.statusText}`);
|
|
4136
4491
|
}
|
|
4137
|
-
|
|
4492
|
+
log5.warn("child-sitemap.http-error", { url, status: res.status, statusText: res.statusText });
|
|
4138
4493
|
return;
|
|
4139
4494
|
}
|
|
4140
4495
|
let xml;
|
|
@@ -4142,7 +4497,7 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
4142
4497
|
xml = await readSitemapBody(res);
|
|
4143
4498
|
} catch (err) {
|
|
4144
4499
|
if (!isChild) throw err;
|
|
4145
|
-
|
|
4500
|
+
log5.warn("child-sitemap.parse-failed", {
|
|
4146
4501
|
url,
|
|
4147
4502
|
error: err instanceof Error ? err.message : String(err)
|
|
4148
4503
|
});
|
|
@@ -4178,16 +4533,16 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
4178
4533
|
}
|
|
4179
4534
|
|
|
4180
4535
|
// src/gsc-inspect-sitemap.ts
|
|
4181
|
-
var
|
|
4536
|
+
var log6 = createLogger("InspectSitemap");
|
|
4182
4537
|
async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
4183
4538
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4184
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
4539
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq5(runs.id, runId)).run();
|
|
4185
4540
|
try {
|
|
4186
4541
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
4187
4542
|
if (!googleClientId || !googleClientSecret) {
|
|
4188
4543
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
4189
4544
|
}
|
|
4190
|
-
const project = db.select().from(projects).where(
|
|
4545
|
+
const project = db.select().from(projects).where(eq5(projects.id, projectId)).get();
|
|
4191
4546
|
if (!project) {
|
|
4192
4547
|
throw new Error(`Project not found: ${projectId}`);
|
|
4193
4548
|
}
|
|
@@ -4212,9 +4567,9 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
4212
4567
|
saveConfigPatch(opts.config);
|
|
4213
4568
|
}
|
|
4214
4569
|
const sitemapUrl = opts.sitemapUrl || conn.sitemapUrl || `https://${project.canonicalDomain}/sitemap.xml`;
|
|
4215
|
-
|
|
4570
|
+
log6.info("sitemap.fetch", { runId, projectId, sitemapUrl });
|
|
4216
4571
|
const urls = await fetchAndParseSitemap(sitemapUrl);
|
|
4217
|
-
|
|
4572
|
+
log6.info("sitemap.parsed", { runId, projectId, urlCount: urls.length, sitemapUrl });
|
|
4218
4573
|
if (urls.length === 0) {
|
|
4219
4574
|
throw new Error("No URLs found in sitemap");
|
|
4220
4575
|
}
|
|
@@ -4229,7 +4584,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
4229
4584
|
const rich = ir.richResultsResult;
|
|
4230
4585
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
4231
4586
|
db.insert(gscUrlInspections).values({
|
|
4232
|
-
id:
|
|
4587
|
+
id: crypto7.randomUUID(),
|
|
4233
4588
|
projectId,
|
|
4234
4589
|
syncRunId: runId,
|
|
4235
4590
|
url: pageUrl,
|
|
@@ -4246,16 +4601,16 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
4246
4601
|
inspectedAt,
|
|
4247
4602
|
createdAt: inspectedAt
|
|
4248
4603
|
}).run();
|
|
4249
|
-
|
|
4604
|
+
log6.info("inspect.url-done", { runId, projectId, url: pageUrl, progress: `${index + 1}/${urls.length}` });
|
|
4250
4605
|
},
|
|
4251
4606
|
onError: (pageUrl, err) => {
|
|
4252
|
-
|
|
4607
|
+
log6.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
|
|
4253
4608
|
}
|
|
4254
4609
|
},
|
|
4255
4610
|
{
|
|
4256
4611
|
log: {
|
|
4257
|
-
info: (action, ctx) =>
|
|
4258
|
-
error: (action, ctx) =>
|
|
4612
|
+
info: (action, ctx) => log6.info(action, { runId, projectId, ...ctx }),
|
|
4613
|
+
error: (action, ctx) => log6.error(action, { runId, projectId, ...ctx })
|
|
4259
4614
|
}
|
|
4260
4615
|
}
|
|
4261
4616
|
);
|
|
@@ -4265,7 +4620,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
4265
4620
|
`URL inspection aborted after ${INSPECT_FAILFAST_THRESHOLD} consecutive rate/access failures (likely GSC URL Inspection quota exhaustion or property access loss). Last error: ${detail}`
|
|
4266
4621
|
);
|
|
4267
4622
|
}
|
|
4268
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
4623
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq5(gscUrlInspections.projectId, projectId)).all();
|
|
4269
4624
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
4270
4625
|
for (const row of allInspections) {
|
|
4271
4626
|
const existing = latestByUrl.get(row.url);
|
|
@@ -4286,9 +4641,9 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
4286
4641
|
}
|
|
4287
4642
|
}
|
|
4288
4643
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
4289
|
-
db.delete(gscCoverageSnapshots).where(and4(
|
|
4644
|
+
db.delete(gscCoverageSnapshots).where(and4(eq5(gscCoverageSnapshots.projectId, projectId), eq5(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
4290
4645
|
db.insert(gscCoverageSnapshots).values({
|
|
4291
|
-
id:
|
|
4646
|
+
id: crypto7.randomUUID(),
|
|
4292
4647
|
projectId,
|
|
4293
4648
|
syncRunId: runId,
|
|
4294
4649
|
date: snapshotDate,
|
|
@@ -4298,20 +4653,20 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
4298
4653
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
4299
4654
|
}).run();
|
|
4300
4655
|
const status = errors > 0 && inspected > 0 ? "partial" : errors === urls.length ? "failed" : "completed";
|
|
4301
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
4302
|
-
|
|
4656
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq5(runs.id, runId)).run();
|
|
4657
|
+
log6.info("inspect.completed", { runId, projectId, inspected, errors, total: urls.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
4303
4658
|
} catch (err) {
|
|
4304
4659
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
4305
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
4306
|
-
|
|
4660
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq5(runs.id, runId)).run();
|
|
4661
|
+
log6.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
4307
4662
|
throw err;
|
|
4308
4663
|
}
|
|
4309
4664
|
}
|
|
4310
4665
|
|
|
4311
4666
|
// src/bing-inspect-sitemap.ts
|
|
4312
|
-
import
|
|
4313
|
-
import { eq as
|
|
4314
|
-
var
|
|
4667
|
+
import crypto8 from "crypto";
|
|
4668
|
+
import { eq as eq6, desc as desc2 } from "drizzle-orm";
|
|
4669
|
+
var log7 = createLogger("BingInspectSitemap");
|
|
4315
4670
|
function parseBingDate(value) {
|
|
4316
4671
|
if (!value) return null;
|
|
4317
4672
|
const match = /\/Date\((-?\d+)(?:[-+]\d+)?\)\//.exec(value);
|
|
@@ -4328,9 +4683,9 @@ function isBlockingIssueType(issueType) {
|
|
|
4328
4683
|
}
|
|
4329
4684
|
async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
4330
4685
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
4331
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
4686
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq6(runs.id, runId)).run();
|
|
4332
4687
|
try {
|
|
4333
|
-
const project = db.select().from(projects).where(
|
|
4688
|
+
const project = db.select().from(projects).where(eq6(projects.id, projectId)).get();
|
|
4334
4689
|
if (!project) {
|
|
4335
4690
|
throw new Error(`Project not found: ${projectId}`);
|
|
4336
4691
|
}
|
|
@@ -4342,16 +4697,16 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4342
4697
|
throw new Error('No Bing site configured. Run "canonry bing set-site <project> <url>" first.');
|
|
4343
4698
|
}
|
|
4344
4699
|
const sitemapUrl = opts.sitemapUrl ?? `https://${project.canonicalDomain}/sitemap.xml`;
|
|
4345
|
-
|
|
4700
|
+
log7.info("sitemap.fetch", { runId, projectId, sitemapUrl });
|
|
4346
4701
|
const sitemapUrls = await fetchAndParseSitemap(sitemapUrl);
|
|
4347
|
-
|
|
4702
|
+
log7.info("sitemap.parsed", { runId, projectId, urlCount: sitemapUrls.length, sitemapUrl });
|
|
4348
4703
|
if (sitemapUrls.length === 0) {
|
|
4349
4704
|
throw new Error("No URLs found in sitemap");
|
|
4350
4705
|
}
|
|
4351
|
-
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(
|
|
4706
|
+
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(eq6(bingUrlInspections.projectId, projectId)).all();
|
|
4352
4707
|
const trackedUrls = new Set(trackedRows.map((r) => r.url));
|
|
4353
4708
|
const discovered = sitemapUrls.filter((u) => !trackedUrls.has(u));
|
|
4354
|
-
|
|
4709
|
+
log7.info("sitemap.diff", {
|
|
4355
4710
|
runId,
|
|
4356
4711
|
projectId,
|
|
4357
4712
|
sitemapTotal: sitemapUrls.length,
|
|
@@ -4366,9 +4721,9 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4366
4721
|
blockedUrls.add(issue.Url);
|
|
4367
4722
|
}
|
|
4368
4723
|
}
|
|
4369
|
-
|
|
4724
|
+
log7.info("crawl-issues.loaded", { runId, projectId, blockedCount: blockedUrls.size });
|
|
4370
4725
|
} catch (err) {
|
|
4371
|
-
|
|
4726
|
+
log7.warn("crawl-issues.lookup-failed", {
|
|
4372
4727
|
runId,
|
|
4373
4728
|
projectId,
|
|
4374
4729
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -4397,7 +4752,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4397
4752
|
derivedInIndex = false;
|
|
4398
4753
|
}
|
|
4399
4754
|
db.insert(bingUrlInspections).values({
|
|
4400
|
-
id:
|
|
4755
|
+
id: crypto8.randomUUID(),
|
|
4401
4756
|
projectId,
|
|
4402
4757
|
url: pageUrl,
|
|
4403
4758
|
httpCode,
|
|
@@ -4412,7 +4767,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4412
4767
|
discoveryDate
|
|
4413
4768
|
}).run();
|
|
4414
4769
|
inspected++;
|
|
4415
|
-
|
|
4770
|
+
log7.info("inspect.url-done", {
|
|
4416
4771
|
runId,
|
|
4417
4772
|
projectId,
|
|
4418
4773
|
url: pageUrl,
|
|
@@ -4420,7 +4775,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4420
4775
|
});
|
|
4421
4776
|
} catch (err) {
|
|
4422
4777
|
errors++;
|
|
4423
|
-
|
|
4778
|
+
log7.error("inspect.url-failed", {
|
|
4424
4779
|
runId,
|
|
4425
4780
|
projectId,
|
|
4426
4781
|
url: pageUrl,
|
|
@@ -4431,7 +4786,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4431
4786
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
4432
4787
|
}
|
|
4433
4788
|
}
|
|
4434
|
-
const allInspections = db.select().from(bingUrlInspections).where(
|
|
4789
|
+
const allInspections = db.select().from(bingUrlInspections).where(eq6(bingUrlInspections.projectId, projectId)).orderBy(desc2(bingUrlInspections.inspectedAt)).all();
|
|
4435
4790
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
4436
4791
|
const definitiveByUrl = /* @__PURE__ */ new Map();
|
|
4437
4792
|
for (const row of allInspections) {
|
|
@@ -4455,7 +4810,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4455
4810
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
4456
4811
|
const snapNow = (/* @__PURE__ */ new Date()).toISOString();
|
|
4457
4812
|
db.insert(bingCoverageSnapshots).values({
|
|
4458
|
-
id:
|
|
4813
|
+
id: crypto8.randomUUID(),
|
|
4459
4814
|
projectId,
|
|
4460
4815
|
syncRunId: runId,
|
|
4461
4816
|
date: snapshotDate,
|
|
@@ -4474,8 +4829,8 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4474
4829
|
}
|
|
4475
4830
|
}).run();
|
|
4476
4831
|
const status = errors === sitemapUrls.length ? RunStatuses.failed : errors > 0 ? RunStatuses.partial : RunStatuses.completed;
|
|
4477
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
4478
|
-
|
|
4832
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq6(runs.id, runId)).run();
|
|
4833
|
+
log7.info("inspect.completed", {
|
|
4479
4834
|
runId,
|
|
4480
4835
|
projectId,
|
|
4481
4836
|
inspected,
|
|
@@ -4488,16 +4843,16 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4488
4843
|
});
|
|
4489
4844
|
} catch (err) {
|
|
4490
4845
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
4491
|
-
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
4492
|
-
|
|
4846
|
+
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq6(runs.id, runId)).run();
|
|
4847
|
+
log7.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
4493
4848
|
throw err;
|
|
4494
4849
|
}
|
|
4495
4850
|
}
|
|
4496
4851
|
|
|
4497
4852
|
// src/coverage-refresh.ts
|
|
4498
|
-
import
|
|
4499
|
-
import { and as and5, desc as desc3, eq as
|
|
4500
|
-
var
|
|
4853
|
+
import crypto9 from "crypto";
|
|
4854
|
+
import { and as and5, desc as desc3, eq as eq7, inArray as inArray3 } from "drizzle-orm";
|
|
4855
|
+
var log8 = createLogger("CoverageRefresh");
|
|
4501
4856
|
var COVERAGE_REFRESH_MIN_INTERVAL_MS = 60 * 60 * 1e3;
|
|
4502
4857
|
var ACTIVE_OR_DONE_STATUSES = [
|
|
4503
4858
|
RunStatuses.queued,
|
|
@@ -4507,7 +4862,7 @@ var ACTIVE_OR_DONE_STATUSES = [
|
|
|
4507
4862
|
];
|
|
4508
4863
|
var defaultDeps = { executeInspectSitemap };
|
|
4509
4864
|
async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps, nowMs = Date.now()) {
|
|
4510
|
-
const project = db.select({ canonicalDomain: projects.canonicalDomain }).from(projects).where(
|
|
4865
|
+
const project = db.select({ canonicalDomain: projects.canonicalDomain }).from(projects).where(eq7(projects.id, projectId)).get();
|
|
4511
4866
|
if (!project) return null;
|
|
4512
4867
|
const { clientId, clientSecret } = getGoogleAuthConfig(config);
|
|
4513
4868
|
if (!clientId || !clientSecret) return null;
|
|
@@ -4515,19 +4870,19 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
4515
4870
|
if (!conn?.refreshToken || !conn.propertyId) return null;
|
|
4516
4871
|
const recent = db.select({ createdAt: runs.createdAt }).from(runs).where(
|
|
4517
4872
|
and5(
|
|
4518
|
-
|
|
4519
|
-
|
|
4873
|
+
eq7(runs.projectId, projectId),
|
|
4874
|
+
eq7(runs.kind, RunKinds["inspect-sitemap"]),
|
|
4520
4875
|
inArray3(runs.status, ACTIVE_OR_DONE_STATUSES)
|
|
4521
4876
|
)
|
|
4522
4877
|
).orderBy(desc3(runs.createdAt)).limit(1).get();
|
|
4523
4878
|
if (recent) {
|
|
4524
4879
|
const ageMs = nowMs - Date.parse(recent.createdAt);
|
|
4525
4880
|
if (Number.isFinite(ageMs) && ageMs < COVERAGE_REFRESH_MIN_INTERVAL_MS) {
|
|
4526
|
-
|
|
4881
|
+
log8.info("skip.recent", { projectId, ageMs });
|
|
4527
4882
|
return null;
|
|
4528
4883
|
}
|
|
4529
4884
|
}
|
|
4530
|
-
const runId =
|
|
4885
|
+
const runId = crypto9.randomUUID();
|
|
4531
4886
|
db.insert(runs).values({
|
|
4532
4887
|
id: runId,
|
|
4533
4888
|
projectId,
|
|
@@ -4536,11 +4891,11 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
4536
4891
|
trigger: RunTriggers.scheduled,
|
|
4537
4892
|
createdAt: new Date(nowMs).toISOString()
|
|
4538
4893
|
}).run();
|
|
4539
|
-
|
|
4894
|
+
log8.info("refresh.start", { projectId, runId });
|
|
4540
4895
|
try {
|
|
4541
4896
|
await deps.executeInspectSitemap(db, runId, projectId, { config });
|
|
4542
4897
|
} catch (err) {
|
|
4543
|
-
|
|
4898
|
+
log8.error("refresh.failed", {
|
|
4544
4899
|
projectId,
|
|
4545
4900
|
runId,
|
|
4546
4901
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -4550,10 +4905,10 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
4550
4905
|
}
|
|
4551
4906
|
|
|
4552
4907
|
// src/commoncrawl-sync.ts
|
|
4553
|
-
import
|
|
4908
|
+
import crypto10 from "crypto";
|
|
4554
4909
|
import path4 from "path";
|
|
4555
|
-
import { and as and6, eq as
|
|
4556
|
-
var
|
|
4910
|
+
import { and as and6, eq as eq8, sql as sql3 } from "drizzle-orm";
|
|
4911
|
+
var log9 = createLogger("CommonCrawlSync");
|
|
4557
4912
|
var INSERT_CHUNK_SIZE = 1e4;
|
|
4558
4913
|
function defaultDeps2() {
|
|
4559
4914
|
return {
|
|
@@ -4578,7 +4933,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4578
4933
|
phaseDetail: "downloading vertices + edges",
|
|
4579
4934
|
updatedAt: downloadStartedAt,
|
|
4580
4935
|
error: null
|
|
4581
|
-
}).where(
|
|
4936
|
+
}).where(eq8(ccReleaseSyncs.id, syncId)).run();
|
|
4582
4937
|
const paths = ccReleasePaths(release);
|
|
4583
4938
|
const releaseCacheDir = path4.join(deps.cacheDir, release);
|
|
4584
4939
|
const vertexPath = path4.join(releaseCacheDir, paths.vertexFilename);
|
|
@@ -4601,7 +4956,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4601
4956
|
vertexSha256: vertex.sha256,
|
|
4602
4957
|
edgesSha256: edges.sha256,
|
|
4603
4958
|
updatedAt: downloadFinishedAt
|
|
4604
|
-
}).where(
|
|
4959
|
+
}).where(eq8(ccReleaseSyncs.id, syncId)).run();
|
|
4605
4960
|
const allProjects = db.select().from(projects).all();
|
|
4606
4961
|
const targets = Array.from(new Set(allProjects.map((p) => p.canonicalDomain)));
|
|
4607
4962
|
let rows = [];
|
|
@@ -4617,15 +4972,15 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4617
4972
|
}
|
|
4618
4973
|
const queriedAt = deps.now().toISOString();
|
|
4619
4974
|
db.transaction((tx) => {
|
|
4620
|
-
tx.delete(backlinkDomains).where(
|
|
4621
|
-
tx.delete(backlinkSummaries).where(
|
|
4975
|
+
tx.delete(backlinkDomains).where(eq8(backlinkDomains.releaseSyncId, syncId)).run();
|
|
4976
|
+
tx.delete(backlinkSummaries).where(eq8(backlinkSummaries.releaseSyncId, syncId)).run();
|
|
4622
4977
|
const expanded = [];
|
|
4623
4978
|
for (const r of rows) {
|
|
4624
4979
|
const projectIds = projectsByDomain.get(r.targetDomain);
|
|
4625
4980
|
if (!projectIds) continue;
|
|
4626
4981
|
for (const projectId of projectIds) {
|
|
4627
4982
|
expanded.push({
|
|
4628
|
-
id:
|
|
4983
|
+
id: crypto10.randomUUID(),
|
|
4629
4984
|
projectId,
|
|
4630
4985
|
releaseSyncId: syncId,
|
|
4631
4986
|
release,
|
|
@@ -4645,7 +5000,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4645
5000
|
const projectRows = rowsByProject.get(p.id) ?? [];
|
|
4646
5001
|
const summary = computeSummary(projectRows);
|
|
4647
5002
|
tx.insert(backlinkSummaries).values({
|
|
4648
|
-
id:
|
|
5003
|
+
id: crypto10.randomUUID(),
|
|
4649
5004
|
projectId: p.id,
|
|
4650
5005
|
releaseSyncId: syncId,
|
|
4651
5006
|
release,
|
|
@@ -4677,8 +5032,8 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4677
5032
|
domainsDiscovered: rows.length,
|
|
4678
5033
|
updatedAt: finishedAt,
|
|
4679
5034
|
error: null
|
|
4680
|
-
}).where(
|
|
4681
|
-
|
|
5035
|
+
}).where(eq8(ccReleaseSyncs.id, syncId)).run();
|
|
5036
|
+
log9.info("sync.completed", {
|
|
4682
5037
|
syncId,
|
|
4683
5038
|
release,
|
|
4684
5039
|
projectsProcessed: allProjects.length,
|
|
@@ -4690,7 +5045,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4690
5045
|
try {
|
|
4691
5046
|
deps.enqueueAutoExtract({ projectId: p.id, release });
|
|
4692
5047
|
} catch (err) {
|
|
4693
|
-
|
|
5048
|
+
log9.error("auto-extract.enqueue-failed", {
|
|
4694
5049
|
syncId,
|
|
4695
5050
|
release,
|
|
4696
5051
|
projectId: p.id,
|
|
@@ -4707,8 +5062,8 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4707
5062
|
error: errorMsg,
|
|
4708
5063
|
phaseDetail: null,
|
|
4709
5064
|
updatedAt: finishedAt
|
|
4710
|
-
}).where(
|
|
4711
|
-
|
|
5065
|
+
}).where(eq8(ccReleaseSyncs.id, syncId)).run();
|
|
5066
|
+
log9.error("sync.failed", { syncId, release, error: errorMsg });
|
|
4712
5067
|
throw err;
|
|
4713
5068
|
}
|
|
4714
5069
|
}
|
|
@@ -4741,10 +5096,10 @@ function computeSummary(rows) {
|
|
|
4741
5096
|
}
|
|
4742
5097
|
|
|
4743
5098
|
// src/backlink-extract.ts
|
|
4744
|
-
import
|
|
5099
|
+
import crypto11 from "crypto";
|
|
4745
5100
|
import fs3 from "fs";
|
|
4746
|
-
import { and as and7, desc as desc4, eq as
|
|
4747
|
-
var
|
|
5101
|
+
import { and as and7, desc as desc4, eq as eq9 } from "drizzle-orm";
|
|
5102
|
+
var log10 = createLogger("BacklinkExtract");
|
|
4748
5103
|
function defaultDeps3() {
|
|
4749
5104
|
return {
|
|
4750
5105
|
queryBacklinks,
|
|
@@ -4755,13 +5110,13 @@ function defaultDeps3() {
|
|
|
4755
5110
|
async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
4756
5111
|
const deps = { ...defaultDeps3(), ...opts.deps };
|
|
4757
5112
|
const startedAt = deps.now().toISOString();
|
|
4758
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
5113
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq9(runs.id, runId)).run();
|
|
4759
5114
|
try {
|
|
4760
|
-
const project = db.select().from(projects).where(
|
|
5115
|
+
const project = db.select().from(projects).where(eq9(projects.id, projectId)).get();
|
|
4761
5116
|
if (!project) {
|
|
4762
5117
|
throw new Error(`Project not found: ${projectId}`);
|
|
4763
5118
|
}
|
|
4764
|
-
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(
|
|
5119
|
+
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(eq9(ccReleaseSyncs.release, opts.release)).get() : db.select().from(ccReleaseSyncs).where(eq9(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).orderBy(desc4(ccReleaseSyncs.createdAt)).limit(1).get();
|
|
4765
5120
|
if (!sync) {
|
|
4766
5121
|
throw new Error("No ready release sync available \u2014 run `canonry backlinks sync` first");
|
|
4767
5122
|
}
|
|
@@ -4789,11 +5144,11 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
4789
5144
|
const targetDomain = project.canonicalDomain;
|
|
4790
5145
|
db.transaction((tx) => {
|
|
4791
5146
|
tx.delete(backlinkDomains).where(
|
|
4792
|
-
and7(
|
|
5147
|
+
and7(eq9(backlinkDomains.projectId, projectId), eq9(backlinkDomains.release, release))
|
|
4793
5148
|
).run();
|
|
4794
5149
|
if (rows.length > 0) {
|
|
4795
5150
|
const values = rows.map((r) => ({
|
|
4796
|
-
id:
|
|
5151
|
+
id: crypto11.randomUUID(),
|
|
4797
5152
|
projectId,
|
|
4798
5153
|
releaseSyncId: syncId,
|
|
4799
5154
|
release,
|
|
@@ -4806,7 +5161,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
4806
5161
|
}
|
|
4807
5162
|
const summary = computeSummary2(rows);
|
|
4808
5163
|
tx.insert(backlinkSummaries).values({
|
|
4809
|
-
id:
|
|
5164
|
+
id: crypto11.randomUUID(),
|
|
4810
5165
|
projectId,
|
|
4811
5166
|
releaseSyncId: syncId,
|
|
4812
5167
|
release,
|
|
@@ -4829,8 +5184,8 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
4829
5184
|
}).run();
|
|
4830
5185
|
});
|
|
4831
5186
|
const finishedAt = deps.now().toISOString();
|
|
4832
|
-
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
4833
|
-
|
|
5187
|
+
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq9(runs.id, runId)).run();
|
|
5188
|
+
log10.info("extract.completed", { runId, projectId, release, rows: rows.length });
|
|
4834
5189
|
} catch (err) {
|
|
4835
5190
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
4836
5191
|
const finishedAt = deps.now().toISOString();
|
|
@@ -4838,8 +5193,8 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
4838
5193
|
status: RunStatuses.failed,
|
|
4839
5194
|
error: errorMsg,
|
|
4840
5195
|
finishedAt
|
|
4841
|
-
}).where(
|
|
4842
|
-
|
|
5196
|
+
}).where(eq9(runs.id, runId)).run();
|
|
5197
|
+
log10.error("extract.failed", { runId, projectId, error: errorMsg });
|
|
4843
5198
|
throw err;
|
|
4844
5199
|
}
|
|
4845
5200
|
}
|
|
@@ -4859,18 +5214,18 @@ function computeSummary2(rows) {
|
|
|
4859
5214
|
}
|
|
4860
5215
|
|
|
4861
5216
|
// src/discovery-run.ts
|
|
4862
|
-
import
|
|
4863
|
-
import { and as and8, eq as
|
|
4864
|
-
var
|
|
5217
|
+
import crypto12 from "crypto";
|
|
5218
|
+
import { and as and8, eq as eq10 } from "drizzle-orm";
|
|
5219
|
+
var log11 = createLogger("DiscoveryRun");
|
|
4865
5220
|
var DEFAULT_SEED_COUNT = 30;
|
|
4866
5221
|
var QUERIES_PER_INTENT_BUCKET = 6;
|
|
4867
5222
|
async function executeDiscoveryRun(opts) {
|
|
4868
5223
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
4869
|
-
opts.db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
5224
|
+
opts.db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq10(runs.id, opts.runId)).run();
|
|
4870
5225
|
try {
|
|
4871
|
-
const projectRow = opts.db.select().from(projects).where(
|
|
5226
|
+
const projectRow = opts.db.select().from(projects).where(eq10(projects.id, opts.projectId)).get();
|
|
4872
5227
|
if (!projectRow) throw new Error(`Project ${opts.projectId} not found`);
|
|
4873
|
-
const projectCompetitors = opts.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
5228
|
+
const projectCompetitors = opts.db.select({ domain: competitors.domain }).from(competitors).where(eq10(competitors.projectId, opts.projectId)).all().map((r) => r.domain.toLowerCase());
|
|
4874
5229
|
const canonicalDomains = effectiveDomains({
|
|
4875
5230
|
canonicalDomain: projectRow.canonicalDomain,
|
|
4876
5231
|
ownedDomains: projectRow.ownedDomains
|
|
@@ -4900,8 +5255,8 @@ async function executeDiscoveryRun(opts) {
|
|
|
4900
5255
|
seedProvider: result.seedProvider,
|
|
4901
5256
|
result
|
|
4902
5257
|
});
|
|
4903
|
-
opts.db.update(runs).set({ status: RunStatuses.completed, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
4904
|
-
|
|
5258
|
+
opts.db.update(runs).set({ status: RunStatuses.completed, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq10(runs.id, opts.runId)).run();
|
|
5259
|
+
log11.info("discovery.completed", {
|
|
4905
5260
|
runId: opts.runId,
|
|
4906
5261
|
sessionId: opts.sessionId,
|
|
4907
5262
|
buckets: result.buckets,
|
|
@@ -4909,13 +5264,13 @@ async function executeDiscoveryRun(opts) {
|
|
|
4909
5264
|
});
|
|
4910
5265
|
} catch (err) {
|
|
4911
5266
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
4912
|
-
|
|
5267
|
+
log11.error("discovery.failed", { runId: opts.runId, sessionId: opts.sessionId, error: errorMsg });
|
|
4913
5268
|
markSessionFailed(opts.db, opts.sessionId, errorMsg);
|
|
4914
5269
|
opts.db.update(runs).set({
|
|
4915
5270
|
status: RunStatuses.failed,
|
|
4916
5271
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4917
5272
|
error: errorMsg
|
|
4918
|
-
}).where(
|
|
5273
|
+
}).where(eq10(runs.id, opts.runId)).run();
|
|
4919
5274
|
}
|
|
4920
5275
|
}
|
|
4921
5276
|
function buildDefaultDeps(registry) {
|
|
@@ -5121,12 +5476,12 @@ function writeDiscoveryInsight(db, input) {
|
|
|
5121
5476
|
});
|
|
5122
5477
|
db.transaction((tx) => {
|
|
5123
5478
|
tx.update(insights).set({ dismissed: true }).where(and8(
|
|
5124
|
-
|
|
5125
|
-
|
|
5126
|
-
|
|
5479
|
+
eq10(insights.projectId, input.projectId),
|
|
5480
|
+
eq10(insights.type, "discovery.basket-divergence"),
|
|
5481
|
+
eq10(insights.dismissed, false)
|
|
5127
5482
|
)).run();
|
|
5128
5483
|
tx.insert(insights).values({
|
|
5129
|
-
id:
|
|
5484
|
+
id: crypto12.randomUUID(),
|
|
5130
5485
|
projectId: input.projectId,
|
|
5131
5486
|
runId: input.runId,
|
|
5132
5487
|
type: "discovery.basket-divergence",
|
|
@@ -5162,10 +5517,10 @@ function buildDiscoveryInsightTitle(input) {
|
|
|
5162
5517
|
}
|
|
5163
5518
|
|
|
5164
5519
|
// src/execute-site-audit.ts
|
|
5165
|
-
import
|
|
5166
|
-
import { eq as
|
|
5520
|
+
import crypto13 from "crypto";
|
|
5521
|
+
import { eq as eq11 } from "drizzle-orm";
|
|
5167
5522
|
import { runSitemapAudit } from "@ainyc/aeo-audit";
|
|
5168
|
-
var
|
|
5523
|
+
var log12 = createLogger("SiteAudit");
|
|
5169
5524
|
var SITE_AUDIT_DEFAULT_PAGE_LIMIT = 500;
|
|
5170
5525
|
var SITE_AUDIT_MAX_PAGE_LIMIT = 2e3;
|
|
5171
5526
|
function toHomepageUrl(canonicalDomain) {
|
|
@@ -5226,15 +5581,15 @@ function computeFactorAverages(pages) {
|
|
|
5226
5581
|
}
|
|
5227
5582
|
async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
5228
5583
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
5229
|
-
db.update(runs).set({ status: "running", startedAt }).where(
|
|
5584
|
+
db.update(runs).set({ status: "running", startedAt }).where(eq11(runs.id, runId)).run();
|
|
5230
5585
|
try {
|
|
5231
|
-
const project = db.select().from(projects).where(
|
|
5586
|
+
const project = db.select().from(projects).where(eq11(projects.id, projectId)).get();
|
|
5232
5587
|
if (!project) {
|
|
5233
5588
|
throw new Error(`Project not found: ${projectId}`);
|
|
5234
5589
|
}
|
|
5235
5590
|
const homepageUrl = toHomepageUrl(project.canonicalDomain);
|
|
5236
5591
|
const limit = clampSiteAuditLimit(opts.limit);
|
|
5237
|
-
|
|
5592
|
+
log12.info("start", { runId, projectId, homepageUrl, sitemapUrl: opts.sitemapUrl ?? null, limit });
|
|
5238
5593
|
await assertSiteAuditUrlAllowed(homepageUrl, "canonicalDomain");
|
|
5239
5594
|
if (opts.sitemapUrl) await assertSiteAuditUrlAllowed(opts.sitemapUrl, "sitemapUrl");
|
|
5240
5595
|
const report = await runSitemapAudit(homepageUrl, { sitemapUrl: opts.sitemapUrl, limit });
|
|
@@ -5242,7 +5597,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
5242
5597
|
const pagesErrored = report.pages.filter((page) => page.status === "error").length;
|
|
5243
5598
|
const auditable = report.pagesDiscovered - report.pagesSkipped;
|
|
5244
5599
|
if (auditable > report.pagesAudited) {
|
|
5245
|
-
|
|
5600
|
+
log12.info("truncated", {
|
|
5246
5601
|
runId,
|
|
5247
5602
|
projectId,
|
|
5248
5603
|
auditable,
|
|
@@ -5261,7 +5616,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
5261
5616
|
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
5262
5617
|
db.transaction((tx) => {
|
|
5263
5618
|
tx.insert(siteAuditSnapshots).values({
|
|
5264
|
-
id:
|
|
5619
|
+
id: crypto13.randomUUID(),
|
|
5265
5620
|
projectId,
|
|
5266
5621
|
runId,
|
|
5267
5622
|
sitemapUrl: report.sitemapUrl,
|
|
@@ -5290,7 +5645,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
5290
5645
|
}).run();
|
|
5291
5646
|
for (const page of report.pages) {
|
|
5292
5647
|
tx.insert(siteAuditPages).values({
|
|
5293
|
-
id:
|
|
5648
|
+
id: crypto13.randomUUID(),
|
|
5294
5649
|
projectId,
|
|
5295
5650
|
runId,
|
|
5296
5651
|
url: page.url,
|
|
@@ -5301,9 +5656,9 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
5301
5656
|
createdAt: finishedAt
|
|
5302
5657
|
}).run();
|
|
5303
5658
|
}
|
|
5304
|
-
tx.update(runs).set({ status, finishedAt }).where(
|
|
5659
|
+
tx.update(runs).set({ status, finishedAt }).where(eq11(runs.id, runId)).run();
|
|
5305
5660
|
});
|
|
5306
|
-
|
|
5661
|
+
log12.info("completed", {
|
|
5307
5662
|
runId,
|
|
5308
5663
|
projectId,
|
|
5309
5664
|
status,
|
|
@@ -5313,14 +5668,14 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
5313
5668
|
});
|
|
5314
5669
|
} catch (err) {
|
|
5315
5670
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
5316
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
5317
|
-
|
|
5671
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq11(runs.id, runId)).run();
|
|
5672
|
+
log12.error("failed", { runId, projectId, error: errorMsg });
|
|
5318
5673
|
throw err;
|
|
5319
5674
|
}
|
|
5320
5675
|
}
|
|
5321
5676
|
|
|
5322
5677
|
// src/commands/backfill.ts
|
|
5323
|
-
import { and as and9, eq as
|
|
5678
|
+
import { and as and9, eq as eq12, inArray as inArray4, isNull, sql as sql4 } from "drizzle-orm";
|
|
5324
5679
|
var SNAPSHOT_BATCH_SIZE = 500;
|
|
5325
5680
|
async function backfillAnswerVisibilityCommand(opts) {
|
|
5326
5681
|
const config = loadConfig();
|
|
@@ -5328,7 +5683,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
5328
5683
|
migrate(db);
|
|
5329
5684
|
const projectFilter = opts?.project?.trim();
|
|
5330
5685
|
const isDryRun = opts?.dryRun === true;
|
|
5331
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
5686
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq12(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
5332
5687
|
let examined = 0;
|
|
5333
5688
|
let updated = 0;
|
|
5334
5689
|
let wouldUpdate = 0;
|
|
@@ -5337,9 +5692,9 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
5337
5692
|
let providerErrors = 0;
|
|
5338
5693
|
if (scopedProjects.length > 0) {
|
|
5339
5694
|
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(and9(
|
|
5340
|
-
|
|
5695
|
+
eq12(runs.kind, RunKinds["answer-visibility"]),
|
|
5341
5696
|
inArray4(runs.projectId, scopedProjects.map((project) => project.id))
|
|
5342
|
-
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
5697
|
+
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(eq12(runs.kind, RunKinds["answer-visibility"])).all();
|
|
5343
5698
|
const runIdsByProject = /* @__PURE__ */ new Map();
|
|
5344
5699
|
for (const run of runRows) {
|
|
5345
5700
|
const existing = runIdsByProject.get(run.projectId);
|
|
@@ -5347,7 +5702,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
5347
5702
|
else runIdsByProject.set(run.projectId, [run.id]);
|
|
5348
5703
|
}
|
|
5349
5704
|
for (const project of scopedProjects) {
|
|
5350
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
5705
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq12(competitors.projectId, project.id)).all().map((row) => row.domain);
|
|
5351
5706
|
const runIds = runIdsByProject.get(project.id) ?? [];
|
|
5352
5707
|
if (runIds.length === 0) continue;
|
|
5353
5708
|
const projectDomains = effectiveDomains({
|
|
@@ -5435,7 +5790,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
5435
5790
|
} else {
|
|
5436
5791
|
db.transaction((tx) => {
|
|
5437
5792
|
for (const update of pendingUpdates) {
|
|
5438
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
5793
|
+
tx.update(querySnapshots).set(update.patch).where(eq12(querySnapshots.id, update.id)).run();
|
|
5439
5794
|
}
|
|
5440
5795
|
});
|
|
5441
5796
|
updated += pendingUpdates.length;
|
|
@@ -5484,7 +5839,7 @@ No DB writes performed. Re-run without --dry-run to apply.`);
|
|
|
5484
5839
|
function backfillNormalizedPaths(db, opts) {
|
|
5485
5840
|
const baseConditions = [];
|
|
5486
5841
|
if (opts?.projectId) {
|
|
5487
|
-
baseConditions.push(
|
|
5842
|
+
baseConditions.push(eq12(gaTrafficSnapshots.projectId, opts.projectId));
|
|
5488
5843
|
}
|
|
5489
5844
|
const rows = db.select({
|
|
5490
5845
|
id: gaTrafficSnapshots.id,
|
|
@@ -5505,7 +5860,7 @@ function backfillNormalizedPaths(db, opts) {
|
|
|
5505
5860
|
unchanged++;
|
|
5506
5861
|
continue;
|
|
5507
5862
|
}
|
|
5508
|
-
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(
|
|
5863
|
+
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(eq12(gaTrafficSnapshots.id, row.id)).run();
|
|
5509
5864
|
updated++;
|
|
5510
5865
|
}
|
|
5511
5866
|
});
|
|
@@ -5519,7 +5874,7 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
5519
5874
|
const projectFilter = opts?.project?.trim();
|
|
5520
5875
|
let projectId;
|
|
5521
5876
|
if (projectFilter) {
|
|
5522
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
5877
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq12(projects.name, projectFilter)).get();
|
|
5523
5878
|
if (!project) {
|
|
5524
5879
|
const result2 = {
|
|
5525
5880
|
project: projectFilter,
|
|
@@ -5556,7 +5911,7 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
5556
5911
|
function backfillAiReferralPaths(db, opts) {
|
|
5557
5912
|
const baseConditions = [];
|
|
5558
5913
|
if (opts?.projectId) {
|
|
5559
|
-
baseConditions.push(
|
|
5914
|
+
baseConditions.push(eq12(gaAiReferrals.projectId, opts.projectId));
|
|
5560
5915
|
}
|
|
5561
5916
|
const rows = db.select({
|
|
5562
5917
|
id: gaAiReferrals.id,
|
|
@@ -5577,7 +5932,7 @@ function backfillAiReferralPaths(db, opts) {
|
|
|
5577
5932
|
unchanged++;
|
|
5578
5933
|
continue;
|
|
5579
5934
|
}
|
|
5580
|
-
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(
|
|
5935
|
+
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(eq12(gaAiReferrals.id, row.id)).run();
|
|
5581
5936
|
updated++;
|
|
5582
5937
|
}
|
|
5583
5938
|
});
|
|
@@ -5591,7 +5946,7 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
5591
5946
|
const projectFilter = opts?.project?.trim();
|
|
5592
5947
|
let projectId;
|
|
5593
5948
|
if (projectFilter) {
|
|
5594
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
5949
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq12(projects.name, projectFilter)).get();
|
|
5595
5950
|
if (!project) {
|
|
5596
5951
|
const result2 = {
|
|
5597
5952
|
project: projectFilter,
|
|
@@ -5627,10 +5982,10 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
5627
5982
|
}
|
|
5628
5983
|
function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
5629
5984
|
const isDryRun = opts?.dryRun === true;
|
|
5630
|
-
const project = db.select().from(projects).where(
|
|
5985
|
+
const project = db.select().from(projects).where(eq12(projects.id, projectId)).get();
|
|
5631
5986
|
if (!project) return { examined: 0, updated: 0, mentioned: 0 };
|
|
5632
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
5633
|
-
const runRows = db.select({ id: runs.id }).from(runs).where(and9(
|
|
5987
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq12(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
5988
|
+
const runRows = db.select({ id: runs.id }).from(runs).where(and9(eq12(runs.kind, RunKinds["answer-visibility"]), eq12(runs.projectId, projectId))).all();
|
|
5634
5989
|
const runIds = runRows.map((r) => r.id);
|
|
5635
5990
|
let examined = 0;
|
|
5636
5991
|
let updated = 0;
|
|
@@ -5702,7 +6057,7 @@ function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
|
5702
6057
|
} else {
|
|
5703
6058
|
db.transaction((tx) => {
|
|
5704
6059
|
for (const update of pendingUpdates) {
|
|
5705
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
6060
|
+
tx.update(querySnapshots).set(update.patch).where(eq12(querySnapshots.id, update.id)).run();
|
|
5706
6061
|
}
|
|
5707
6062
|
});
|
|
5708
6063
|
updated += pendingUpdates.length;
|
|
@@ -5717,7 +6072,7 @@ async function backfillAnswerMentionsCommand(opts) {
|
|
|
5717
6072
|
migrate(db);
|
|
5718
6073
|
const projectFilter = opts?.project?.trim();
|
|
5719
6074
|
const isDryRun = opts?.dryRun === true;
|
|
5720
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
6075
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq12(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
5721
6076
|
let examined = 0;
|
|
5722
6077
|
let updated = 0;
|
|
5723
6078
|
let wouldUpdate = 0;
|
|
@@ -5777,7 +6132,7 @@ function readStoredGroundingSources(rawResponse) {
|
|
|
5777
6132
|
return result;
|
|
5778
6133
|
}
|
|
5779
6134
|
async function backfillInsightsCommand(project, opts) {
|
|
5780
|
-
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-
|
|
6135
|
+
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-ZWW3I3NL.js");
|
|
5781
6136
|
const config = loadConfig();
|
|
5782
6137
|
const db = createClient(config.database);
|
|
5783
6138
|
migrate(db);
|
|
@@ -5936,7 +6291,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
5936
6291
|
const config = loadConfig();
|
|
5937
6292
|
const db = createClient(config.database);
|
|
5938
6293
|
migrate(db);
|
|
5939
|
-
const project = db.select().from(projects).where(
|
|
6294
|
+
const project = db.select().from(projects).where(eq12(projects.name, opts.project)).get();
|
|
5940
6295
|
if (!project) {
|
|
5941
6296
|
throw new Error(`Project "${opts.project}" not found`);
|
|
5942
6297
|
}
|
|
@@ -5948,7 +6303,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
5948
6303
|
`);
|
|
5949
6304
|
}
|
|
5950
6305
|
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(and9(
|
|
5951
|
-
|
|
6306
|
+
eq12(auditLog.projectId, project.id),
|
|
5952
6307
|
inArray4(auditLog.action, ["keywords.appended", "keywords.deleted", "queries.appended", "queries.deleted", "queries.replaced"])
|
|
5953
6308
|
)).orderBy(auditLog.createdAt).all();
|
|
5954
6309
|
const history = replayQueryAuditLog(events);
|
|
@@ -5956,8 +6311,8 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
5956
6311
|
runId: runs.id,
|
|
5957
6312
|
createdAt: runs.createdAt,
|
|
5958
6313
|
location: runs.location
|
|
5959
|
-
}).from(runs).innerJoin(querySnapshots,
|
|
5960
|
-
|
|
6314
|
+
}).from(runs).innerJoin(querySnapshots, eq12(querySnapshots.runId, runs.id)).where(and9(
|
|
6315
|
+
eq12(runs.projectId, project.id),
|
|
5961
6316
|
isNull(querySnapshots.queryId),
|
|
5962
6317
|
isNull(querySnapshots.queryText)
|
|
5963
6318
|
)).groupBy(runs.id).orderBy(runs.createdAt).all();
|
|
@@ -5980,7 +6335,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
5980
6335
|
createdAt: querySnapshots.createdAt,
|
|
5981
6336
|
answerText: querySnapshots.answerText
|
|
5982
6337
|
}).from(querySnapshots).where(and9(
|
|
5983
|
-
|
|
6338
|
+
eq12(querySnapshots.runId, run.runId),
|
|
5984
6339
|
isNull(querySnapshots.queryId),
|
|
5985
6340
|
isNull(querySnapshots.queryText)
|
|
5986
6341
|
)).orderBy(querySnapshots.provider, querySnapshots.createdAt).all();
|
|
@@ -6046,7 +6401,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
6046
6401
|
if (!isDryRun && updates.length > 0) {
|
|
6047
6402
|
db.transaction((tx) => {
|
|
6048
6403
|
for (const u of updates) {
|
|
6049
|
-
tx.update(querySnapshots).set({ queryText: u.queryText }).where(
|
|
6404
|
+
tx.update(querySnapshots).set({ queryText: u.queryText }).where(eq12(querySnapshots.id, u.id)).run();
|
|
6050
6405
|
}
|
|
6051
6406
|
});
|
|
6052
6407
|
}
|
|
@@ -6120,7 +6475,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
6120
6475
|
const projectFilter = opts?.project?.trim();
|
|
6121
6476
|
const isDryRun = opts?.dryRun === true;
|
|
6122
6477
|
const isJson = isMachineFormat(opts?.format);
|
|
6123
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
6478
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq12(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
6124
6479
|
if (scopedProjects.length === 0) {
|
|
6125
6480
|
if (projectFilter && !isJson) {
|
|
6126
6481
|
process.stderr.write(`No project named "${projectFilter}".
|
|
@@ -6146,7 +6501,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
6146
6501
|
byBot: {}
|
|
6147
6502
|
};
|
|
6148
6503
|
const unknownCountRow = db.select({ n: sql4`count(*)` }).from(rawEventSamples).where(and9(
|
|
6149
|
-
|
|
6504
|
+
eq12(rawEventSamples.eventType, "unknown"),
|
|
6150
6505
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
6151
6506
|
)).get();
|
|
6152
6507
|
result.unknownBefore = Number(unknownCountRow?.n ?? 0);
|
|
@@ -6159,7 +6514,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
6159
6514
|
pathNormalized: rawEventSamples.pathNormalized,
|
|
6160
6515
|
status: rawEventSamples.status
|
|
6161
6516
|
}).from(rawEventSamples).where(and9(
|
|
6162
|
-
|
|
6517
|
+
eq12(rawEventSamples.eventType, "unknown"),
|
|
6163
6518
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
6164
6519
|
)).all();
|
|
6165
6520
|
result.examined = unknownSamples.length;
|
|
@@ -6198,7 +6553,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
6198
6553
|
result.reclassified++;
|
|
6199
6554
|
result.byBot[classified.botId] = (result.byBot[classified.botId] ?? 0) + 1;
|
|
6200
6555
|
if (isDryRun) continue;
|
|
6201
|
-
db.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(
|
|
6556
|
+
db.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(eq12(rawEventSamples.id, snap.id)).run();
|
|
6202
6557
|
const tsHour = new Date(snap.ts);
|
|
6203
6558
|
tsHour.setUTCMinutes(0, 0, 0);
|
|
6204
6559
|
if (userFetch) {
|
|
@@ -6263,7 +6618,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
6263
6618
|
}
|
|
6264
6619
|
if (!isDryRun) {
|
|
6265
6620
|
const afterRow = db.select({ n: sql4`count(*)` }).from(rawEventSamples).where(and9(
|
|
6266
|
-
|
|
6621
|
+
eq12(rawEventSamples.eventType, "unknown"),
|
|
6267
6622
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
6268
6623
|
)).get();
|
|
6269
6624
|
result.unknownAfter = Number(afterRow?.n ?? 0);
|
|
@@ -6298,7 +6653,7 @@ No DB writes performed. Re-run without --dry-run to apply.`);
|
|
|
6298
6653
|
}
|
|
6299
6654
|
|
|
6300
6655
|
// src/commands/skills.ts
|
|
6301
|
-
import
|
|
6656
|
+
import crypto14 from "crypto";
|
|
6302
6657
|
import fs4 from "fs";
|
|
6303
6658
|
import os4 from "os";
|
|
6304
6659
|
import path5 from "path";
|
|
@@ -6353,7 +6708,7 @@ function walkRelative(dir, prefix = "") {
|
|
|
6353
6708
|
return out.sort();
|
|
6354
6709
|
}
|
|
6355
6710
|
function sha256File(filePath) {
|
|
6356
|
-
return
|
|
6711
|
+
return crypto14.createHash("sha256").update(fs4.readFileSync(filePath)).digest("hex");
|
|
6357
6712
|
}
|
|
6358
6713
|
function readSkillManifest(skillDir) {
|
|
6359
6714
|
try {
|
|
@@ -6676,10 +7031,10 @@ var ProviderRegistry = class {
|
|
|
6676
7031
|
};
|
|
6677
7032
|
|
|
6678
7033
|
// src/scheduler.ts
|
|
6679
|
-
import
|
|
7034
|
+
import crypto15 from "crypto";
|
|
6680
7035
|
import cron from "node-cron";
|
|
6681
|
-
import { and as and10, eq as
|
|
6682
|
-
var
|
|
7036
|
+
import { and as and10, eq as eq13, inArray as inArray5 } from "drizzle-orm";
|
|
7037
|
+
var log13 = createLogger("Scheduler");
|
|
6683
7038
|
function taskKey(projectId, kind) {
|
|
6684
7039
|
return `${projectId}::${kind}`;
|
|
6685
7040
|
}
|
|
@@ -6693,16 +7048,16 @@ var Scheduler = class {
|
|
|
6693
7048
|
}
|
|
6694
7049
|
/** Load all enabled schedules from DB and register cron jobs. */
|
|
6695
7050
|
start() {
|
|
6696
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
7051
|
+
const allSchedules = this.db.select().from(schedules).where(eq13(schedules.enabled, true)).all();
|
|
6697
7052
|
for (const schedule of allSchedules) {
|
|
6698
7053
|
const missedRunAt = schedule.nextRunAt;
|
|
6699
7054
|
this.registerCronTask(schedule);
|
|
6700
7055
|
if (missedRunAt && new Date(missedRunAt) < /* @__PURE__ */ new Date()) {
|
|
6701
|
-
|
|
7056
|
+
log13.info("run.catch-up", { projectId: schedule.projectId, kind: schedule.kind, missedRunAt });
|
|
6702
7057
|
this.triggerRun(schedule.id, schedule.projectId, schedule.kind);
|
|
6703
7058
|
}
|
|
6704
7059
|
}
|
|
6705
|
-
|
|
7060
|
+
log13.info("started", { scheduleCount: allSchedules.length });
|
|
6706
7061
|
}
|
|
6707
7062
|
/** Stop all cron tasks for graceful shutdown. */
|
|
6708
7063
|
stop() {
|
|
@@ -6723,7 +7078,7 @@ var Scheduler = class {
|
|
|
6723
7078
|
this.stopTask(key, existing, "Stopped");
|
|
6724
7079
|
this.tasks.delete(key);
|
|
6725
7080
|
}
|
|
6726
|
-
const schedule = this.db.select().from(schedules).where(and10(
|
|
7081
|
+
const schedule = this.db.select().from(schedules).where(and10(eq13(schedules.projectId, projectId), eq13(schedules.kind, kind))).get();
|
|
6727
7082
|
if (schedule && schedule.enabled) {
|
|
6728
7083
|
this.registerCronTask(schedule);
|
|
6729
7084
|
}
|
|
@@ -6746,13 +7101,13 @@ var Scheduler = class {
|
|
|
6746
7101
|
stopTask(key, task, verb) {
|
|
6747
7102
|
void task.stop();
|
|
6748
7103
|
void task.destroy();
|
|
6749
|
-
|
|
7104
|
+
log13.info(`task.${verb.toLowerCase()}`, { key });
|
|
6750
7105
|
}
|
|
6751
7106
|
registerCronTask(schedule) {
|
|
6752
7107
|
const { id: scheduleId, projectId, cronExpr, timezone } = schedule;
|
|
6753
7108
|
const kind = schedule.kind;
|
|
6754
7109
|
if (!cron.validate(cronExpr)) {
|
|
6755
|
-
|
|
7110
|
+
log13.error("cron.invalid", { projectId, kind, cronExpr });
|
|
6756
7111
|
return;
|
|
6757
7112
|
}
|
|
6758
7113
|
const task = cron.schedule(cronExpr, () => {
|
|
@@ -6764,51 +7119,51 @@ var Scheduler = class {
|
|
|
6764
7119
|
this.db.update(schedules).set({
|
|
6765
7120
|
nextRunAt: nextRunFromCron(cronExpr, timezone),
|
|
6766
7121
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
6767
|
-
}).where(
|
|
7122
|
+
}).where(eq13(schedules.id, scheduleId)).run();
|
|
6768
7123
|
const label = schedule.preset ?? cronExpr;
|
|
6769
|
-
|
|
7124
|
+
log13.info("cron.registered", { projectId, kind, schedule: label, timezone });
|
|
6770
7125
|
}
|
|
6771
7126
|
triggerRun(scheduleId, projectId, kind) {
|
|
6772
7127
|
try {
|
|
6773
7128
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
6774
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
7129
|
+
const currentSchedule = this.db.select().from(schedules).where(eq13(schedules.id, scheduleId)).get();
|
|
6775
7130
|
if (!currentSchedule || !currentSchedule.enabled) {
|
|
6776
|
-
|
|
7131
|
+
log13.warn("schedule.stale", { scheduleId, projectId, kind, msg: "schedule no longer exists or is disabled" });
|
|
6777
7132
|
this.remove(projectId, kind);
|
|
6778
7133
|
return;
|
|
6779
7134
|
}
|
|
6780
7135
|
const nextRunAt = nextRunFromCron(currentSchedule.cronExpr, currentSchedule.timezone);
|
|
6781
|
-
const project = this.db.select().from(projects).where(
|
|
7136
|
+
const project = this.db.select().from(projects).where(eq13(projects.id, projectId)).get();
|
|
6782
7137
|
if (!project) {
|
|
6783
|
-
|
|
7138
|
+
log13.error("project.not-found", { projectId, kind, msg: "skipping scheduled run" });
|
|
6784
7139
|
this.remove(projectId, kind);
|
|
6785
7140
|
return;
|
|
6786
7141
|
}
|
|
6787
7142
|
if (kind === SchedulableRunKinds["traffic-sync"]) {
|
|
6788
7143
|
const sourceId = currentSchedule.sourceId;
|
|
6789
7144
|
if (!sourceId) {
|
|
6790
|
-
|
|
7145
|
+
log13.warn("traffic-sync.missing-source", { scheduleId, projectId });
|
|
6791
7146
|
return;
|
|
6792
7147
|
}
|
|
6793
7148
|
if (!this.callbacks.onTrafficSyncRequested) {
|
|
6794
|
-
|
|
7149
|
+
log13.warn("traffic-sync.no-callback", { scheduleId, projectId, msg: "host did not register onTrafficSyncRequested" });
|
|
6795
7150
|
return;
|
|
6796
7151
|
}
|
|
6797
7152
|
this.db.update(schedules).set({
|
|
6798
7153
|
lastRunAt: now,
|
|
6799
7154
|
nextRunAt,
|
|
6800
7155
|
updatedAt: now
|
|
6801
|
-
}).where(
|
|
6802
|
-
|
|
7156
|
+
}).where(eq13(schedules.id, currentSchedule.id)).run();
|
|
7157
|
+
log13.info("traffic-sync.triggered", { projectName: project.name, sourceId });
|
|
6803
7158
|
this.callbacks.onTrafficSyncRequested(project.name, sourceId);
|
|
6804
7159
|
return;
|
|
6805
7160
|
}
|
|
6806
7161
|
if (kind === SchedulableRunKinds["gbp-sync"]) {
|
|
6807
7162
|
if (!this.callbacks.onGbpSyncRequested) {
|
|
6808
|
-
|
|
7163
|
+
log13.warn("gbp-sync.no-callback", { scheduleId, projectId, msg: "host did not register onGbpSyncRequested" });
|
|
6809
7164
|
return;
|
|
6810
7165
|
}
|
|
6811
|
-
const runId2 =
|
|
7166
|
+
const runId2 = crypto15.randomUUID();
|
|
6812
7167
|
this.db.insert(runs).values({
|
|
6813
7168
|
id: runId2,
|
|
6814
7169
|
projectId,
|
|
@@ -6821,55 +7176,88 @@ var Scheduler = class {
|
|
|
6821
7176
|
lastRunAt: now,
|
|
6822
7177
|
nextRunAt,
|
|
6823
7178
|
updatedAt: now
|
|
6824
|
-
}).where(
|
|
6825
|
-
|
|
7179
|
+
}).where(eq13(schedules.id, currentSchedule.id)).run();
|
|
7180
|
+
log13.info("gbp-sync.triggered", { runId: runId2, projectName: project.name });
|
|
6826
7181
|
this.callbacks.onGbpSyncRequested(runId2, projectId);
|
|
6827
7182
|
return;
|
|
6828
7183
|
}
|
|
7184
|
+
if (kind === SchedulableRunKinds["ads-sync"]) {
|
|
7185
|
+
if (!this.callbacks.onAdsSyncRequested) {
|
|
7186
|
+
log13.warn("ads-sync.no-callback", { scheduleId, projectId, msg: "host did not register onAdsSyncRequested" });
|
|
7187
|
+
return;
|
|
7188
|
+
}
|
|
7189
|
+
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(and10(
|
|
7190
|
+
eq13(runs.projectId, projectId),
|
|
7191
|
+
eq13(runs.kind, RunKinds["ads-sync"]),
|
|
7192
|
+
inArray5(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
7193
|
+
)).get();
|
|
7194
|
+
if (activeAdsRun) {
|
|
7195
|
+
log13.info("ads-sync.skipped-active", { projectName: project.name, activeRunId: activeAdsRun.id });
|
|
7196
|
+
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq13(schedules.id, currentSchedule.id)).run();
|
|
7197
|
+
return;
|
|
7198
|
+
}
|
|
7199
|
+
const runId2 = crypto15.randomUUID();
|
|
7200
|
+
this.db.insert(runs).values({
|
|
7201
|
+
id: runId2,
|
|
7202
|
+
projectId,
|
|
7203
|
+
kind: RunKinds["ads-sync"],
|
|
7204
|
+
status: RunStatuses.queued,
|
|
7205
|
+
trigger: RunTriggers.scheduled,
|
|
7206
|
+
createdAt: now
|
|
7207
|
+
}).run();
|
|
7208
|
+
this.db.update(schedules).set({
|
|
7209
|
+
lastRunAt: now,
|
|
7210
|
+
nextRunAt,
|
|
7211
|
+
updatedAt: now
|
|
7212
|
+
}).where(eq13(schedules.id, currentSchedule.id)).run();
|
|
7213
|
+
log13.info("ads-sync.triggered", { runId: runId2, projectName: project.name });
|
|
7214
|
+
this.callbacks.onAdsSyncRequested(runId2, projectId);
|
|
7215
|
+
return;
|
|
7216
|
+
}
|
|
6829
7217
|
if (kind === SchedulableRunKinds["data-refresh"]) {
|
|
6830
7218
|
if (!this.callbacks.onDataRefreshRequested) {
|
|
6831
|
-
|
|
7219
|
+
log13.warn("data-refresh.no-callback", { scheduleId, projectId, msg: "host did not register onDataRefreshRequested" });
|
|
6832
7220
|
return;
|
|
6833
7221
|
}
|
|
6834
7222
|
this.db.update(schedules).set({
|
|
6835
7223
|
lastRunAt: now,
|
|
6836
7224
|
nextRunAt,
|
|
6837
7225
|
updatedAt: now
|
|
6838
|
-
}).where(
|
|
6839
|
-
|
|
7226
|
+
}).where(eq13(schedules.id, currentSchedule.id)).run();
|
|
7227
|
+
log13.info("data-refresh.triggered", { projectName: project.name });
|
|
6840
7228
|
this.callbacks.onDataRefreshRequested(project.name);
|
|
6841
7229
|
return;
|
|
6842
7230
|
}
|
|
6843
7231
|
if (kind === SchedulableRunKinds["backlinks-sync"]) {
|
|
6844
7232
|
if (!this.callbacks.onBacklinksSyncRequested) {
|
|
6845
|
-
|
|
7233
|
+
log13.warn("backlinks-sync.no-callback", { scheduleId, projectId, msg: "host did not register onBacklinksSyncRequested" });
|
|
6846
7234
|
return;
|
|
6847
7235
|
}
|
|
6848
7236
|
this.db.update(schedules).set({
|
|
6849
7237
|
lastRunAt: now,
|
|
6850
7238
|
nextRunAt,
|
|
6851
7239
|
updatedAt: now
|
|
6852
|
-
}).where(
|
|
6853
|
-
|
|
7240
|
+
}).where(eq13(schedules.id, currentSchedule.id)).run();
|
|
7241
|
+
log13.info("backlinks-sync.triggered", { projectName: project.name });
|
|
6854
7242
|
this.callbacks.onBacklinksSyncRequested(project.name);
|
|
6855
7243
|
return;
|
|
6856
7244
|
}
|
|
6857
7245
|
if (kind === SchedulableRunKinds["site-audit"]) {
|
|
6858
7246
|
if (!this.callbacks.onSiteAuditRequested) {
|
|
6859
|
-
|
|
7247
|
+
log13.warn("site-audit.no-callback", { scheduleId, projectId, msg: "host did not register onSiteAuditRequested" });
|
|
6860
7248
|
return;
|
|
6861
7249
|
}
|
|
6862
7250
|
const active = this.db.select({ id: runs.id }).from(runs).where(and10(
|
|
6863
|
-
|
|
6864
|
-
|
|
7251
|
+
eq13(runs.projectId, projectId),
|
|
7252
|
+
eq13(runs.kind, RunKinds["site-audit"]),
|
|
6865
7253
|
inArray5(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
6866
7254
|
)).get();
|
|
6867
7255
|
if (active) {
|
|
6868
|
-
|
|
6869
|
-
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(
|
|
7256
|
+
log13.info("site-audit.skipped-active", { projectName: project.name, activeRunId: active.id });
|
|
7257
|
+
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq13(schedules.id, currentSchedule.id)).run();
|
|
6870
7258
|
return;
|
|
6871
7259
|
}
|
|
6872
|
-
const runId2 =
|
|
7260
|
+
const runId2 = crypto15.randomUUID();
|
|
6873
7261
|
this.db.insert(runs).values({
|
|
6874
7262
|
id: runId2,
|
|
6875
7263
|
projectId,
|
|
@@ -6882,8 +7270,8 @@ var Scheduler = class {
|
|
|
6882
7270
|
lastRunAt: now,
|
|
6883
7271
|
nextRunAt,
|
|
6884
7272
|
updatedAt: now
|
|
6885
|
-
}).where(
|
|
6886
|
-
|
|
7273
|
+
}).where(eq13(schedules.id, currentSchedule.id)).run();
|
|
7274
|
+
log13.info("site-audit.triggered", { runId: runId2, projectName: project.name });
|
|
6887
7275
|
this.callbacks.onSiteAuditRequested(runId2, projectId);
|
|
6888
7276
|
return;
|
|
6889
7277
|
}
|
|
@@ -6892,7 +7280,7 @@ var Scheduler = class {
|
|
|
6892
7280
|
if (project.defaultLocation) {
|
|
6893
7281
|
const loc = projectLocations.find((l) => l.label === project.defaultLocation);
|
|
6894
7282
|
if (!loc) {
|
|
6895
|
-
|
|
7283
|
+
log13.warn("default-location.stale", { scheduleId, projectId, label: project.defaultLocation });
|
|
6896
7284
|
return;
|
|
6897
7285
|
}
|
|
6898
7286
|
resolvedLocation = loc;
|
|
@@ -6906,11 +7294,11 @@ var Scheduler = class {
|
|
|
6906
7294
|
location: locationLabel
|
|
6907
7295
|
});
|
|
6908
7296
|
if (queueResult.conflict) {
|
|
6909
|
-
|
|
7297
|
+
log13.info("run.skipped-active", { projectName: project.name, activeRunId: queueResult.activeRunId });
|
|
6910
7298
|
this.db.update(schedules).set({
|
|
6911
7299
|
nextRunAt,
|
|
6912
7300
|
updatedAt: now
|
|
6913
|
-
}).where(
|
|
7301
|
+
}).where(eq13(schedules.id, currentSchedule.id)).run();
|
|
6914
7302
|
return;
|
|
6915
7303
|
}
|
|
6916
7304
|
const runId = queueResult.runId;
|
|
@@ -6918,43 +7306,44 @@ var Scheduler = class {
|
|
|
6918
7306
|
lastRunAt: now,
|
|
6919
7307
|
nextRunAt,
|
|
6920
7308
|
updatedAt: now
|
|
6921
|
-
}).where(
|
|
7309
|
+
}).where(eq13(schedules.id, currentSchedule.id)).run();
|
|
6922
7310
|
const scheduleProviders = currentSchedule.providers;
|
|
6923
7311
|
const providers = scheduleProviders.length > 0 ? scheduleProviders : void 0;
|
|
6924
|
-
|
|
7312
|
+
log13.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
6925
7313
|
this.callbacks.onRunCreated(runId, projectId, providers, resolvedLocation);
|
|
6926
7314
|
} catch (err) {
|
|
6927
|
-
|
|
7315
|
+
log13.error("trigger.error", { scheduleId, projectId, kind, error: err instanceof Error ? err.message : String(err) });
|
|
6928
7316
|
}
|
|
6929
7317
|
}
|
|
6930
7318
|
};
|
|
6931
7319
|
|
|
6932
7320
|
// src/data-refresh.ts
|
|
6933
|
-
var
|
|
7321
|
+
var log14 = createLogger("DataRefresh");
|
|
6934
7322
|
async function refreshAllIntegrations(client, projectName) {
|
|
6935
7323
|
const integrations = [
|
|
6936
7324
|
{ name: "gsc", run: () => client.gscSync(projectName, {}) },
|
|
6937
7325
|
{ name: "bing", run: () => client.bingInspectSitemap(projectName, {}) },
|
|
6938
7326
|
{ name: "ga", run: () => client.gaSync(projectName, { days: 30 }) },
|
|
6939
|
-
{ name: "gbp", run: () => client.triggerGbpSync(projectName, {}) }
|
|
7327
|
+
{ name: "gbp", run: () => client.triggerGbpSync(projectName, {}) },
|
|
7328
|
+
{ name: "ads", run: () => client.triggerAdsSync(projectName) }
|
|
6940
7329
|
];
|
|
6941
7330
|
const results = await Promise.allSettled(integrations.map((i) => i.run()));
|
|
6942
7331
|
results.forEach((result, idx) => {
|
|
6943
7332
|
const integration = integrations[idx].name;
|
|
6944
7333
|
if (result.status === "fulfilled") {
|
|
6945
|
-
|
|
7334
|
+
log14.info("integration.refreshed", { projectName, integration });
|
|
6946
7335
|
} else {
|
|
6947
7336
|
const reason = result.reason;
|
|
6948
7337
|
const message = reason instanceof Error ? reason.message : String(reason);
|
|
6949
|
-
|
|
7338
|
+
log14.warn("integration.refresh-failed", { projectName, integration, error: message });
|
|
6950
7339
|
}
|
|
6951
7340
|
});
|
|
6952
7341
|
}
|
|
6953
7342
|
|
|
6954
7343
|
// src/notifier.ts
|
|
6955
|
-
import { eq as
|
|
6956
|
-
import
|
|
6957
|
-
var
|
|
7344
|
+
import { eq as eq14, desc as desc5, and as and11, inArray as inArray6, or } from "drizzle-orm";
|
|
7345
|
+
import crypto16 from "crypto";
|
|
7346
|
+
var log15 = createLogger("Notifier");
|
|
6958
7347
|
var Notifier = class {
|
|
6959
7348
|
db;
|
|
6960
7349
|
serverUrl;
|
|
@@ -6964,26 +7353,26 @@ var Notifier = class {
|
|
|
6964
7353
|
}
|
|
6965
7354
|
/** Called after a run completes (success, partial, or failed). */
|
|
6966
7355
|
async onRunCompleted(runId, projectId) {
|
|
6967
|
-
|
|
6968
|
-
const notifs = this.db.select().from(notifications).where(
|
|
7356
|
+
log15.info("run.completed", { runId, projectId });
|
|
7357
|
+
const notifs = this.db.select().from(notifications).where(eq14(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
6969
7358
|
if (notifs.length === 0) {
|
|
6970
|
-
|
|
7359
|
+
log15.info("notifications.none-enabled", { projectId });
|
|
6971
7360
|
return;
|
|
6972
7361
|
}
|
|
6973
|
-
|
|
6974
|
-
const run = this.db.select().from(runs).where(
|
|
7362
|
+
log15.info("notifications.found", { projectId, count: notifs.length });
|
|
7363
|
+
const run = this.db.select().from(runs).where(eq14(runs.id, runId)).get();
|
|
6975
7364
|
if (!run) {
|
|
6976
|
-
|
|
7365
|
+
log15.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
6977
7366
|
return;
|
|
6978
7367
|
}
|
|
6979
|
-
const project = this.db.select().from(projects).where(
|
|
7368
|
+
const project = this.db.select().from(projects).where(eq14(projects.id, projectId)).get();
|
|
6980
7369
|
if (!project) {
|
|
6981
|
-
|
|
7370
|
+
log15.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
6982
7371
|
return;
|
|
6983
7372
|
}
|
|
6984
7373
|
const transitions = this.computeTransitions(runId, projectId);
|
|
6985
7374
|
const events = [];
|
|
6986
|
-
|
|
7375
|
+
log15.info("run.status", { runId: run.id, status: run.status, projectId });
|
|
6987
7376
|
if (run.status === "completed" || run.status === "partial") {
|
|
6988
7377
|
events.push("run.completed");
|
|
6989
7378
|
}
|
|
@@ -6999,7 +7388,7 @@ var Notifier = class {
|
|
|
6999
7388
|
if (!config.url) continue;
|
|
7000
7389
|
const subscribedEvents = config.events;
|
|
7001
7390
|
const matchingEvents = events.filter((e) => subscribedEvents.includes(e));
|
|
7002
|
-
|
|
7391
|
+
log15.info("notification.match", { notificationId: notif.id, subscribedEvents, matchedEvents: matchingEvents });
|
|
7003
7392
|
if (matchingEvents.length === 0) continue;
|
|
7004
7393
|
for (const event of matchingEvents) {
|
|
7005
7394
|
const relevantTransitions = event === "citation.lost" ? lostTransitions : event === "citation.gained" ? gainedTransitions : transitions;
|
|
@@ -7023,11 +7412,11 @@ var Notifier = class {
|
|
|
7023
7412
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
7024
7413
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
7025
7414
|
if (insightEvents.length === 0) return;
|
|
7026
|
-
const notifs = this.db.select().from(notifications).where(
|
|
7415
|
+
const notifs = this.db.select().from(notifications).where(eq14(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
7027
7416
|
if (notifs.length === 0) return;
|
|
7028
|
-
const run = this.db.select().from(runs).where(
|
|
7417
|
+
const run = this.db.select().from(runs).where(eq14(runs.id, runId)).get();
|
|
7029
7418
|
if (!run) return;
|
|
7030
|
-
const project = this.db.select().from(projects).where(
|
|
7419
|
+
const project = this.db.select().from(projects).where(eq14(projects.id, projectId)).get();
|
|
7031
7420
|
if (!project) return;
|
|
7032
7421
|
for (const notif of notifs) {
|
|
7033
7422
|
const config = notif.config;
|
|
@@ -7057,12 +7446,12 @@ var Notifier = class {
|
|
|
7057
7446
|
}
|
|
7058
7447
|
}
|
|
7059
7448
|
computeTransitions(runId, projectId) {
|
|
7060
|
-
const thisRun = this.db.select().from(runs).where(
|
|
7449
|
+
const thisRun = this.db.select().from(runs).where(eq14(runs.id, runId)).get();
|
|
7061
7450
|
if (!thisRun) return [];
|
|
7062
7451
|
const groupSiblings = this.db.select().from(runs).where(and11(
|
|
7063
|
-
|
|
7064
|
-
|
|
7065
|
-
|
|
7452
|
+
eq14(runs.projectId, projectId),
|
|
7453
|
+
eq14(runs.kind, thisRun.kind),
|
|
7454
|
+
eq14(runs.createdAt, thisRun.createdAt)
|
|
7066
7455
|
)).all();
|
|
7067
7456
|
const stillPending = groupSiblings.some((r) => r.status === "queued" || r.status === "running");
|
|
7068
7457
|
if (stillPending) return [];
|
|
@@ -7078,7 +7467,7 @@ var Notifier = class {
|
|
|
7078
7467
|
return candidate.id > best.id ? candidate : best;
|
|
7079
7468
|
});
|
|
7080
7469
|
if (winner.id !== runId) return [];
|
|
7081
|
-
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
7470
|
+
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(eq14(projects.id, projectId)).get();
|
|
7082
7471
|
const locationCount = Math.max(
|
|
7083
7472
|
1,
|
|
7084
7473
|
(projectLocations?.locations ?? []).length
|
|
@@ -7086,9 +7475,9 @@ var Notifier = class {
|
|
|
7086
7475
|
const RECENT_FETCH_LIMIT = Math.max(8, locationCount * 4);
|
|
7087
7476
|
const recentRuns = this.db.select().from(runs).where(
|
|
7088
7477
|
and11(
|
|
7089
|
-
|
|
7090
|
-
|
|
7091
|
-
or(
|
|
7478
|
+
eq14(runs.projectId, projectId),
|
|
7479
|
+
eq14(runs.kind, thisRun.kind),
|
|
7480
|
+
or(eq14(runs.status, "completed"), eq14(runs.status, "partial"))
|
|
7092
7481
|
)
|
|
7093
7482
|
).orderBy(desc5(runs.createdAt), desc5(runs.id)).limit(RECENT_FETCH_LIMIT).all();
|
|
7094
7483
|
const groups = groupRunsByCreatedAt(recentRuns);
|
|
@@ -7105,7 +7494,7 @@ var Notifier = class {
|
|
|
7105
7494
|
provider: querySnapshots.provider,
|
|
7106
7495
|
location: querySnapshots.location,
|
|
7107
7496
|
citationState: querySnapshots.citationState
|
|
7108
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
7497
|
+
}).from(querySnapshots).leftJoin(queries, eq14(querySnapshots.queryId, queries.id)).where(inArray6(querySnapshots.runId, currentRunIds)).all();
|
|
7109
7498
|
const previousSnapshots = this.db.select({
|
|
7110
7499
|
queryId: querySnapshots.queryId,
|
|
7111
7500
|
provider: querySnapshots.provider,
|
|
@@ -7138,23 +7527,23 @@ var Notifier = class {
|
|
|
7138
7527
|
const targetLabel = redactNotificationUrl(url).urlDisplay;
|
|
7139
7528
|
const targetCheck = await resolveWebhookTarget(url);
|
|
7140
7529
|
if (!targetCheck.ok) {
|
|
7141
|
-
|
|
7530
|
+
log15.error("webhook.ssrf-blocked", { url: targetLabel, reason: targetCheck.message });
|
|
7142
7531
|
this.logDelivery(projectId, notificationId, payload.event, "failed", `SSRF: ${targetCheck.message}`);
|
|
7143
7532
|
return;
|
|
7144
7533
|
}
|
|
7145
|
-
|
|
7534
|
+
log15.info("webhook.send", { event: payload.event, url: targetLabel });
|
|
7146
7535
|
const maxRetries = 3;
|
|
7147
7536
|
const delays = [1e3, 4e3, 16e3];
|
|
7148
7537
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
7149
7538
|
try {
|
|
7150
7539
|
const response = await deliverWebhook(targetCheck.target, payload, webhookSecret);
|
|
7151
7540
|
if (response.status >= 200 && response.status < 300) {
|
|
7152
|
-
|
|
7541
|
+
log15.info("webhook.delivered", { event: payload.event, url: targetLabel, httpStatus: response.status });
|
|
7153
7542
|
this.logDelivery(projectId, notificationId, payload.event, "sent", null);
|
|
7154
7543
|
return;
|
|
7155
7544
|
}
|
|
7156
7545
|
const errorDetail = response.error ?? `HTTP ${response.status}`;
|
|
7157
|
-
|
|
7546
|
+
log15.warn("webhook.attempt-failed", { event: payload.event, url: targetLabel, attempt: attempt + 1, maxRetries, httpStatus: response.status, error: errorDetail });
|
|
7158
7547
|
if (attempt === maxRetries - 1) {
|
|
7159
7548
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
7160
7549
|
}
|
|
@@ -7162,7 +7551,7 @@ var Notifier = class {
|
|
|
7162
7551
|
const errorDetail = err instanceof Error ? err.message : String(err);
|
|
7163
7552
|
if (attempt === maxRetries - 1) {
|
|
7164
7553
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
7165
|
-
|
|
7554
|
+
log15.error("webhook.exhausted", { event: payload.event, url: targetLabel, maxRetries, error: errorDetail });
|
|
7166
7555
|
}
|
|
7167
7556
|
}
|
|
7168
7557
|
if (attempt < maxRetries - 1) {
|
|
@@ -7172,7 +7561,7 @@ var Notifier = class {
|
|
|
7172
7561
|
}
|
|
7173
7562
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
7174
7563
|
this.db.insert(auditLog).values({
|
|
7175
|
-
id:
|
|
7564
|
+
id: crypto16.randomUUID(),
|
|
7176
7565
|
projectId,
|
|
7177
7566
|
actor: "scheduler",
|
|
7178
7567
|
action: `notification.${status}`,
|
|
@@ -7185,8 +7574,8 @@ var Notifier = class {
|
|
|
7185
7574
|
};
|
|
7186
7575
|
|
|
7187
7576
|
// src/run-coordinator.ts
|
|
7188
|
-
import { eq as
|
|
7189
|
-
var
|
|
7577
|
+
import { eq as eq15 } from "drizzle-orm";
|
|
7578
|
+
var log16 = createLogger("RunCoordinator");
|
|
7190
7579
|
var RunCoordinator = class {
|
|
7191
7580
|
constructor(db, notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
7192
7581
|
this.db = db;
|
|
@@ -7201,10 +7590,10 @@ var RunCoordinator = class {
|
|
|
7201
7590
|
onInsightsGenerated;
|
|
7202
7591
|
onAeroEvent;
|
|
7203
7592
|
async onRunCompleted(runId, projectId) {
|
|
7204
|
-
const runRow = this.db.select().from(runs).where(
|
|
7593
|
+
const runRow = this.db.select().from(runs).where(eq15(runs.id, runId)).get();
|
|
7205
7594
|
const kind = runRow?.kind ?? RunKinds["answer-visibility"];
|
|
7206
7595
|
if (runRow?.trigger === RunTriggers.probe) {
|
|
7207
|
-
|
|
7596
|
+
log16.info("probe.skip-side-effects", { runId, projectId, kind });
|
|
7208
7597
|
return;
|
|
7209
7598
|
}
|
|
7210
7599
|
let insightCount = 0;
|
|
@@ -7221,12 +7610,12 @@ var RunCoordinator = class {
|
|
|
7221
7610
|
try {
|
|
7222
7611
|
await this.onInsightsGenerated(runId, projectId, result);
|
|
7223
7612
|
} catch (err) {
|
|
7224
|
-
|
|
7613
|
+
log16.error("insight-webhook.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
7225
7614
|
}
|
|
7226
7615
|
}
|
|
7227
7616
|
}
|
|
7228
7617
|
} catch (err) {
|
|
7229
|
-
|
|
7618
|
+
log16.error("intelligence.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
7230
7619
|
}
|
|
7231
7620
|
} else if (kind === RunKinds["gbp-sync"]) {
|
|
7232
7621
|
try {
|
|
@@ -7239,17 +7628,17 @@ var RunCoordinator = class {
|
|
|
7239
7628
|
try {
|
|
7240
7629
|
await this.onInsightsGenerated(runId, projectId, analysisResultFromInsights(gbpInsights));
|
|
7241
7630
|
} catch (err) {
|
|
7242
|
-
|
|
7631
|
+
log16.error("gbp-insight-webhook.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
7243
7632
|
}
|
|
7244
7633
|
}
|
|
7245
7634
|
} catch (err) {
|
|
7246
|
-
|
|
7635
|
+
log16.error("gbp-intelligence.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
7247
7636
|
}
|
|
7248
7637
|
}
|
|
7249
7638
|
try {
|
|
7250
7639
|
await this.notifier.onRunCompleted(runId, projectId);
|
|
7251
7640
|
} catch (err) {
|
|
7252
|
-
|
|
7641
|
+
log16.error("notifier.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
7253
7642
|
}
|
|
7254
7643
|
if (this.onAeroEvent) {
|
|
7255
7644
|
try {
|
|
@@ -7262,7 +7651,7 @@ var RunCoordinator = class {
|
|
|
7262
7651
|
};
|
|
7263
7652
|
await this.onAeroEvent(ctx);
|
|
7264
7653
|
} catch (err) {
|
|
7265
|
-
|
|
7654
|
+
log16.error("aero.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
7266
7655
|
}
|
|
7267
7656
|
}
|
|
7268
7657
|
}
|
|
@@ -7277,7 +7666,7 @@ var RunCoordinator = class {
|
|
|
7277
7666
|
* so the Aero queue is never starved of a follow-up.
|
|
7278
7667
|
*/
|
|
7279
7668
|
buildDiscoveryAeroContext(runId, projectId, status, error) {
|
|
7280
|
-
const session = this.db.select().from(discoverySessions).where(
|
|
7669
|
+
const session = this.db.select().from(discoverySessions).where(eq15(discoverySessions.runId, runId)).get();
|
|
7281
7670
|
const competitorMap = session ? session.competitorMap : [];
|
|
7282
7671
|
return {
|
|
7283
7672
|
kind: RunKinds["aeo-discover-probe"],
|
|
@@ -7317,8 +7706,8 @@ function analysisResultFromInsights(insights2) {
|
|
|
7317
7706
|
}
|
|
7318
7707
|
|
|
7319
7708
|
// src/agent/session-registry.ts
|
|
7320
|
-
import
|
|
7321
|
-
import { eq as
|
|
7709
|
+
import crypto18 from "crypto";
|
|
7710
|
+
import { eq as eq17 } from "drizzle-orm";
|
|
7322
7711
|
|
|
7323
7712
|
// src/agent/session.ts
|
|
7324
7713
|
import fs7 from "fs";
|
|
@@ -7706,8 +8095,8 @@ function resolveSessionProviderAndModel(config, opts) {
|
|
|
7706
8095
|
}
|
|
7707
8096
|
|
|
7708
8097
|
// src/agent/memory-store.ts
|
|
7709
|
-
import
|
|
7710
|
-
import { and as and12, desc as desc6, eq as
|
|
8098
|
+
import crypto17 from "crypto";
|
|
8099
|
+
import { and as and12, desc as desc6, eq as eq16, like, sql as sql5 } from "drizzle-orm";
|
|
7711
8100
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
7712
8101
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
7713
8102
|
function rowToDto(row) {
|
|
@@ -7721,7 +8110,7 @@ function rowToDto(row) {
|
|
|
7721
8110
|
};
|
|
7722
8111
|
}
|
|
7723
8112
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
7724
|
-
const query = db.select().from(agentMemory).where(
|
|
8113
|
+
const query = db.select().from(agentMemory).where(eq16(agentMemory.projectId, projectId)).orderBy(desc6(agentMemory.updatedAt));
|
|
7725
8114
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
7726
8115
|
return rows.map(rowToDto);
|
|
7727
8116
|
}
|
|
@@ -7735,7 +8124,7 @@ function upsertMemoryEntry(db, args) {
|
|
|
7735
8124
|
throw new Error(`memory key prefix "${COMPACTION_KEY_PREFIX}" is reserved for compaction notes`);
|
|
7736
8125
|
}
|
|
7737
8126
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7738
|
-
const id =
|
|
8127
|
+
const id = crypto17.randomUUID();
|
|
7739
8128
|
db.insert(agentMemory).values({
|
|
7740
8129
|
id,
|
|
7741
8130
|
projectId: args.projectId,
|
|
@@ -7752,12 +8141,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
7752
8141
|
updatedAt: now
|
|
7753
8142
|
}
|
|
7754
8143
|
}).run();
|
|
7755
|
-
const row = db.select().from(agentMemory).where(and12(
|
|
8144
|
+
const row = db.select().from(agentMemory).where(and12(eq16(agentMemory.projectId, args.projectId), eq16(agentMemory.key, args.key))).get();
|
|
7756
8145
|
if (!row) throw new Error("memory upsert produced no row");
|
|
7757
8146
|
return rowToDto(row);
|
|
7758
8147
|
}
|
|
7759
8148
|
function deleteMemoryEntry(db, projectId, key) {
|
|
7760
|
-
const result = db.delete(agentMemory).where(and12(
|
|
8149
|
+
const result = db.delete(agentMemory).where(and12(eq16(agentMemory.projectId, projectId), eq16(agentMemory.key, key))).run();
|
|
7761
8150
|
const changes = result.changes ?? 0;
|
|
7762
8151
|
return changes > 0;
|
|
7763
8152
|
}
|
|
@@ -7772,7 +8161,7 @@ function writeCompactionNote(db, args) {
|
|
|
7772
8161
|
}
|
|
7773
8162
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7774
8163
|
const key = `${COMPACTION_KEY_PREFIX}${args.sessionId}:${now}`;
|
|
7775
|
-
const id =
|
|
8164
|
+
const id = crypto17.randomUUID();
|
|
7776
8165
|
let inserted;
|
|
7777
8166
|
db.transaction((tx) => {
|
|
7778
8167
|
tx.insert(agentMemory).values({
|
|
@@ -7787,7 +8176,7 @@ function writeCompactionNote(db, args) {
|
|
|
7787
8176
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
7788
8177
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
7789
8178
|
and12(
|
|
7790
|
-
|
|
8179
|
+
eq16(agentMemory.projectId, args.projectId),
|
|
7791
8180
|
like(agentMemory.key, `${sessionPrefix}%`)
|
|
7792
8181
|
)
|
|
7793
8182
|
).orderBy(desc6(agentMemory.updatedAt)).all();
|
|
@@ -7795,7 +8184,7 @@ function writeCompactionNote(db, args) {
|
|
|
7795
8184
|
if (stale.length > 0) {
|
|
7796
8185
|
tx.delete(agentMemory).where(sql5`${agentMemory.id} IN (${sql5.join(stale.map((s) => sql5`${s}`), sql5`, `)})`).run();
|
|
7797
8186
|
}
|
|
7798
|
-
const row = tx.select().from(agentMemory).where(and12(
|
|
8187
|
+
const row = tx.select().from(agentMemory).where(and12(eq16(agentMemory.projectId, args.projectId), eq16(agentMemory.key, key))).get();
|
|
7799
8188
|
if (row) inserted = rowToDto(row);
|
|
7800
8189
|
});
|
|
7801
8190
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -7928,7 +8317,7 @@ async function compactMessages(args) {
|
|
|
7928
8317
|
}
|
|
7929
8318
|
|
|
7930
8319
|
// src/agent/session-registry.ts
|
|
7931
|
-
var
|
|
8320
|
+
var log17 = createLogger("SessionRegistry");
|
|
7932
8321
|
var MAX_HYDRATE_NOTES = 20;
|
|
7933
8322
|
var MAX_HYDRATE_BYTES = 32 * 1024;
|
|
7934
8323
|
function escapeMemoryFragment(value) {
|
|
@@ -7977,7 +8366,7 @@ var SessionRegistry = class {
|
|
|
7977
8366
|
modelProvider: effectiveProvider,
|
|
7978
8367
|
modelId: effectiveModelId,
|
|
7979
8368
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
7980
|
-
}).where(
|
|
8369
|
+
}).where(eq17(agentSessions.projectId, projectId)).run();
|
|
7981
8370
|
}
|
|
7982
8371
|
const agent2 = createAeroSession({
|
|
7983
8372
|
projectName,
|
|
@@ -8155,13 +8544,13 @@ ${lines.join("\n")}
|
|
|
8155
8544
|
agent.state.messages = result.messages;
|
|
8156
8545
|
agent.state.systemPrompt = this.buildHydratedSystemPrompt(projectId, row.systemPrompt);
|
|
8157
8546
|
this.save(projectName);
|
|
8158
|
-
|
|
8547
|
+
log17.info("compaction.completed", {
|
|
8159
8548
|
projectName,
|
|
8160
8549
|
removedCount: result.removedCount,
|
|
8161
8550
|
summaryBytes: Buffer.byteLength(result.summary, "utf8")
|
|
8162
8551
|
});
|
|
8163
8552
|
} catch (err) {
|
|
8164
|
-
|
|
8553
|
+
log17.error("compaction.failed", {
|
|
8165
8554
|
projectName,
|
|
8166
8555
|
error: err instanceof Error ? err.message : String(err)
|
|
8167
8556
|
});
|
|
@@ -8191,7 +8580,7 @@ ${lines.join("\n")}
|
|
|
8191
8580
|
modelProvider: nextProvider,
|
|
8192
8581
|
modelId: nextModelId,
|
|
8193
8582
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
8194
|
-
}).where(
|
|
8583
|
+
}).where(eq17(agentSessions.projectId, projectId)).run();
|
|
8195
8584
|
}
|
|
8196
8585
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
8197
8586
|
save(projectName) {
|
|
@@ -8258,7 +8647,7 @@ ${lines.join("\n")}
|
|
|
8258
8647
|
await agent.prompt(msgs);
|
|
8259
8648
|
this.save(projectName);
|
|
8260
8649
|
} catch (err) {
|
|
8261
|
-
|
|
8650
|
+
log17.error("drain.failed", {
|
|
8262
8651
|
projectName,
|
|
8263
8652
|
error: err instanceof Error ? err.message : String(err)
|
|
8264
8653
|
});
|
|
@@ -8353,17 +8742,17 @@ ${lines.join("\n")}
|
|
|
8353
8742
|
return id;
|
|
8354
8743
|
}
|
|
8355
8744
|
tryResolveProjectId(projectName) {
|
|
8356
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
8745
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq17(projects.name, projectName)).get();
|
|
8357
8746
|
return row?.id;
|
|
8358
8747
|
}
|
|
8359
8748
|
loadRow(projectId) {
|
|
8360
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
8749
|
+
const row = this.opts.db.select().from(agentSessions).where(eq17(agentSessions.projectId, projectId)).get();
|
|
8361
8750
|
return row ?? null;
|
|
8362
8751
|
}
|
|
8363
8752
|
insertRow(params) {
|
|
8364
8753
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8365
8754
|
this.opts.db.insert(agentSessions).values({
|
|
8366
|
-
id:
|
|
8755
|
+
id: crypto18.randomUUID(),
|
|
8367
8756
|
projectId: params.projectId,
|
|
8368
8757
|
systemPrompt: params.systemPrompt,
|
|
8369
8758
|
modelProvider: params.provider ?? params.modelProvider ?? AgentProviderIds.claude,
|
|
@@ -8376,14 +8765,14 @@ ${lines.join("\n")}
|
|
|
8376
8765
|
}
|
|
8377
8766
|
updateRow(projectId, patch) {
|
|
8378
8767
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8379
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
8768
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq17(agentSessions.projectId, projectId)).run();
|
|
8380
8769
|
}
|
|
8381
8770
|
};
|
|
8382
8771
|
|
|
8383
8772
|
// src/agent/agent-routes.ts
|
|
8384
|
-
import { eq as
|
|
8773
|
+
import { eq as eq18 } from "drizzle-orm";
|
|
8385
8774
|
function resolveProject(db, name) {
|
|
8386
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
8775
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq18(projects.name, name)).get();
|
|
8387
8776
|
if (!row) throw notFound("project", name);
|
|
8388
8777
|
return row;
|
|
8389
8778
|
}
|
|
@@ -8392,7 +8781,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
8392
8781
|
"/projects/:name/agent/transcript",
|
|
8393
8782
|
async (request) => {
|
|
8394
8783
|
const project = resolveProject(opts.db, request.params.name);
|
|
8395
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
8784
|
+
const row = opts.db.select().from(agentSessions).where(eq18(agentSessions.projectId, project.id)).get();
|
|
8396
8785
|
if (!row) {
|
|
8397
8786
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
8398
8787
|
}
|
|
@@ -8416,7 +8805,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
8416
8805
|
async (request) => {
|
|
8417
8806
|
const project = resolveProject(opts.db, request.params.name);
|
|
8418
8807
|
opts.sessionRegistry.reset(project.name);
|
|
8419
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
8808
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq18(agentSessions.projectId, project.id)).run();
|
|
8420
8809
|
return { status: "reset" };
|
|
8421
8810
|
}
|
|
8422
8811
|
);
|
|
@@ -8856,7 +9245,7 @@ function formatAuditFactorScore(factor) {
|
|
|
8856
9245
|
}
|
|
8857
9246
|
|
|
8858
9247
|
// src/snapshot-service.ts
|
|
8859
|
-
var
|
|
9248
|
+
var log18 = createLogger("Snapshot");
|
|
8860
9249
|
var ANALYSIS_PROVIDER_PRIORITY = ["openai", "claude", "gemini", "perplexity", "local"];
|
|
8861
9250
|
var SNAPSHOT_QUERY_COUNT = 6;
|
|
8862
9251
|
var ProviderExecutionGate2 = class {
|
|
@@ -9002,7 +9391,7 @@ var SnapshotService = class {
|
|
|
9002
9391
|
return mapAuditReport(report);
|
|
9003
9392
|
} catch (err) {
|
|
9004
9393
|
const message = err instanceof Error ? err.message : String(err);
|
|
9005
|
-
|
|
9394
|
+
log18.warn("audit.failed", { homepageUrl, error: message });
|
|
9006
9395
|
return {
|
|
9007
9396
|
url: homepageUrl,
|
|
9008
9397
|
finalUrl: homepageUrl,
|
|
@@ -9031,7 +9420,7 @@ var SnapshotService = class {
|
|
|
9031
9420
|
queries: parsedQueries
|
|
9032
9421
|
};
|
|
9033
9422
|
} catch (err) {
|
|
9034
|
-
|
|
9423
|
+
log18.warn("profile.generation-failed", {
|
|
9035
9424
|
domain: ctx.domain,
|
|
9036
9425
|
provider: ctx.analysisProvider.adapter.name,
|
|
9037
9426
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -9173,7 +9562,7 @@ var SnapshotService = class {
|
|
|
9173
9562
|
recommendedActions: uniqueStrings(parsed.recommendedActions ?? []).slice(0, 4)
|
|
9174
9563
|
};
|
|
9175
9564
|
} catch (err) {
|
|
9176
|
-
|
|
9565
|
+
log18.warn("response.analysis-failed", {
|
|
9177
9566
|
provider: ctx.analysisProvider.adapter.name,
|
|
9178
9567
|
error: err instanceof Error ? err.message : String(err)
|
|
9179
9568
|
});
|
|
@@ -9455,7 +9844,7 @@ function clipText(value, length) {
|
|
|
9455
9844
|
// src/server.ts
|
|
9456
9845
|
var _require3 = createRequire3(import.meta.url);
|
|
9457
9846
|
var { version: PKG_VERSION2 } = _require3("../package.json");
|
|
9458
|
-
var
|
|
9847
|
+
var log19 = createLogger("Server");
|
|
9459
9848
|
var DEFAULT_QUOTA = {
|
|
9460
9849
|
maxConcurrency: 2,
|
|
9461
9850
|
maxRequestsPerMinute: 10,
|
|
@@ -9490,14 +9879,14 @@ function summarizeProviderConfig(config) {
|
|
|
9490
9879
|
};
|
|
9491
9880
|
}
|
|
9492
9881
|
function hashApiKey(key) {
|
|
9493
|
-
return
|
|
9882
|
+
return crypto19.createHash("sha256").update(key).digest("hex");
|
|
9494
9883
|
}
|
|
9495
9884
|
var DASHBOARD_SCRYPT_KEYLEN = 64;
|
|
9496
9885
|
var DASHBOARD_SCRYPT_COST = 1 << 15;
|
|
9497
9886
|
var DASHBOARD_SCRYPT_MAXMEM = 64 * 1024 * 1024;
|
|
9498
9887
|
function hashDashboardPassword(password) {
|
|
9499
|
-
const salt =
|
|
9500
|
-
const derived =
|
|
9888
|
+
const salt = crypto19.randomBytes(16);
|
|
9889
|
+
const derived = crypto19.scryptSync(password, salt, DASHBOARD_SCRYPT_KEYLEN, {
|
|
9501
9890
|
N: DASHBOARD_SCRYPT_COST,
|
|
9502
9891
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
9503
9892
|
});
|
|
@@ -9518,18 +9907,18 @@ function verifyDashboardPassword(password, storedHash) {
|
|
|
9518
9907
|
} catch {
|
|
9519
9908
|
return { ok: false, needsRehash: false };
|
|
9520
9909
|
}
|
|
9521
|
-
const derived =
|
|
9910
|
+
const derived = crypto19.scryptSync(password, salt, expected.length, {
|
|
9522
9911
|
N: DASHBOARD_SCRYPT_COST,
|
|
9523
9912
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
9524
9913
|
});
|
|
9525
9914
|
if (derived.length !== expected.length) return { ok: false, needsRehash: false };
|
|
9526
|
-
return { ok:
|
|
9915
|
+
return { ok: crypto19.timingSafeEqual(derived, expected), needsRehash: false };
|
|
9527
9916
|
}
|
|
9528
9917
|
if (/^[a-f0-9]{64}$/i.test(storedHash)) {
|
|
9529
9918
|
const candidate = Buffer.from(hashApiKey(password), "hex");
|
|
9530
9919
|
const expected = Buffer.from(storedHash, "hex");
|
|
9531
9920
|
if (candidate.length !== expected.length) return { ok: false, needsRehash: false };
|
|
9532
|
-
const ok =
|
|
9921
|
+
const ok = crypto19.timingSafeEqual(candidate, expected);
|
|
9533
9922
|
return { ok, needsRehash: ok };
|
|
9534
9923
|
}
|
|
9535
9924
|
return { ok: false, needsRehash: false };
|
|
@@ -9588,7 +9977,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
9588
9977
|
}
|
|
9589
9978
|
if (migratedGoogle > 0) {
|
|
9590
9979
|
saveConfigPatch({ google: config.google });
|
|
9591
|
-
|
|
9980
|
+
log19.info("credentials.migrated", { type: "google", count: migratedGoogle });
|
|
9592
9981
|
}
|
|
9593
9982
|
let migratedGa4 = 0;
|
|
9594
9983
|
for (const row of rows.ga4) {
|
|
@@ -9606,7 +9995,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
9606
9995
|
}
|
|
9607
9996
|
if (migratedGa4 > 0) {
|
|
9608
9997
|
saveConfigPatch({ ga4: config.ga4 });
|
|
9609
|
-
|
|
9998
|
+
log19.info("credentials.migrated", { type: "ga4", count: migratedGa4 });
|
|
9610
9999
|
}
|
|
9611
10000
|
}
|
|
9612
10001
|
function isLoopbackBindHost(host) {
|
|
@@ -9645,11 +10034,11 @@ async function createServer(opts) {
|
|
|
9645
10034
|
applyLegacyCredentials(legacyRows, opts.config);
|
|
9646
10035
|
dropLegacyCredentialColumns(opts.db);
|
|
9647
10036
|
} catch (err) {
|
|
9648
|
-
|
|
10037
|
+
log19.warn("credentials.migration.failed", {
|
|
9649
10038
|
error: err instanceof Error ? err.message : String(err)
|
|
9650
10039
|
});
|
|
9651
10040
|
}
|
|
9652
|
-
|
|
10041
|
+
log19.info("providers.configured", { providers: Object.keys(providers).filter((k) => {
|
|
9653
10042
|
const p = providers[k];
|
|
9654
10043
|
return p?.apiKey || p?.baseUrl || p?.vertexProject;
|
|
9655
10044
|
}) });
|
|
@@ -9698,7 +10087,7 @@ async function createServer(opts) {
|
|
|
9698
10087
|
intelligenceService,
|
|
9699
10088
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
9700
10089
|
async (ctx) => {
|
|
9701
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
10090
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq19(projects.id, ctx.projectId)).get();
|
|
9702
10091
|
if (!project) return;
|
|
9703
10092
|
let content;
|
|
9704
10093
|
if (ctx.kind === RunKinds["aeo-discover-probe"]) {
|
|
@@ -9741,11 +10130,41 @@ async function createServer(opts) {
|
|
|
9741
10130
|
app.log.error({ runId, err }, "GBP sync failed");
|
|
9742
10131
|
});
|
|
9743
10132
|
};
|
|
10133
|
+
const runAdsSync = (runId, projectId) => {
|
|
10134
|
+
executeAdsSync(opts.db, runId, projectId, { config: opts.config }).then(() => runCoordinator.onRunCompleted(runId, projectId)).catch((err) => {
|
|
10135
|
+
app.log.error({ runId, err }, "Ads sync failed");
|
|
10136
|
+
});
|
|
10137
|
+
};
|
|
9744
10138
|
const runSiteAudit = (runId, projectId, auditOpts) => {
|
|
9745
10139
|
executeSiteAudit(opts.db, runId, projectId, auditOpts ?? {}).then(() => runCoordinator.onRunCompleted(runId, projectId)).catch((err) => {
|
|
9746
10140
|
app.log.error({ runId, err }, "Site audit failed");
|
|
9747
10141
|
});
|
|
9748
10142
|
};
|
|
10143
|
+
const adsCredentialStore = {
|
|
10144
|
+
getConnection: (projectName) => {
|
|
10145
|
+
return getOpenAiAdsConnection(opts.config, projectName);
|
|
10146
|
+
},
|
|
10147
|
+
upsertConnection: (connection) => {
|
|
10148
|
+
const updated = upsertOpenAiAdsConnection(opts.config, connection);
|
|
10149
|
+
saveConfigPatch(opts.config);
|
|
10150
|
+
return updated;
|
|
10151
|
+
},
|
|
10152
|
+
removeConnection: (projectName) => {
|
|
10153
|
+
const removed = removeOpenAiAdsConnection(opts.config, projectName);
|
|
10154
|
+
if (removed) saveConfigPatch(opts.config);
|
|
10155
|
+
return removed;
|
|
10156
|
+
}
|
|
10157
|
+
};
|
|
10158
|
+
const verifyAdsAccount = async (apiKey) => {
|
|
10159
|
+
const account = await getAdAccount(apiKey);
|
|
10160
|
+
return {
|
|
10161
|
+
id: account.id,
|
|
10162
|
+
name: account.name,
|
|
10163
|
+
status: account.status,
|
|
10164
|
+
currencyCode: account.currency_code ?? null,
|
|
10165
|
+
timezone: account.timezone ?? null
|
|
10166
|
+
};
|
|
10167
|
+
};
|
|
9749
10168
|
const scheduler = new Scheduler(opts.db, {
|
|
9750
10169
|
onRunCreated: (runId, projectId, providers2, location) => {
|
|
9751
10170
|
jobRunner.executeRun(runId, projectId, providers2, location).catch((err) => {
|
|
@@ -9760,6 +10179,9 @@ async function createServer(opts) {
|
|
|
9760
10179
|
onGbpSyncRequested: (runId, projectId) => {
|
|
9761
10180
|
runGbpSync(runId, projectId);
|
|
9762
10181
|
},
|
|
10182
|
+
onAdsSyncRequested: (runId, projectId) => {
|
|
10183
|
+
runAdsSync(runId, projectId);
|
|
10184
|
+
},
|
|
9763
10185
|
onDataRefreshRequested: (projectName) => {
|
|
9764
10186
|
void refreshAllIntegrations(aeroClient, projectName);
|
|
9765
10187
|
},
|
|
@@ -9771,8 +10193,8 @@ async function createServer(opts) {
|
|
|
9771
10193
|
});
|
|
9772
10194
|
if (!probed) return;
|
|
9773
10195
|
const alreadySynced = opts.db.select().from(ccReleaseSyncs).where(and13(
|
|
9774
|
-
|
|
9775
|
-
|
|
10196
|
+
eq19(ccReleaseSyncs.release, probed.release),
|
|
10197
|
+
eq19(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)
|
|
9776
10198
|
)).limit(1).get();
|
|
9777
10199
|
if (alreadySynced) {
|
|
9778
10200
|
app.log.info({ projectName, release: probed.release }, "Scheduled backlinks sync: already up to date, skipping");
|
|
@@ -9905,7 +10327,7 @@ async function createServer(opts) {
|
|
|
9905
10327
|
return removed;
|
|
9906
10328
|
}
|
|
9907
10329
|
};
|
|
9908
|
-
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ??
|
|
10330
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto19.randomBytes(32).toString("hex");
|
|
9909
10331
|
const googleConnectionStore = {
|
|
9910
10332
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
9911
10333
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -9951,11 +10373,11 @@ async function createServer(opts) {
|
|
|
9951
10373
|
const apiPrefix = basePath ? `${basePath}api/v1` : "/api/v1";
|
|
9952
10374
|
if (opts.config.apiKey) {
|
|
9953
10375
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
9954
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
10376
|
+
const existing = opts.db.select().from(apiKeys).where(eq19(apiKeys.keyHash, keyHash)).get();
|
|
9955
10377
|
if (!existing) {
|
|
9956
10378
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
9957
10379
|
opts.db.insert(apiKeys).values({
|
|
9958
|
-
id: `key_${
|
|
10380
|
+
id: `key_${crypto19.randomBytes(8).toString("hex")}`,
|
|
9959
10381
|
name: "default",
|
|
9960
10382
|
keyHash,
|
|
9961
10383
|
keyPrefix: prefix,
|
|
@@ -9979,7 +10401,7 @@ async function createServer(opts) {
|
|
|
9979
10401
|
};
|
|
9980
10402
|
const createSession = (apiKeyId) => {
|
|
9981
10403
|
pruneExpiredSessions();
|
|
9982
|
-
const sessionId =
|
|
10404
|
+
const sessionId = crypto19.randomBytes(32).toString("hex");
|
|
9983
10405
|
sessions.set(sessionId, {
|
|
9984
10406
|
apiKeyId,
|
|
9985
10407
|
expiresAt: Date.now() + SESSION_TTL_MS
|
|
@@ -10003,7 +10425,7 @@ async function createServer(opts) {
|
|
|
10003
10425
|
};
|
|
10004
10426
|
const getDefaultApiKey = () => {
|
|
10005
10427
|
if (!opts.config.apiKey) return void 0;
|
|
10006
|
-
return opts.db.select().from(apiKeys).where(
|
|
10428
|
+
return opts.db.select().from(apiKeys).where(eq19(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
10007
10429
|
};
|
|
10008
10430
|
const createPasswordSession = (reply) => {
|
|
10009
10431
|
const key = getDefaultApiKey();
|
|
@@ -10024,7 +10446,7 @@ async function createServer(opts) {
|
|
|
10024
10446
|
if (!header) return false;
|
|
10025
10447
|
const parts = header.split(" ");
|
|
10026
10448
|
if (parts.length !== 2 || parts[0] !== "Bearer") return false;
|
|
10027
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
10449
|
+
const key = opts.db.select().from(apiKeys).where(eq19(apiKeys.keyHash, hashApiKey(parts[1]))).get();
|
|
10028
10450
|
return Boolean(key && !key.revokedAt);
|
|
10029
10451
|
};
|
|
10030
10452
|
app.get(apiPrefix + "/session", async (request, reply) => {
|
|
@@ -10078,12 +10500,12 @@ async function createServer(opts) {
|
|
|
10078
10500
|
return reply.send({ authenticated: true });
|
|
10079
10501
|
}
|
|
10080
10502
|
if (apiKey) {
|
|
10081
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
10503
|
+
const key = opts.db.select().from(apiKeys).where(eq19(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
10082
10504
|
if (!key || key.revokedAt) {
|
|
10083
10505
|
const err2 = authInvalid();
|
|
10084
10506
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
10085
10507
|
}
|
|
10086
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
10508
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq19(apiKeys.id, key.id)).run();
|
|
10087
10509
|
const sessionId = createSession(key.id);
|
|
10088
10510
|
reply.header("set-cookie", serializeSessionCookie({
|
|
10089
10511
|
name: SESSION_COOKIE_NAME,
|
|
@@ -10216,6 +10638,11 @@ async function createServer(opts) {
|
|
|
10216
10638
|
onGbpSyncRequested: (runId, projectId, syncOpts) => {
|
|
10217
10639
|
runGbpSync(runId, projectId, syncOpts);
|
|
10218
10640
|
},
|
|
10641
|
+
adsCredentialStore,
|
|
10642
|
+
verifyAdsAccount,
|
|
10643
|
+
onAdsSyncRequested: (runId, projectId) => {
|
|
10644
|
+
runAdsSync(runId, projectId);
|
|
10645
|
+
},
|
|
10219
10646
|
getBacklinksStatus: () => ({
|
|
10220
10647
|
duckdbInstalled: isDuckdbInstalled(),
|
|
10221
10648
|
duckdbVersion: readInstalledVersion() ?? void 0,
|
|
@@ -10237,7 +10664,7 @@ async function createServer(opts) {
|
|
|
10237
10664
|
deps: {
|
|
10238
10665
|
enqueueAutoExtract: ({ projectId, release: r }) => {
|
|
10239
10666
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
10240
|
-
const runId =
|
|
10667
|
+
const runId = crypto19.randomUUID();
|
|
10241
10668
|
opts.db.insert(runs).values({
|
|
10242
10669
|
id: runId,
|
|
10243
10670
|
projectId,
|
|
@@ -10323,7 +10750,7 @@ async function createServer(opts) {
|
|
|
10323
10750
|
...inspectOpts,
|
|
10324
10751
|
config: opts.config
|
|
10325
10752
|
}).then(() => {
|
|
10326
|
-
const finished = opts.db.select({ status: runs.status }).from(runs).where(
|
|
10753
|
+
const finished = opts.db.select({ status: runs.status }).from(runs).where(eq19(runs.id, runId)).get();
|
|
10327
10754
|
if (finished?.status === RunStatuses.completed || finished?.status === RunStatuses.partial) {
|
|
10328
10755
|
return maybeRefreshGscCoverage(opts.db, opts.config, projectId);
|
|
10329
10756
|
}
|
|
@@ -10411,7 +10838,7 @@ async function createServer(opts) {
|
|
|
10411
10838
|
const targetProjectIds = affectedProjectIds.length > 0 ? affectedProjectIds : [null];
|
|
10412
10839
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
10413
10840
|
opts.db.insert(auditLog).values(targetProjectIds.map((projectId) => ({
|
|
10414
|
-
id:
|
|
10841
|
+
id: crypto19.randomUUID(),
|
|
10415
10842
|
projectId,
|
|
10416
10843
|
actor: "api",
|
|
10417
10844
|
action: existing ? "provider.updated" : "provider.created",
|